src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 127 {
 128    Builder bld(ctx->program, ctx->block);
 129
 130    if (!dst.id())
 131       dst = bld.tmp(src.regClass());
 132
 133    assert(src.size() == dst.size());
 134
 135    if (ctx->stage != fragment_fs) {
 136       if (!dst.id())
 137          return src;
 138
 139       bld.copy(Definition(dst), src);
 140       return dst;
 141    }
 142
 143    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 144    ctx->program->needs_wqm |= program_needs_wqm;
 145    return dst;
 146 }
 147
 148 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 149 {
 150    if (index.regClass() == s1)
 151       return bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), data, index);
 152
 153    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 154
 155    /* Currently not implemented on GFX6-7 */
 156    assert(ctx->options->chip_class >= GFX8);
 157
 158    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 159       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 160    }
 161
 162    /* GFX10, wave64 mode:
 163     * The bpermute instruction is limited to half-wave operation, which means that it can't
 164     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 165     * emulate it here.
 166     */
 167    if (!ctx->has_gfx10_wave64_bpermute) {
 168       ctx->has_gfx10_wave64_bpermute = true;
 169       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 170       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 171    }
 172
 173    Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u));
 174    lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1), lane_id);
 175    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 176    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 177    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 178
 179    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 180                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 181 }
 182
 183 Temp as_vgpr(isel_context *ctx, Temp val)
 184 {
 185    if (val.type() == RegType::sgpr) {
 186       Builder bld(ctx->program, ctx->block);
 187       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 188    }
 189    assert(val.type() == RegType::vgpr);
 190    return val;
 191 }
 192
 193 //assumes a != 0xffffffff
 194 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 195 {
 196    assert(b != 0);
 197    Builder bld(ctx->program, ctx->block);
 198
 199    if (util_is_power_of_two_or_zero(b)) {
 200       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 201       return;
 202    }
 203
 204    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 205
 206    assert(info.multiplier <= 0xffffffff);
 207
 208    bool pre_shift = info.pre_shift != 0;
 209    bool increment = info.increment != 0;
 210    bool multiply = true;
 211    bool post_shift = info.post_shift != 0;
 212
 213    if (!pre_shift && !increment && !multiply && !post_shift) {
 214       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 215       return;
 216    }
 217
 218    Temp pre_shift_dst = a;
 219    if (pre_shift) {
 220       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 221       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 222    }
 223
 224    Temp increment_dst = pre_shift_dst;
 225    if (increment) {
 226       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 227       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 228    }
 229
 230    Temp multiply_dst = increment_dst;
 231    if (multiply) {
 232       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 233       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 234                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 235    }
 236
 237    if (post_shift) {
 238       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 239    }
 240 }
 241
 242 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 243 {
 244    Builder bld(ctx->program, ctx->block);
 245    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 246 }
 247
 248
 249 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 250 {
 251    /* no need to extract the whole vector */
 252    if (src.regClass() == dst_rc) {
 253       assert(idx == 0);
 254       return src;
 255    }
 256    assert(src.size() > idx);
 257    Builder bld(ctx->program, ctx->block);
 258    auto it = ctx->allocated_vec.find(src.id());
 259    /* the size check needs to be early because elements other than 0 may be garbage */
 260    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 261       if (it->second[idx].regClass() == dst_rc) {
 262          return it->second[idx];
 263       } else {
 264          assert(dst_rc.size() == it->second[idx].regClass().size());
 265          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 266          return bld.copy(bld.def(dst_rc), it->second[idx]);
 267       }
 268    }
 269
 270    if (src.size() == dst_rc.size()) {
 271       assert(idx == 0);
 272       return bld.copy(bld.def(dst_rc), src);
 273    } else {
 274       Temp dst = bld.tmp(dst_rc);
 275       emit_extract_vector(ctx, src, idx, dst);
 276       return dst;
 277    }
 278 }
 279
 280 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 281 {
 282    if (num_components == 1)
 283       return;
 284    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 285       return;
 286    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 287    split->operands[0] = Operand(vec_src);
 288    std::array<Temp,4> elems;
 289    for (unsigned i = 0; i < num_components; i++) {
 290       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 291       split->definitions[i] = Definition(elems[i]);
 292    }
 293    ctx->block->instructions.emplace_back(std::move(split));
 294    ctx->allocated_vec.emplace(vec_src.id(), elems);
 295 }
 296
 297 /* This vector expansion uses a mask to determine which elements in the new vector
 298  * come from the original vector. The other elements are undefined. */
 299 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 300 {
 301    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 302
 303    if (vec_src == dst)
 304       return;
 305
 306    Builder bld(ctx->program, ctx->block);
 307    if (num_components == 1) {
 308       if (dst.type() == RegType::sgpr)
 309          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 310       else
 311          bld.copy(Definition(dst), vec_src);
 312       return;
 313    }
 314
 315    unsigned component_size = dst.size() / num_components;
 316    std::array<Temp,4> elems;
 317
 318    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 319    vec->definitions[0] = Definition(dst);
 320    unsigned k = 0;
 321    for (unsigned i = 0; i < num_components; i++) {
 322       if (mask & (1 << i)) {
 323          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 324          if (dst.type() == RegType::sgpr)
 325             src = bld.as_uniform(src);
 326          vec->operands[i] = Operand(src);
 327       } else {
 328          vec->operands[i] = Operand(0u);
 329       }
 330       elems[i] = vec->operands[i].getTemp();
 331    }
 332    ctx->block->instructions.emplace_back(std::move(vec));
 333    ctx->allocated_vec.emplace(dst.id(), elems);
 334 }
 335
 336 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 337 {
 338    Builder bld(ctx->program, ctx->block);
 339    if (!dst.id())
 340       dst = bld.tmp(s2);
 341
 342    assert(val.regClass() == s1);
 343    assert(dst.regClass() == s2);
 344
 345    return bld.sop2(aco_opcode::s_cselect_b64, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 346 }
 347
 348 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 349 {
 350    Builder bld(ctx->program, ctx->block);
 351    if (!dst.id())
 352       dst = bld.tmp(s1);
 353
 354    assert(val.regClass() == s2);
 355    assert(dst.regClass() == s1);
 356
 357    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 358    Temp tmp = bld.tmp(s1);
 359    bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(Definition(tmp)), val, Operand(exec, s2));
 360    return emit_wqm(ctx, tmp, dst);
 361 }
 362
 363 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 364 {
 365    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 366       return get_ssa_temp(ctx, src.src.ssa);
 367
 368    if (src.src.ssa->num_components == size) {
 369       bool identity_swizzle = true;
 370       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 371          if (src.swizzle[i] != i)
 372             identity_swizzle = false;
 373       }
 374       if (identity_swizzle)
 375          return get_ssa_temp(ctx, src.src.ssa);
 376    }
 377
 378    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 379    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 380    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 381    assert(vec.size() % elem_size == 0);
 382
 383    RegClass elem_rc = RegClass(vec.type(), elem_size);
 384    if (size == 1) {
 385       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 386    } else {
 387       assert(size <= 4);
 388       std::array<Temp,4> elems;
 389       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 390       for (unsigned i = 0; i < size; ++i) {
 391          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 392          vec_instr->operands[i] = Operand{elems[i]};
 393       }
 394       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 395       vec_instr->definitions[0] = Definition(dst);
 396       ctx->block->instructions.emplace_back(std::move(vec_instr));
 397       ctx->allocated_vec.emplace(dst.id(), elems);
 398       return dst;
 399    }
 400 }
 401
 402 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 403 {
 404    if (ptr.size() == 2)
 405       return ptr;
 406    Builder bld(ctx->program, ctx->block);
 407    if (ptr.type() == RegType::vgpr)
 408       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 409    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 410                      ptr, Operand((unsigned)ctx->options->address32_hi));
 411 }
 412
 413 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 414 {
 415    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 416    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 417    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 418    sop2->definitions[0] = Definition(dst);
 419    if (writes_scc)
 420       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 421    ctx->block->instructions.emplace_back(std::move(sop2));
 422 }
 423
 424 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 425 {
 426    Builder bld(ctx->program, ctx->block);
 427    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 428    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 429    if (src1.type() == RegType::sgpr) {
 430       if (commutative && src0.type() == RegType::vgpr) {
 431          Temp t = src0;
 432          src0 = src1;
 433          src1 = t;
 434       } else if (src0.type() == RegType::vgpr &&
 435                  op != aco_opcode::v_madmk_f32 &&
 436                  op != aco_opcode::v_madak_f32 &&
 437                  op != aco_opcode::v_madmk_f16 &&
 438                  op != aco_opcode::v_madak_f16) {
 439          /* If the instruction is not commutative, we emit a VOP3A instruction */
 440          bld.vop2_e64(op, Definition(dst), src0, src1);
 441          return;
 442       } else {
 443          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 444       }
 445    }
 446    bld.vop2(op, Definition(dst), src0, src1);
 447 }
 448
 449 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 450 {
 451    Temp src0 = get_alu_src(ctx, instr->src[0]);
 452    Temp src1 = get_alu_src(ctx, instr->src[1]);
 453    Temp src2 = get_alu_src(ctx, instr->src[2]);
 454
 455    /* ensure that the instruction has at most 1 sgpr operand
 456     * The optimizer will inline constants for us */
 457    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 458       src0 = as_vgpr(ctx, src0);
 459    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 460       src1 = as_vgpr(ctx, src1);
 461    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 462       src2 = as_vgpr(ctx, src2);
 463
 464    Builder bld(ctx->program, ctx->block);
 465    bld.vop3(op, Definition(dst), src0, src1, src2);
 466 }
 467
 468 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 469 {
 470    Builder bld(ctx->program, ctx->block);
 471    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 472 }
 473
 474 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 475 {
 476    Temp src0 = get_alu_src(ctx, instr->src[0]);
 477    Temp src1 = get_alu_src(ctx, instr->src[1]);
 478    aco_ptr<Instruction> vopc;
 479    if (src1.type() == RegType::sgpr) {
 480       if (src0.type() == RegType::vgpr) {
 481          /* to swap the operands, we might also have to change the opcode */
 482          switch (op) {
 483             case aco_opcode::v_cmp_lt_f32:
 484                op = aco_opcode::v_cmp_gt_f32;
 485                break;
 486             case aco_opcode::v_cmp_ge_f32:
 487                op = aco_opcode::v_cmp_le_f32;
 488                break;
 489             case aco_opcode::v_cmp_lt_i32:
 490                op = aco_opcode::v_cmp_gt_i32;
 491                break;
 492             case aco_opcode::v_cmp_ge_i32:
 493                op = aco_opcode::v_cmp_le_i32;
 494                break;
 495             case aco_opcode::v_cmp_lt_u32:
 496                op = aco_opcode::v_cmp_gt_u32;
 497                break;
 498             case aco_opcode::v_cmp_ge_u32:
 499                op = aco_opcode::v_cmp_le_u32;
 500                break;
 501             case aco_opcode::v_cmp_lt_f64:
 502                op = aco_opcode::v_cmp_gt_f64;
 503                break;
 504             case aco_opcode::v_cmp_ge_f64:
 505                op = aco_opcode::v_cmp_le_f64;
 506                break;
 507             case aco_opcode::v_cmp_lt_i64:
 508                op = aco_opcode::v_cmp_gt_i64;
 509                break;
 510             case aco_opcode::v_cmp_ge_i64:
 511                op = aco_opcode::v_cmp_le_i64;
 512                break;
 513             case aco_opcode::v_cmp_lt_u64:
 514                op = aco_opcode::v_cmp_gt_u64;
 515                break;
 516             case aco_opcode::v_cmp_ge_u64:
 517                op = aco_opcode::v_cmp_le_u64;
 518                break;
 519             default: /* eq and ne are commutative */
 520                break;
 521          }
 522          Temp t = src0;
 523          src0 = src1;
 524          src1 = t;
 525       } else {
 526          src1 = as_vgpr(ctx, src1);
 527       }
 528    }
 529
 530    Builder bld(ctx->program, ctx->block);
 531    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 532 }
 533
 534 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 535 {
 536    Temp src0 = get_alu_src(ctx, instr->src[0]);
 537    Temp src1 = get_alu_src(ctx, instr->src[1]);
 538
 539    assert(dst.regClass() == s2);
 540    assert(src0.type() == RegType::sgpr);
 541    assert(src1.type() == RegType::sgpr);
 542
 543    Builder bld(ctx->program, ctx->block);
 544    /* Emit the SALU comparison instruction */
 545    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 546    /* Turn the result into a per-lane bool */
 547    bool_to_vector_condition(ctx, cmp, dst);
 548 }
 549
 550 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 551                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::last_opcode, aco_opcode s64_op = aco_opcode::last_opcode)
 552 {
 553    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 554    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 555    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 556    bool use_valu = s_op == aco_opcode::last_opcode ||
 557                    divergent_vals ||
 558                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 559                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 560    aco_opcode op = use_valu ? v_op : s_op;
 561    assert(op != aco_opcode::last_opcode);
 562
 563    if (use_valu)
 564       emit_vopc_instruction(ctx, instr, op, dst);
 565    else
 566       emit_sopc_instruction(ctx, instr, op, dst);
 567 }
 568
 569 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 570 {
 571    Builder bld(ctx->program, ctx->block);
 572    Temp src0 = get_alu_src(ctx, instr->src[0]);
 573    Temp src1 = get_alu_src(ctx, instr->src[1]);
 574
 575    assert(dst.regClass() == s2);
 576    assert(src0.regClass() == s2);
 577    assert(src1.regClass() == s2);
 578
 579    bld.sop2(op64, Definition(dst), bld.def(s1, scc), src0, src1);
 580 }
 581
 582 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 583 {
 584    Builder bld(ctx->program, ctx->block);
 585    Temp cond = get_alu_src(ctx, instr->src[0]);
 586    Temp then = get_alu_src(ctx, instr->src[1]);
 587    Temp els = get_alu_src(ctx, instr->src[2]);
 588
 589    assert(cond.regClass() == s2);
 590
 591    if (dst.type() == RegType::vgpr) {
 592       aco_ptr<Instruction> bcsel;
 593       if (dst.size() == 1) {
 594          then = as_vgpr(ctx, then);
 595          els = as_vgpr(ctx, els);
 596
 597          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 598       } else if (dst.size() == 2) {
 599          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 600          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 601          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 602          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 603
 604          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 605          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 606
 607          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 608       } else {
 609          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 610          nir_print_instr(&instr->instr, stderr);
 611          fprintf(stderr, "\n");
 612       }
 613       return;
 614    }
 615
 616    if (instr->dest.dest.ssa.bit_size == 1) {
 617       assert(dst.regClass() == s2);
 618       assert(then.regClass() == s2);
 619       assert(els.regClass() == s2);
 620    }
 621
 622    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 623       if (dst.regClass() == s1 || dst.regClass() == s2) {
 624          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 625          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 626          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 627       } else {
 628          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 629          nir_print_instr(&instr->instr, stderr);
 630          fprintf(stderr, "\n");
 631       }
 632       return;
 633    }
 634
 635    /* divergent boolean bcsel
 636     * this implements bcsel on bools: dst = s0 ? s1 : s2
 637     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 638    assert(instr->dest.dest.ssa.bit_size == 1);
 639
 640    if (cond.id() != then.id())
 641       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 642
 643    if (cond.id() == els.id())
 644       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 645    else
 646       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 647                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 648 }
 649
 650 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 651 {
 652    if (!instr->dest.dest.is_ssa) {
 653       fprintf(stderr, "nir alu dst not in ssa: ");
 654       nir_print_instr(&instr->instr, stderr);
 655       fprintf(stderr, "\n");
 656       abort();
 657    }
 658    Builder bld(ctx->program, ctx->block);
 659    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 660    switch(instr->op) {
 661    case nir_op_vec2:
 662    case nir_op_vec3:
 663    case nir_op_vec4: {
 664       std::array<Temp,4> elems;
 665       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 666       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 667          elems[i] = get_alu_src(ctx, instr->src[i]);
 668          vec->operands[i] = Operand{elems[i]};
 669       }
 670       vec->definitions[0] = Definition(dst);
 671       ctx->block->instructions.emplace_back(std::move(vec));
 672       ctx->allocated_vec.emplace(dst.id(), elems);
 673       break;
 674    }
 675    case nir_op_mov: {
 676       Temp src = get_alu_src(ctx, instr->src[0]);
 677       aco_ptr<Instruction> mov;
 678       if (dst.type() == RegType::sgpr) {
 679          if (src.type() == RegType::vgpr)
 680             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 681          else if (src.regClass() == s1)
 682             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 683          else if (src.regClass() == s2)
 684             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 685          else
 686             unreachable("wrong src register class for nir_op_imov");
 687       } else if (dst.regClass() == v1) {
 688          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 689       } else if (dst.regClass() == v2) {
 690          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 691       } else {
 692          nir_print_instr(&instr->instr, stderr);
 693          unreachable("Should have been lowered to scalar.");
 694       }
 695       break;
 696    }
 697    case nir_op_inot: {
 698       Temp src = get_alu_src(ctx, instr->src[0]);
 699       if (instr->dest.dest.ssa.bit_size == 1) {
 700          assert(src.regClass() == s2);
 701          assert(dst.regClass() == s2);
 702          bld.sop2(aco_opcode::s_andn2_b64, Definition(dst), bld.def(s1, scc), Operand(exec, s2), src);
 703       } else if (dst.regClass() == v1) {
 704          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 705       } else if (dst.type() == RegType::sgpr) {
 706          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 707          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 708       } else {
 709          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 710          nir_print_instr(&instr->instr, stderr);
 711          fprintf(stderr, "\n");
 712       }
 713       break;
 714    }
 715    case nir_op_ineg: {
 716       Temp src = get_alu_src(ctx, instr->src[0]);
 717       if (dst.regClass() == v1) {
 718          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 719       } else if (dst.regClass() == s1) {
 720          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 721       } else if (dst.size() == 2) {
 722          Temp src0 = bld.tmp(dst.type(), 1);
 723          Temp src1 = bld.tmp(dst.type(), 1);
 724          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 725
 726          if (dst.regClass() == s2) {
 727             Temp carry = bld.tmp(s1);
 728             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 729             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 730             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 731          } else {
 732             Temp lower = bld.tmp(v1);
 733             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 734             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 735             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 736          }
 737       } else {
 738          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 739          nir_print_instr(&instr->instr, stderr);
 740          fprintf(stderr, "\n");
 741       }
 742       break;
 743    }
 744    case nir_op_iabs: {
 745       if (dst.regClass() == s1) {
 746          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 747       } else if (dst.regClass() == v1) {
 748          Temp src = get_alu_src(ctx, instr->src[0]);
 749          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 750       } else {
 751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 752          nir_print_instr(&instr->instr, stderr);
 753          fprintf(stderr, "\n");
 754       }
 755       break;
 756    }
 757    case nir_op_isign: {
 758       Temp src = get_alu_src(ctx, instr->src[0]);
 759       if (dst.regClass() == s1) {
 760          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 761          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 762          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 763       } else if (dst.regClass() == s2) {
 764          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 765          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 766          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 767       } else if (dst.regClass() == v1) {
 768          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 769          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 770          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 771       } else if (dst.regClass() == v2) {
 772          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 773          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 774          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 775          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 776          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 777          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 778       } else {
 779          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 780          nir_print_instr(&instr->instr, stderr);
 781          fprintf(stderr, "\n");
 782       }
 783       break;
 784    }
 785    case nir_op_imax: {
 786       if (dst.regClass() == v1) {
 787          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 788       } else if (dst.regClass() == s1) {
 789          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 790       } else {
 791          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 792          nir_print_instr(&instr->instr, stderr);
 793          fprintf(stderr, "\n");
 794       }
 795       break;
 796    }
 797    case nir_op_umax: {
 798       if (dst.regClass() == v1) {
 799          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 800       } else if (dst.regClass() == s1) {
 801          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 802       } else {
 803          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 804          nir_print_instr(&instr->instr, stderr);
 805          fprintf(stderr, "\n");
 806       }
 807       break;
 808    }
 809    case nir_op_imin: {
 810       if (dst.regClass() == v1) {
 811          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 812       } else if (dst.regClass() == s1) {
 813          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 814       } else {
 815          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 816          nir_print_instr(&instr->instr, stderr);
 817          fprintf(stderr, "\n");
 818       }
 819       break;
 820    }
 821    case nir_op_umin: {
 822       if (dst.regClass() == v1) {
 823          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 824       } else if (dst.regClass() == s1) {
 825          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 826       } else {
 827          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 828          nir_print_instr(&instr->instr, stderr);
 829          fprintf(stderr, "\n");
 830       }
 831       break;
 832    }
 833    case nir_op_ior: {
 834       if (instr->dest.dest.ssa.bit_size == 1) {
 835          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 836       } else if (dst.regClass() == v1) {
 837          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 838       } else if (dst.regClass() == s1) {
 839          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 840       } else if (dst.regClass() == s2) {
 841          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 842       } else {
 843          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 844          nir_print_instr(&instr->instr, stderr);
 845          fprintf(stderr, "\n");
 846       }
 847       break;
 848    }
 849    case nir_op_iand: {
 850       if (instr->dest.dest.ssa.bit_size == 1) {
 851          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 852       } else if (dst.regClass() == v1) {
 853          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 854       } else if (dst.regClass() == s1) {
 855          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 856       } else if (dst.regClass() == s2) {
 857          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 858       } else {
 859          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 860          nir_print_instr(&instr->instr, stderr);
 861          fprintf(stderr, "\n");
 862       }
 863       break;
 864    }
 865    case nir_op_ixor: {
 866       if (instr->dest.dest.ssa.bit_size == 1) {
 867          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 868       } else if (dst.regClass() == v1) {
 869          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 870       } else if (dst.regClass() == s1) {
 871          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 872       } else if (dst.regClass() == s2) {
 873          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 874       } else {
 875          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 876          nir_print_instr(&instr->instr, stderr);
 877          fprintf(stderr, "\n");
 878       }
 879       break;
 880    }
 881    case nir_op_ushr: {
 882       if (dst.regClass() == v1) {
 883          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 884       } else if (dst.regClass() == v2) {
 885          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 886                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 887       } else if (dst.regClass() == s2) {
 888          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 889       } else if (dst.regClass() == s1) {
 890          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 891       } else {
 892          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 893          nir_print_instr(&instr->instr, stderr);
 894          fprintf(stderr, "\n");
 895       }
 896       break;
 897    }
 898    case nir_op_ishl: {
 899       if (dst.regClass() == v1) {
 900          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 901       } else if (dst.regClass() == v2) {
 902          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 903                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 904       } else if (dst.regClass() == s1) {
 905          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 906       } else if (dst.regClass() == s2) {
 907          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 908       } else {
 909          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 910          nir_print_instr(&instr->instr, stderr);
 911          fprintf(stderr, "\n");
 912       }
 913       break;
 914    }
 915    case nir_op_ishr: {
 916       if (dst.regClass() == v1) {
 917          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 918       } else if (dst.regClass() == v2) {
 919          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 920                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 921       } else if (dst.regClass() == s1) {
 922          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 923       } else if (dst.regClass() == s2) {
 924          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 925       } else {
 926          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 927          nir_print_instr(&instr->instr, stderr);
 928          fprintf(stderr, "\n");
 929       }
 930       break;
 931    }
 932    case nir_op_find_lsb: {
 933       Temp src = get_alu_src(ctx, instr->src[0]);
 934       if (src.regClass() == s1) {
 935          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 936       } else if (src.regClass() == v1) {
 937          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 938       } else if (src.regClass() == s2) {
 939          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 940       } else {
 941          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 942          nir_print_instr(&instr->instr, stderr);
 943          fprintf(stderr, "\n");
 944       }
 945       break;
 946    }
 947    case nir_op_ufind_msb:
 948    case nir_op_ifind_msb: {
 949       Temp src = get_alu_src(ctx, instr->src[0]);
 950       if (src.regClass() == s1 || src.regClass() == s2) {
 951          aco_opcode op = src.regClass() == s2 ?
 952                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 953                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 954          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 955
 956          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 957                                         Operand(src.size() * 32u - 1u), msb_rev);
 958          Temp msb = sub.def(0).getTemp();
 959          Temp carry = sub.def(1).getTemp();
 960
 961          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 962       } else if (src.regClass() == v1) {
 963          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 964          Temp msb_rev = bld.tmp(v1);
 965          emit_vop1_instruction(ctx, instr, op, msb_rev);
 966          Temp msb = bld.tmp(v1);
 967          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 968          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 969       } else {
 970          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 971          nir_print_instr(&instr->instr, stderr);
 972          fprintf(stderr, "\n");
 973       }
 974       break;
 975    }
 976    case nir_op_bitfield_reverse: {
 977       if (dst.regClass() == s1) {
 978          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 979       } else if (dst.regClass() == v1) {
 980          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 981       } else {
 982          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 983          nir_print_instr(&instr->instr, stderr);
 984          fprintf(stderr, "\n");
 985       }
 986       break;
 987    }
 988    case nir_op_iadd: {
 989       if (dst.regClass() == s1) {
 990          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
 991          break;
 992       }
 993
 994       Temp src0 = get_alu_src(ctx, instr->src[0]);
 995       Temp src1 = get_alu_src(ctx, instr->src[1]);
 996       if (dst.regClass() == v1) {
 997          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
 998          break;
 999       }
1000
1001       assert(src0.size() == 2 && src1.size() == 2);
1002       Temp src00 = bld.tmp(src0.type(), 1);
1003       Temp src01 = bld.tmp(dst.type(), 1);
1004       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1005       Temp src10 = bld.tmp(src1.type(), 1);
1006       Temp src11 = bld.tmp(dst.type(), 1);
1007       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1008
1009       if (dst.regClass() == s2) {
1010          Temp carry = bld.tmp(s1);
1011          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1012          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1013          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1014       } else if (dst.regClass() == v2) {
1015          Temp dst0 = bld.tmp(v1);
1016          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1017          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1018          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1019       } else {
1020          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1021          nir_print_instr(&instr->instr, stderr);
1022          fprintf(stderr, "\n");
1023       }
1024       break;
1025    }
1026    case nir_op_uadd_sat: {
1027       Temp src0 = get_alu_src(ctx, instr->src[0]);
1028       Temp src1 = get_alu_src(ctx, instr->src[1]);
1029       if (dst.regClass() == s1) {
1030          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1031          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1032                   src0, src1);
1033          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1034       } else if (dst.regClass() == v1) {
1035          if (ctx->options->chip_class >= GFX9) {
1036             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1037             add->operands[0] = Operand(src0);
1038             add->operands[1] = Operand(src1);
1039             add->definitions[0] = Definition(dst);
1040             add->clamp = 1;
1041             ctx->block->instructions.emplace_back(std::move(add));
1042          } else {
1043             if (src1.regClass() != v1)
1044                std::swap(src0, src1);
1045             assert(src1.regClass() == v1);
1046             Temp tmp = bld.tmp(v1);
1047             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1048             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1049          }
1050       } else {
1051          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1052          nir_print_instr(&instr->instr, stderr);
1053          fprintf(stderr, "\n");
1054       }
1055       break;
1056    }
1057    case nir_op_uadd_carry: {
1058       Temp src0 = get_alu_src(ctx, instr->src[0]);
1059       Temp src1 = get_alu_src(ctx, instr->src[1]);
1060       if (dst.regClass() == s1) {
1061          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1062          break;
1063       }
1064       if (dst.regClass() == v1) {
1065          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1066          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1067          break;
1068       }
1069
1070       Temp src00 = bld.tmp(src0.type(), 1);
1071       Temp src01 = bld.tmp(dst.type(), 1);
1072       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1073       Temp src10 = bld.tmp(src1.type(), 1);
1074       Temp src11 = bld.tmp(dst.type(), 1);
1075       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1076       if (dst.regClass() == s2) {
1077          Temp carry = bld.tmp(s1);
1078          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1079          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1080          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1081       } else if (dst.regClass() == v2) {
1082          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1083          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1084          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1085          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1086       } else {
1087          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1088          nir_print_instr(&instr->instr, stderr);
1089          fprintf(stderr, "\n");
1090       }
1091       break;
1092    }
1093    case nir_op_isub: {
1094       if (dst.regClass() == s1) {
1095          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1096          break;
1097       }
1098
1099       Temp src0 = get_alu_src(ctx, instr->src[0]);
1100       Temp src1 = get_alu_src(ctx, instr->src[1]);
1101       if (dst.regClass() == v1) {
1102          bld.vsub32(Definition(dst), src0, src1);
1103          break;
1104       }
1105
1106       Temp src00 = bld.tmp(src0.type(), 1);
1107       Temp src01 = bld.tmp(dst.type(), 1);
1108       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1109       Temp src10 = bld.tmp(src1.type(), 1);
1110       Temp src11 = bld.tmp(dst.type(), 1);
1111       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1112       if (dst.regClass() == s2) {
1113          Temp carry = bld.tmp(s1);
1114          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1115          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1116          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1117       } else if (dst.regClass() == v2) {
1118          Temp lower = bld.tmp(v1);
1119          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1120          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1121          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1122       } else {
1123          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1124          nir_print_instr(&instr->instr, stderr);
1125          fprintf(stderr, "\n");
1126       }
1127       break;
1128    }
1129    case nir_op_usub_borrow: {
1130       Temp src0 = get_alu_src(ctx, instr->src[0]);
1131       Temp src1 = get_alu_src(ctx, instr->src[1]);
1132       if (dst.regClass() == s1) {
1133          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1134          break;
1135       } else if (dst.regClass() == v1) {
1136          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1137          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1138          break;
1139       }
1140
1141       Temp src00 = bld.tmp(src0.type(), 1);
1142       Temp src01 = bld.tmp(dst.type(), 1);
1143       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1144       Temp src10 = bld.tmp(src1.type(), 1);
1145       Temp src11 = bld.tmp(dst.type(), 1);
1146       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1147       if (dst.regClass() == s2) {
1148          Temp borrow = bld.tmp(s1);
1149          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1150          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1151          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1152       } else if (dst.regClass() == v2) {
1153          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1154          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1155          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1156          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1157       } else {
1158          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1159          nir_print_instr(&instr->instr, stderr);
1160          fprintf(stderr, "\n");
1161       }
1162       break;
1163    }
1164    case nir_op_imul: {
1165       if (dst.regClass() == v1) {
1166          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1167                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1168       } else if (dst.regClass() == s1) {
1169          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1170       } else {
1171          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1172          nir_print_instr(&instr->instr, stderr);
1173          fprintf(stderr, "\n");
1174       }
1175       break;
1176    }
1177    case nir_op_umul_high: {
1178       if (dst.regClass() == v1) {
1179          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1180       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1181          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1182       } else if (dst.regClass() == s1) {
1183          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1184                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1185          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1186       } else {
1187          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1188          nir_print_instr(&instr->instr, stderr);
1189          fprintf(stderr, "\n");
1190       }
1191       break;
1192    }
1193    case nir_op_imul_high: {
1194       if (dst.regClass() == v1) {
1195          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1196       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1197          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1198       } else if (dst.regClass() == s1) {
1199          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1200                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1201          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1202       } else {
1203          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1204          nir_print_instr(&instr->instr, stderr);
1205          fprintf(stderr, "\n");
1206       }
1207       break;
1208    }
1209    case nir_op_fmul: {
1210       if (dst.size() == 1) {
1211          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1212       } else if (dst.size() == 2) {
1213          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1214                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1215       } else {
1216          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1217          nir_print_instr(&instr->instr, stderr);
1218          fprintf(stderr, "\n");
1219       }
1220       break;
1221    }
1222    case nir_op_fadd: {
1223       if (dst.size() == 1) {
1224          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1225       } else if (dst.size() == 2) {
1226          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1227                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1228       } else {
1229          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1230          nir_print_instr(&instr->instr, stderr);
1231          fprintf(stderr, "\n");
1232       }
1233       break;
1234    }
1235    case nir_op_fsub: {
1236       Temp src0 = get_alu_src(ctx, instr->src[0]);
1237       Temp src1 = get_alu_src(ctx, instr->src[1]);
1238       if (dst.size() == 1) {
1239          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1240             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1241          else
1242             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1243       } else if (dst.size() == 2) {
1244          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1245                                      get_alu_src(ctx, instr->src[0]),
1246                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1247          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1248          sub->neg[1] = true;
1249       } else {
1250          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1251          nir_print_instr(&instr->instr, stderr);
1252          fprintf(stderr, "\n");
1253       }
1254       break;
1255    }
1256    case nir_op_fmax: {
1257       if (dst.size() == 1) {
1258          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1259       } else if (dst.size() == 2) {
1260          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1261                   get_alu_src(ctx, instr->src[0]),
1262                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1263       } else {
1264          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1265          nir_print_instr(&instr->instr, stderr);
1266          fprintf(stderr, "\n");
1267       }
1268       break;
1269    }
1270    case nir_op_fmin: {
1271       if (dst.size() == 1) {
1272          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1273       } else if (dst.size() == 2) {
1274          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1275                   get_alu_src(ctx, instr->src[0]),
1276                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1277       } else {
1278          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1279          nir_print_instr(&instr->instr, stderr);
1280          fprintf(stderr, "\n");
1281       }
1282       break;
1283    }
1284    case nir_op_fmax3: {
1285       if (dst.size() == 1) {
1286          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1287       } else {
1288          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1289          nir_print_instr(&instr->instr, stderr);
1290          fprintf(stderr, "\n");
1291       }
1292       break;
1293    }
1294    case nir_op_fmin3: {
1295       if (dst.size() == 1) {
1296          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1297       } else {
1298          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1299          nir_print_instr(&instr->instr, stderr);
1300          fprintf(stderr, "\n");
1301       }
1302       break;
1303    }
1304    case nir_op_fmed3: {
1305       if (dst.size() == 1) {
1306          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1307       } else {
1308          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1309          nir_print_instr(&instr->instr, stderr);
1310          fprintf(stderr, "\n");
1311       }
1312       break;
1313    }
1314    case nir_op_umax3: {
1315       if (dst.size() == 1) {
1316          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1317       } else {
1318          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1319          nir_print_instr(&instr->instr, stderr);
1320          fprintf(stderr, "\n");
1321       }
1322       break;
1323    }
1324    case nir_op_umin3: {
1325       if (dst.size() == 1) {
1326          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1327       } else {
1328          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1329          nir_print_instr(&instr->instr, stderr);
1330          fprintf(stderr, "\n");
1331       }
1332       break;
1333    }
1334    case nir_op_umed3: {
1335       if (dst.size() == 1) {
1336          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1337       } else {
1338          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1339          nir_print_instr(&instr->instr, stderr);
1340          fprintf(stderr, "\n");
1341       }
1342       break;
1343    }
1344    case nir_op_imax3: {
1345       if (dst.size() == 1) {
1346          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1347       } else {
1348          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1349          nir_print_instr(&instr->instr, stderr);
1350          fprintf(stderr, "\n");
1351       }
1352       break;
1353    }
1354    case nir_op_imin3: {
1355       if (dst.size() == 1) {
1356          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1357       } else {
1358          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1359          nir_print_instr(&instr->instr, stderr);
1360          fprintf(stderr, "\n");
1361       }
1362       break;
1363    }
1364    case nir_op_imed3: {
1365       if (dst.size() == 1) {
1366          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_cube_face_coord: {
1375       Temp in = get_alu_src(ctx, instr->src[0], 3);
1376       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1377                       emit_extract_vector(ctx, in, 1, v1),
1378                       emit_extract_vector(ctx, in, 2, v1) };
1379       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1380       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1381       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1382       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1383       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1384       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1385       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1386       break;
1387    }
1388    case nir_op_cube_face_index: {
1389       Temp in = get_alu_src(ctx, instr->src[0], 3);
1390       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1391                       emit_extract_vector(ctx, in, 1, v1),
1392                       emit_extract_vector(ctx, in, 2, v1) };
1393       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1394       break;
1395    }
1396    case nir_op_bcsel: {
1397       emit_bcsel(ctx, instr, dst);
1398       break;
1399    }
1400    case nir_op_frsq: {
1401       if (dst.size() == 1) {
1402          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1403       } else if (dst.size() == 2) {
1404          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1405       } else {
1406          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1407          nir_print_instr(&instr->instr, stderr);
1408          fprintf(stderr, "\n");
1409       }
1410       break;
1411    }
1412    case nir_op_fneg: {
1413       Temp src = get_alu_src(ctx, instr->src[0]);
1414       if (dst.size() == 1) {
1415          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1416       } else if (dst.size() == 2) {
1417          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1418          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1419          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1420          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1421       } else {
1422          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1423          nir_print_instr(&instr->instr, stderr);
1424          fprintf(stderr, "\n");
1425       }
1426       break;
1427    }
1428    case nir_op_fabs: {
1429       Temp src = get_alu_src(ctx, instr->src[0]);
1430       if (dst.size() == 1) {
1431          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1432       } else if (dst.size() == 2) {
1433          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1434          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1435          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1436          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1437       } else {
1438          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1439          nir_print_instr(&instr->instr, stderr);
1440          fprintf(stderr, "\n");
1441       }
1442       break;
1443    }
1444    case nir_op_fsat: {
1445       Temp src = get_alu_src(ctx, instr->src[0]);
1446       if (dst.size() == 1) {
1447          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1448       } else if (dst.size() == 2) {
1449          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1450          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1451          vop3->clamp = true;
1452       } else {
1453          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1454          nir_print_instr(&instr->instr, stderr);
1455          fprintf(stderr, "\n");
1456       }
1457       break;
1458    }
1459    case nir_op_flog2: {
1460       if (dst.size() == 1) {
1461          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1462       } else {
1463          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1464          nir_print_instr(&instr->instr, stderr);
1465          fprintf(stderr, "\n");
1466       }
1467       break;
1468    }
1469    case nir_op_frcp: {
1470       if (dst.size() == 1) {
1471          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1472       } else if (dst.size() == 2) {
1473          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_fexp2: {
1482       if (dst.size() == 1) {
1483          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fsqrt: {
1492       if (dst.size() == 1) {
1493          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1494       } else if (dst.size() == 2) {
1495          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1496       } else {
1497          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1498          nir_print_instr(&instr->instr, stderr);
1499          fprintf(stderr, "\n");
1500       }
1501       break;
1502    }
1503    case nir_op_ffract: {
1504       if (dst.size() == 1) {
1505          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1506       } else if (dst.size() == 2) {
1507          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1508       } else {
1509          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1510          nir_print_instr(&instr->instr, stderr);
1511          fprintf(stderr, "\n");
1512       }
1513       break;
1514    }
1515    case nir_op_ffloor: {
1516       if (dst.size() == 1) {
1517          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1518       } else if (dst.size() == 2) {
1519          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1520       } else {
1521          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1522          nir_print_instr(&instr->instr, stderr);
1523          fprintf(stderr, "\n");
1524       }
1525       break;
1526    }
1527    case nir_op_fceil: {
1528       if (dst.size() == 1) {
1529          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1530       } else if (dst.size() == 2) {
1531          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_ftrunc: {
1540       if (dst.size() == 1) {
1541          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1542       } else if (dst.size() == 2) {
1543          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_fround_even: {
1552       if (dst.size() == 1) {
1553          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1554       } else if (dst.size() == 2) {
1555          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1556       } else {
1557          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1558          nir_print_instr(&instr->instr, stderr);
1559          fprintf(stderr, "\n");
1560       }
1561       break;
1562    }
1563    case nir_op_fsin:
1564    case nir_op_fcos: {
1565       Temp src = get_alu_src(ctx, instr->src[0]);
1566       aco_ptr<Instruction> norm;
1567       if (dst.size() == 1) {
1568          Temp tmp;
1569          Operand half_pi(0x3e22f983u);
1570          if (src.type() == RegType::sgpr)
1571             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1572          else
1573             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1574
1575          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1576          if (ctx->options->chip_class < GFX9)
1577             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1578
1579          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1580          bld.vop1(opcode, Definition(dst), tmp);
1581       } else {
1582          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1583          nir_print_instr(&instr->instr, stderr);
1584          fprintf(stderr, "\n");
1585       }
1586       break;
1587    }
1588    case nir_op_ldexp: {
1589       if (dst.size() == 1) {
1590          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1591                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1592                   get_alu_src(ctx, instr->src[1]));
1593       } else if (dst.size() == 2) {
1594          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1595                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1596                   get_alu_src(ctx, instr->src[1]));
1597       } else {
1598          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1599          nir_print_instr(&instr->instr, stderr);
1600          fprintf(stderr, "\n");
1601       }
1602       break;
1603    }
1604    case nir_op_frexp_sig: {
1605       if (dst.size() == 1) {
1606          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1607                   get_alu_src(ctx, instr->src[0]));
1608       } else if (dst.size() == 2) {
1609          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1610                   get_alu_src(ctx, instr->src[0]));
1611       } else {
1612          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1613          nir_print_instr(&instr->instr, stderr);
1614          fprintf(stderr, "\n");
1615       }
1616       break;
1617    }
1618    case nir_op_frexp_exp: {
1619       if (instr->src[0].src.ssa->bit_size == 32) {
1620          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1621                   get_alu_src(ctx, instr->src[0]));
1622       } else if (instr->src[0].src.ssa->bit_size == 64) {
1623          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1624                   get_alu_src(ctx, instr->src[0]));
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_fsign: {
1633       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1634       if (dst.size() == 1) {
1635          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1636          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1637          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1638          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1639       } else if (dst.size() == 2) {
1640          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1641          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1642          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1643
1644          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1645          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1646          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1647
1648          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1649       } else {
1650          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1651          nir_print_instr(&instr->instr, stderr);
1652          fprintf(stderr, "\n");
1653       }
1654       break;
1655    }
1656    case nir_op_f2f32: {
1657       if (instr->src[0].src.ssa->bit_size == 64) {
1658          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_f2f64: {
1667       if (instr->src[0].src.ssa->bit_size == 32) {
1668          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_i2f32: {
1677       assert(dst.size() == 1);
1678       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1679       break;
1680    }
1681    case nir_op_i2f64: {
1682       if (instr->src[0].src.ssa->bit_size == 32) {
1683          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1684       } else if (instr->src[0].src.ssa->bit_size == 64) {
1685          Temp src = get_alu_src(ctx, instr->src[0]);
1686          RegClass rc = RegClass(src.type(), 1);
1687          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1688          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1689          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1690          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1691          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1692          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1693
1694       } else {
1695          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1696          nir_print_instr(&instr->instr, stderr);
1697          fprintf(stderr, "\n");
1698       }
1699       break;
1700    }
1701    case nir_op_u2f32: {
1702       assert(dst.size() == 1);
1703       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1704       break;
1705    }
1706    case nir_op_u2f64: {
1707       if (instr->src[0].src.ssa->bit_size == 32) {
1708          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1709       } else if (instr->src[0].src.ssa->bit_size == 64) {
1710          Temp src = get_alu_src(ctx, instr->src[0]);
1711          RegClass rc = RegClass(src.type(), 1);
1712          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1713          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1714          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1715          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1716          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1717          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1718       } else {
1719          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1720          nir_print_instr(&instr->instr, stderr);
1721          fprintf(stderr, "\n");
1722       }
1723       break;
1724    }
1725    case nir_op_f2i32: {
1726       Temp src = get_alu_src(ctx, instr->src[0]);
1727       if (instr->src[0].src.ssa->bit_size == 32) {
1728          if (dst.type() == RegType::vgpr)
1729             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1730          else
1731             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1732                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1733
1734       } else if (instr->src[0].src.ssa->bit_size == 64) {
1735          if (dst.type() == RegType::vgpr)
1736             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1737          else
1738             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1739                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1740
1741       } else {
1742          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1743          nir_print_instr(&instr->instr, stderr);
1744          fprintf(stderr, "\n");
1745       }
1746       break;
1747    }
1748    case nir_op_f2u32: {
1749       Temp src = get_alu_src(ctx, instr->src[0]);
1750       if (instr->src[0].src.ssa->bit_size == 32) {
1751          if (dst.type() == RegType::vgpr)
1752             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1753          else
1754             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1755                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1756
1757       } else if (instr->src[0].src.ssa->bit_size == 64) {
1758          if (dst.type() == RegType::vgpr)
1759             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1760          else
1761             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1762                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1763
1764       } else {
1765          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1766          nir_print_instr(&instr->instr, stderr);
1767          fprintf(stderr, "\n");
1768       }
1769       break;
1770    }
1771    case nir_op_f2i64: {
1772       Temp src = get_alu_src(ctx, instr->src[0]);
1773       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1774          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1775          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1776          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1777          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1778          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1779          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1780          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1781          Temp new_exponent = bld.tmp(v1);
1782          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1783          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1784          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1785          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1786          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1787          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1788          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1789          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1790          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1791          Temp new_lower = bld.tmp(v1);
1792          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1793          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1794          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1795
1796       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1797          if (src.type() == RegType::vgpr)
1798             src = bld.as_uniform(src);
1799          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1800          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1801          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1802          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1803          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1804          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1805          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1806          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1807          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1808          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1809          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1810          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1811          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1812          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1813          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1814          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1815          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1816          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1817          Temp borrow = bld.tmp(s1);
1818          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1819          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1820          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1821
1822       } else if (instr->src[0].src.ssa->bit_size == 64) {
1823          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1824          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1825          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1826          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1827          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1828          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1829          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1830          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1831          if (dst.type() == RegType::sgpr) {
1832             lower = bld.as_uniform(lower);
1833             upper = bld.as_uniform(upper);
1834          }
1835          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1836
1837       } else {
1838          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1839          nir_print_instr(&instr->instr, stderr);
1840          fprintf(stderr, "\n");
1841       }
1842       break;
1843    }
1844    case nir_op_f2u64: {
1845       Temp src = get_alu_src(ctx, instr->src[0]);
1846       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1847          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1848          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1849          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1850          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1851          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1852          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1853          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1854          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1855          Temp new_exponent = bld.tmp(v1);
1856          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1857          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1858          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1859          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1860          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1861          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1862          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1863          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1864          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1865
1866       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1867          if (src.type() == RegType::vgpr)
1868             src = bld.as_uniform(src);
1869          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1870          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1871          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1872          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1873          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1874          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1875          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1876          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1877          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1878          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1879          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1880          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1881          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1882          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1883          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1884          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1885          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1886          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1887
1888       } else if (instr->src[0].src.ssa->bit_size == 64) {
1889          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1890          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1891          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1892          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1893          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1894          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1895          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1896          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1897          if (dst.type() == RegType::sgpr) {
1898             lower = bld.as_uniform(lower);
1899             upper = bld.as_uniform(upper);
1900          }
1901          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1902
1903       } else {
1904          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1905          nir_print_instr(&instr->instr, stderr);
1906          fprintf(stderr, "\n");
1907       }
1908       break;
1909    }
1910    case nir_op_b2f32: {
1911       Temp src = get_alu_src(ctx, instr->src[0]);
1912       assert(src.regClass() == s2);
1913
1914       if (dst.regClass() == s1) {
1915          src = bool_to_scalar_condition(ctx, src);
1916          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1917       } else if (dst.regClass() == v1) {
1918          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1919       } else {
1920          unreachable("Wrong destination register class for nir_op_b2f32.");
1921       }
1922       break;
1923    }
1924    case nir_op_b2f64: {
1925       Temp src = get_alu_src(ctx, instr->src[0]);
1926       assert(src.regClass() == s2);
1927
1928       if (dst.regClass() == s2) {
1929          src = bool_to_scalar_condition(ctx, src);
1930          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1931       } else if (dst.regClass() == v2) {
1932          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1933          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
1934          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1935       } else {
1936          unreachable("Wrong destination register class for nir_op_b2f64.");
1937       }
1938       break;
1939    }
1940    case nir_op_i2i32: {
1941       Temp src = get_alu_src(ctx, instr->src[0]);
1942       if (instr->src[0].src.ssa->bit_size == 64) {
1943          /* we can actually just say dst = src, as it would map the lower register */
1944          emit_extract_vector(ctx, src, 0, dst);
1945       } else {
1946          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1947          nir_print_instr(&instr->instr, stderr);
1948          fprintf(stderr, "\n");
1949       }
1950       break;
1951    }
1952    case nir_op_u2u32: {
1953       Temp src = get_alu_src(ctx, instr->src[0]);
1954       if (instr->src[0].src.ssa->bit_size == 16) {
1955          if (dst.regClass() == s1) {
1956             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1957          } else {
1958             // TODO: do better with SDWA
1959             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1960          }
1961       } else if (instr->src[0].src.ssa->bit_size == 64) {
1962          /* we can actually just say dst = src, as it would map the lower register */
1963          emit_extract_vector(ctx, src, 0, dst);
1964       } else {
1965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1966          nir_print_instr(&instr->instr, stderr);
1967          fprintf(stderr, "\n");
1968       }
1969       break;
1970    }
1971    case nir_op_i2i64: {
1972       Temp src = get_alu_src(ctx, instr->src[0]);
1973       if (instr->src[0].src.ssa->bit_size == 32) {
1974          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1975       } else {
1976          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1977          nir_print_instr(&instr->instr, stderr);
1978          fprintf(stderr, "\n");
1979       }
1980       break;
1981    }
1982    case nir_op_u2u64: {
1983       Temp src = get_alu_src(ctx, instr->src[0]);
1984       if (instr->src[0].src.ssa->bit_size == 32) {
1985          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1986       } else {
1987          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1988          nir_print_instr(&instr->instr, stderr);
1989          fprintf(stderr, "\n");
1990       }
1991       break;
1992    }
1993    case nir_op_b2i32: {
1994       Temp src = get_alu_src(ctx, instr->src[0]);
1995       assert(src.regClass() == s2);
1996
1997       if (dst.regClass() == s1) {
1998          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
1999          bool_to_scalar_condition(ctx, src, dst);
2000       } else if (dst.regClass() == v1) {
2001          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2002       } else {
2003          unreachable("Invalid register class for b2i32");
2004       }
2005       break;
2006    }
2007    case nir_op_i2b1: {
2008       Temp src = get_alu_src(ctx, instr->src[0]);
2009       assert(dst.regClass() == s2);
2010
2011       if (src.type() == RegType::vgpr) {
2012          assert(src.regClass() == v1 || src.regClass() == v2);
2013          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2014                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2015       } else {
2016          assert(src.regClass() == s1 || src.regClass() == s2);
2017          Temp tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2018                              bld.scc(bld.def(s1)), Operand(0u), src);
2019          bool_to_vector_condition(ctx, tmp, dst);
2020       }
2021       break;
2022    }
2023    case nir_op_pack_64_2x32_split: {
2024       Temp src0 = get_alu_src(ctx, instr->src[0]);
2025       Temp src1 = get_alu_src(ctx, instr->src[1]);
2026
2027       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2028       break;
2029    }
2030    case nir_op_unpack_64_2x32_split_x:
2031       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2032       break;
2033    case nir_op_unpack_64_2x32_split_y:
2034       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2035       break;
2036    case nir_op_pack_half_2x16: {
2037       Temp src = get_alu_src(ctx, instr->src[0], 2);
2038
2039       if (dst.regClass() == v1) {
2040          Temp src0 = bld.tmp(v1);
2041          Temp src1 = bld.tmp(v1);
2042          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2043          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2044
2045       } else {
2046          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2047          nir_print_instr(&instr->instr, stderr);
2048          fprintf(stderr, "\n");
2049       }
2050       break;
2051    }
2052    case nir_op_unpack_half_2x16_split_x: {
2053       if (dst.regClass() == v1) {
2054          Builder bld(ctx->program, ctx->block);
2055          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2056       } else {
2057          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2058          nir_print_instr(&instr->instr, stderr);
2059          fprintf(stderr, "\n");
2060       }
2061       break;
2062    }
2063    case nir_op_unpack_half_2x16_split_y: {
2064       if (dst.regClass() == v1) {
2065          Builder bld(ctx->program, ctx->block);
2066          /* TODO: use SDWA here */
2067          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2068                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2069       } else {
2070          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2071          nir_print_instr(&instr->instr, stderr);
2072          fprintf(stderr, "\n");
2073       }
2074       break;
2075    }
2076    case nir_op_fquantize2f16: {
2077       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2078
2079       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2080
2081       Temp cmp_res = bld.tmp(s2);
2082       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2083
2084       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2085
2086       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2087       break;
2088    }
2089    case nir_op_bfm: {
2090       Temp bits = get_alu_src(ctx, instr->src[0]);
2091       Temp offset = get_alu_src(ctx, instr->src[1]);
2092
2093       if (dst.regClass() == s1) {
2094          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2095       } else if (dst.regClass() == v1) {
2096          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2097       } else {
2098          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2099          nir_print_instr(&instr->instr, stderr);
2100          fprintf(stderr, "\n");
2101       }
2102       break;
2103    }
2104    case nir_op_bitfield_select: {
2105       /* (mask & insert) | (~mask & base) */
2106       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2107       Temp insert = get_alu_src(ctx, instr->src[1]);
2108       Temp base = get_alu_src(ctx, instr->src[2]);
2109
2110       /* dst = (insert & bitmask) | (base & ~bitmask) */
2111       if (dst.regClass() == s1) {
2112          aco_ptr<Instruction> sop2;
2113          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2114          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2115          Operand lhs;
2116          if (const_insert && const_bitmask) {
2117             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2118          } else {
2119             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2120             lhs = Operand(insert);
2121          }
2122
2123          Operand rhs;
2124          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2125          if (const_base && const_bitmask) {
2126             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2127          } else {
2128             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2129             rhs = Operand(base);
2130          }
2131
2132          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2133
2134       } else if (dst.regClass() == v1) {
2135          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2136             base = as_vgpr(ctx, base);
2137          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2138             insert = as_vgpr(ctx, insert);
2139
2140          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2141
2142       } else {
2143          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2144          nir_print_instr(&instr->instr, stderr);
2145          fprintf(stderr, "\n");
2146       }
2147       break;
2148    }
2149    case nir_op_ubfe:
2150    case nir_op_ibfe: {
2151       Temp base = get_alu_src(ctx, instr->src[0]);
2152       Temp offset = get_alu_src(ctx, instr->src[1]);
2153       Temp bits = get_alu_src(ctx, instr->src[2]);
2154
2155       if (dst.type() == RegType::sgpr) {
2156          Operand extract;
2157          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2158          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2159          if (const_offset && const_bits) {
2160             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2161             extract = Operand(const_extract);
2162          } else {
2163             Operand width;
2164             if (const_bits) {
2165                width = Operand(const_bits->u32 << 16);
2166             } else {
2167                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2168             }
2169             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2170          }
2171
2172          aco_opcode opcode;
2173          if (dst.regClass() == s1) {
2174             if (instr->op == nir_op_ubfe)
2175                opcode = aco_opcode::s_bfe_u32;
2176             else
2177                opcode = aco_opcode::s_bfe_i32;
2178          } else if (dst.regClass() == s2) {
2179             if (instr->op == nir_op_ubfe)
2180                opcode = aco_opcode::s_bfe_u64;
2181             else
2182                opcode = aco_opcode::s_bfe_i64;
2183          } else {
2184             unreachable("Unsupported BFE bit size");
2185          }
2186
2187          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2188
2189       } else {
2190          aco_opcode opcode;
2191          if (dst.regClass() == v1) {
2192             if (instr->op == nir_op_ubfe)
2193                opcode = aco_opcode::v_bfe_u32;
2194             else
2195                opcode = aco_opcode::v_bfe_i32;
2196          } else {
2197             unreachable("Unsupported BFE bit size");
2198          }
2199
2200          emit_vop3a_instruction(ctx, instr, opcode, dst);
2201       }
2202       break;
2203    }
2204    case nir_op_bit_count: {
2205       Temp src = get_alu_src(ctx, instr->src[0]);
2206       if (src.regClass() == s1) {
2207          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2208       } else if (src.regClass() == v1) {
2209          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2210       } else if (src.regClass() == v2) {
2211          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2212                   emit_extract_vector(ctx, src, 1, v1),
2213                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2214                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2215       } else if (src.regClass() == s2) {
2216          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2217       } else {
2218          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2219          nir_print_instr(&instr->instr, stderr);
2220          fprintf(stderr, "\n");
2221       }
2222       break;
2223    }
2224    case nir_op_flt: {
2225       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2226       break;
2227    }
2228    case nir_op_fge: {
2229       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2230       break;
2231    }
2232    case nir_op_feq: {
2233       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2234       break;
2235    }
2236    case nir_op_fne: {
2237       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2238       break;
2239    }
2240    case nir_op_ilt: {
2241       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2242       break;
2243    }
2244    case nir_op_ige: {
2245       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2246       break;
2247    }
2248    case nir_op_ieq: {
2249       if (instr->src[0].src.ssa->bit_size == 1)
2250          emit_boolean_logic(ctx, instr, aco_opcode::s_xnor_b32, aco_opcode::s_xnor_b64, dst);
2251       else
2252          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32, aco_opcode::s_cmp_eq_u64);
2253       break;
2254    }
2255    case nir_op_ine: {
2256       if (instr->src[0].src.ssa->bit_size == 1)
2257          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
2258       else
2259          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32, aco_opcode::s_cmp_lg_u64);
2260       break;
2261    }
2262    case nir_op_ult: {
2263       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2264       break;
2265    }
2266    case nir_op_uge: {
2267       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2268       break;
2269    }
2270    case nir_op_fddx:
2271    case nir_op_fddy:
2272    case nir_op_fddx_fine:
2273    case nir_op_fddy_fine:
2274    case nir_op_fddx_coarse:
2275    case nir_op_fddy_coarse: {
2276       Definition tl = bld.def(v1);
2277       uint16_t dpp_ctrl;
2278       if (instr->op == nir_op_fddx_fine) {
2279          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2280          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2281       } else if (instr->op == nir_op_fddy_fine) {
2282          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2283          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2284       } else {
2285          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2286          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2287             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2288          else
2289             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2290       }
2291
2292       Definition tmp = bld.def(v1);
2293       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2294       emit_wqm(ctx, tmp.getTemp(), dst, true);
2295       break;
2296    }
2297    default:
2298       fprintf(stderr, "Unknown NIR ALU instr: ");
2299       nir_print_instr(&instr->instr, stderr);
2300       fprintf(stderr, "\n");
2301    }
2302 }
2303
2304 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2305 {
2306    Temp dst = get_ssa_temp(ctx, &instr->def);
2307
2308    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2309    // which get truncated the lsb if double and msb if int
2310    // for now, we only use s_mov_b64 with 64bit inline constants
2311    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2312    assert(dst.type() == RegType::sgpr);
2313
2314    Builder bld(ctx->program, ctx->block);
2315
2316    if (instr->def.bit_size == 1) {
2317       assert(dst.regClass() == s2);
2318       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), Operand((uint64_t)(instr->value[0].b ? -1 : 0)));
2319    } else if (dst.size() == 1) {
2320       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2321    } else {
2322       assert(dst.size() != 1);
2323       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2324       if (instr->def.bit_size == 64)
2325          for (unsigned i = 0; i < dst.size(); i++)
2326             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2327       else {
2328          for (unsigned i = 0; i < dst.size(); i++)
2329             vec->operands[i] = Operand{instr->value[i].u32};
2330       }
2331       vec->definitions[0] = Definition(dst);
2332       ctx->block->instructions.emplace_back(std::move(vec));
2333    }
2334 }
2335
2336 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2337 {
2338    uint32_t new_mask = 0;
2339    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2340       if (mask & (1u << i))
2341          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2342    return new_mask;
2343 }
2344
2345 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2346 {
2347    /* This wouldn't work inside control flow or with indirect offsets but
2348     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2349
2350    unsigned write_mask = nir_intrinsic_write_mask(instr);
2351    unsigned component = nir_intrinsic_component(instr);
2352    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2353    unsigned idx = nir_intrinsic_base(instr) + component;
2354
2355    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2356    if (off_instr->type != nir_instr_type_load_const) {
2357       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2358       nir_print_instr(off_instr, stderr);
2359       fprintf(stderr, "\n");
2360    }
2361    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2362
2363    if (instr->src[0].ssa->bit_size == 64)
2364       write_mask = widen_mask(write_mask, 2);
2365
2366    for (unsigned i = 0; i < 8; ++i) {
2367       if (write_mask & (1 << i)) {
2368          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2369          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2370       }
2371       idx++;
2372    }
2373 }
2374
2375 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2376 {
2377    unsigned write_mask = nir_intrinsic_write_mask(instr);
2378    Operand values[4];
2379    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2380    for (unsigned i = 0; i < 4; ++i) {
2381       if (write_mask & (1 << i)) {
2382          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2383          values[i] = Operand(tmp);
2384       } else {
2385          values[i] = Operand(v1);
2386       }
2387    }
2388
2389    unsigned index = nir_intrinsic_base(instr) / 4;
2390    unsigned target, col_format;
2391    unsigned enabled_channels = 0xF;
2392    aco_opcode compr_op = (aco_opcode)0;
2393
2394    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2395    assert(offset && "Non-const offsets on exports not yet supported");
2396    index += offset->u32;
2397
2398    assert(index != FRAG_RESULT_COLOR);
2399
2400    /* Unlike vertex shader exports, it's fine to use multiple exports to
2401     * export separate channels of one target. So shaders which export both
2402     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2403     * TODO: combine the exports in those cases and create better code
2404     */
2405
2406    if (index == FRAG_RESULT_SAMPLE_MASK) {
2407
2408       if (ctx->program->info->ps.writes_z) {
2409          target = V_008DFC_SQ_EXP_MRTZ;
2410          enabled_channels = 0x4;
2411          col_format = (unsigned) -1;
2412
2413          values[2] = values[0];
2414          values[0] = Operand(v1);
2415       } else {
2416          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2417          exp->valid_mask = false;
2418          exp->done = false;
2419          exp->compressed = true;
2420          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2421          exp->enabled_mask = 0xc;
2422          for (int i = 0; i < 4; i++)
2423             exp->operands[i] = Operand(v1);
2424          exp->operands[1] = Operand(values[0]);
2425          ctx->block->instructions.emplace_back(std::move(exp));
2426          return;
2427       }
2428
2429    } else if (index == FRAG_RESULT_DEPTH) {
2430
2431       target = V_008DFC_SQ_EXP_MRTZ;
2432       enabled_channels = 0x1;
2433       col_format = (unsigned) -1;
2434
2435    } else if (index == FRAG_RESULT_STENCIL) {
2436
2437       if (ctx->program->info->ps.writes_z) {
2438          target = V_008DFC_SQ_EXP_MRTZ;
2439          enabled_channels = 0x2;
2440          col_format = (unsigned) -1;
2441
2442          values[1] = values[0];
2443          values[0] = Operand(v1);
2444       } else {
2445          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2446          shift->operands[0] = Operand((uint32_t) 16);
2447          shift->operands[1] = values[0];
2448          Temp tmp = {ctx->program->allocateId(), v1};
2449          shift->definitions[0] = Definition(tmp);
2450          ctx->block->instructions.emplace_back(std::move(shift));
2451
2452          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2453          exp->valid_mask = false;
2454          exp->done = false;
2455          exp->compressed = true;
2456          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2457          exp->enabled_mask = 0x3;
2458          exp->operands[0] = Operand(tmp);
2459          for (int i = 1; i < 4; i++)
2460             exp->operands[i] = Operand(v1);
2461          ctx->block->instructions.emplace_back(std::move(exp));
2462          return;
2463       }
2464
2465    } else {
2466       index -= FRAG_RESULT_DATA0;
2467       target = V_008DFC_SQ_EXP_MRT + index;
2468       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2469    }
2470    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2471    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2472    assert(!is_int8 && !is_int10);
2473
2474    switch (col_format)
2475    {
2476    case V_028714_SPI_SHADER_ZERO:
2477       enabled_channels = 0; /* writemask */
2478       target = V_008DFC_SQ_EXP_NULL;
2479       break;
2480
2481    case V_028714_SPI_SHADER_32_R:
2482       enabled_channels = 1;
2483       break;
2484
2485    case V_028714_SPI_SHADER_32_GR:
2486       enabled_channels = 0x3;
2487       break;
2488
2489    case V_028714_SPI_SHADER_32_AR:
2490       if (ctx->options->chip_class >= GFX10) {
2491          /* Special case: on GFX10, the outputs are different for 32_AR */
2492          enabled_channels = 0x3;
2493          values[1] = values[3];
2494       } else {
2495          enabled_channels = 0x9;
2496       }
2497       break;
2498
2499    case V_028714_SPI_SHADER_FP16_ABGR:
2500       enabled_channels = 0x5;
2501       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2502       break;
2503
2504    case V_028714_SPI_SHADER_UNORM16_ABGR:
2505       enabled_channels = 0x5;
2506       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2507       break;
2508
2509    case V_028714_SPI_SHADER_SNORM16_ABGR:
2510       enabled_channels = 0x5;
2511       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2512       break;
2513
2514    case V_028714_SPI_SHADER_UINT16_ABGR:
2515       enabled_channels = 0x5;
2516       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2517       break;
2518
2519    case V_028714_SPI_SHADER_SINT16_ABGR:
2520       enabled_channels = 0x5;
2521       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2522       break;
2523
2524    case V_028714_SPI_SHADER_32_ABGR:
2525       enabled_channels = 0xF;
2526       break;
2527
2528    default:
2529       break;
2530    }
2531
2532    if (target == V_008DFC_SQ_EXP_NULL)
2533       return;
2534
2535    if ((bool)compr_op)
2536    {
2537       for (int i = 0; i < 2; i++)
2538       {
2539          /* check if at least one of the values to be compressed is enabled */
2540          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2541          if (enabled) {
2542             enabled_channels |= enabled << (i*2);
2543             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2544             Temp tmp{ctx->program->allocateId(), v1};
2545             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2546             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2547             compr->definitions[0] = Definition(tmp);
2548             values[i] = Operand(tmp);
2549             ctx->block->instructions.emplace_back(std::move(compr));
2550          } else {
2551             values[i] = Operand(v1);
2552          }
2553       }
2554    }
2555
2556    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2557    exp->valid_mask = false;
2558    exp->done = false;
2559    exp->compressed = (bool) compr_op;
2560    exp->dest = target;
2561    exp->enabled_mask = enabled_channels;
2562    if ((bool) compr_op) {
2563       for (int i = 0; i < 2; i++)
2564          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2565       exp->operands[2] = Operand(v1);
2566       exp->operands[3] = Operand(v1);
2567    } else {
2568       for (int i = 0; i < 4; i++)
2569          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2570    }
2571
2572    ctx->block->instructions.emplace_back(std::move(exp));
2573 }
2574
2575 Operand load_lds_size_m0(isel_context *ctx)
2576 {
2577    /* TODO: m0 does not need to be initialized on GFX9+ */
2578    Builder bld(ctx->program, ctx->block);
2579    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2580 }
2581
2582 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2583               Temp address, unsigned base_offset, unsigned align)
2584 {
2585    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2586
2587    Builder bld(ctx->program, ctx->block);
2588
2589    Operand m = load_lds_size_m0(ctx);
2590
2591    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2592    unsigned bytes_read = 0;
2593    unsigned result_size = 0;
2594    unsigned total_bytes = num_components * elem_size_bytes;
2595    std::array<Temp, 4> result;
2596
2597    while (bytes_read < total_bytes) {
2598       unsigned todo = total_bytes - bytes_read;
2599       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2600       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2601
2602       aco_opcode op = aco_opcode::last_opcode;
2603       bool read2 = false;
2604       if (todo >= 16 && aligned16) {
2605          op = aco_opcode::ds_read_b128;
2606          todo = 16;
2607       } else if (todo >= 16 && aligned8) {
2608          op = aco_opcode::ds_read2_b64;
2609          read2 = true;
2610          todo = 16;
2611       } else if (todo >= 12 && aligned16) {
2612          op = aco_opcode::ds_read_b96;
2613          todo = 12;
2614       } else if (todo >= 8 && aligned8) {
2615          op = aco_opcode::ds_read_b64;
2616          todo = 8;
2617       } else if (todo >= 8) {
2618          op = aco_opcode::ds_read2_b32;
2619          read2 = true;
2620          todo = 8;
2621       } else if (todo >= 4) {
2622          op = aco_opcode::ds_read_b32;
2623          todo = 4;
2624       } else {
2625          assert(false);
2626       }
2627       assert(todo % elem_size_bytes == 0);
2628       unsigned num_elements = todo / elem_size_bytes;
2629       unsigned offset = base_offset + bytes_read;
2630       unsigned max_offset = read2 ? 1019 : 65535;
2631
2632       Temp address_offset = address;
2633       if (offset > max_offset) {
2634          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2635          offset = bytes_read;
2636       }
2637       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2638
2639       Temp res;
2640       if (num_components == 1 && dst.type() == RegType::vgpr)
2641          res = dst;
2642       else
2643          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2644
2645       if (read2)
2646          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2647       else
2648          res = bld.ds(op, Definition(res), address_offset, m, offset);
2649
2650       if (num_components == 1) {
2651          assert(todo == total_bytes);
2652          if (dst.type() == RegType::sgpr)
2653             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2654          return;
2655       }
2656
2657       if (dst.type() == RegType::sgpr)
2658          res = bld.as_uniform(res);
2659
2660       if (num_elements == 1) {
2661          result[result_size++] = res;
2662       } else {
2663          assert(res != dst && res.size() % num_elements == 0);
2664          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2665          split->operands[0] = Operand(res);
2666          for (unsigned i = 0; i < num_elements; i++)
2667             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2668          ctx->block->instructions.emplace_back(std::move(split));
2669       }
2670
2671       bytes_read += todo;
2672    }
2673
2674    assert(result_size == num_components && result_size > 1);
2675    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2676    for (unsigned i = 0; i < result_size; i++)
2677       vec->operands[i] = Operand(result[i]);
2678    vec->definitions[0] = Definition(dst);
2679    ctx->block->instructions.emplace_back(std::move(vec));
2680    ctx->allocated_vec.emplace(dst.id(), result);
2681 }
2682
2683 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2684 {
2685    if (start == 0 && size == data.size())
2686       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2687
2688    unsigned size_hint = 1;
2689    auto it = ctx->allocated_vec.find(data.id());
2690    if (it != ctx->allocated_vec.end())
2691       size_hint = it->second[0].size();
2692    if (size % size_hint || start % size_hint)
2693       size_hint = 1;
2694
2695    start /= size_hint;
2696    size /= size_hint;
2697
2698    Temp elems[size];
2699    for (unsigned i = 0; i < size; i++)
2700       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2701
2702    if (size == 1)
2703       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2704
2705    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2706    for (unsigned i = 0; i < size; i++)
2707       vec->operands[i] = Operand(elems[i]);
2708    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2709    vec->definitions[0] = Definition(res);
2710    ctx->block->instructions.emplace_back(std::move(vec));
2711    return res;
2712 }
2713
2714 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2715 {
2716    Builder bld(ctx->program, ctx->block);
2717    unsigned bytes_written = 0;
2718    while (bytes_written < total_size * 4) {
2719       unsigned todo = total_size * 4 - bytes_written;
2720       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2721       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2722
2723       aco_opcode op = aco_opcode::last_opcode;
2724       bool write2 = false;
2725       unsigned size = 0;
2726       if (todo >= 16 && aligned16) {
2727          op = aco_opcode::ds_write_b128;
2728          size = 4;
2729       } else if (todo >= 16 && aligned8) {
2730          op = aco_opcode::ds_write2_b64;
2731          write2 = true;
2732          size = 4;
2733       } else if (todo >= 12 && aligned16) {
2734          op = aco_opcode::ds_write_b96;
2735          size = 3;
2736       } else if (todo >= 8 && aligned8) {
2737          op = aco_opcode::ds_write_b64;
2738          size = 2;
2739       } else if (todo >= 8) {
2740          op = aco_opcode::ds_write2_b32;
2741          write2 = true;
2742          size = 2;
2743       } else if (todo >= 4) {
2744          op = aco_opcode::ds_write_b32;
2745          size = 1;
2746       } else {
2747          assert(false);
2748       }
2749
2750       unsigned offset = offset0 + offset1 + bytes_written;
2751       unsigned max_offset = write2 ? 1020 : 65535;
2752       Temp address_offset = address;
2753       if (offset > max_offset) {
2754          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2755          offset = offset1 + bytes_written;
2756       }
2757       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2758
2759       if (write2) {
2760          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2761          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2762          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2763       } else {
2764          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2765          bld.ds(op, address_offset, val, m, offset);
2766       }
2767
2768       bytes_written += size * 4;
2769    }
2770 }
2771
2772 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2773                Temp address, unsigned base_offset, unsigned align)
2774 {
2775    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2776
2777    Operand m = load_lds_size_m0(ctx);
2778
2779    /* we need at most two stores for 32bit variables */
2780    int start[2], count[2];
2781    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2782    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2783    assert(wrmask == 0);
2784
2785    /* one combined store is sufficient */
2786    if (count[0] == count[1]) {
2787       Builder bld(ctx->program, ctx->block);
2788
2789       Temp address_offset = address;
2790       if ((base_offset >> 2) + start[1] > 255) {
2791          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2792          base_offset = 0;
2793       }
2794
2795       assert(count[0] == 1);
2796       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2797       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2798       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2799       base_offset = base_offset / elem_size_bytes;
2800       bld.ds(op, address_offset, val0, val1, m,
2801              base_offset + start[0], base_offset + start[1]);
2802       return;
2803    }
2804
2805    for (unsigned i = 0; i < 2; i++) {
2806       if (count[i] == 0)
2807          continue;
2808
2809       unsigned elem_size_words = elem_size_bytes / 4;
2810       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2811                       base_offset, start[i] * elem_size_bytes, align);
2812    }
2813    return;
2814 }
2815
2816 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2817 {
2818    if (ctx->stage == vertex_vs) {
2819       visit_store_vs_output(ctx, instr);
2820    } else if (ctx->stage == fragment_fs) {
2821       visit_store_fs_output(ctx, instr);
2822    } else {
2823       unreachable("Shader stage not implemented");
2824    }
2825 }
2826
2827 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2828 {
2829    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2830    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2831
2832    Builder bld(ctx->program, ctx->block);
2833    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2834    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2835 }
2836
2837 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2838 {
2839    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2840    for (unsigned i = 0; i < num_components; i++)
2841       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2842
2843    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2844       assert(num_components == 4);
2845       Builder bld(ctx->program, ctx->block);
2846       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2847    }
2848
2849    for (Operand& op : vec->operands)
2850       op = op.isUndefined() ? Operand(0u) : op;
2851
2852    vec->definitions[0] = Definition(dst);
2853    ctx->block->instructions.emplace_back(std::move(vec));
2854    emit_split_vector(ctx, dst, num_components);
2855    return;
2856 }
2857
2858 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2859 {
2860    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2861    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2862    unsigned idx = nir_intrinsic_base(instr);
2863    unsigned component = nir_intrinsic_component(instr);
2864    Temp prim_mask = ctx->prim_mask;
2865
2866    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2867    if (offset) {
2868       assert(offset->u32 == 0);
2869    } else {
2870       /* the lower 15bit of the prim_mask contain the offset into LDS
2871        * while the upper bits contain the number of prims */
2872       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2873       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2874       Builder bld(ctx->program, ctx->block);
2875       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2876       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2877       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2878       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2879       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2880    }
2881
2882    if (instr->dest.ssa.num_components == 1) {
2883       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2884    } else {
2885       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2886       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2887       {
2888          Temp tmp = {ctx->program->allocateId(), v1};
2889          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2890          vec->operands[i] = Operand(tmp);
2891       }
2892       vec->definitions[0] = Definition(dst);
2893       ctx->block->instructions.emplace_back(std::move(vec));
2894    }
2895 }
2896
2897 unsigned get_num_channels_from_data_format(unsigned data_format)
2898 {
2899    switch (data_format) {
2900    case V_008F0C_BUF_DATA_FORMAT_8:
2901    case V_008F0C_BUF_DATA_FORMAT_16:
2902    case V_008F0C_BUF_DATA_FORMAT_32:
2903       return 1;
2904    case V_008F0C_BUF_DATA_FORMAT_8_8:
2905    case V_008F0C_BUF_DATA_FORMAT_16_16:
2906    case V_008F0C_BUF_DATA_FORMAT_32_32:
2907       return 2;
2908    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2909    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2910    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2911       return 3;
2912    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2913    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2914    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2915    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2916    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2917       return 4;
2918    default:
2919       break;
2920    }
2921
2922    return 4;
2923 }
2924
2925 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2926  * so we may need to fix it up. */
2927 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
2928 {
2929    Builder bld(ctx->program, ctx->block);
2930
2931    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
2932       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
2933
2934    /* For the integer-like cases, do a natural sign extension.
2935     *
2936     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
2937     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
2938     * exponent.
2939     */
2940    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
2941    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
2942
2943    /* Convert back to the right type. */
2944    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
2945       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2946       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
2947       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
2948    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
2949       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2950    }
2951
2952    return alpha;
2953 }
2954
2955 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
2956 {
2957    Builder bld(ctx->program, ctx->block);
2958    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2959    if (ctx->stage & sw_vs) {
2960
2961       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2962       if (off_instr->type != nir_instr_type_load_const) {
2963          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2964          nir_print_instr(off_instr, stderr);
2965          fprintf(stderr, "\n");
2966       }
2967       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
2968
2969       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
2970
2971       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
2972       unsigned component = nir_intrinsic_component(instr);
2973       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
2974       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
2975       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
2976       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
2977
2978       unsigned dfmt = attrib_format & 0xf;
2979
2980       unsigned nfmt = (attrib_format >> 4) & 0x7;
2981       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
2982       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
2983       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
2984       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
2985       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
2986       if (post_shuffle)
2987          num_channels = MAX2(num_channels, 3);
2988
2989       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
2990
2991       Temp index;
2992       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
2993          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
2994          if (divisor) {
2995             ctx->needs_instance_id = true;
2996
2997             if (divisor != 1) {
2998                Temp divided = bld.tmp(v1);
2999                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
3000                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
3001             } else {
3002                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
3003             }
3004          } else {
3005             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
3006          }
3007       } else {
3008          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
3009       }
3010
3011       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3012          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3013          attrib_offset = attrib_offset % attrib_stride;
3014       }
3015
3016       Operand soffset(0u);
3017       if (attrib_offset >= 4096) {
3018          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3019          attrib_offset = 0;
3020       }
3021
3022       aco_opcode opcode;
3023       switch (num_channels) {
3024       case 1:
3025          opcode = aco_opcode::tbuffer_load_format_x;
3026          break;
3027       case 2:
3028          opcode = aco_opcode::tbuffer_load_format_xy;
3029          break;
3030       case 3:
3031          opcode = aco_opcode::tbuffer_load_format_xyz;
3032          break;
3033       case 4:
3034          opcode = aco_opcode::tbuffer_load_format_xyzw;
3035          break;
3036       default:
3037          unreachable("Unimplemented load_input vector size");
3038       }
3039
3040       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3041
3042       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3043       mubuf->operands[0] = Operand(index);
3044       mubuf->operands[1] = Operand(list);
3045       mubuf->operands[2] = soffset;
3046       mubuf->definitions[0] = Definition(tmp);
3047       mubuf->idxen = true;
3048       mubuf->can_reorder = true;
3049       mubuf->dfmt = dfmt;
3050       mubuf->nfmt = nfmt;
3051       assert(attrib_offset < 4096);
3052       mubuf->offset = attrib_offset;
3053       ctx->block->instructions.emplace_back(std::move(mubuf));
3054
3055       emit_split_vector(ctx, tmp, tmp.size());
3056
3057       if (tmp.id() != dst.id()) {
3058          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3059                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3060
3061          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3062          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3063          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3064
3065          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3066          for (unsigned i = 0; i < dst.size(); i++) {
3067             unsigned idx = i + component;
3068             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3069                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3070                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3071             } else if (idx < num_channels) {
3072                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3073             } else if (is_float && idx == 3) {
3074                vec->operands[i] = Operand(0x3f800000u);
3075             } else if (!is_float && idx == 3) {
3076                vec->operands[i] = Operand(1u);
3077             } else {
3078                vec->operands[i] = Operand(0u);
3079             }
3080          }
3081          vec->definitions[0] = Definition(dst);
3082          ctx->block->instructions.emplace_back(std::move(vec));
3083          emit_split_vector(ctx, dst, dst.size());
3084       }
3085
3086    } else if (ctx->stage == fragment_fs) {
3087       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3088       if (off_instr->type != nir_instr_type_load_const ||
3089           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3090          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3091          nir_print_instr(off_instr, stderr);
3092          fprintf(stderr, "\n");
3093       }
3094
3095       Temp prim_mask = ctx->prim_mask;
3096       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3097       if (offset) {
3098          assert(offset->u32 == 0);
3099       } else {
3100          /* the lower 15bit of the prim_mask contain the offset into LDS
3101           * while the upper bits contain the number of prims */
3102          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3103          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3104          Builder bld(ctx->program, ctx->block);
3105          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3106          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3107          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3108          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3109          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3110       }
3111
3112       unsigned idx = nir_intrinsic_base(instr);
3113       unsigned component = nir_intrinsic_component(instr);
3114
3115       if (dst.size() == 1) {
3116          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3117       } else {
3118          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3119          for (unsigned i = 0; i < dst.size(); i++)
3120             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3121          vec->definitions[0] = Definition(dst);
3122          bld.insert(std::move(vec));
3123       }
3124
3125    } else {
3126       unreachable("Shader stage not implemented");
3127    }
3128 }
3129
3130 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3131 {
3132    if (ctx->program->info->need_indirect_descriptor_sets) {
3133       Builder bld(ctx->program, ctx->block);
3134       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
3135       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3136    }
3137
3138    return ctx->descriptor_sets[desc_set];
3139 }
3140
3141
3142 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3143 {
3144    Builder bld(ctx->program, ctx->block);
3145    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3146    if (!ctx->divergent_vals[instr->dest.ssa.index])
3147       index = bld.as_uniform(index);
3148    unsigned desc_set = nir_intrinsic_desc_set(instr);
3149    unsigned binding = nir_intrinsic_binding(instr);
3150
3151    Temp desc_ptr;
3152    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3153    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3154    unsigned offset = layout->binding[binding].offset;
3155    unsigned stride;
3156    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3157        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3158       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3159       desc_ptr = ctx->push_constants;
3160       offset = pipeline_layout->push_constant_size + 16 * idx;
3161       stride = 16;
3162    } else {
3163       desc_ptr = load_desc_ptr(ctx, desc_set);
3164       stride = layout->binding[binding].size;
3165    }
3166
3167    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3168    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3169    if (stride != 1) {
3170       if (nir_const_index) {
3171          const_index = const_index * stride;
3172       } else if (index.type() == RegType::vgpr) {
3173          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3174          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3175       } else {
3176          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3177       }
3178    }
3179    if (offset) {
3180       if (nir_const_index) {
3181          const_index = const_index + offset;
3182       } else if (index.type() == RegType::vgpr) {
3183          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3184       } else {
3185          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3186       }
3187    }
3188
3189    if (nir_const_index && const_index == 0) {
3190       index = desc_ptr;
3191    } else if (index.type() == RegType::vgpr) {
3192       index = bld.vadd32(bld.def(v1),
3193                          nir_const_index ? Operand(const_index) : Operand(index),
3194                          Operand(desc_ptr));
3195    } else {
3196       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3197                        nir_const_index ? Operand(const_index) : Operand(index),
3198                        Operand(desc_ptr));
3199    }
3200
3201    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3202 }
3203
3204 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3205                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3206 {
3207    Builder bld(ctx->program, ctx->block);
3208
3209    unsigned num_bytes = dst.size() * 4;
3210    bool dlc = glc && ctx->options->chip_class >= GFX10;
3211
3212    aco_opcode op;
3213    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3214       if (ctx->options->chip_class < GFX8)
3215          offset = as_vgpr(ctx, offset);
3216
3217       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3218       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3219       unsigned const_offset = 0;
3220
3221       Temp lower = Temp();
3222       if (num_bytes > 16) {
3223          assert(num_components == 3 || num_components == 4);
3224          op = aco_opcode::buffer_load_dwordx4;
3225          lower = bld.tmp(v4);
3226          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3227          mubuf->definitions[0] = Definition(lower);
3228          mubuf->operands[0] = vaddr;
3229          mubuf->operands[1] = Operand(rsrc);
3230          mubuf->operands[2] = soffset;
3231          mubuf->offen = (offset.type() == RegType::vgpr);
3232          mubuf->glc = glc;
3233          mubuf->dlc = dlc;
3234          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3235          mubuf->can_reorder = readonly;
3236          bld.insert(std::move(mubuf));
3237          emit_split_vector(ctx, lower, 2);
3238          num_bytes -= 16;
3239          const_offset = 16;
3240       }
3241
3242       switch (num_bytes) {
3243          case 4:
3244             op = aco_opcode::buffer_load_dword;
3245             break;
3246          case 8:
3247             op = aco_opcode::buffer_load_dwordx2;
3248             break;
3249          case 12:
3250             op = aco_opcode::buffer_load_dwordx3;
3251             break;
3252          case 16:
3253             op = aco_opcode::buffer_load_dwordx4;
3254             break;
3255          default:
3256             unreachable("Load SSBO not implemented for this size.");
3257       }
3258       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3259       mubuf->operands[0] = vaddr;
3260       mubuf->operands[1] = Operand(rsrc);
3261       mubuf->operands[2] = soffset;
3262       mubuf->offen = (offset.type() == RegType::vgpr);
3263       mubuf->glc = glc;
3264       mubuf->dlc = dlc;
3265       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3266       mubuf->can_reorder = readonly;
3267       mubuf->offset = const_offset;
3268       aco_ptr<Instruction> instr = std::move(mubuf);
3269
3270       if (dst.size() > 4) {
3271          assert(lower != Temp());
3272          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3273          instr->definitions[0] = Definition(upper);
3274          bld.insert(std::move(instr));
3275          if (dst.size() == 8)
3276             emit_split_vector(ctx, upper, 2);
3277          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3278          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3279          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3280          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3281          if (dst.size() == 8)
3282             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3283       }
3284
3285       if (dst.type() == RegType::sgpr) {
3286          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3287          instr->definitions[0] = Definition(vec);
3288          bld.insert(std::move(instr));
3289          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3290       } else {
3291          instr->definitions[0] = Definition(dst);
3292          bld.insert(std::move(instr));
3293       }
3294    } else {
3295       switch (num_bytes) {
3296          case 4:
3297             op = aco_opcode::s_buffer_load_dword;
3298             break;
3299          case 8:
3300             op = aco_opcode::s_buffer_load_dwordx2;
3301             break;
3302          case 12:
3303          case 16:
3304             op = aco_opcode::s_buffer_load_dwordx4;
3305             break;
3306          case 24:
3307          case 32:
3308             op = aco_opcode::s_buffer_load_dwordx8;
3309             break;
3310          default:
3311             unreachable("Load SSBO not implemented for this size.");
3312       }
3313       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3314       load->operands[0] = Operand(rsrc);
3315       load->operands[1] = Operand(bld.as_uniform(offset));
3316       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3317       load->definitions[0] = Definition(dst);
3318       load->glc = glc;
3319       load->dlc = dlc;
3320       load->barrier = readonly ? barrier_none : barrier_buffer;
3321       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3322       assert(ctx->options->chip_class >= GFX8 || !glc);
3323
3324       /* trim vector */
3325       if (dst.size() == 3) {
3326          Temp vec = bld.tmp(s4);
3327          load->definitions[0] = Definition(vec);
3328          bld.insert(std::move(load));
3329          emit_split_vector(ctx, vec, 4);
3330
3331          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3332                     emit_extract_vector(ctx, vec, 0, s1),
3333                     emit_extract_vector(ctx, vec, 1, s1),
3334                     emit_extract_vector(ctx, vec, 2, s1));
3335       } else if (dst.size() == 6) {
3336          Temp vec = bld.tmp(s8);
3337          load->definitions[0] = Definition(vec);
3338          bld.insert(std::move(load));
3339          emit_split_vector(ctx, vec, 4);
3340
3341          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3342                     emit_extract_vector(ctx, vec, 0, s2),
3343                     emit_extract_vector(ctx, vec, 1, s2),
3344                     emit_extract_vector(ctx, vec, 2, s2));
3345       } else {
3346          bld.insert(std::move(load));
3347       }
3348
3349    }
3350    emit_split_vector(ctx, dst, num_components);
3351 }
3352
3353 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3354 {
3355    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3356    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3357
3358    Builder bld(ctx->program, ctx->block);
3359
3360    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3361    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3362    unsigned binding = nir_intrinsic_binding(idx_instr);
3363    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3364
3365    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3366       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3367                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3368                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3369                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3370       if (ctx->options->chip_class >= GFX10) {
3371          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3372                       S_008F0C_OOB_SELECT(3) |
3373                       S_008F0C_RESOURCE_LEVEL(1);
3374       } else {
3375          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3376                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3377       }
3378       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3379                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3380                                      Operand(0xFFFFFFFFu),
3381                                      Operand(desc_type));
3382       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3383                         rsrc, upper_dwords);
3384    } else {
3385       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3386       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3387    }
3388
3389    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3390 }
3391
3392 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3393 {
3394    Builder bld(ctx->program, ctx->block);
3395    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3396
3397    unsigned offset = nir_intrinsic_base(instr);
3398    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3399    if (index_cv && instr->dest.ssa.bit_size == 32) {
3400
3401       unsigned count = instr->dest.ssa.num_components;
3402       unsigned start = (offset + index_cv->u32) / 4u;
3403       start -= ctx->base_inline_push_consts;
3404       if (start + count <= ctx->num_inline_push_consts) {
3405          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3406          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3407          for (unsigned i = 0; i < count; ++i) {
3408             elems[i] = ctx->inline_push_consts[start + i];
3409             vec->operands[i] = Operand{elems[i]};
3410          }
3411          vec->definitions[0] = Definition(dst);
3412          ctx->block->instructions.emplace_back(std::move(vec));
3413          ctx->allocated_vec.emplace(dst.id(), elems);
3414          return;
3415       }
3416    }
3417
3418    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3419    if (offset != 0) // TODO check if index != 0 as well
3420       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3421    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3422    Temp vec = dst;
3423    bool trim = false;
3424    aco_opcode op;
3425
3426    switch (dst.size()) {
3427    case 1:
3428       op = aco_opcode::s_load_dword;
3429       break;
3430    case 2:
3431       op = aco_opcode::s_load_dwordx2;
3432       break;
3433    case 3:
3434       vec = bld.tmp(s4);
3435       trim = true;
3436    case 4:
3437       op = aco_opcode::s_load_dwordx4;
3438       break;
3439    case 6:
3440       vec = bld.tmp(s8);
3441       trim = true;
3442    case 8:
3443       op = aco_opcode::s_load_dwordx8;
3444       break;
3445    default:
3446       unreachable("unimplemented or forbidden load_push_constant.");
3447    }
3448
3449    bld.smem(op, Definition(vec), ptr, index);
3450
3451    if (trim) {
3452       emit_split_vector(ctx, vec, 4);
3453       RegClass rc = dst.size() == 3 ? s1 : s2;
3454       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3455                  emit_extract_vector(ctx, vec, 0, rc),
3456                  emit_extract_vector(ctx, vec, 1, rc),
3457                  emit_extract_vector(ctx, vec, 2, rc));
3458
3459    }
3460    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3461 }
3462
3463 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3464 {
3465    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3466
3467    Builder bld(ctx->program, ctx->block);
3468
3469    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3470                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3471                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3472                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3473    if (ctx->options->chip_class >= GFX10) {
3474       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3475                    S_008F0C_OOB_SELECT(3) |
3476                    S_008F0C_RESOURCE_LEVEL(1);
3477    } else {
3478       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3479                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3480    }
3481
3482    unsigned base = nir_intrinsic_base(instr);
3483    unsigned range = nir_intrinsic_range(instr);
3484
3485    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3486    if (base && offset.type() == RegType::sgpr)
3487       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3488    else if (base && offset.type() == RegType::vgpr)
3489       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3490
3491    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3492                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3493                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3494                           Operand(desc_type));
3495
3496    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3497 }
3498
3499 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3500 {
3501    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3502       ctx->cf_info.exec_potentially_empty = true;
3503
3504    ctx->program->needs_exact = true;
3505
3506    // TODO: optimize uniform conditions
3507    Builder bld(ctx->program, ctx->block);
3508    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3509    assert(src.regClass() == s2);
3510    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3511    bld.pseudo(aco_opcode::p_discard_if, src);
3512    ctx->block->kind |= block_kind_uses_discard_if;
3513    return;
3514 }
3515
3516 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3517 {
3518    Builder bld(ctx->program, ctx->block);
3519
3520    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3521       ctx->cf_info.exec_potentially_empty = true;
3522
3523    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3524                     ctx->cf_info.parent_loop.has_divergent_continue;
3525
3526    if (ctx->block->loop_nest_depth &&
3527        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3528       /* we handle discards the same way as jump instructions */
3529       append_logical_end(ctx->block);
3530
3531       /* in loops, discard behaves like break */
3532       Block *linear_target = ctx->cf_info.parent_loop.exit;
3533       ctx->block->kind |= block_kind_discard;
3534
3535       if (!divergent) {
3536          /* uniform discard - loop ends here */
3537          assert(nir_instr_is_last(&instr->instr));
3538          ctx->block->kind |= block_kind_uniform;
3539          ctx->cf_info.has_branch = true;
3540          bld.branch(aco_opcode::p_branch);
3541          add_linear_edge(ctx->block->index, linear_target);
3542          return;
3543       }
3544
3545       /* we add a break right behind the discard() instructions */
3546       ctx->block->kind |= block_kind_break;
3547       unsigned idx = ctx->block->index;
3548
3549       /* remove critical edges from linear CFG */
3550       bld.branch(aco_opcode::p_branch);
3551       Block* break_block = ctx->program->create_and_insert_block();
3552       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3553       break_block->kind |= block_kind_uniform;
3554       add_linear_edge(idx, break_block);
3555       add_linear_edge(break_block->index, linear_target);
3556       bld.reset(break_block);
3557       bld.branch(aco_opcode::p_branch);
3558
3559       Block* continue_block = ctx->program->create_and_insert_block();
3560       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3561       add_linear_edge(idx, continue_block);
3562       append_logical_start(continue_block);
3563       ctx->block = continue_block;
3564
3565       return;
3566    }
3567
3568    /* it can currently happen that NIR doesn't remove the unreachable code */
3569    if (!nir_instr_is_last(&instr->instr)) {
3570       ctx->program->needs_exact = true;
3571       /* save exec somewhere temporarily so that it doesn't get
3572        * overwritten before the discard from outer exec masks */
3573       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3574       bld.pseudo(aco_opcode::p_discard_if, cond);
3575       ctx->block->kind |= block_kind_uses_discard_if;
3576       return;
3577    }
3578
3579    /* This condition is incorrect for uniformly branched discards in a loop
3580     * predicated by a divergent condition, but the above code catches that case
3581     * and the discard would end up turning into a discard_if.
3582     * For example:
3583     * if (divergent) {
3584     *    while (...) {
3585     *       if (uniform) {
3586     *          discard;
3587     *       }
3588     *    }
3589     * }
3590     */
3591    if (!ctx->cf_info.parent_if.is_divergent) {
3592       /* program just ends here */
3593       ctx->block->kind |= block_kind_uniform;
3594       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3595               0 /* enabled mask */, 9 /* dest */,
3596               false /* compressed */, true/* done */, true /* valid mask */);
3597       bld.sopp(aco_opcode::s_endpgm);
3598       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3599    } else {
3600       ctx->block->kind |= block_kind_discard;
3601       /* branch and linear edge is added by visit_if() */
3602    }
3603 }
3604
3605 enum aco_descriptor_type {
3606    ACO_DESC_IMAGE,
3607    ACO_DESC_FMASK,
3608    ACO_DESC_SAMPLER,
3609    ACO_DESC_BUFFER,
3610    ACO_DESC_PLANE_0,
3611    ACO_DESC_PLANE_1,
3612    ACO_DESC_PLANE_2,
3613 };
3614
3615 static bool
3616 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3617    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3618       return false;
3619    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3620    return dim == ac_image_cube ||
3621           dim == ac_image_1darray ||
3622           dim == ac_image_2darray ||
3623           dim == ac_image_2darraymsaa;
3624 }
3625
3626 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3627                       enum aco_descriptor_type desc_type,
3628                       const nir_tex_instr *tex_instr, bool image, bool write)
3629 {
3630 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3631    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3632    if (it != ctx->tex_desc.end())
3633       return it->second;
3634 */
3635    Temp index = Temp();
3636    bool index_set = false;
3637    unsigned constant_index = 0;
3638    unsigned descriptor_set;
3639    unsigned base_index;
3640    Builder bld(ctx->program, ctx->block);
3641
3642    if (!deref_instr) {
3643       assert(tex_instr && !image);
3644       descriptor_set = 0;
3645       base_index = tex_instr->sampler_index;
3646    } else {
3647       while(deref_instr->deref_type != nir_deref_type_var) {
3648          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3649          if (!array_size)
3650             array_size = 1;
3651
3652          assert(deref_instr->deref_type == nir_deref_type_array);
3653          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3654          if (const_value) {
3655             constant_index += array_size * const_value->u32;
3656          } else {
3657             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3658             if (indirect.type() == RegType::vgpr)
3659                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3660
3661             if (array_size != 1)
3662                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3663
3664             if (!index_set) {
3665                index = indirect;
3666                index_set = true;
3667             } else {
3668                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3669             }
3670          }
3671
3672          deref_instr = nir_src_as_deref(deref_instr->parent);
3673       }
3674       descriptor_set = deref_instr->var->data.descriptor_set;
3675       base_index = deref_instr->var->data.binding;
3676    }
3677
3678    Temp list = load_desc_ptr(ctx, descriptor_set);
3679    list = convert_pointer_to_64_bit(ctx, list);
3680
3681    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3682    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3683    unsigned offset = binding->offset;
3684    unsigned stride = binding->size;
3685    aco_opcode opcode;
3686    RegClass type;
3687
3688    assert(base_index < layout->binding_count);
3689
3690    switch (desc_type) {
3691    case ACO_DESC_IMAGE:
3692       type = s8;
3693       opcode = aco_opcode::s_load_dwordx8;
3694       break;
3695    case ACO_DESC_FMASK:
3696       type = s8;
3697       opcode = aco_opcode::s_load_dwordx8;
3698       offset += 32;
3699       break;
3700    case ACO_DESC_SAMPLER:
3701       type = s4;
3702       opcode = aco_opcode::s_load_dwordx4;
3703       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3704          offset += radv_combined_image_descriptor_sampler_offset(binding);
3705       break;
3706    case ACO_DESC_BUFFER:
3707       type = s4;
3708       opcode = aco_opcode::s_load_dwordx4;
3709       break;
3710    case ACO_DESC_PLANE_0:
3711    case ACO_DESC_PLANE_1:
3712       type = s8;
3713       opcode = aco_opcode::s_load_dwordx8;
3714       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3715       break;
3716    case ACO_DESC_PLANE_2:
3717       type = s4;
3718       opcode = aco_opcode::s_load_dwordx4;
3719       offset += 64;
3720       break;
3721    default:
3722       unreachable("invalid desc_type\n");
3723    }
3724
3725    offset += constant_index * stride;
3726
3727    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3728       (!index_set || binding->immutable_samplers_equal)) {
3729       if (binding->immutable_samplers_equal)
3730          constant_index = 0;
3731
3732       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3733       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3734                         Operand(samplers[constant_index * 4 + 0]),
3735                         Operand(samplers[constant_index * 4 + 1]),
3736                         Operand(samplers[constant_index * 4 + 2]),
3737                         Operand(samplers[constant_index * 4 + 3]));
3738    }
3739
3740    Operand off;
3741    if (!index_set) {
3742       off = Operand(offset);
3743    } else {
3744       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3745                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3746    }
3747
3748    Temp res = bld.smem(opcode, bld.def(type), list, off);
3749
3750    if (desc_type == ACO_DESC_PLANE_2) {
3751       Temp components[8];
3752       for (unsigned i = 0; i < 8; i++)
3753          components[i] = bld.tmp(s1);
3754       bld.pseudo(aco_opcode::p_split_vector,
3755                  Definition(components[0]),
3756                  Definition(components[1]),
3757                  Definition(components[2]),
3758                  Definition(components[3]),
3759                  res);
3760
3761       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3762       bld.pseudo(aco_opcode::p_split_vector,
3763                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3764                  Definition(components[4]),
3765                  Definition(components[5]),
3766                  Definition(components[6]),
3767                  Definition(components[7]),
3768                  desc2);
3769
3770       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3771                        components[0], components[1], components[2], components[3],
3772                        components[4], components[5], components[6], components[7]);
3773    }
3774
3775    return res;
3776 }
3777
3778 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3779 {
3780    switch (dim) {
3781    case GLSL_SAMPLER_DIM_BUF:
3782       return 1;
3783    case GLSL_SAMPLER_DIM_1D:
3784       return array ? 2 : 1;
3785    case GLSL_SAMPLER_DIM_2D:
3786       return array ? 3 : 2;
3787    case GLSL_SAMPLER_DIM_MS:
3788       return array ? 4 : 3;
3789    case GLSL_SAMPLER_DIM_3D:
3790    case GLSL_SAMPLER_DIM_CUBE:
3791       return 3;
3792    case GLSL_SAMPLER_DIM_RECT:
3793    case GLSL_SAMPLER_DIM_SUBPASS:
3794       return 2;
3795    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3796       return 3;
3797    default:
3798       break;
3799    }
3800    return 0;
3801 }
3802
3803
3804 /* Adjust the sample index according to FMASK.
3805  *
3806  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3807  * which is the identity mapping. Each nibble says which physical sample
3808  * should be fetched to get that sample.
3809  *
3810  * For example, 0x11111100 means there are only 2 samples stored and
3811  * the second sample covers 3/4 of the pixel. When reading samples 0
3812  * and 1, return physical sample 0 (determined by the first two 0s
3813  * in FMASK), otherwise return physical sample 1.
3814  *
3815  * The sample index should be adjusted as follows:
3816  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3817  */
3818 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3819 {
3820    Builder bld(ctx->program, ctx->block);
3821    Temp fmask = bld.tmp(v1);
3822    unsigned dim = ctx->options->chip_class >= GFX10
3823                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3824                   : 0;
3825
3826    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3827    load->operands[0] = Operand(coords);
3828    load->operands[1] = Operand(fmask_desc_ptr);
3829    load->definitions[0] = Definition(fmask);
3830    load->glc = false;
3831    load->dlc = false;
3832    load->dmask = 0x1;
3833    load->unrm = true;
3834    load->da = da;
3835    load->dim = dim;
3836    load->can_reorder = true; /* fmask images shouldn't be modified */
3837    ctx->block->instructions.emplace_back(std::move(load));
3838
3839    Operand sample_index4;
3840    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3841       sample_index4 = Operand(sample_index.constantValue() << 2);
3842    } else if (sample_index.regClass() == s1) {
3843       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3844    } else {
3845       assert(sample_index.regClass() == v1);
3846       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3847    }
3848
3849    Temp final_sample;
3850    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3851       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3852    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3853       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3854    else
3855       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3856
3857    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3858     * resource descriptor is 0 (invalid),
3859     */
3860    Temp compare = bld.tmp(s2);
3861    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3862                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3863
3864    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3865
3866    /* Replace the MSAA sample index. */
3867    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3868 }
3869
3870 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3871 {
3872
3873    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3874    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3875    bool is_array = glsl_sampler_type_is_array(type);
3876    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3877    assert(!add_frag_pos && "Input attachments should be lowered.");
3878    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3879    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3880    int count = image_type_to_components_count(dim, is_array);
3881    std::vector<Operand> coords(count);
3882
3883    if (is_ms) {
3884       Operand sample_index;
3885       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3886       if (sample_cv)
3887          sample_index = Operand(sample_cv->u32);
3888       else
3889          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3890
3891       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3892          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3893          for (unsigned i = 0; i < vec->operands.size(); i++)
3894             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3895          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3896          vec->definitions[0] = Definition(fmask_load_address);
3897          ctx->block->instructions.emplace_back(std::move(vec));
3898
3899          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3900          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3901       }
3902       count--;
3903       coords[count] = sample_index;
3904    }
3905
3906    if (count == 1 && !gfx9_1d)
3907       return emit_extract_vector(ctx, src0, 0, v1);
3908
3909    if (gfx9_1d) {
3910       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3911       coords.resize(coords.size() + 1);
3912       coords[1] = Operand((uint32_t) 0);
3913       if (is_array)
3914          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3915    } else {
3916       for (int i = 0; i < count; i++)
3917          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3918    }
3919
3920    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3921    for (unsigned i = 0; i < coords.size(); i++)
3922       vec->operands[i] = coords[i];
3923    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3924    vec->definitions[0] = Definition(res);
3925    ctx->block->instructions.emplace_back(std::move(vec));
3926    return res;
3927 }
3928
3929
3930 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
3931 {
3932    Builder bld(ctx->program, ctx->block);
3933    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3934    const struct glsl_type *type = glsl_without_array(var->type);
3935    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3936    bool is_array = glsl_sampler_type_is_array(type);
3937    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3938
3939    if (dim == GLSL_SAMPLER_DIM_BUF) {
3940       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
3941       unsigned num_channels = util_last_bit(mask);
3942       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3943       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3944
3945       aco_opcode opcode;
3946       switch (num_channels) {
3947       case 1:
3948          opcode = aco_opcode::buffer_load_format_x;
3949          break;
3950       case 2:
3951          opcode = aco_opcode::buffer_load_format_xy;
3952          break;
3953       case 3:
3954          opcode = aco_opcode::buffer_load_format_xyz;
3955          break;
3956       case 4:
3957          opcode = aco_opcode::buffer_load_format_xyzw;
3958          break;
3959       default:
3960          unreachable(">4 channel buffer image load");
3961       }
3962       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
3963       load->operands[0] = Operand(vindex);
3964       load->operands[1] = Operand(rsrc);
3965       load->operands[2] = Operand((uint32_t) 0);
3966       Temp tmp;
3967       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3968          tmp = dst;
3969       else
3970          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
3971       load->definitions[0] = Definition(tmp);
3972       load->idxen = true;
3973       load->barrier = barrier_image;
3974       ctx->block->instructions.emplace_back(std::move(load));
3975
3976       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
3977       return;
3978    }
3979
3980    Temp coords = get_image_coords(ctx, instr, type);
3981    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3982
3983    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
3984    unsigned num_components = util_bitcount(dmask);
3985    Temp tmp;
3986    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3987       tmp = dst;
3988    else
3989       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
3990
3991    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3992    load->operands[0] = Operand(coords);
3993    load->operands[1] = Operand(resource);
3994    load->definitions[0] = Definition(tmp);
3995    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
3996    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
3997    load->dmask = dmask;
3998    load->unrm = true;
3999    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4000    load->barrier = barrier_image;
4001    ctx->block->instructions.emplace_back(std::move(load));
4002
4003    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4004    return;
4005 }
4006
4007 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4008 {
4009    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4010    const struct glsl_type *type = glsl_without_array(var->type);
4011    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4012    bool is_array = glsl_sampler_type_is_array(type);
4013    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4014
4015    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4016
4017    if (dim == GLSL_SAMPLER_DIM_BUF) {
4018       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4019       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4020       aco_opcode opcode;
4021       switch (data.size()) {
4022       case 1:
4023          opcode = aco_opcode::buffer_store_format_x;
4024          break;
4025       case 2:
4026          opcode = aco_opcode::buffer_store_format_xy;
4027          break;
4028       case 3:
4029          opcode = aco_opcode::buffer_store_format_xyz;
4030          break;
4031       case 4:
4032          opcode = aco_opcode::buffer_store_format_xyzw;
4033          break;
4034       default:
4035          unreachable(">4 channel buffer image store");
4036       }
4037       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4038       store->operands[0] = Operand(vindex);
4039       store->operands[1] = Operand(rsrc);
4040       store->operands[2] = Operand((uint32_t) 0);
4041       store->operands[3] = Operand(data);
4042       store->idxen = true;
4043       store->glc = glc;
4044       store->dlc = false;
4045       store->disable_wqm = true;
4046       store->barrier = barrier_image;
4047       ctx->program->needs_exact = true;
4048       ctx->block->instructions.emplace_back(std::move(store));
4049       return;
4050    }
4051
4052    assert(data.type() == RegType::vgpr);
4053    Temp coords = get_image_coords(ctx, instr, type);
4054    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4055
4056    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4057    store->operands[0] = Operand(coords);
4058    store->operands[1] = Operand(resource);
4059    store->operands[2] = Operand(s4);
4060    store->operands[3] = Operand(data);
4061    store->glc = glc;
4062    store->dlc = false;
4063    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4064    store->dmask = (1 << data.size()) - 1;
4065    store->unrm = true;
4066    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4067    store->disable_wqm = true;
4068    store->barrier = barrier_image;
4069    ctx->program->needs_exact = true;
4070    ctx->block->instructions.emplace_back(std::move(store));
4071    return;
4072 }
4073
4074 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4075 {
4076    /* return the previous value if dest is ever used */
4077    bool return_previous = false;
4078    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4079       return_previous = true;
4080       break;
4081    }
4082    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4083       return_previous = true;
4084       break;
4085    }
4086
4087    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4088    const struct glsl_type *type = glsl_without_array(var->type);
4089    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4090    bool is_array = glsl_sampler_type_is_array(type);
4091    Builder bld(ctx->program, ctx->block);
4092
4093    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4094    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4095
4096    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4097       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4098
4099    aco_opcode buf_op, image_op;
4100    switch (instr->intrinsic) {
4101       case nir_intrinsic_image_deref_atomic_add:
4102          buf_op = aco_opcode::buffer_atomic_add;
4103          image_op = aco_opcode::image_atomic_add;
4104          break;
4105       case nir_intrinsic_image_deref_atomic_umin:
4106          buf_op = aco_opcode::buffer_atomic_umin;
4107          image_op = aco_opcode::image_atomic_umin;
4108          break;
4109       case nir_intrinsic_image_deref_atomic_imin:
4110          buf_op = aco_opcode::buffer_atomic_smin;
4111          image_op = aco_opcode::image_atomic_smin;
4112          break;
4113       case nir_intrinsic_image_deref_atomic_umax:
4114          buf_op = aco_opcode::buffer_atomic_umax;
4115          image_op = aco_opcode::image_atomic_umax;
4116          break;
4117       case nir_intrinsic_image_deref_atomic_imax:
4118          buf_op = aco_opcode::buffer_atomic_smax;
4119          image_op = aco_opcode::image_atomic_smax;
4120          break;
4121       case nir_intrinsic_image_deref_atomic_and:
4122          buf_op = aco_opcode::buffer_atomic_and;
4123          image_op = aco_opcode::image_atomic_and;
4124          break;
4125       case nir_intrinsic_image_deref_atomic_or:
4126          buf_op = aco_opcode::buffer_atomic_or;
4127          image_op = aco_opcode::image_atomic_or;
4128          break;
4129       case nir_intrinsic_image_deref_atomic_xor:
4130          buf_op = aco_opcode::buffer_atomic_xor;
4131          image_op = aco_opcode::image_atomic_xor;
4132          break;
4133       case nir_intrinsic_image_deref_atomic_exchange:
4134          buf_op = aco_opcode::buffer_atomic_swap;
4135          image_op = aco_opcode::image_atomic_swap;
4136          break;
4137       case nir_intrinsic_image_deref_atomic_comp_swap:
4138          buf_op = aco_opcode::buffer_atomic_cmpswap;
4139          image_op = aco_opcode::image_atomic_cmpswap;
4140          break;
4141       default:
4142          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4143    }
4144
4145    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4146
4147    if (dim == GLSL_SAMPLER_DIM_BUF) {
4148       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4149       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4150       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4151       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4152       mubuf->operands[0] = Operand(vindex);
4153       mubuf->operands[1] = Operand(resource);
4154       mubuf->operands[2] = Operand((uint32_t)0);
4155       mubuf->operands[3] = Operand(data);
4156       if (return_previous)
4157          mubuf->definitions[0] = Definition(dst);
4158       mubuf->offset = 0;
4159       mubuf->idxen = true;
4160       mubuf->glc = return_previous;
4161       mubuf->dlc = false; /* Not needed for atomics */
4162       mubuf->disable_wqm = true;
4163       mubuf->barrier = barrier_image;
4164       ctx->program->needs_exact = true;
4165       ctx->block->instructions.emplace_back(std::move(mubuf));
4166       return;
4167    }
4168
4169    Temp coords = get_image_coords(ctx, instr, type);
4170    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4171    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4172    mimg->operands[0] = Operand(coords);
4173    mimg->operands[1] = Operand(resource);
4174    mimg->operands[2] = Operand(s4); /* no sampler */
4175    mimg->operands[3] = Operand(data);
4176    if (return_previous)
4177       mimg->definitions[0] = Definition(dst);
4178    mimg->glc = return_previous;
4179    mimg->dlc = false; /* Not needed for atomics */
4180    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4181    mimg->dmask = (1 << data.size()) - 1;
4182    mimg->unrm = true;
4183    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4184    mimg->disable_wqm = true;
4185    mimg->barrier = barrier_image;
4186    ctx->program->needs_exact = true;
4187    ctx->block->instructions.emplace_back(std::move(mimg));
4188    return;
4189 }
4190
4191 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4192 {
4193    if (in_elements && ctx->options->chip_class == GFX8) {
4194       Builder bld(ctx->program, ctx->block);
4195
4196       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4197       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4198       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4199       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4200
4201       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4202       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4203
4204       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4205       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4206       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4207
4208       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4209       /* idea
4210        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4211        * in case 12 (or 3?), we have to divide by 3:
4212        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4213        * use v_mul_hi_u32 with magic number to divide
4214        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4215        * disable v_skip
4216        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4217        */
4218
4219    } else {
4220       emit_extract_vector(ctx, desc, 2, dst);
4221    }
4222 }
4223
4224 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4225 {
4226    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4227    const struct glsl_type *type = glsl_without_array(var->type);
4228    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4229    bool is_array = glsl_sampler_type_is_array(type);
4230    Builder bld(ctx->program, ctx->block);
4231
4232    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4233       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4234       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4235    }
4236
4237    /* LOD */
4238    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4239
4240    /* Resource */
4241    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4242
4243    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4244
4245    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4246    mimg->operands[0] = Operand(lod);
4247    mimg->operands[1] = Operand(resource);
4248    unsigned& dmask = mimg->dmask;
4249    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4250    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4251    mimg->da = glsl_sampler_type_is_array(type);
4252    mimg->can_reorder = true;
4253    Definition& def = mimg->definitions[0];
4254    ctx->block->instructions.emplace_back(std::move(mimg));
4255
4256    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4257        glsl_sampler_type_is_array(type)) {
4258
4259       assert(instr->dest.ssa.num_components == 3);
4260       Temp tmp = {ctx->program->allocateId(), v3};
4261       def = Definition(tmp);
4262       emit_split_vector(ctx, tmp, 3);
4263
4264       /* divide 3rd value by 6 by multiplying with magic number */
4265       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4266       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4267
4268       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4269                  emit_extract_vector(ctx, tmp, 0, v1),
4270                  emit_extract_vector(ctx, tmp, 1, v1),
4271                  by_6);
4272
4273    } else if (ctx->options->chip_class == GFX9 &&
4274               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4275               glsl_sampler_type_is_array(type)) {
4276       assert(instr->dest.ssa.num_components == 2);
4277       def = Definition(dst);
4278       dmask = 0x5;
4279    } else {
4280       def = Definition(dst);
4281    }
4282
4283    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4284 }
4285
4286 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4287 {
4288    Builder bld(ctx->program, ctx->block);
4289    unsigned num_components = instr->num_components;
4290
4291    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4292    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4293    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4294
4295    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4296    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4297 }
4298
4299 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4300 {
4301    Builder bld(ctx->program, ctx->block);
4302    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4303    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4304    unsigned writemask = nir_intrinsic_write_mask(instr);
4305
4306    Temp offset;
4307    if (ctx->options->chip_class < GFX8)
4308       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4309    else
4310       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4311
4312    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4313    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4314
4315    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4316                ctx->options->chip_class >= GFX8;
4317    if (smem)
4318       offset = bld.as_uniform(offset);
4319    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4320
4321    while (writemask) {
4322       int start, count;
4323       u_bit_scan_consecutive_range(&writemask, &start, &count);
4324       if (count == 3 && smem) {
4325          writemask |= 1u << (start + 2);
4326          count = 2;
4327       }
4328       int num_bytes = count * elem_size_bytes;
4329
4330       if (num_bytes > 16) {
4331          assert(elem_size_bytes == 8);
4332          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4333          count = 2;
4334          num_bytes = 16;
4335       }
4336
4337       // TODO: check alignment of sub-dword stores
4338       // TODO: split 3 bytes. there is no store instruction for that
4339
4340       Temp write_data;
4341       if (count != instr->num_components) {
4342          emit_split_vector(ctx, data, instr->num_components);
4343          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4344          for (int i = 0; i < count; i++) {
4345             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4346             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4347          }
4348          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4349          vec->definitions[0] = Definition(write_data);
4350          ctx->block->instructions.emplace_back(std::move(vec));
4351       } else if (!smem && data.type() != RegType::vgpr) {
4352          assert(num_bytes % 4 == 0);
4353          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4354       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4355          assert(num_bytes % 4 == 0);
4356          write_data = bld.as_uniform(data);
4357       } else {
4358          write_data = data;
4359       }
4360
4361       aco_opcode vmem_op, smem_op;
4362       switch (num_bytes) {
4363          case 4:
4364             vmem_op = aco_opcode::buffer_store_dword;
4365             smem_op = aco_opcode::s_buffer_store_dword;
4366             break;
4367          case 8:
4368             vmem_op = aco_opcode::buffer_store_dwordx2;
4369             smem_op = aco_opcode::s_buffer_store_dwordx2;
4370             break;
4371          case 12:
4372             vmem_op = aco_opcode::buffer_store_dwordx3;
4373             smem_op = aco_opcode::last_opcode;
4374             assert(!smem);
4375             break;
4376          case 16:
4377             vmem_op = aco_opcode::buffer_store_dwordx4;
4378             smem_op = aco_opcode::s_buffer_store_dwordx4;
4379             break;
4380          default:
4381             unreachable("Store SSBO not implemented for this size.");
4382       }
4383       if (ctx->stage == fragment_fs)
4384          smem_op = aco_opcode::p_fs_buffer_store_smem;
4385
4386       if (smem) {
4387          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4388          store->operands[0] = Operand(rsrc);
4389          if (start) {
4390             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4391                                 offset, Operand(start * elem_size_bytes));
4392             store->operands[1] = Operand(off);
4393          } else {
4394             store->operands[1] = Operand(offset);
4395          }
4396          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4397             store->operands[1].setFixed(m0);
4398          store->operands[2] = Operand(write_data);
4399          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4400          store->dlc = false;
4401          store->disable_wqm = true;
4402          store->barrier = barrier_buffer;
4403          ctx->block->instructions.emplace_back(std::move(store));
4404          ctx->program->wb_smem_l1_on_end = true;
4405          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4406             ctx->block->kind |= block_kind_needs_lowering;
4407             ctx->program->needs_exact = true;
4408          }
4409       } else {
4410          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4411          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4412          store->operands[1] = Operand(rsrc);
4413          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4414          store->operands[3] = Operand(write_data);
4415          store->offset = start * elem_size_bytes;
4416          store->offen = (offset.type() == RegType::vgpr);
4417          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4418          store->dlc = false;
4419          store->disable_wqm = true;
4420          store->barrier = barrier_buffer;
4421          ctx->program->needs_exact = true;
4422          ctx->block->instructions.emplace_back(std::move(store));
4423       }
4424    }
4425 }
4426
4427 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4428 {
4429    /* return the previous value if dest is ever used */
4430    bool return_previous = false;
4431    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4432       return_previous = true;
4433       break;
4434    }
4435    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4436       return_previous = true;
4437       break;
4438    }
4439
4440    Builder bld(ctx->program, ctx->block);
4441    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4442
4443    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4444       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4445                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4446
4447    Temp offset;
4448    if (ctx->options->chip_class < GFX8)
4449       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4450    else
4451       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4452
4453    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4454    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4455
4456    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4457
4458    aco_opcode op32, op64;
4459    switch (instr->intrinsic) {
4460       case nir_intrinsic_ssbo_atomic_add:
4461          op32 = aco_opcode::buffer_atomic_add;
4462          op64 = aco_opcode::buffer_atomic_add_x2;
4463          break;
4464       case nir_intrinsic_ssbo_atomic_imin:
4465          op32 = aco_opcode::buffer_atomic_smin;
4466          op64 = aco_opcode::buffer_atomic_smin_x2;
4467          break;
4468       case nir_intrinsic_ssbo_atomic_umin:
4469          op32 = aco_opcode::buffer_atomic_umin;
4470          op64 = aco_opcode::buffer_atomic_umin_x2;
4471          break;
4472       case nir_intrinsic_ssbo_atomic_imax:
4473          op32 = aco_opcode::buffer_atomic_smax;
4474          op64 = aco_opcode::buffer_atomic_smax_x2;
4475          break;
4476       case nir_intrinsic_ssbo_atomic_umax:
4477          op32 = aco_opcode::buffer_atomic_umax;
4478          op64 = aco_opcode::buffer_atomic_umax_x2;
4479          break;
4480       case nir_intrinsic_ssbo_atomic_and:
4481          op32 = aco_opcode::buffer_atomic_and;
4482          op64 = aco_opcode::buffer_atomic_and_x2;
4483          break;
4484       case nir_intrinsic_ssbo_atomic_or:
4485          op32 = aco_opcode::buffer_atomic_or;
4486          op64 = aco_opcode::buffer_atomic_or_x2;
4487          break;
4488       case nir_intrinsic_ssbo_atomic_xor:
4489          op32 = aco_opcode::buffer_atomic_xor;
4490          op64 = aco_opcode::buffer_atomic_xor_x2;
4491          break;
4492       case nir_intrinsic_ssbo_atomic_exchange:
4493          op32 = aco_opcode::buffer_atomic_swap;
4494          op64 = aco_opcode::buffer_atomic_swap_x2;
4495          break;
4496       case nir_intrinsic_ssbo_atomic_comp_swap:
4497          op32 = aco_opcode::buffer_atomic_cmpswap;
4498          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4499          break;
4500       default:
4501          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4502    }
4503    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4504    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4505    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4506    mubuf->operands[1] = Operand(rsrc);
4507    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4508    mubuf->operands[3] = Operand(data);
4509    if (return_previous)
4510       mubuf->definitions[0] = Definition(dst);
4511    mubuf->offset = 0;
4512    mubuf->offen = (offset.type() == RegType::vgpr);
4513    mubuf->glc = return_previous;
4514    mubuf->dlc = false; /* Not needed for atomics */
4515    mubuf->disable_wqm = true;
4516    mubuf->barrier = barrier_buffer;
4517    ctx->program->needs_exact = true;
4518    ctx->block->instructions.emplace_back(std::move(mubuf));
4519 }
4520
4521 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4522
4523    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4524    Builder bld(ctx->program, ctx->block);
4525    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4526    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4527 }
4528
4529 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4530 {
4531    Builder bld(ctx->program, ctx->block);
4532    unsigned num_components = instr->num_components;
4533    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4534
4535    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4536    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4537
4538    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4539    bool dlc = glc && ctx->options->chip_class >= GFX10;
4540    aco_opcode op;
4541    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4542       bool global = ctx->options->chip_class >= GFX9;
4543       aco_opcode op;
4544       switch (num_bytes) {
4545       case 4:
4546          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4547          break;
4548       case 8:
4549          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4550          break;
4551       case 12:
4552          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4553          break;
4554       case 16:
4555          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4556          break;
4557       default:
4558          unreachable("load_global not implemented for this size.");
4559       }
4560       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4561       flat->operands[0] = Operand(addr);
4562       flat->operands[1] = Operand(s1);
4563       flat->glc = glc;
4564       flat->dlc = dlc;
4565
4566       if (dst.type() == RegType::sgpr) {
4567          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4568          flat->definitions[0] = Definition(vec);
4569          ctx->block->instructions.emplace_back(std::move(flat));
4570          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4571       } else {
4572          flat->definitions[0] = Definition(dst);
4573          ctx->block->instructions.emplace_back(std::move(flat));
4574       }
4575       emit_split_vector(ctx, dst, num_components);
4576    } else {
4577       switch (num_bytes) {
4578          case 4:
4579             op = aco_opcode::s_load_dword;
4580             break;
4581          case 8:
4582             op = aco_opcode::s_load_dwordx2;
4583             break;
4584          case 12:
4585          case 16:
4586             op = aco_opcode::s_load_dwordx4;
4587             break;
4588          default:
4589             unreachable("load_global not implemented for this size.");
4590       }
4591       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4592       load->operands[0] = Operand(addr);
4593       load->operands[1] = Operand(0u);
4594       load->definitions[0] = Definition(dst);
4595       load->glc = glc;
4596       load->dlc = dlc;
4597       load->barrier = barrier_buffer;
4598       assert(ctx->options->chip_class >= GFX8 || !glc);
4599
4600       if (dst.size() == 3) {
4601          /* trim vector */
4602          Temp vec = bld.tmp(s4);
4603          load->definitions[0] = Definition(vec);
4604          ctx->block->instructions.emplace_back(std::move(load));
4605          emit_split_vector(ctx, vec, 4);
4606
4607          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4608                     emit_extract_vector(ctx, vec, 0, s1),
4609                     emit_extract_vector(ctx, vec, 1, s1),
4610                     emit_extract_vector(ctx, vec, 2, s1));
4611       } else {
4612          ctx->block->instructions.emplace_back(std::move(load));
4613       }
4614    }
4615 }
4616
4617 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4618 {
4619    Builder bld(ctx->program, ctx->block);
4620    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4621
4622    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4623    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4624
4625    unsigned writemask = nir_intrinsic_write_mask(instr);
4626    while (writemask) {
4627       int start, count;
4628       u_bit_scan_consecutive_range(&writemask, &start, &count);
4629       unsigned num_bytes = count * elem_size_bytes;
4630
4631       Temp write_data = data;
4632       if (count != instr->num_components) {
4633          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4634          for (int i = 0; i < count; i++)
4635             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4636          write_data = bld.tmp(RegType::vgpr, count);
4637          vec->definitions[0] = Definition(write_data);
4638          ctx->block->instructions.emplace_back(std::move(vec));
4639       }
4640
4641       unsigned offset = start * elem_size_bytes;
4642       if (offset > 0 && ctx->options->chip_class < GFX9) {
4643          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4644          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4645          Temp carry = bld.tmp(s2);
4646          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4647
4648          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4649                   Operand(offset), addr0);
4650          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4651                   Operand(0u), addr1,
4652                   carry).def(1).setHint(vcc);
4653
4654          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4655
4656          offset = 0;
4657       }
4658
4659       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4660       bool global = ctx->options->chip_class >= GFX9;
4661       aco_opcode op;
4662       switch (num_bytes) {
4663       case 4:
4664          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4665          break;
4666       case 8:
4667          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4668          break;
4669       case 12:
4670          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4671          break;
4672       case 16:
4673          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4674          break;
4675       default:
4676          unreachable("store_global not implemented for this size.");
4677       }
4678       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4679       flat->operands[0] = Operand(addr);
4680       flat->operands[1] = Operand(s1);
4681       flat->operands[2] = Operand(data);
4682       flat->glc = glc;
4683       flat->dlc = false;
4684       flat->offset = offset;
4685       ctx->block->instructions.emplace_back(std::move(flat));
4686    }
4687 }
4688
4689 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4690    Builder bld(ctx->program, ctx->block);
4691    switch(instr->intrinsic) {
4692       case nir_intrinsic_group_memory_barrier:
4693       case nir_intrinsic_memory_barrier:
4694          bld.barrier(aco_opcode::p_memory_barrier_all);
4695          break;
4696       case nir_intrinsic_memory_barrier_atomic_counter:
4697          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4698          break;
4699       case nir_intrinsic_memory_barrier_buffer:
4700          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4701          break;
4702       case nir_intrinsic_memory_barrier_image:
4703          bld.barrier(aco_opcode::p_memory_barrier_image);
4704          break;
4705       case nir_intrinsic_memory_barrier_shared:
4706          bld.barrier(aco_opcode::p_memory_barrier_shared);
4707          break;
4708       default:
4709          unreachable("Unimplemented memory barrier intrinsic");
4710          break;
4711    }
4712 }
4713
4714 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4715 {
4716    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4717    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4718    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4719    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4720    Builder bld(ctx->program, ctx->block);
4721
4722    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4723    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4724    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4725 }
4726
4727 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4728 {
4729    unsigned writemask = nir_intrinsic_write_mask(instr);
4730    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4731    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4732    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4733    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4734
4735    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4736    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4737 }
4738
4739 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4740 {
4741    unsigned offset = nir_intrinsic_base(instr);
4742    Operand m = load_lds_size_m0(ctx);
4743    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4744    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4745
4746    unsigned num_operands = 3;
4747    aco_opcode op32, op64, op32_rtn, op64_rtn;
4748    switch(instr->intrinsic) {
4749       case nir_intrinsic_shared_atomic_add:
4750          op32 = aco_opcode::ds_add_u32;
4751          op64 = aco_opcode::ds_add_u64;
4752          op32_rtn = aco_opcode::ds_add_rtn_u32;
4753          op64_rtn = aco_opcode::ds_add_rtn_u64;
4754          break;
4755       case nir_intrinsic_shared_atomic_imin:
4756          op32 = aco_opcode::ds_min_i32;
4757          op64 = aco_opcode::ds_min_i64;
4758          op32_rtn = aco_opcode::ds_min_rtn_i32;
4759          op64_rtn = aco_opcode::ds_min_rtn_i64;
4760          break;
4761       case nir_intrinsic_shared_atomic_umin:
4762          op32 = aco_opcode::ds_min_u32;
4763          op64 = aco_opcode::ds_min_u64;
4764          op32_rtn = aco_opcode::ds_min_rtn_u32;
4765          op64_rtn = aco_opcode::ds_min_rtn_u64;
4766          break;
4767       case nir_intrinsic_shared_atomic_imax:
4768          op32 = aco_opcode::ds_max_i32;
4769          op64 = aco_opcode::ds_max_i64;
4770          op32_rtn = aco_opcode::ds_max_rtn_i32;
4771          op64_rtn = aco_opcode::ds_max_rtn_i64;
4772          break;
4773       case nir_intrinsic_shared_atomic_umax:
4774          op32 = aco_opcode::ds_max_u32;
4775          op64 = aco_opcode::ds_max_u64;
4776          op32_rtn = aco_opcode::ds_max_rtn_u32;
4777          op64_rtn = aco_opcode::ds_max_rtn_u64;
4778          break;
4779       case nir_intrinsic_shared_atomic_and:
4780          op32 = aco_opcode::ds_and_b32;
4781          op64 = aco_opcode::ds_and_b64;
4782          op32_rtn = aco_opcode::ds_and_rtn_b32;
4783          op64_rtn = aco_opcode::ds_and_rtn_b64;
4784          break;
4785       case nir_intrinsic_shared_atomic_or:
4786          op32 = aco_opcode::ds_or_b32;
4787          op64 = aco_opcode::ds_or_b64;
4788          op32_rtn = aco_opcode::ds_or_rtn_b32;
4789          op64_rtn = aco_opcode::ds_or_rtn_b64;
4790          break;
4791       case nir_intrinsic_shared_atomic_xor:
4792          op32 = aco_opcode::ds_xor_b32;
4793          op64 = aco_opcode::ds_xor_b64;
4794          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4795          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4796          break;
4797       case nir_intrinsic_shared_atomic_exchange:
4798          op32 = aco_opcode::ds_write_b32;
4799          op64 = aco_opcode::ds_write_b64;
4800          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4801          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4802          break;
4803       case nir_intrinsic_shared_atomic_comp_swap:
4804          op32 = aco_opcode::ds_cmpst_b32;
4805          op64 = aco_opcode::ds_cmpst_b64;
4806          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4807          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4808          num_operands = 4;
4809          break;
4810       default:
4811          unreachable("Unhandled shared atomic intrinsic");
4812    }
4813
4814    /* return the previous value if dest is ever used */
4815    bool return_previous = false;
4816    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4817       return_previous = true;
4818       break;
4819    }
4820    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4821       return_previous = true;
4822       break;
4823    }
4824
4825    aco_opcode op;
4826    if (data.size() == 1) {
4827       assert(instr->dest.ssa.bit_size == 32);
4828       op = return_previous ? op32_rtn : op32;
4829    } else {
4830       assert(instr->dest.ssa.bit_size == 64);
4831       op = return_previous ? op64_rtn : op64;
4832    }
4833
4834    if (offset > 65535) {
4835       Builder bld(ctx->program, ctx->block);
4836       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4837       offset = 0;
4838    }
4839
4840    aco_ptr<DS_instruction> ds;
4841    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4842    ds->operands[0] = Operand(address);
4843    ds->operands[1] = Operand(data);
4844    if (num_operands == 4)
4845       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4846    ds->operands[num_operands - 1] = m;
4847    ds->offset0 = offset;
4848    if (return_previous)
4849       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4850    ctx->block->instructions.emplace_back(std::move(ds));
4851 }
4852
4853 Temp get_scratch_resource(isel_context *ctx)
4854 {
4855    Builder bld(ctx->program, ctx->block);
4856    Temp scratch_addr = ctx->program->private_segment_buffer;
4857    if (ctx->stage != compute_cs)
4858       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
4859
4860    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
4861                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
4862
4863    if (ctx->program->chip_class >= GFX10) {
4864       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
4865                    S_008F0C_OOB_SELECT(3) |
4866                    S_008F0C_RESOURCE_LEVEL(1);
4867    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
4868       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
4869                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
4870    }
4871
4872    /* older generations need element size = 16 bytes. element size removed in GFX9 */
4873    if (ctx->program->chip_class <= GFX8)
4874       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
4875
4876    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4877 }
4878
4879 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4880    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4881    Builder bld(ctx->program, ctx->block);
4882    Temp rsrc = get_scratch_resource(ctx);
4883    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4884    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4885
4886    aco_opcode op;
4887    switch (dst.size()) {
4888       case 1:
4889          op = aco_opcode::buffer_load_dword;
4890          break;
4891       case 2:
4892          op = aco_opcode::buffer_load_dwordx2;
4893          break;
4894       case 3:
4895          op = aco_opcode::buffer_load_dwordx3;
4896          break;
4897       case 4:
4898          op = aco_opcode::buffer_load_dwordx4;
4899          break;
4900       case 6:
4901       case 8: {
4902          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4903          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4904                                 bld.def(v4), offset, rsrc,
4905                                 ctx->program->scratch_offset, 0, true);
4906          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4907                                                   aco_opcode::buffer_load_dwordx4,
4908                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4909                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
4910          emit_split_vector(ctx, lower, 2);
4911          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4912          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4913          if (dst.size() == 8) {
4914             emit_split_vector(ctx, upper, 2);
4915             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4916             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4917          } else {
4918             elems[2] = upper;
4919          }
4920
4921          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4922                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4923          for (unsigned i = 0; i < dst.size() / 2; i++)
4924             vec->operands[i] = Operand(elems[i]);
4925          vec->definitions[0] = Definition(dst);
4926          bld.insert(std::move(vec));
4927          ctx->allocated_vec.emplace(dst.id(), elems);
4928          return;
4929       }
4930       default:
4931          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
4932    }
4933
4934    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
4935    emit_split_vector(ctx, dst, instr->num_components);
4936 }
4937
4938 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4939    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
4940    Builder bld(ctx->program, ctx->block);
4941    Temp rsrc = get_scratch_resource(ctx);
4942    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4943    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4944
4945    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4946    unsigned writemask = nir_intrinsic_write_mask(instr);
4947
4948    while (writemask) {
4949       int start, count;
4950       u_bit_scan_consecutive_range(&writemask, &start, &count);
4951       int num_bytes = count * elem_size_bytes;
4952
4953       if (num_bytes > 16) {
4954          assert(elem_size_bytes == 8);
4955          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4956          count = 2;
4957          num_bytes = 16;
4958       }
4959
4960       // TODO: check alignment of sub-dword stores
4961       // TODO: split 3 bytes. there is no store instruction for that
4962
4963       Temp write_data;
4964       if (count != instr->num_components) {
4965          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4966          for (int i = 0; i < count; i++) {
4967             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
4968             vec->operands[i] = Operand(elem);
4969          }
4970          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
4971          vec->definitions[0] = Definition(write_data);
4972          ctx->block->instructions.emplace_back(std::move(vec));
4973       } else {
4974          write_data = data;
4975       }
4976
4977       aco_opcode op;
4978       switch (num_bytes) {
4979          case 4:
4980             op = aco_opcode::buffer_store_dword;
4981             break;
4982          case 8:
4983             op = aco_opcode::buffer_store_dwordx2;
4984             break;
4985          case 12:
4986             op = aco_opcode::buffer_store_dwordx3;
4987             break;
4988          case 16:
4989             op = aco_opcode::buffer_store_dwordx4;
4990             break;
4991          default:
4992             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
4993       }
4994
4995       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
4996    }
4997 }
4998
4999 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5000    uint8_t log2_ps_iter_samples;
5001    if (ctx->program->info->ps.force_persample) {
5002       log2_ps_iter_samples =
5003          util_logbase2(ctx->options->key.fs.num_samples);
5004    } else {
5005       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5006    }
5007
5008    /* The bit pattern matches that used by fixed function fragment
5009     * processing. */
5010    static const unsigned ps_iter_masks[] = {
5011       0xffff, /* not used */
5012       0x5555,
5013       0x1111,
5014       0x0101,
5015       0x0001,
5016    };
5017    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5018
5019    Builder bld(ctx->program, ctx->block);
5020
5021    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
5022    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5023    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5024    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5025    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
5026 }
5027
5028 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5029 {
5030    Builder bld(ctx->program, ctx->block);
5031
5032    if (cluster_size == 1) {
5033       return src;
5034    } if (op == nir_op_iand && cluster_size == 4) {
5035       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5036       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5037       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
5038                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
5039    } else if (op == nir_op_ior && cluster_size == 4) {
5040       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5041       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
5042                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
5043    } else if (op == nir_op_iand && cluster_size == 64) {
5044       //subgroupAnd(val) -> (exec & ~val) == 0
5045       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5046       return bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), Operand(0u), Operand(-1u), bld.scc(tmp));
5047    } else if (op == nir_op_ior && cluster_size == 64) {
5048       //subgroupOr(val) -> (val & exec) != 0
5049       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
5050       return bool_to_vector_condition(ctx, tmp);
5051    } else if (op == nir_op_ixor && cluster_size == 64) {
5052       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5053       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5054       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
5055       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5056       return bool_to_vector_condition(ctx, tmp);
5057    } else {
5058       //subgroupClustered{And,Or,Xor}(val, n) ->
5059       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5060       //cluster_offset = ~(n - 1) & lane_id
5061       //cluster_mask = ((1 << n) - 1)
5062       //subgroupClusteredAnd():
5063       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5064       //subgroupClusteredOr():
5065       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5066       //subgroupClusteredXor():
5067       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5068       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5069                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5070       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5071
5072       Temp tmp;
5073       if (op == nir_op_iand)
5074          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5075       else
5076          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5077
5078       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5079       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5080       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5081       if (cluster_mask != 0xffffffff)
5082          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5083
5084       Definition cmp_def = Definition();
5085       if (op == nir_op_iand) {
5086          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5087       } else if (op == nir_op_ior) {
5088          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5089       } else if (op == nir_op_ixor) {
5090          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5091                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5092          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5093       }
5094       cmp_def.setHint(vcc);
5095       return cmp_def.getTemp();
5096    }
5097 }
5098
5099 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5100 {
5101    Builder bld(ctx->program, ctx->block);
5102
5103    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5104    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5105    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5106    Temp tmp;
5107    if (op == nir_op_iand)
5108       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5109    else
5110       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5111
5112    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5113    Temp lo = lohi.def(0).getTemp();
5114    Temp hi = lohi.def(1).getTemp();
5115    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5116                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5117
5118    Definition cmp_def = Definition();
5119    if (op == nir_op_iand)
5120       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5121    else if (op == nir_op_ior)
5122       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5123    else if (op == nir_op_ixor)
5124       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5125                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5126    cmp_def.setHint(vcc);
5127    return cmp_def.getTemp();
5128 }
5129
5130 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5131 {
5132    Builder bld(ctx->program, ctx->block);
5133
5134    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5135    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5136    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5137    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5138    if (op == nir_op_iand)
5139       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5140    else if (op == nir_op_ior)
5141       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5142    else if (op == nir_op_ixor)
5143       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5144
5145    assert(false);
5146    return Temp();
5147 }
5148
5149 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5150 {
5151    Builder bld(ctx->program, ctx->block);
5152    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5153    if (src.regClass().type() == RegType::vgpr) {
5154       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5155    } else if (src.regClass() == s1) {
5156       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5157    } else if (src.regClass() == s2) {
5158       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5159    } else {
5160       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5161       nir_print_instr(&instr->instr, stderr);
5162       fprintf(stderr, "\n");
5163    }
5164 }
5165
5166 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5167 {
5168    Builder bld(ctx->program, ctx->block);
5169    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5170    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5171
5172    /* Build DD X/Y */
5173    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5174    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5175    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5176    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5177    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5178    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5179
5180    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5181    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5182    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5183    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5184    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5185    Temp wqm1 = bld.tmp(v1);
5186    emit_wqm(ctx, tmp1, wqm1, true);
5187    Temp wqm2 = bld.tmp(v1);
5188    emit_wqm(ctx, tmp2, wqm2, true);
5189    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5190    return;
5191 }
5192
5193 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5194 {
5195    Builder bld(ctx->program, ctx->block);
5196    switch(instr->intrinsic) {
5197    case nir_intrinsic_load_barycentric_sample:
5198    case nir_intrinsic_load_barycentric_pixel:
5199    case nir_intrinsic_load_barycentric_centroid: {
5200       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5201       fs_input input = get_interp_input(instr->intrinsic, mode);
5202
5203       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5204       if (input == fs_input::max_inputs) {
5205          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5206                     Operand(0u), Operand(0u));
5207       } else {
5208          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5209                     ctx->fs_inputs[input],
5210                     ctx->fs_inputs[input + 1]);
5211       }
5212       emit_split_vector(ctx, dst, 2);
5213       break;
5214    }
5215    case nir_intrinsic_load_barycentric_at_sample: {
5216       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5217       switch (ctx->options->key.fs.num_samples) {
5218          case 2: sample_pos_offset += 1 << 3; break;
5219          case 4: sample_pos_offset += 3 << 3; break;
5220          case 8: sample_pos_offset += 7 << 3; break;
5221          default: break;
5222       }
5223       Temp sample_pos;
5224       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5225       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5226       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5227       if (addr.type() == RegType::sgpr) {
5228          Operand offset;
5229          if (const_addr) {
5230             sample_pos_offset += const_addr->u32 << 3;
5231             offset = Operand(sample_pos_offset);
5232          } else if (ctx->options->chip_class >= GFX9) {
5233             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5234          } else {
5235             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5236             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5237          }
5238          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5239
5240       } else if (ctx->options->chip_class >= GFX9) {
5241          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5242          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5243       } else {
5244          /* addr += private_segment_buffer + sample_pos_offset */
5245          Temp tmp0 = bld.tmp(s1);
5246          Temp tmp1 = bld.tmp(s1);
5247          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5248          Definition scc_tmp = bld.def(s1, scc);
5249          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5250          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5251          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5252          Temp pck0 = bld.tmp(v1);
5253          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5254          tmp1 = as_vgpr(ctx, tmp1);
5255          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5256          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5257
5258          /* sample_pos = flat_load_dwordx2 addr */
5259          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5260       }
5261
5262       /* sample_pos -= 0.5 */
5263       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5264       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5265       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5266       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5267       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5268
5269       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5270       break;
5271    }
5272    case nir_intrinsic_load_barycentric_at_offset: {
5273       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5274       RegClass rc = RegClass(offset.type(), 1);
5275       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5276       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5277       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5278       break;
5279    }
5280    case nir_intrinsic_load_front_face: {
5281       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5282                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5283       break;
5284    }
5285    case nir_intrinsic_load_view_index:
5286    case nir_intrinsic_load_layer_id: {
5287       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5288          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5289          bld.copy(Definition(dst), Operand(ctx->view_index));
5290          break;
5291       }
5292
5293       unsigned idx = nir_intrinsic_base(instr);
5294       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5295                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5296       break;
5297    }
5298    case nir_intrinsic_load_frag_coord: {
5299       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5300       break;
5301    }
5302    case nir_intrinsic_load_sample_pos: {
5303       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5304       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5305       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5306                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5307                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5308       break;
5309    }
5310    case nir_intrinsic_load_interpolated_input:
5311       visit_load_interpolated_input(ctx, instr);
5312       break;
5313    case nir_intrinsic_store_output:
5314       visit_store_output(ctx, instr);
5315       break;
5316    case nir_intrinsic_load_input:
5317       visit_load_input(ctx, instr);
5318       break;
5319    case nir_intrinsic_load_ubo:
5320       visit_load_ubo(ctx, instr);
5321       break;
5322    case nir_intrinsic_load_push_constant:
5323       visit_load_push_constant(ctx, instr);
5324       break;
5325    case nir_intrinsic_load_constant:
5326       visit_load_constant(ctx, instr);
5327       break;
5328    case nir_intrinsic_vulkan_resource_index:
5329       visit_load_resource(ctx, instr);
5330       break;
5331    case nir_intrinsic_discard:
5332       visit_discard(ctx, instr);
5333       break;
5334    case nir_intrinsic_discard_if:
5335       visit_discard_if(ctx, instr);
5336       break;
5337    case nir_intrinsic_load_shared:
5338       visit_load_shared(ctx, instr);
5339       break;
5340    case nir_intrinsic_store_shared:
5341       visit_store_shared(ctx, instr);
5342       break;
5343    case nir_intrinsic_shared_atomic_add:
5344    case nir_intrinsic_shared_atomic_imin:
5345    case nir_intrinsic_shared_atomic_umin:
5346    case nir_intrinsic_shared_atomic_imax:
5347    case nir_intrinsic_shared_atomic_umax:
5348    case nir_intrinsic_shared_atomic_and:
5349    case nir_intrinsic_shared_atomic_or:
5350    case nir_intrinsic_shared_atomic_xor:
5351    case nir_intrinsic_shared_atomic_exchange:
5352    case nir_intrinsic_shared_atomic_comp_swap:
5353       visit_shared_atomic(ctx, instr);
5354       break;
5355    case nir_intrinsic_image_deref_load:
5356       visit_image_load(ctx, instr);
5357       break;
5358    case nir_intrinsic_image_deref_store:
5359       visit_image_store(ctx, instr);
5360       break;
5361    case nir_intrinsic_image_deref_atomic_add:
5362    case nir_intrinsic_image_deref_atomic_umin:
5363    case nir_intrinsic_image_deref_atomic_imin:
5364    case nir_intrinsic_image_deref_atomic_umax:
5365    case nir_intrinsic_image_deref_atomic_imax:
5366    case nir_intrinsic_image_deref_atomic_and:
5367    case nir_intrinsic_image_deref_atomic_or:
5368    case nir_intrinsic_image_deref_atomic_xor:
5369    case nir_intrinsic_image_deref_atomic_exchange:
5370    case nir_intrinsic_image_deref_atomic_comp_swap:
5371       visit_image_atomic(ctx, instr);
5372       break;
5373    case nir_intrinsic_image_deref_size:
5374       visit_image_size(ctx, instr);
5375       break;
5376    case nir_intrinsic_load_ssbo:
5377       visit_load_ssbo(ctx, instr);
5378       break;
5379    case nir_intrinsic_store_ssbo:
5380       visit_store_ssbo(ctx, instr);
5381       break;
5382    case nir_intrinsic_load_global:
5383       visit_load_global(ctx, instr);
5384       break;
5385    case nir_intrinsic_store_global:
5386       visit_store_global(ctx, instr);
5387       break;
5388    case nir_intrinsic_ssbo_atomic_add:
5389    case nir_intrinsic_ssbo_atomic_imin:
5390    case nir_intrinsic_ssbo_atomic_umin:
5391    case nir_intrinsic_ssbo_atomic_imax:
5392    case nir_intrinsic_ssbo_atomic_umax:
5393    case nir_intrinsic_ssbo_atomic_and:
5394    case nir_intrinsic_ssbo_atomic_or:
5395    case nir_intrinsic_ssbo_atomic_xor:
5396    case nir_intrinsic_ssbo_atomic_exchange:
5397    case nir_intrinsic_ssbo_atomic_comp_swap:
5398       visit_atomic_ssbo(ctx, instr);
5399       break;
5400    case nir_intrinsic_load_scratch:
5401       visit_load_scratch(ctx, instr);
5402       break;
5403    case nir_intrinsic_store_scratch:
5404       visit_store_scratch(ctx, instr);
5405       break;
5406    case nir_intrinsic_get_buffer_size:
5407       visit_get_buffer_size(ctx, instr);
5408       break;
5409    case nir_intrinsic_barrier: {
5410       unsigned* bsize = ctx->program->info->cs.block_size;
5411       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5412       if (workgroup_size > 64)
5413          bld.sopp(aco_opcode::s_barrier);
5414       break;
5415    }
5416    case nir_intrinsic_group_memory_barrier:
5417    case nir_intrinsic_memory_barrier:
5418    case nir_intrinsic_memory_barrier_atomic_counter:
5419    case nir_intrinsic_memory_barrier_buffer:
5420    case nir_intrinsic_memory_barrier_image:
5421    case nir_intrinsic_memory_barrier_shared:
5422       emit_memory_barrier(ctx, instr);
5423       break;
5424    case nir_intrinsic_load_num_work_groups:
5425    case nir_intrinsic_load_work_group_id:
5426    case nir_intrinsic_load_local_invocation_id: {
5427       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5428       Temp* ids;
5429       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5430          ids = ctx->num_workgroups;
5431       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5432          ids = ctx->workgroup_ids;
5433       else
5434          ids = ctx->local_invocation_ids;
5435       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5436                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5437                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5438                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5439       emit_split_vector(ctx, dst, 3);
5440       break;
5441    }
5442    case nir_intrinsic_load_local_invocation_index: {
5443       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5444                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5445       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5446       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5447       break;
5448    }
5449    case nir_intrinsic_load_subgroup_id: {
5450       if (ctx->stage == compute_cs) {
5451          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5452          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5453       } else {
5454          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5455       }
5456       break;
5457    }
5458    case nir_intrinsic_load_subgroup_invocation: {
5459       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5460                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5461       break;
5462    }
5463    case nir_intrinsic_load_num_subgroups: {
5464       if (ctx->stage == compute_cs)
5465          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5466       else
5467          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5468       break;
5469    }
5470    case nir_intrinsic_ballot: {
5471       Definition tmp = bld.def(s2);
5472       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5473       if (instr->src[0].ssa->bit_size == 1) {
5474          assert(src.regClass() == s2);
5475          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5476       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5477          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5478       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5479          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5480       } else {
5481          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5482          nir_print_instr(&instr->instr, stderr);
5483          fprintf(stderr, "\n");
5484       }
5485       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5486       break;
5487    }
5488    case nir_intrinsic_shuffle:
5489    case nir_intrinsic_read_invocation: {
5490       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5491       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5492          emit_uniform_subgroup(ctx, instr, src);
5493       } else {
5494          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5495          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5496             tid = bld.as_uniform(tid);
5497          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5498          if (src.regClass() == v1) {
5499             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5500          } else if (src.regClass() == v2) {
5501             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5502             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5503             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5504             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5505             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5506             emit_split_vector(ctx, dst, 2);
5507          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5508             assert(src.regClass() == s2);
5509             Temp tmp = bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, tid);
5510             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5511          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5512             assert(src.regClass() == s2);
5513             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5514             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5515             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5516             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5517          } else {
5518             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5519             nir_print_instr(&instr->instr, stderr);
5520             fprintf(stderr, "\n");
5521          }
5522       }
5523       break;
5524    }
5525    case nir_intrinsic_load_sample_id: {
5526       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5527                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5528       break;
5529    }
5530    case nir_intrinsic_load_sample_mask_in: {
5531       visit_load_sample_mask_in(ctx, instr);
5532       break;
5533    }
5534    case nir_intrinsic_read_first_invocation: {
5535       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5536       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5537       if (src.regClass() == v1) {
5538          emit_wqm(ctx,
5539                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5540                   dst);
5541       } else if (src.regClass() == v2) {
5542          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5543          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5544          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5545          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5546          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5547          emit_split_vector(ctx, dst, 2);
5548       } else if (instr->dest.ssa.bit_size == 1) {
5549          assert(src.regClass() == s2);
5550          Temp tmp = bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5551                              bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)));
5552          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5553       } else if (src.regClass() == s1) {
5554          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5555       } else if (src.regClass() == s2) {
5556          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5557       } else {
5558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5559          nir_print_instr(&instr->instr, stderr);
5560          fprintf(stderr, "\n");
5561       }
5562       break;
5563    }
5564    case nir_intrinsic_vote_all: {
5565       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5566       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5567       assert(src.regClass() == s2);
5568       assert(dst.regClass() == s2);
5569
5570       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5571       Temp val = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), Operand(0u), Operand(-1u), bld.scc(tmp));
5572       emit_wqm(ctx, val, dst);
5573       break;
5574    }
5575    case nir_intrinsic_vote_any: {
5576       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5577       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5578       assert(src.regClass() == s2);
5579       assert(dst.regClass() == s2);
5580
5581       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5582       Temp val = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), Operand(-1u), Operand(0u), bld.scc(tmp));
5583       emit_wqm(ctx, val, dst);
5584       break;
5585    }
5586    case nir_intrinsic_reduce:
5587    case nir_intrinsic_inclusive_scan:
5588    case nir_intrinsic_exclusive_scan: {
5589       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5590       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5591       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5592       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5593          nir_intrinsic_cluster_size(instr) : 0;
5594       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5595
5596       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5597          emit_uniform_subgroup(ctx, instr, src);
5598       } else if (instr->dest.ssa.bit_size == 1) {
5599          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5600             op = nir_op_iand;
5601          else if (op == nir_op_iadd)
5602             op = nir_op_ixor;
5603          else if (op == nir_op_umax || op == nir_op_imax)
5604             op = nir_op_ior;
5605          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5606
5607          switch (instr->intrinsic) {
5608          case nir_intrinsic_reduce:
5609             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5610             break;
5611          case nir_intrinsic_exclusive_scan:
5612             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5613             break;
5614          case nir_intrinsic_inclusive_scan:
5615             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5616             break;
5617          default:
5618             assert(false);
5619          }
5620       } else if (cluster_size == 1) {
5621          bld.copy(Definition(dst), src);
5622       } else {
5623          src = as_vgpr(ctx, src);
5624
5625          ReduceOp reduce_op;
5626          switch (op) {
5627          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5628             CASE(iadd)
5629             CASE(imul)
5630             CASE(fadd)
5631             CASE(fmul)
5632             CASE(imin)
5633             CASE(umin)
5634             CASE(fmin)
5635             CASE(imax)
5636             CASE(umax)
5637             CASE(fmax)
5638             CASE(iand)
5639             CASE(ior)
5640             CASE(ixor)
5641             default:
5642                unreachable("unknown reduction op");
5643          #undef CASE
5644          }
5645
5646          aco_opcode aco_op;
5647          switch (instr->intrinsic) {
5648             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5649             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5650             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5651             default:
5652                unreachable("unknown reduce intrinsic");
5653          }
5654
5655          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5656          reduce->operands[0] = Operand(src);
5657          // filled in by aco_reduce_assign.cpp, used internally as part of the
5658          // reduce sequence
5659          assert(dst.size() == 1 || dst.size() == 2);
5660          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5661          reduce->operands[2] = Operand(v1.as_linear());
5662
5663          Temp tmp_dst = bld.tmp(dst.regClass());
5664          reduce->definitions[0] = Definition(tmp_dst);
5665          reduce->definitions[1] = bld.def(s2); // used internally
5666          reduce->definitions[2] = Definition();
5667          reduce->definitions[3] = Definition(scc, s1);
5668          reduce->definitions[4] = Definition();
5669          reduce->reduce_op = reduce_op;
5670          reduce->cluster_size = cluster_size;
5671          ctx->block->instructions.emplace_back(std::move(reduce));
5672
5673          emit_wqm(ctx, tmp_dst, dst);
5674       }
5675       break;
5676    }
5677    case nir_intrinsic_quad_broadcast: {
5678       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5679       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5680          emit_uniform_subgroup(ctx, instr, src);
5681       } else {
5682          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5683          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5684          if (instr->dest.ssa.bit_size == 1) {
5685             assert(src.regClass() == s2);
5686             uint32_t half_mask = 0x11111111u << lane;
5687             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5688             Temp tmp = bld.tmp(s2);
5689             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5690                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5691                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5692             emit_wqm(ctx, tmp, dst);
5693          } else if (instr->dest.ssa.bit_size == 32) {
5694             emit_wqm(ctx,
5695                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5696                                   dpp_quad_perm(lane, lane, lane, lane)),
5697                      dst);
5698          } else if (instr->dest.ssa.bit_size == 64) {
5699             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5700             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5701             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5702             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5703             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5704             emit_split_vector(ctx, dst, 2);
5705          } else {
5706             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5707             nir_print_instr(&instr->instr, stderr);
5708             fprintf(stderr, "\n");
5709          }
5710       }
5711       break;
5712    }
5713    case nir_intrinsic_quad_swap_horizontal:
5714    case nir_intrinsic_quad_swap_vertical:
5715    case nir_intrinsic_quad_swap_diagonal:
5716    case nir_intrinsic_quad_swizzle_amd: {
5717       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5718       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5719          emit_uniform_subgroup(ctx, instr, src);
5720          break;
5721       }
5722       uint16_t dpp_ctrl = 0;
5723       switch (instr->intrinsic) {
5724       case nir_intrinsic_quad_swap_horizontal:
5725          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5726          break;
5727       case nir_intrinsic_quad_swap_vertical:
5728          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5729          break;
5730       case nir_intrinsic_quad_swap_diagonal:
5731          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5732          break;
5733       case nir_intrinsic_quad_swizzle_amd: {
5734          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5735          break;
5736       }
5737       default:
5738          break;
5739       }
5740
5741       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5742       if (instr->dest.ssa.bit_size == 1) {
5743          assert(src.regClass() == s2);
5744          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5745          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5746          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5747          emit_wqm(ctx, tmp, dst);
5748       } else if (instr->dest.ssa.bit_size == 32) {
5749          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5750          emit_wqm(ctx, tmp, dst);
5751       } else if (instr->dest.ssa.bit_size == 64) {
5752          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5753          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5754          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5755          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5756          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5757          emit_split_vector(ctx, dst, 2);
5758       } else {
5759          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5760          nir_print_instr(&instr->instr, stderr);
5761          fprintf(stderr, "\n");
5762       }
5763       break;
5764    }
5765    case nir_intrinsic_masked_swizzle_amd: {
5766       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5767       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5768          emit_uniform_subgroup(ctx, instr, src);
5769          break;
5770       }
5771       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5772       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5773       if (dst.regClass() == v1) {
5774          emit_wqm(ctx,
5775                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5776                   dst);
5777       } else if (dst.regClass() == v2) {
5778          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5779          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5780          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5781          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5782          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5783          emit_split_vector(ctx, dst, 2);
5784       } else {
5785          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5786          nir_print_instr(&instr->instr, stderr);
5787          fprintf(stderr, "\n");
5788       }
5789       break;
5790    }
5791    case nir_intrinsic_write_invocation_amd: {
5792       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5793       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5794       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5795       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5796       if (dst.regClass() == v1) {
5797          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5798          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5799       } else if (dst.regClass() == v2) {
5800          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5801          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5802          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5803          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5804          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5805          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5806          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5807          emit_split_vector(ctx, dst, 2);
5808       } else {
5809          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5810          nir_print_instr(&instr->instr, stderr);
5811          fprintf(stderr, "\n");
5812       }
5813       break;
5814    }
5815    case nir_intrinsic_mbcnt_amd: {
5816       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5817       RegClass rc = RegClass(src.type(), 1);
5818       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5819       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5820       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5821       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5822       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5823       emit_wqm(ctx, wqm_tmp, dst);
5824       break;
5825    }
5826    case nir_intrinsic_load_helper_invocation: {
5827       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5828       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5829       ctx->block->kind |= block_kind_needs_lowering;
5830       ctx->program->needs_exact = true;
5831       break;
5832    }
5833    case nir_intrinsic_is_helper_invocation: {
5834       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5835       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5836       ctx->block->kind |= block_kind_needs_lowering;
5837       ctx->program->needs_exact = true;
5838       break;
5839    }
5840    case nir_intrinsic_demote:
5841       bld.pseudo(aco_opcode::p_demote_to_helper);
5842       ctx->block->kind |= block_kind_uses_demote;
5843       ctx->program->needs_exact = true;
5844       break;
5845    case nir_intrinsic_demote_if: {
5846       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5847       assert(src.regClass() == s2);
5848       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5849       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5850       ctx->block->kind |= block_kind_uses_demote;
5851       ctx->program->needs_exact = true;
5852       break;
5853    }
5854    case nir_intrinsic_first_invocation: {
5855       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5856                get_ssa_temp(ctx, &instr->dest.ssa));
5857       break;
5858    }
5859    case nir_intrinsic_shader_clock:
5860       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
5861       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
5862       break;
5863    case nir_intrinsic_load_vertex_id_zero_base: {
5864       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5865       bld.copy(Definition(dst), ctx->vertex_id);
5866       break;
5867    }
5868    case nir_intrinsic_load_first_vertex: {
5869       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5870       bld.copy(Definition(dst), ctx->base_vertex);
5871       break;
5872    }
5873    case nir_intrinsic_load_base_instance: {
5874       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5875       bld.copy(Definition(dst), ctx->start_instance);
5876       break;
5877    }
5878    case nir_intrinsic_load_instance_id: {
5879       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5880       bld.copy(Definition(dst), ctx->instance_id);
5881       break;
5882    }
5883    case nir_intrinsic_load_draw_id: {
5884       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5885       bld.copy(Definition(dst), ctx->draw_id);
5886       break;
5887    }
5888    default:
5889       fprintf(stderr, "Unimplemented intrinsic instr: ");
5890       nir_print_instr(&instr->instr, stderr);
5891       fprintf(stderr, "\n");
5892       abort();
5893
5894       break;
5895    }
5896 }
5897
5898
5899 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5900                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5901                     enum glsl_base_type *stype)
5902 {
5903    nir_deref_instr *texture_deref_instr = NULL;
5904    nir_deref_instr *sampler_deref_instr = NULL;
5905    int plane = -1;
5906
5907    for (unsigned i = 0; i < instr->num_srcs; i++) {
5908       switch (instr->src[i].src_type) {
5909       case nir_tex_src_texture_deref:
5910          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
5911          break;
5912       case nir_tex_src_sampler_deref:
5913          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
5914          break;
5915       case nir_tex_src_plane:
5916          plane = nir_src_as_int(instr->src[i].src);
5917          break;
5918       default:
5919          break;
5920       }
5921    }
5922
5923    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
5924
5925    if (!sampler_deref_instr)
5926       sampler_deref_instr = texture_deref_instr;
5927
5928    if (plane >= 0) {
5929       assert(instr->op != nir_texop_txf_ms &&
5930              instr->op != nir_texop_samples_identical);
5931       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
5932       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
5933    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
5934       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
5935    } else {
5936       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
5937    }
5938    if (samp_ptr) {
5939       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
5940       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
5941          fprintf(stderr, "Unimplemented sampler descriptor: ");
5942          nir_print_instr(&instr->instr, stderr);
5943          fprintf(stderr, "\n");
5944          abort();
5945          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
5946       }
5947    }
5948    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
5949                      instr->op == nir_texop_samples_identical))
5950       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
5951 }
5952
5953 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
5954                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
5955 {
5956    Builder bld(ctx->program, ctx->block);
5957
5958    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
5959    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
5960    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
5961
5962    Operand neg_one(0xbf800000u);
5963    Operand one(0x3f800000u);
5964    Operand two(0x40000000u);
5965    Operand four(0x40800000u);
5966
5967    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
5968    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
5969    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
5970
5971    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
5972    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
5973    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
5974    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
5975
5976    // select sc
5977    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
5978    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
5979                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
5980                        one, is_ma_y);
5981    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5982
5983    // select tc
5984    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
5985    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
5986    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5987
5988    // select ma
5989    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
5990                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
5991                   deriv_z, is_ma_z);
5992    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
5993    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
5994 }
5995
5996 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
5997 {
5998    Builder bld(ctx->program, ctx->block);
5999    Temp coord_args[4], ma, tc, sc, id;
6000    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6001       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6002
6003    if (is_array) {
6004       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6005
6006       // see comment in ac_prepare_cube_coords()
6007       if (ctx->options->chip_class <= GFX8)
6008          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6009    }
6010
6011    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6012
6013    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6014    vop3a->operands[0] = Operand(ma);
6015    vop3a->abs[0] = true;
6016    Temp invma = bld.tmp(v1);
6017    vop3a->definitions[0] = Definition(invma);
6018    ctx->block->instructions.emplace_back(std::move(vop3a));
6019
6020    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6021    if (!is_deriv)
6022       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6023
6024    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6025    if (!is_deriv)
6026       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6027
6028    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6029
6030    if (is_deriv) {
6031       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6032       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6033
6034       for (unsigned i = 0; i < 2; i++) {
6035          // see comment in ac_prepare_cube_coords()
6036          Temp deriv_ma;
6037          Temp deriv_sc, deriv_tc;
6038          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6039                            &deriv_ma, &deriv_sc, &deriv_tc);
6040
6041          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6042
6043          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6044                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6045                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6046          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6047                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6048                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6049          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6050       }
6051
6052       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6053       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6054    }
6055
6056    if (is_array)
6057       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6058    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6059
6060 }
6061
6062 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6063 {
6064    Temp coord_vec[3];
6065    for (unsigned i = 0; i < coords.size(); i++)
6066       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6067
6068    Builder bld(ctx->program, ctx->block);
6069    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6070
6071    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6072    for (unsigned i = 0; i < coords.size(); i++)
6073       vec->operands[i] = Operand(coord_vec[i]);
6074    Temp res = bld.tmp(RegType::vgpr, coords.size());
6075    vec->definitions[0] = Definition(res);
6076    ctx->block->instructions.emplace_back(std::move(vec));
6077    return res;
6078 }
6079
6080 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6081 {
6082    if (vec->parent_instr->type != nir_instr_type_alu)
6083       return;
6084    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6085    if (vec_instr->op != nir_op_vec(vec->num_components))
6086       return;
6087
6088    for (unsigned i = 0; i < vec->num_components; i++) {
6089       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6090               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6091    }
6092 }
6093
6094 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6095 {
6096    Builder bld(ctx->program, ctx->block);
6097    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6098         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6099    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6100         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6101    nir_const_value *sample_index_cv = NULL;
6102    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6103    enum glsl_base_type stype;
6104    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6105
6106    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6107                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6108    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6109                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6110
6111    for (unsigned i = 0; i < instr->num_srcs; i++) {
6112       switch (instr->src[i].src_type) {
6113       case nir_tex_src_coord:
6114          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6115          break;
6116       case nir_tex_src_bias:
6117          if (instr->op == nir_texop_txb) {
6118             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6119             has_bias = true;
6120          }
6121          break;
6122       case nir_tex_src_lod: {
6123          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6124
6125          if (val && val->f32 <= 0.0) {
6126             level_zero = true;
6127          } else {
6128             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6129             has_lod = true;
6130          }
6131          break;
6132       }
6133       case nir_tex_src_comparator:
6134          if (instr->is_shadow) {
6135             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6136             has_compare = true;
6137          }
6138          break;
6139       case nir_tex_src_offset:
6140          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6141          get_const_vec(instr->src[i].src.ssa, const_offset);
6142          has_offset = true;
6143          break;
6144       case nir_tex_src_ddx:
6145          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6146          has_ddx = true;
6147          break;
6148       case nir_tex_src_ddy:
6149          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6150          has_ddy = true;
6151          break;
6152       case nir_tex_src_ms_index:
6153          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6154          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6155          has_sample_index = true;
6156          break;
6157       case nir_tex_src_texture_offset:
6158       case nir_tex_src_sampler_offset:
6159       default:
6160          break;
6161       }
6162    }
6163 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6164    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6165       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6166
6167    if (instr->op == nir_texop_texture_samples) {
6168       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6169
6170       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6171       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6172       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6173       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6174
6175       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6176                samples, Operand(1u), bld.scc(is_msaa));
6177       return;
6178    }
6179
6180    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6181       aco_ptr<Instruction> tmp_instr;
6182       Temp acc, pack = Temp();
6183
6184       uint32_t pack_const = 0;
6185       for (unsigned i = 0; i < offset.size(); i++) {
6186          if (!const_offset[i])
6187             continue;
6188          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6189       }
6190
6191       if (offset.type() == RegType::sgpr) {
6192          for (unsigned i = 0; i < offset.size(); i++) {
6193             if (const_offset[i])
6194                continue;
6195
6196             acc = emit_extract_vector(ctx, offset, i, s1);
6197             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6198
6199             if (i) {
6200                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6201             }
6202
6203             if (pack == Temp()) {
6204                pack = acc;
6205             } else {
6206                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6207             }
6208          }
6209
6210          if (pack_const && pack != Temp())
6211             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6212       } else {
6213          for (unsigned i = 0; i < offset.size(); i++) {
6214             if (const_offset[i])
6215                continue;
6216
6217             acc = emit_extract_vector(ctx, offset, i, v1);
6218             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6219
6220             if (i) {
6221                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6222             }
6223
6224             if (pack == Temp()) {
6225                pack = acc;
6226             } else {
6227                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6228             }
6229          }
6230
6231          if (pack_const && pack != Temp())
6232             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6233       }
6234       if (pack_const && pack == Temp())
6235          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6236       else if (pack == Temp())
6237          has_offset = false;
6238       else
6239          offset = pack;
6240    }
6241
6242    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6243       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6244
6245    /* pack derivatives */
6246    if (has_ddx || has_ddy) {
6247       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6248          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6249                              ddx, Operand(0u), ddy, Operand(0u));
6250       } else {
6251          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6252       }
6253       has_derivs = true;
6254    }
6255
6256    if (instr->coord_components > 1 &&
6257        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6258        instr->is_array &&
6259        instr->op != nir_texop_txf)
6260       coords = apply_round_slice(ctx, coords, 1);
6261
6262    if (instr->coord_components > 2 &&
6263       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6264        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6265        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6266        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6267        instr->is_array &&
6268        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6269       coords = apply_round_slice(ctx, coords, 2);
6270
6271    if (ctx->options->chip_class == GFX9 &&
6272        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6273        instr->op != nir_texop_lod && instr->coord_components) {
6274       assert(coords.size() > 0 && coords.size() < 3);
6275
6276       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6277       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6278       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6279       if (coords.size() > 1)
6280          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6281       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6282       vec->definitions[0] = Definition(coords);
6283       ctx->block->instructions.emplace_back(std::move(vec));
6284    }
6285
6286    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6287
6288    if (instr->op == nir_texop_samples_identical)
6289       resource = fmask_ptr;
6290
6291    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6292              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6293             instr->op != nir_texop_txs) {
6294       assert(has_sample_index);
6295       Operand op(sample_index);
6296       if (sample_index_cv)
6297          op = Operand(sample_index_cv->u32);
6298       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6299    }
6300
6301    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6302       Temp split_coords[coords.size()];
6303       emit_split_vector(ctx, coords, coords.size());
6304       for (unsigned i = 0; i < coords.size(); i++)
6305          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6306
6307       unsigned i = 0;
6308       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6309          Temp off = emit_extract_vector(ctx, offset, i, v1);
6310          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6311       }
6312
6313       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6314       for (unsigned i = 0; i < coords.size(); i++)
6315          vec->operands[i] = Operand(split_coords[i]);
6316       coords = bld.tmp(coords.regClass());
6317       vec->definitions[0] = Definition(coords);
6318       ctx->block->instructions.emplace_back(std::move(vec));
6319
6320       has_offset = false;
6321    }
6322
6323    /* Build tex instruction */
6324    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6325    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6326                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6327                   : 0;
6328    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6329    Temp tmp_dst = dst;
6330
6331    /* gather4 selects the component by dmask and always returns vec4 */
6332    if (instr->op == nir_texop_tg4) {
6333       assert(instr->dest.ssa.num_components == 4);
6334       if (instr->is_shadow)
6335          dmask = 1;
6336       else
6337          dmask = 1 << instr->component;
6338       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6339          tmp_dst = bld.tmp(v4);
6340    } else if (instr->op == nir_texop_samples_identical) {
6341       tmp_dst = bld.tmp(v1);
6342    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6343       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6344    }
6345
6346    aco_ptr<MIMG_instruction> tex;
6347    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6348       if (!has_lod)
6349          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6350
6351       bool div_by_6 = instr->op == nir_texop_txs &&
6352                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6353                       instr->is_array &&
6354                       (dmask & (1 << 2));
6355       if (tmp_dst.id() == dst.id() && div_by_6)
6356          tmp_dst = bld.tmp(tmp_dst.regClass());
6357
6358       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6359       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6360       tex->operands[1] = Operand(resource);
6361       if (ctx->options->chip_class == GFX9 &&
6362           instr->op == nir_texop_txs &&
6363           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6364           instr->is_array) {
6365          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6366       } else if (instr->op == nir_texop_query_levels) {
6367          tex->dmask = 1 << 3;
6368       } else {
6369          tex->dmask = dmask;
6370       }
6371       tex->da = da;
6372       tex->definitions[0] = Definition(tmp_dst);
6373       tex->dim = dim;
6374       tex->can_reorder = true;
6375       ctx->block->instructions.emplace_back(std::move(tex));
6376
6377       if (div_by_6) {
6378          /* divide 3rd value by 6 by multiplying with magic number */
6379          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6380          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6381          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6382          assert(instr->dest.ssa.num_components == 3);
6383          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6384          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6385                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6386                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6387                               by_6);
6388
6389       }
6390
6391       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6392       return;
6393    }
6394
6395    Temp tg4_compare_cube_wa64 = Temp();
6396
6397    if (tg4_integer_workarounds) {
6398       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6399       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6400       tex->operands[1] = Operand(resource);
6401       tex->dim = dim;
6402       tex->dmask = 0x3;
6403       tex->da = da;
6404       Temp size = bld.tmp(v2);
6405       tex->definitions[0] = Definition(size);
6406       tex->can_reorder = true;
6407       ctx->block->instructions.emplace_back(std::move(tex));
6408       emit_split_vector(ctx, size, size.size());
6409
6410       Temp half_texel[2];
6411       for (unsigned i = 0; i < 2; i++) {
6412          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6413          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6414          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6415          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6416       }
6417
6418       Temp orig_coords[2] = {
6419          emit_extract_vector(ctx, coords, 0, v1),
6420          emit_extract_vector(ctx, coords, 1, v1)};
6421       Temp new_coords[2] = {
6422          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6423          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6424       };
6425
6426       if (tg4_integer_cube_workaround) {
6427          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6428          Temp desc[resource.size()];
6429          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6430                                                                            Format::PSEUDO, 1, resource.size())};
6431          split->operands[0] = Operand(resource);
6432          for (unsigned i = 0; i < resource.size(); i++) {
6433             desc[i] = bld.tmp(s1);
6434             split->definitions[i] = Definition(desc[i]);
6435          }
6436          ctx->block->instructions.emplace_back(std::move(split));
6437
6438          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6439          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6440                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6441
6442          Temp nfmt;
6443          if (stype == GLSL_TYPE_UINT) {
6444             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6445                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6446                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6447                             bld.scc(compare_cube_wa));
6448          } else {
6449             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6450                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6451                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6452                             bld.scc(compare_cube_wa));
6453          }
6454          tg4_compare_cube_wa64 = bld.tmp(s2);
6455          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6456
6457          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6458
6459          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6460                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6461          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6462
6463          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6464                                                                          Format::PSEUDO, resource.size(), 1)};
6465          for (unsigned i = 0; i < resource.size(); i++)
6466             vec->operands[i] = Operand(desc[i]);
6467          resource = bld.tmp(resource.regClass());
6468          vec->definitions[0] = Definition(resource);
6469          ctx->block->instructions.emplace_back(std::move(vec));
6470
6471          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6472                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6473          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6474                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6475       }
6476
6477       if (coords.size() == 3) {
6478          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6479                              new_coords[0], new_coords[1],
6480                              emit_extract_vector(ctx, coords, 2, v1));
6481       } else {
6482          assert(coords.size() == 2);
6483          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6484                              new_coords[0], new_coords[1]);
6485       }
6486    }
6487
6488    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6489        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6490        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6491       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6492
6493    std::vector<Operand> args;
6494    if (has_offset)
6495       args.emplace_back(Operand(offset));
6496    if (has_bias)
6497       args.emplace_back(Operand(bias));
6498    if (has_compare)
6499       args.emplace_back(Operand(compare));
6500    if (has_derivs)
6501       args.emplace_back(Operand(derivs));
6502    args.emplace_back(Operand(coords));
6503    if (has_sample_index)
6504       args.emplace_back(Operand(sample_index));
6505    if (has_lod)
6506       args.emplace_back(lod);
6507
6508    Operand arg;
6509    if (args.size() > 1) {
6510       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6511       unsigned size = 0;
6512       for (unsigned i = 0; i < args.size(); i++) {
6513          size += args[i].size();
6514          vec->operands[i] = args[i];
6515       }
6516       RegClass rc = RegClass(RegType::vgpr, size);
6517       Temp tmp = bld.tmp(rc);
6518       vec->definitions[0] = Definition(tmp);
6519       ctx->block->instructions.emplace_back(std::move(vec));
6520       arg = Operand(tmp);
6521    } else {
6522       assert(args[0].isTemp());
6523       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6524    }
6525
6526    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6527       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6528
6529       assert(coords.size() == 1);
6530       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6531       aco_opcode op;
6532       switch (last_bit) {
6533       case 1:
6534          op = aco_opcode::buffer_load_format_x; break;
6535       case 2:
6536          op = aco_opcode::buffer_load_format_xy; break;
6537       case 3:
6538          op = aco_opcode::buffer_load_format_xyz; break;
6539       case 4:
6540          op = aco_opcode::buffer_load_format_xyzw; break;
6541       default:
6542          unreachable("Tex instruction loads more than 4 components.");
6543       }
6544
6545       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6546       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6547          tmp_dst = dst;
6548       else
6549          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6550
6551       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6552       mubuf->operands[0] = Operand(coords);
6553       mubuf->operands[1] = Operand(resource);
6554       mubuf->operands[2] = Operand((uint32_t) 0);
6555       mubuf->definitions[0] = Definition(tmp_dst);
6556       mubuf->idxen = true;
6557       mubuf->can_reorder = true;
6558       ctx->block->instructions.emplace_back(std::move(mubuf));
6559
6560       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6561       return;
6562    }
6563
6564
6565    if (instr->op == nir_texop_txf ||
6566        instr->op == nir_texop_txf_ms ||
6567        instr->op == nir_texop_samples_identical) {
6568       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6569       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6570       tex->operands[0] = Operand(arg);
6571       tex->operands[1] = Operand(resource);
6572       tex->dim = dim;
6573       tex->dmask = dmask;
6574       tex->unrm = true;
6575       tex->da = da;
6576       tex->definitions[0] = Definition(tmp_dst);
6577       tex->can_reorder = true;
6578       ctx->block->instructions.emplace_back(std::move(tex));
6579
6580       if (instr->op == nir_texop_samples_identical) {
6581          assert(dmask == 1 && dst.regClass() == v1);
6582          assert(dst.id() != tmp_dst.id());
6583
6584          Temp tmp = bld.tmp(s2);
6585          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6586          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6587
6588       } else {
6589          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6590       }
6591       return;
6592    }
6593
6594    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6595    aco_opcode opcode = aco_opcode::image_sample;
6596    if (has_offset) { /* image_sample_*_o */
6597       if (has_compare) {
6598          opcode = aco_opcode::image_sample_c_o;
6599          if (has_derivs)
6600             opcode = aco_opcode::image_sample_c_d_o;
6601          if (has_bias)
6602             opcode = aco_opcode::image_sample_c_b_o;
6603          if (level_zero)
6604             opcode = aco_opcode::image_sample_c_lz_o;
6605          if (has_lod)
6606             opcode = aco_opcode::image_sample_c_l_o;
6607       } else {
6608          opcode = aco_opcode::image_sample_o;
6609          if (has_derivs)
6610             opcode = aco_opcode::image_sample_d_o;
6611          if (has_bias)
6612             opcode = aco_opcode::image_sample_b_o;
6613          if (level_zero)
6614             opcode = aco_opcode::image_sample_lz_o;
6615          if (has_lod)
6616             opcode = aco_opcode::image_sample_l_o;
6617       }
6618    } else { /* no offset */
6619       if (has_compare) {
6620          opcode = aco_opcode::image_sample_c;
6621          if (has_derivs)
6622             opcode = aco_opcode::image_sample_c_d;
6623          if (has_bias)
6624             opcode = aco_opcode::image_sample_c_b;
6625          if (level_zero)
6626             opcode = aco_opcode::image_sample_c_lz;
6627          if (has_lod)
6628             opcode = aco_opcode::image_sample_c_l;
6629       } else {
6630          opcode = aco_opcode::image_sample;
6631          if (has_derivs)
6632             opcode = aco_opcode::image_sample_d;
6633          if (has_bias)
6634             opcode = aco_opcode::image_sample_b;
6635          if (level_zero)
6636             opcode = aco_opcode::image_sample_lz;
6637          if (has_lod)
6638             opcode = aco_opcode::image_sample_l;
6639       }
6640    }
6641
6642    if (instr->op == nir_texop_tg4) {
6643       if (has_offset) {
6644          opcode = aco_opcode::image_gather4_lz_o;
6645          if (has_compare)
6646             opcode = aco_opcode::image_gather4_c_lz_o;
6647       } else {
6648          opcode = aco_opcode::image_gather4_lz;
6649          if (has_compare)
6650             opcode = aco_opcode::image_gather4_c_lz;
6651       }
6652    } else if (instr->op == nir_texop_lod) {
6653       opcode = aco_opcode::image_get_lod;
6654    }
6655
6656    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6657    tex->operands[0] = arg;
6658    tex->operands[1] = Operand(resource);
6659    tex->operands[2] = Operand(sampler);
6660    tex->dim = dim;
6661    tex->dmask = dmask;
6662    tex->da = da;
6663    tex->definitions[0] = Definition(tmp_dst);
6664    tex->can_reorder = true;
6665    ctx->block->instructions.emplace_back(std::move(tex));
6666
6667    if (tg4_integer_cube_workaround) {
6668       assert(tmp_dst.id() != dst.id());
6669       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6670
6671       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6672       Temp val[4];
6673       for (unsigned i = 0; i < dst.size(); i++) {
6674          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6675          Temp cvt_val;
6676          if (stype == GLSL_TYPE_UINT)
6677             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6678          else
6679             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6680          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6681       }
6682       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6683       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6684                            val[0], val[1], val[2], val[3]);
6685    }
6686    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6687    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6688
6689 }
6690
6691
6692 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6693 {
6694    Temp tmp = get_ssa_temp(ctx, ssa);
6695    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6696       return Operand(tmp.regClass());
6697    else
6698       return Operand(tmp);
6699 }
6700
6701 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6702 {
6703    aco_ptr<Pseudo_instruction> phi;
6704    unsigned num_src = exec_list_length(&instr->srcs);
6705    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6706    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == s2);
6707
6708    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6709
6710    std::map<unsigned, nir_ssa_def*> phi_src;
6711    bool all_undef = true;
6712    nir_foreach_phi_src(src, instr) {
6713       phi_src[src->pred->index] = src->src.ssa;
6714       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6715          all_undef = false;
6716    }
6717    if (all_undef) {
6718       Builder bld(ctx->program, ctx->block);
6719       if (dst.regClass() == s1) {
6720          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6721       } else if (dst.regClass() == v1) {
6722          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6723       } else {
6724          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6725          for (unsigned i = 0; i < dst.size(); i++)
6726             vec->operands[i] = Operand(0u);
6727          vec->definitions[0] = Definition(dst);
6728          ctx->block->instructions.emplace_back(std::move(vec));
6729       }
6730       return;
6731    }
6732
6733    /* try to scalarize vector phis */
6734    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
6735       // TODO: scalarize linear phis on divergent ifs
6736       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6737       std::array<Temp, 4> new_vec;
6738       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6739          Operand src = get_phi_operand(ctx, pair.second);
6740          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6741             can_scalarize = false;
6742             break;
6743          }
6744       }
6745       if (can_scalarize) {
6746          unsigned num_components = instr->dest.ssa.num_components;
6747          assert(dst.size() % num_components == 0);
6748          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6749
6750          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6751          for (unsigned k = 0; k < num_components; k++) {
6752             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6753             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6754             for (unsigned i = 0; i < num_src; i++) {
6755                Operand src = get_phi_operand(ctx, it->second);
6756                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6757                ++it;
6758             }
6759             Temp phi_dst = {ctx->program->allocateId(), rc};
6760             phi->definitions[0] = Definition(phi_dst);
6761             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6762             new_vec[k] = phi_dst;
6763             vec->operands[k] = Operand(phi_dst);
6764          }
6765          vec->definitions[0] = Definition(dst);
6766          ctx->block->instructions.emplace_back(std::move(vec));
6767          ctx->allocated_vec.emplace(dst.id(), new_vec);
6768          return;
6769       }
6770    }
6771
6772    unsigned extra_src = 0;
6773    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6774        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6775       extra_src++;
6776    }
6777
6778    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6779
6780    /* if we have a linear phi on a divergent if, we know that one src is undef */
6781    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6782       assert(extra_src == 0);
6783       Block* block;
6784       /* we place the phi either in the invert-block or in the current block */
6785       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6786          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6787          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6788          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6789          assert(block->kind & block_kind_invert);
6790          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6791       } else {
6792          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6793          block = ctx->block;
6794          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6795       }
6796       phi->operands[1] = Operand(dst.regClass());
6797       phi->definitions[0] = Definition(dst);
6798       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6799       return;
6800    }
6801
6802    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6803    for (unsigned i = 0; i < num_src; i++) {
6804       phi->operands[i] = get_phi_operand(ctx, it->second);
6805       ++it;
6806    }
6807    for (unsigned i = 0; i < extra_src; i++)
6808       phi->operands[num_src + i] = Operand(dst.regClass());
6809    phi->definitions[0] = Definition(dst);
6810    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6811 }
6812
6813
6814 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6815 {
6816    Temp dst = get_ssa_temp(ctx, &instr->def);
6817
6818    assert(dst.type() == RegType::sgpr);
6819
6820    if (dst.size() == 1) {
6821       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6822    } else {
6823       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6824       for (unsigned i = 0; i < dst.size(); i++)
6825          vec->operands[i] = Operand(0u);
6826       vec->definitions[0] = Definition(dst);
6827       ctx->block->instructions.emplace_back(std::move(vec));
6828    }
6829 }
6830
6831 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6832 {
6833    Builder bld(ctx->program, ctx->block);
6834    Block *logical_target;
6835    append_logical_end(ctx->block);
6836    unsigned idx = ctx->block->index;
6837
6838    switch (instr->type) {
6839    case nir_jump_break:
6840       logical_target = ctx->cf_info.parent_loop.exit;
6841       add_logical_edge(idx, logical_target);
6842       ctx->block->kind |= block_kind_break;
6843
6844       if (!ctx->cf_info.parent_if.is_divergent &&
6845           !ctx->cf_info.parent_loop.has_divergent_continue) {
6846          /* uniform break - directly jump out of the loop */
6847          ctx->block->kind |= block_kind_uniform;
6848          ctx->cf_info.has_branch = true;
6849          bld.branch(aco_opcode::p_branch);
6850          add_linear_edge(idx, logical_target);
6851          return;
6852       }
6853       ctx->cf_info.parent_loop.has_divergent_branch = true;
6854       break;
6855    case nir_jump_continue:
6856       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6857       add_logical_edge(idx, logical_target);
6858       ctx->block->kind |= block_kind_continue;
6859
6860       if (ctx->cf_info.parent_if.is_divergent) {
6861          /* for potential uniform breaks after this continue,
6862             we must ensure that they are handled correctly */
6863          ctx->cf_info.parent_loop.has_divergent_continue = true;
6864          ctx->cf_info.parent_loop.has_divergent_branch = true;
6865       } else {
6866          /* uniform continue - directly jump to the loop header */
6867          ctx->block->kind |= block_kind_uniform;
6868          ctx->cf_info.has_branch = true;
6869          bld.branch(aco_opcode::p_branch);
6870          add_linear_edge(idx, logical_target);
6871          return;
6872       }
6873       break;
6874    default:
6875       fprintf(stderr, "Unknown NIR jump instr: ");
6876       nir_print_instr(&instr->instr, stderr);
6877       fprintf(stderr, "\n");
6878       abort();
6879    }
6880
6881    /* remove critical edges from linear CFG */
6882    bld.branch(aco_opcode::p_branch);
6883    Block* break_block = ctx->program->create_and_insert_block();
6884    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6885    break_block->kind |= block_kind_uniform;
6886    add_linear_edge(idx, break_block);
6887    /* the loop_header pointer might be invalidated by this point */
6888    if (instr->type == nir_jump_continue)
6889       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6890    add_linear_edge(break_block->index, logical_target);
6891    bld.reset(break_block);
6892    bld.branch(aco_opcode::p_branch);
6893
6894    Block* continue_block = ctx->program->create_and_insert_block();
6895    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6896    add_linear_edge(idx, continue_block);
6897    append_logical_start(continue_block);
6898    ctx->block = continue_block;
6899    return;
6900 }
6901
6902 void visit_block(isel_context *ctx, nir_block *block)
6903 {
6904    nir_foreach_instr(instr, block) {
6905       switch (instr->type) {
6906       case nir_instr_type_alu:
6907          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6908          break;
6909       case nir_instr_type_load_const:
6910          visit_load_const(ctx, nir_instr_as_load_const(instr));
6911          break;
6912       case nir_instr_type_intrinsic:
6913          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
6914          break;
6915       case nir_instr_type_tex:
6916          visit_tex(ctx, nir_instr_as_tex(instr));
6917          break;
6918       case nir_instr_type_phi:
6919          visit_phi(ctx, nir_instr_as_phi(instr));
6920          break;
6921       case nir_instr_type_ssa_undef:
6922          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
6923          break;
6924       case nir_instr_type_deref:
6925          break;
6926       case nir_instr_type_jump:
6927          visit_jump(ctx, nir_instr_as_jump(instr));
6928          break;
6929       default:
6930          fprintf(stderr, "Unknown NIR instr type: ");
6931          nir_print_instr(instr, stderr);
6932          fprintf(stderr, "\n");
6933          //abort();
6934       }
6935    }
6936 }
6937
6938
6939
6940 static void visit_loop(isel_context *ctx, nir_loop *loop)
6941 {
6942    append_logical_end(ctx->block);
6943    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
6944    Builder bld(ctx->program, ctx->block);
6945    bld.branch(aco_opcode::p_branch);
6946    unsigned loop_preheader_idx = ctx->block->index;
6947
6948    Block loop_exit = Block();
6949    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
6950    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
6951
6952    Block* loop_header = ctx->program->create_and_insert_block();
6953    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
6954    loop_header->kind |= block_kind_loop_header;
6955    add_edge(loop_preheader_idx, loop_header);
6956    ctx->block = loop_header;
6957
6958    /* emit loop body */
6959    unsigned loop_header_idx = loop_header->index;
6960    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
6961    append_logical_start(ctx->block);
6962    visit_cf_list(ctx, &loop->body);
6963
6964    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
6965    if (!ctx->cf_info.has_branch) {
6966       append_logical_end(ctx->block);
6967       if (ctx->cf_info.exec_potentially_empty) {
6968          /* Discards can result in code running with an empty exec mask.
6969           * This would result in divergent breaks not ever being taken. As a
6970           * workaround, break the loop when the loop mask is empty instead of
6971           * always continuing. */
6972          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
6973          unsigned block_idx = ctx->block->index;
6974
6975          /* create helper blocks to avoid critical edges */
6976          Block *break_block = ctx->program->create_and_insert_block();
6977          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6978          break_block->kind = block_kind_uniform;
6979          bld.reset(break_block);
6980          bld.branch(aco_opcode::p_branch);
6981          add_linear_edge(block_idx, break_block);
6982          add_linear_edge(break_block->index, &loop_exit);
6983
6984          Block *continue_block = ctx->program->create_and_insert_block();
6985          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6986          continue_block->kind = block_kind_uniform;
6987          bld.reset(continue_block);
6988          bld.branch(aco_opcode::p_branch);
6989          add_linear_edge(block_idx, continue_block);
6990          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
6991
6992          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
6993          ctx->block = &ctx->program->blocks[block_idx];
6994       } else {
6995          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
6996          if (!ctx->cf_info.parent_loop.has_divergent_branch)
6997             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6998          else
6999             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7000       }
7001
7002       bld.reset(ctx->block);
7003       bld.branch(aco_opcode::p_branch);
7004    }
7005
7006    /* fixup phis in loop header from unreachable blocks */
7007    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7008       bool linear = ctx->cf_info.has_branch;
7009       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7010       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7011          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7012              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7013             /* the last operand should be the one that needs to be removed */
7014             instr->operands.pop_back();
7015          } else if (!is_phi(instr)) {
7016             break;
7017          }
7018       }
7019    }
7020
7021    ctx->cf_info.has_branch = false;
7022
7023    // TODO: if the loop has not a single exit, we must add one °°
7024    /* emit loop successor block */
7025    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7026    append_logical_start(ctx->block);
7027
7028    #if 0
7029    // TODO: check if it is beneficial to not branch on continues
7030    /* trim linear phis in loop header */
7031    for (auto&& instr : loop_entry->instructions) {
7032       if (instr->opcode == aco_opcode::p_linear_phi) {
7033          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7034          new_phi->definitions[0] = instr->definitions[0];
7035          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7036             new_phi->operands[i] = instr->operands[i];
7037          /* check that the remaining operands are all the same */
7038          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7039             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7040          instr.swap(new_phi);
7041       } else if (instr->opcode == aco_opcode::p_phi) {
7042          continue;
7043       } else {
7044          break;
7045       }
7046    }
7047    #endif
7048 }
7049
7050 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7051 {
7052    ic->cond = cond;
7053
7054    append_logical_end(ctx->block);
7055    ctx->block->kind |= block_kind_branch;
7056
7057    /* branch to linear then block */
7058    assert(cond.regClass() == s2);
7059    aco_ptr<Pseudo_branch_instruction> branch;
7060    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7061    branch->operands[0] = Operand(cond);
7062    ctx->block->instructions.push_back(std::move(branch));
7063
7064    ic->BB_if_idx = ctx->block->index;
7065    ic->BB_invert = Block();
7066    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7067    /* Invert blocks are intentionally not marked as top level because they
7068     * are not part of the logical cfg. */
7069    ic->BB_invert.kind |= block_kind_invert;
7070    ic->BB_endif = Block();
7071    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7072    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7073
7074    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7075    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7076    ctx->cf_info.parent_if.is_divergent = true;
7077    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7078
7079    /** emit logical then block */
7080    Block* BB_then_logical = ctx->program->create_and_insert_block();
7081    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7082    add_edge(ic->BB_if_idx, BB_then_logical);
7083    ctx->block = BB_then_logical;
7084    append_logical_start(BB_then_logical);
7085 }
7086
7087 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7088 {
7089    Block *BB_then_logical = ctx->block;
7090    append_logical_end(BB_then_logical);
7091     /* branch from logical then block to invert block */
7092    aco_ptr<Pseudo_branch_instruction> branch;
7093    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7094    BB_then_logical->instructions.emplace_back(std::move(branch));
7095    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7096    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7097       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7098    BB_then_logical->kind |= block_kind_uniform;
7099    assert(!ctx->cf_info.has_branch);
7100    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7101    ctx->cf_info.parent_loop.has_divergent_branch = false;
7102
7103    /** emit linear then block */
7104    Block* BB_then_linear = ctx->program->create_and_insert_block();
7105    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7106    BB_then_linear->kind |= block_kind_uniform;
7107    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7108    /* branch from linear then block to invert block */
7109    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7110    BB_then_linear->instructions.emplace_back(std::move(branch));
7111    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7112
7113    /** emit invert merge block */
7114    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7115    ic->invert_idx = ctx->block->index;
7116
7117    /* branch to linear else block (skip else) */
7118    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7119    branch->operands[0] = Operand(ic->cond);
7120    ctx->block->instructions.push_back(std::move(branch));
7121
7122    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7123    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7124
7125    /** emit logical else block */
7126    Block* BB_else_logical = ctx->program->create_and_insert_block();
7127    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7128    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7129    add_linear_edge(ic->invert_idx, BB_else_logical);
7130    ctx->block = BB_else_logical;
7131    append_logical_start(BB_else_logical);
7132 }
7133
7134 static void end_divergent_if(isel_context *ctx, if_context *ic)
7135 {
7136    Block *BB_else_logical = ctx->block;
7137    append_logical_end(BB_else_logical);
7138
7139    /* branch from logical else block to endif block */
7140    aco_ptr<Pseudo_branch_instruction> branch;
7141    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7142    BB_else_logical->instructions.emplace_back(std::move(branch));
7143    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7144    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7145       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7146    BB_else_logical->kind |= block_kind_uniform;
7147
7148    assert(!ctx->cf_info.has_branch);
7149    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7150
7151
7152    /** emit linear else block */
7153    Block* BB_else_linear = ctx->program->create_and_insert_block();
7154    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7155    BB_else_linear->kind |= block_kind_uniform;
7156    add_linear_edge(ic->invert_idx, BB_else_linear);
7157
7158    /* branch from linear else block to endif block */
7159    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7160    BB_else_linear->instructions.emplace_back(std::move(branch));
7161    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7162
7163
7164    /** emit endif merge block */
7165    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7166    append_logical_start(ctx->block);
7167
7168
7169    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7170    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7171    /* uniform control flow never has an empty exec-mask */
7172    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7173       ctx->cf_info.exec_potentially_empty = false;
7174 }
7175
7176 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7177 {
7178    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7179    Builder bld(ctx->program, ctx->block);
7180    aco_ptr<Pseudo_branch_instruction> branch;
7181
7182    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7183       /**
7184        * Uniform conditionals are represented in the following way*) :
7185        *
7186        * The linear and logical CFG:
7187        *                        BB_IF
7188        *                        /    \
7189        *       BB_THEN (logical)      BB_ELSE (logical)
7190        *                        \    /
7191        *                        BB_ENDIF
7192        *
7193        * *) Exceptions may be due to break and continue statements within loops
7194        *    If a break/continue happens within uniform control flow, it branches
7195        *    to the loop exit/entry block. Otherwise, it branches to the next
7196        *    merge block.
7197        **/
7198       append_logical_end(ctx->block);
7199       ctx->block->kind |= block_kind_uniform;
7200
7201       /* emit branch */
7202       assert(cond.regClass() == s2);
7203       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7204       cond = bool_to_scalar_condition(ctx, cond);
7205
7206       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7207       branch->operands[0] = Operand(cond);
7208       branch->operands[0].setFixed(scc);
7209       ctx->block->instructions.emplace_back(std::move(branch));
7210
7211       unsigned BB_if_idx = ctx->block->index;
7212       Block BB_endif = Block();
7213       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7214       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7215
7216       /** emit then block */
7217       Block* BB_then = ctx->program->create_and_insert_block();
7218       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7219       add_edge(BB_if_idx, BB_then);
7220       append_logical_start(BB_then);
7221       ctx->block = BB_then;
7222       visit_cf_list(ctx, &if_stmt->then_list);
7223       BB_then = ctx->block;
7224       bool then_branch = ctx->cf_info.has_branch;
7225       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7226
7227       if (!then_branch) {
7228          append_logical_end(BB_then);
7229          /* branch from then block to endif block */
7230          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7231          BB_then->instructions.emplace_back(std::move(branch));
7232          add_linear_edge(BB_then->index, &BB_endif);
7233          if (!then_branch_divergent)
7234             add_logical_edge(BB_then->index, &BB_endif);
7235          BB_then->kind |= block_kind_uniform;
7236       }
7237
7238       ctx->cf_info.has_branch = false;
7239       ctx->cf_info.parent_loop.has_divergent_branch = false;
7240
7241       /** emit else block */
7242       Block* BB_else = ctx->program->create_and_insert_block();
7243       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7244       add_edge(BB_if_idx, BB_else);
7245       append_logical_start(BB_else);
7246       ctx->block = BB_else;
7247       visit_cf_list(ctx, &if_stmt->else_list);
7248       BB_else = ctx->block;
7249
7250       if (!ctx->cf_info.has_branch) {
7251          append_logical_end(BB_else);
7252          /* branch from then block to endif block */
7253          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7254          BB_else->instructions.emplace_back(std::move(branch));
7255          add_linear_edge(BB_else->index, &BB_endif);
7256          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7257             add_logical_edge(BB_else->index, &BB_endif);
7258          BB_else->kind |= block_kind_uniform;
7259       }
7260
7261       ctx->cf_info.has_branch &= then_branch;
7262       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7263
7264       /** emit endif merge block */
7265       if (!ctx->cf_info.has_branch) {
7266          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7267          append_logical_start(ctx->block);
7268       }
7269    } else { /* non-uniform condition */
7270       /**
7271        * To maintain a logical and linear CFG without critical edges,
7272        * non-uniform conditionals are represented in the following way*) :
7273        *
7274        * The linear CFG:
7275        *                        BB_IF
7276        *                        /    \
7277        *       BB_THEN (logical)      BB_THEN (linear)
7278        *                        \    /
7279        *                        BB_INVERT (linear)
7280        *                        /    \
7281        *       BB_ELSE (logical)      BB_ELSE (linear)
7282        *                        \    /
7283        *                        BB_ENDIF
7284        *
7285        * The logical CFG:
7286        *                        BB_IF
7287        *                        /    \
7288        *       BB_THEN (logical)      BB_ELSE (logical)
7289        *                        \    /
7290        *                        BB_ENDIF
7291        *
7292        * *) Exceptions may be due to break and continue statements within loops
7293        **/
7294
7295       if_context ic;
7296
7297       begin_divergent_if_then(ctx, &ic, cond);
7298       visit_cf_list(ctx, &if_stmt->then_list);
7299
7300       begin_divergent_if_else(ctx, &ic);
7301       visit_cf_list(ctx, &if_stmt->else_list);
7302
7303       end_divergent_if(ctx, &ic);
7304    }
7305 }
7306
7307 static void visit_cf_list(isel_context *ctx,
7308                           struct exec_list *list)
7309 {
7310    foreach_list_typed(nir_cf_node, node, node, list) {
7311       switch (node->type) {
7312       case nir_cf_node_block:
7313          visit_block(ctx, nir_cf_node_as_block(node));
7314          break;
7315       case nir_cf_node_if:
7316          visit_if(ctx, nir_cf_node_as_if(node));
7317          break;
7318       case nir_cf_node_loop:
7319          visit_loop(ctx, nir_cf_node_as_loop(node));
7320          break;
7321       default:
7322          unreachable("unimplemented cf list type");
7323       }
7324    }
7325 }
7326
7327 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7328 {
7329    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7330    uint64_t mask = ctx->vs_output.mask[slot];
7331    if (!is_pos && !mask)
7332       return;
7333    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7334       return;
7335    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7336    exp->enabled_mask = mask;
7337    for (unsigned i = 0; i < 4; ++i) {
7338       if (mask & (1 << i))
7339          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7340       else
7341          exp->operands[i] = Operand(v1);
7342    }
7343    exp->valid_mask = false;
7344    exp->done = false;
7345    exp->compressed = false;
7346    if (is_pos)
7347       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7348    else
7349       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7350    ctx->block->instructions.emplace_back(std::move(exp));
7351 }
7352
7353 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7354 {
7355    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7356    exp->enabled_mask = 0;
7357    for (unsigned i = 0; i < 4; ++i)
7358       exp->operands[i] = Operand(v1);
7359    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7360       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7361       exp->enabled_mask |= 0x1;
7362    }
7363    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7364       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7365       exp->enabled_mask |= 0x4;
7366    }
7367    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7368       if (ctx->options->chip_class < GFX9) {
7369          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7370          exp->enabled_mask |= 0x8;
7371       } else {
7372          Builder bld(ctx->program, ctx->block);
7373
7374          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7375                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7376          if (exp->operands[2].isTemp())
7377             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7378
7379          exp->operands[2] = Operand(out);
7380          exp->enabled_mask |= 0x4;
7381       }
7382    }
7383    exp->valid_mask = false;
7384    exp->done = false;
7385    exp->compressed = false;
7386    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7387    ctx->block->instructions.emplace_back(std::move(exp));
7388 }
7389
7390 static void create_vs_exports(isel_context *ctx)
7391 {
7392    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7393
7394    if (outinfo->export_prim_id) {
7395       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7396       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7397    }
7398
7399    if (ctx->options->key.has_multiview_view_index) {
7400       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7401       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7402    }
7403
7404    /* the order these position exports are created is important */
7405    int next_pos = 0;
7406    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7407    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7408       export_vs_psiz_layer_viewport(ctx, &next_pos);
7409    }
7410    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7411       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7412    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7413       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7414
7415    if (ctx->options->key.vs_common_out.export_clip_dists) {
7416       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7417          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7418       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7419          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7420    }
7421
7422    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7423       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7424           i != VARYING_SLOT_PRIMITIVE_ID)
7425          continue;
7426
7427       export_vs_varying(ctx, i, false, NULL);
7428    }
7429 }
7430
7431 static void emit_stream_output(isel_context *ctx,
7432                                Temp const *so_buffers,
7433                                Temp const *so_write_offset,
7434                                const struct radv_stream_output *output)
7435 {
7436    unsigned num_comps = util_bitcount(output->component_mask);
7437    unsigned loc = output->location;
7438    unsigned buf = output->buffer;
7439    unsigned offset = output->offset;
7440
7441    assert(num_comps && num_comps <= 4);
7442    if (!num_comps || num_comps > 4)
7443       return;
7444
7445    unsigned start = ffs(output->component_mask) - 1;
7446
7447    Temp out[4];
7448    bool all_undef = true;
7449    assert(ctx->stage == vertex_vs);
7450    for (unsigned i = 0; i < num_comps; i++) {
7451       out[i] = ctx->vs_output.outputs[loc][start + i];
7452       all_undef = all_undef && !out[i].id();
7453    }
7454    if (all_undef)
7455       return;
7456
7457    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7458    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7459    for (unsigned i = 0; i < num_comps; ++i)
7460       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7461    vec->definitions[0] = Definition(write_data);
7462    ctx->block->instructions.emplace_back(std::move(vec));
7463
7464    aco_opcode opcode;
7465    switch (num_comps) {
7466    case 1:
7467       opcode = aco_opcode::buffer_store_dword;
7468       break;
7469    case 2:
7470       opcode = aco_opcode::buffer_store_dwordx2;
7471       break;
7472    case 3:
7473       opcode = aco_opcode::buffer_store_dwordx3;
7474       break;
7475    case 4:
7476       opcode = aco_opcode::buffer_store_dwordx4;
7477       break;
7478    }
7479
7480    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7481    store->operands[0] = Operand(so_write_offset[buf]);
7482    store->operands[1] = Operand(so_buffers[buf]);
7483    store->operands[2] = Operand((uint32_t) 0);
7484    store->operands[3] = Operand(write_data);
7485    if (offset > 4095) {
7486       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7487       Builder bld(ctx->program, ctx->block);
7488       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7489    } else {
7490       store->offset = offset;
7491    }
7492    store->offen = true;
7493    store->glc = true;
7494    store->dlc = false;
7495    store->slc = true;
7496    store->can_reorder = true;
7497    ctx->block->instructions.emplace_back(std::move(store));
7498 }
7499
7500 static void emit_streamout(isel_context *ctx, unsigned stream)
7501 {
7502    Builder bld(ctx->program, ctx->block);
7503
7504    Temp so_buffers[4];
7505    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7506    for (unsigned i = 0; i < 4; i++) {
7507       unsigned stride = ctx->program->info->so.strides[i];
7508       if (!stride)
7509          continue;
7510
7511       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7512    }
7513
7514    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7515                                 ctx->streamout_config, Operand(0x70010u));
7516
7517    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7518                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7519
7520    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7521
7522    if_context ic;
7523    begin_divergent_if_then(ctx, &ic, can_emit);
7524
7525    bld.reset(ctx->block);
7526
7527    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7528
7529    Temp so_write_offset[4];
7530
7531    for (unsigned i = 0; i < 4; i++) {
7532       unsigned stride = ctx->program->info->so.strides[i];
7533       if (!stride)
7534          continue;
7535
7536       if (stride == 1) {
7537          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7538                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7539          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7540
7541          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7542       } else {
7543          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7544          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7545          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7546       }
7547    }
7548
7549    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7550       struct radv_stream_output *output =
7551          &ctx->program->info->so.outputs[i];
7552       if (stream != output->stream)
7553          continue;
7554
7555       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7556    }
7557
7558    begin_divergent_if_else(ctx, &ic);
7559    end_divergent_if(ctx, &ic);
7560 }
7561
7562 } /* end namespace */
7563
7564 void handle_bc_optimize(isel_context *ctx)
7565 {
7566    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7567    Builder bld(ctx->program, ctx->block);
7568    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7569    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7570    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7571    if (uses_center && uses_centroid) {
7572       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7573
7574       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7575          for (unsigned i = 0; i < 2; i++) {
7576             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7577                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7578                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7579                                       sel);
7580             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7581          }
7582       }
7583
7584       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7585          for (unsigned i = 0; i < 2; i++) {
7586             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7587                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7588                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7589                                       sel);
7590             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7591          }
7592       }
7593    }
7594 }
7595
7596 void select_program(Program *program,
7597                     unsigned shader_count,
7598                     struct nir_shader *const *shaders,
7599                     ac_shader_config* config,
7600                     struct radv_shader_info *info,
7601                     struct radv_nir_compiler_options *options)
7602 {
7603    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7604
7605    for (unsigned i = 0; i < shader_count; i++) {
7606       nir_shader *nir = shaders[i];
7607       init_context(&ctx, nir);
7608
7609       if (!i) {
7610          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7611          append_logical_start(ctx.block);
7612       }
7613
7614       if_context ic;
7615       if (shader_count >= 2) {
7616          Builder bld(ctx.program, ctx.block);
7617          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7618          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7619                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7620          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7621
7622          begin_divergent_if_then(&ctx, &ic, cond);
7623       }
7624
7625       if (i) {
7626          Builder bld(ctx.program, ctx.block);
7627          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7628          bld.sopp(aco_opcode::s_barrier);
7629       }
7630
7631       if (ctx.stage == fragment_fs)
7632          handle_bc_optimize(&ctx);
7633
7634       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7635       visit_cf_list(&ctx, &func->body);
7636
7637       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7638          emit_streamout(&ctx, 0);
7639
7640       if (ctx.stage == vertex_vs)
7641          create_vs_exports(&ctx);
7642
7643       if (shader_count >= 2) {
7644          begin_divergent_if_else(&ctx, &ic);
7645          end_divergent_if(&ctx, &ic);
7646       }
7647
7648       ralloc_free(ctx.divergent_vals);
7649    }
7650
7651    append_logical_end(ctx.block);
7652    ctx.block->kind |= block_kind_uniform;
7653    Builder bld(ctx.program, ctx.block);
7654    if (ctx.program->wb_smem_l1_on_end)
7655       bld.smem(aco_opcode::s_dcache_wb, false);
7656    bld.sopp(aco_opcode::s_endpgm);
7657
7658    /* cleanup CFG */
7659    for (Block& BB : program->blocks) {
7660       for (unsigned idx : BB.linear_preds)
7661          program->blocks[idx].linear_succs.emplace_back(BB.index);
7662       for (unsigned idx : BB.logical_preds)
7663          program->blocks[idx].logical_succs.emplace_back(BB.index);
7664    }
7665 }
7666 }