src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,4> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,4> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,4> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 439 {
 440    Builder bld(ctx->program, ctx->block);
 441    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 442    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 443    if (src1.type() == RegType::sgpr) {
 444       if (commutative && src0.type() == RegType::vgpr) {
 445          Temp t = src0;
 446          src0 = src1;
 447          src1 = t;
 448       } else if (src0.type() == RegType::vgpr &&
 449                  op != aco_opcode::v_madmk_f32 &&
 450                  op != aco_opcode::v_madak_f32 &&
 451                  op != aco_opcode::v_madmk_f16 &&
 452                  op != aco_opcode::v_madak_f16) {
 453          /* If the instruction is not commutative, we emit a VOP3A instruction */
 454          bld.vop2_e64(op, Definition(dst), src0, src1);
 455          return;
 456       } else {
 457          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 458       }
 459    }
 460    bld.vop2(op, Definition(dst), src0, src1);
 461 }
 462
 463 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 464 {
 465    Temp src0 = get_alu_src(ctx, instr->src[0]);
 466    Temp src1 = get_alu_src(ctx, instr->src[1]);
 467    Temp src2 = get_alu_src(ctx, instr->src[2]);
 468
 469    /* ensure that the instruction has at most 1 sgpr operand
 470     * The optimizer will inline constants for us */
 471    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 472       src0 = as_vgpr(ctx, src0);
 473    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 474       src1 = as_vgpr(ctx, src1);
 475    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 476       src2 = as_vgpr(ctx, src2);
 477
 478    Builder bld(ctx->program, ctx->block);
 479    bld.vop3(op, Definition(dst), src0, src1, src2);
 480 }
 481
 482 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 483 {
 484    Builder bld(ctx->program, ctx->block);
 485    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 486 }
 487
 488 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 489 {
 490    Temp src0 = get_alu_src(ctx, instr->src[0]);
 491    Temp src1 = get_alu_src(ctx, instr->src[1]);
 492    assert(src0.size() == src1.size());
 493
 494    aco_ptr<Instruction> vopc;
 495    if (src1.type() == RegType::sgpr) {
 496       if (src0.type() == RegType::vgpr) {
 497          /* to swap the operands, we might also have to change the opcode */
 498          switch (op) {
 499             case aco_opcode::v_cmp_lt_f32:
 500                op = aco_opcode::v_cmp_gt_f32;
 501                break;
 502             case aco_opcode::v_cmp_ge_f32:
 503                op = aco_opcode::v_cmp_le_f32;
 504                break;
 505             case aco_opcode::v_cmp_lt_i32:
 506                op = aco_opcode::v_cmp_gt_i32;
 507                break;
 508             case aco_opcode::v_cmp_ge_i32:
 509                op = aco_opcode::v_cmp_le_i32;
 510                break;
 511             case aco_opcode::v_cmp_lt_u32:
 512                op = aco_opcode::v_cmp_gt_u32;
 513                break;
 514             case aco_opcode::v_cmp_ge_u32:
 515                op = aco_opcode::v_cmp_le_u32;
 516                break;
 517             case aco_opcode::v_cmp_lt_f64:
 518                op = aco_opcode::v_cmp_gt_f64;
 519                break;
 520             case aco_opcode::v_cmp_ge_f64:
 521                op = aco_opcode::v_cmp_le_f64;
 522                break;
 523             case aco_opcode::v_cmp_lt_i64:
 524                op = aco_opcode::v_cmp_gt_i64;
 525                break;
 526             case aco_opcode::v_cmp_ge_i64:
 527                op = aco_opcode::v_cmp_le_i64;
 528                break;
 529             case aco_opcode::v_cmp_lt_u64:
 530                op = aco_opcode::v_cmp_gt_u64;
 531                break;
 532             case aco_opcode::v_cmp_ge_u64:
 533                op = aco_opcode::v_cmp_le_u64;
 534                break;
 535             default: /* eq and ne are commutative */
 536                break;
 537          }
 538          Temp t = src0;
 539          src0 = src1;
 540          src1 = t;
 541       } else {
 542          src1 = as_vgpr(ctx, src1);
 543       }
 544    }
 545
 546    Builder bld(ctx->program, ctx->block);
 547    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 548 }
 549
 550 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 551 {
 552    Temp src0 = get_alu_src(ctx, instr->src[0]);
 553    Temp src1 = get_alu_src(ctx, instr->src[1]);
 554    Builder bld(ctx->program, ctx->block);
 555
 556    assert(dst.regClass() == bld.lm);
 557    assert(src0.type() == RegType::sgpr);
 558    assert(src1.type() == RegType::sgpr);
 559    assert(src0.regClass() == src1.regClass());
 560
 561    /* Emit the SALU comparison instruction */
 562    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 563    /* Turn the result into a per-lane bool */
 564    bool_to_vector_condition(ctx, cmp, dst);
 565 }
 566
 567 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 568                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::last_opcode, aco_opcode s64_op = aco_opcode::last_opcode)
 569 {
 570    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 571    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 572    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 573    bool use_valu = s_op == aco_opcode::last_opcode ||
 574                    divergent_vals ||
 575                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 576                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 577    aco_opcode op = use_valu ? v_op : s_op;
 578    assert(op != aco_opcode::last_opcode);
 579
 580    if (use_valu)
 581       emit_vopc_instruction(ctx, instr, op, dst);
 582    else
 583       emit_sopc_instruction(ctx, instr, op, dst);
 584 }
 585
 586 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 587 {
 588    Builder bld(ctx->program, ctx->block);
 589    Temp src0 = get_alu_src(ctx, instr->src[0]);
 590    Temp src1 = get_alu_src(ctx, instr->src[1]);
 591
 592    assert(dst.regClass() == bld.lm);
 593    assert(src0.regClass() == bld.lm);
 594    assert(src1.regClass() == bld.lm);
 595
 596    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 597 }
 598
 599 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 600 {
 601    Builder bld(ctx->program, ctx->block);
 602    Temp cond = get_alu_src(ctx, instr->src[0]);
 603    Temp then = get_alu_src(ctx, instr->src[1]);
 604    Temp els = get_alu_src(ctx, instr->src[2]);
 605
 606    assert(cond.regClass() == bld.lm);
 607
 608    if (dst.type() == RegType::vgpr) {
 609       aco_ptr<Instruction> bcsel;
 610       if (dst.size() == 1) {
 611          then = as_vgpr(ctx, then);
 612          els = as_vgpr(ctx, els);
 613
 614          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 615       } else if (dst.size() == 2) {
 616          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 617          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 618          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 619          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 620
 621          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 622          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 623
 624          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 625       } else {
 626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 627          nir_print_instr(&instr->instr, stderr);
 628          fprintf(stderr, "\n");
 629       }
 630       return;
 631    }
 632
 633    if (instr->dest.dest.ssa.bit_size == 1) {
 634       assert(dst.regClass() == bld.lm);
 635       assert(then.regClass() == bld.lm);
 636       assert(els.regClass() == bld.lm);
 637    }
 638
 639    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 640       if (dst.regClass() == s1 || dst.regClass() == s2) {
 641          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 642          assert(dst.size() == then.size());
 643          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 644          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 645       } else {
 646          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 647          nir_print_instr(&instr->instr, stderr);
 648          fprintf(stderr, "\n");
 649       }
 650       return;
 651    }
 652
 653    /* divergent boolean bcsel
 654     * this implements bcsel on bools: dst = s0 ? s1 : s2
 655     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 656    assert(instr->dest.dest.ssa.bit_size == 1);
 657
 658    if (cond.id() != then.id())
 659       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 660
 661    if (cond.id() == els.id())
 662       bld.sop1(Builder::s_mov, Definition(dst), then);
 663    else
 664       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 665                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 666 }
 667
 668 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 669                     aco_opcode op, uint32_t undo)
 670 {
 671    /* multiply by 16777216 to handle denormals */
 672    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 673                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 674    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 675    scaled = bld.vop1(op, bld.def(v1), scaled);
 676    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 677
 678    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 679
 680    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 681 }
 682
 683 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 684 {
 685    if (ctx->block->fp_mode.denorm32 == 0) {
 686       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 687       return;
 688    }
 689
 690    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 691 }
 692
 693 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 694 {
 695    if (ctx->block->fp_mode.denorm32 == 0) {
 696       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 697       return;
 698    }
 699
 700    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 701 }
 702
 703 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 704 {
 705    if (ctx->block->fp_mode.denorm32 == 0) {
 706       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 707       return;
 708    }
 709
 710    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 711 }
 712
 713 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 714 {
 715    if (ctx->block->fp_mode.denorm32 == 0) {
 716       bld.vop1(aco_opcode::v_log_f32, dst, val);
 717       return;
 718    }
 719
 720    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 721 }
 722
 723 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 724 {
 725    if (!instr->dest.dest.is_ssa) {
 726       fprintf(stderr, "nir alu dst not in ssa: ");
 727       nir_print_instr(&instr->instr, stderr);
 728       fprintf(stderr, "\n");
 729       abort();
 730    }
 731    Builder bld(ctx->program, ctx->block);
 732    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 733    switch(instr->op) {
 734    case nir_op_vec2:
 735    case nir_op_vec3:
 736    case nir_op_vec4: {
 737       std::array<Temp,4> elems;
 738       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 739       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 740          elems[i] = get_alu_src(ctx, instr->src[i]);
 741          vec->operands[i] = Operand{elems[i]};
 742       }
 743       vec->definitions[0] = Definition(dst);
 744       ctx->block->instructions.emplace_back(std::move(vec));
 745       ctx->allocated_vec.emplace(dst.id(), elems);
 746       break;
 747    }
 748    case nir_op_mov: {
 749       Temp src = get_alu_src(ctx, instr->src[0]);
 750       aco_ptr<Instruction> mov;
 751       if (dst.type() == RegType::sgpr) {
 752          if (src.type() == RegType::vgpr)
 753             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 754          else if (src.regClass() == s1)
 755             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 756          else if (src.regClass() == s2)
 757             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 758          else
 759             unreachable("wrong src register class for nir_op_imov");
 760       } else if (dst.regClass() == v1) {
 761          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 762       } else if (dst.regClass() == v2) {
 763          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 764       } else {
 765          nir_print_instr(&instr->instr, stderr);
 766          unreachable("Should have been lowered to scalar.");
 767       }
 768       break;
 769    }
 770    case nir_op_inot: {
 771       Temp src = get_alu_src(ctx, instr->src[0]);
 772       if (instr->dest.dest.ssa.bit_size == 1) {
 773          assert(src.regClass() == bld.lm);
 774          assert(dst.regClass() == bld.lm);
 775          bld.sop2(Builder::s_andn2, Definition(dst), bld.def(s1, scc), Operand(exec, bld.lm), src);
 776       } else if (dst.regClass() == v1) {
 777          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 778       } else if (dst.type() == RegType::sgpr) {
 779          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 780          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 781       } else {
 782          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 783          nir_print_instr(&instr->instr, stderr);
 784          fprintf(stderr, "\n");
 785       }
 786       break;
 787    }
 788    case nir_op_ineg: {
 789       Temp src = get_alu_src(ctx, instr->src[0]);
 790       if (dst.regClass() == v1) {
 791          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 792       } else if (dst.regClass() == s1) {
 793          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 794       } else if (dst.size() == 2) {
 795          Temp src0 = bld.tmp(dst.type(), 1);
 796          Temp src1 = bld.tmp(dst.type(), 1);
 797          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 798
 799          if (dst.regClass() == s2) {
 800             Temp carry = bld.tmp(s1);
 801             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 802             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 804          } else {
 805             Temp lower = bld.tmp(v1);
 806             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 807             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 808             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 809          }
 810       } else {
 811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 812          nir_print_instr(&instr->instr, stderr);
 813          fprintf(stderr, "\n");
 814       }
 815       break;
 816    }
 817    case nir_op_iabs: {
 818       if (dst.regClass() == s1) {
 819          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 820       } else if (dst.regClass() == v1) {
 821          Temp src = get_alu_src(ctx, instr->src[0]);
 822          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 823       } else {
 824          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 825          nir_print_instr(&instr->instr, stderr);
 826          fprintf(stderr, "\n");
 827       }
 828       break;
 829    }
 830    case nir_op_isign: {
 831       Temp src = get_alu_src(ctx, instr->src[0]);
 832       if (dst.regClass() == s1) {
 833          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 834          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 835          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 836       } else if (dst.regClass() == s2) {
 837          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 838          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 839          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 840       } else if (dst.regClass() == v1) {
 841          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 842          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 843          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 844       } else if (dst.regClass() == v2) {
 845          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 846          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 847          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 848          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 849          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 850          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 851       } else {
 852          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 853          nir_print_instr(&instr->instr, stderr);
 854          fprintf(stderr, "\n");
 855       }
 856       break;
 857    }
 858    case nir_op_imax: {
 859       if (dst.regClass() == v1) {
 860          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 861       } else if (dst.regClass() == s1) {
 862          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 863       } else {
 864          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 865          nir_print_instr(&instr->instr, stderr);
 866          fprintf(stderr, "\n");
 867       }
 868       break;
 869    }
 870    case nir_op_umax: {
 871       if (dst.regClass() == v1) {
 872          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 873       } else if (dst.regClass() == s1) {
 874          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 875       } else {
 876          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 877          nir_print_instr(&instr->instr, stderr);
 878          fprintf(stderr, "\n");
 879       }
 880       break;
 881    }
 882    case nir_op_imin: {
 883       if (dst.regClass() == v1) {
 884          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 885       } else if (dst.regClass() == s1) {
 886          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 887       } else {
 888          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 889          nir_print_instr(&instr->instr, stderr);
 890          fprintf(stderr, "\n");
 891       }
 892       break;
 893    }
 894    case nir_op_umin: {
 895       if (dst.regClass() == v1) {
 896          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 897       } else if (dst.regClass() == s1) {
 898          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 899       } else {
 900          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 901          nir_print_instr(&instr->instr, stderr);
 902          fprintf(stderr, "\n");
 903       }
 904       break;
 905    }
 906    case nir_op_ior: {
 907       if (instr->dest.dest.ssa.bit_size == 1) {
 908          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 909       } else if (dst.regClass() == v1) {
 910          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 911       } else if (dst.regClass() == s1) {
 912          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 913       } else if (dst.regClass() == s2) {
 914          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 915       } else {
 916          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 917          nir_print_instr(&instr->instr, stderr);
 918          fprintf(stderr, "\n");
 919       }
 920       break;
 921    }
 922    case nir_op_iand: {
 923       if (instr->dest.dest.ssa.bit_size == 1) {
 924          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 925       } else if (dst.regClass() == v1) {
 926          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 927       } else if (dst.regClass() == s1) {
 928          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 929       } else if (dst.regClass() == s2) {
 930          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 931       } else {
 932          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 933          nir_print_instr(&instr->instr, stderr);
 934          fprintf(stderr, "\n");
 935       }
 936       break;
 937    }
 938    case nir_op_ixor: {
 939       if (instr->dest.dest.ssa.bit_size == 1) {
 940          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 941       } else if (dst.regClass() == v1) {
 942          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 943       } else if (dst.regClass() == s1) {
 944          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 945       } else if (dst.regClass() == s2) {
 946          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 947       } else {
 948          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 949          nir_print_instr(&instr->instr, stderr);
 950          fprintf(stderr, "\n");
 951       }
 952       break;
 953    }
 954    case nir_op_ushr: {
 955       if (dst.regClass() == v1) {
 956          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 957       } else if (dst.regClass() == v2) {
 958          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 959                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 960       } else if (dst.regClass() == s2) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 962       } else if (dst.regClass() == s1) {
 963          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 964       } else {
 965          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 966          nir_print_instr(&instr->instr, stderr);
 967          fprintf(stderr, "\n");
 968       }
 969       break;
 970    }
 971    case nir_op_ishl: {
 972       if (dst.regClass() == v1) {
 973          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 974       } else if (dst.regClass() == v2) {
 975          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 976                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 977       } else if (dst.regClass() == s1) {
 978          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 979       } else if (dst.regClass() == s2) {
 980          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 981       } else {
 982          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 983          nir_print_instr(&instr->instr, stderr);
 984          fprintf(stderr, "\n");
 985       }
 986       break;
 987    }
 988    case nir_op_ishr: {
 989       if (dst.regClass() == v1) {
 990          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 991       } else if (dst.regClass() == v2) {
 992          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 993                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 994       } else if (dst.regClass() == s1) {
 995          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 996       } else if (dst.regClass() == s2) {
 997          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 998       } else {
 999          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1000          nir_print_instr(&instr->instr, stderr);
1001          fprintf(stderr, "\n");
1002       }
1003       break;
1004    }
1005    case nir_op_find_lsb: {
1006       Temp src = get_alu_src(ctx, instr->src[0]);
1007       if (src.regClass() == s1) {
1008          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1009       } else if (src.regClass() == v1) {
1010          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1011       } else if (src.regClass() == s2) {
1012          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1013       } else {
1014          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1015          nir_print_instr(&instr->instr, stderr);
1016          fprintf(stderr, "\n");
1017       }
1018       break;
1019    }
1020    case nir_op_ufind_msb:
1021    case nir_op_ifind_msb: {
1022       Temp src = get_alu_src(ctx, instr->src[0]);
1023       if (src.regClass() == s1 || src.regClass() == s2) {
1024          aco_opcode op = src.regClass() == s2 ?
1025                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1026                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1027          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1028
1029          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1030                                         Operand(src.size() * 32u - 1u), msb_rev);
1031          Temp msb = sub.def(0).getTemp();
1032          Temp carry = sub.def(1).getTemp();
1033
1034          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1035       } else if (src.regClass() == v1) {
1036          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1037          Temp msb_rev = bld.tmp(v1);
1038          emit_vop1_instruction(ctx, instr, op, msb_rev);
1039          Temp msb = bld.tmp(v1);
1040          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1041          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1042       } else {
1043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1044          nir_print_instr(&instr->instr, stderr);
1045          fprintf(stderr, "\n");
1046       }
1047       break;
1048    }
1049    case nir_op_bitfield_reverse: {
1050       if (dst.regClass() == s1) {
1051          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1052       } else if (dst.regClass() == v1) {
1053          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1054       } else {
1055          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1056          nir_print_instr(&instr->instr, stderr);
1057          fprintf(stderr, "\n");
1058       }
1059       break;
1060    }
1061    case nir_op_iadd: {
1062       if (dst.regClass() == s1) {
1063          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1064          break;
1065       }
1066
1067       Temp src0 = get_alu_src(ctx, instr->src[0]);
1068       Temp src1 = get_alu_src(ctx, instr->src[1]);
1069       if (dst.regClass() == v1) {
1070          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1071          break;
1072       }
1073
1074       assert(src0.size() == 2 && src1.size() == 2);
1075       Temp src00 = bld.tmp(src0.type(), 1);
1076       Temp src01 = bld.tmp(dst.type(), 1);
1077       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1078       Temp src10 = bld.tmp(src1.type(), 1);
1079       Temp src11 = bld.tmp(dst.type(), 1);
1080       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1081
1082       if (dst.regClass() == s2) {
1083          Temp carry = bld.tmp(s1);
1084          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1085          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1086          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1087       } else if (dst.regClass() == v2) {
1088          Temp dst0 = bld.tmp(v1);
1089          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1090          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1091          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1092       } else {
1093          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1094          nir_print_instr(&instr->instr, stderr);
1095          fprintf(stderr, "\n");
1096       }
1097       break;
1098    }
1099    case nir_op_uadd_sat: {
1100       Temp src0 = get_alu_src(ctx, instr->src[0]);
1101       Temp src1 = get_alu_src(ctx, instr->src[1]);
1102       if (dst.regClass() == s1) {
1103          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1104          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1105                   src0, src1);
1106          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1107       } else if (dst.regClass() == v1) {
1108          if (ctx->options->chip_class >= GFX9) {
1109             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1110             add->operands[0] = Operand(src0);
1111             add->operands[1] = Operand(src1);
1112             add->definitions[0] = Definition(dst);
1113             add->clamp = 1;
1114             ctx->block->instructions.emplace_back(std::move(add));
1115          } else {
1116             if (src1.regClass() != v1)
1117                std::swap(src0, src1);
1118             assert(src1.regClass() == v1);
1119             Temp tmp = bld.tmp(v1);
1120             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1121             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1122          }
1123       } else {
1124          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1125          nir_print_instr(&instr->instr, stderr);
1126          fprintf(stderr, "\n");
1127       }
1128       break;
1129    }
1130    case nir_op_uadd_carry: {
1131       Temp src0 = get_alu_src(ctx, instr->src[0]);
1132       Temp src1 = get_alu_src(ctx, instr->src[1]);
1133       if (dst.regClass() == s1) {
1134          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1135          break;
1136       }
1137       if (dst.regClass() == v1) {
1138          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1139          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1140          break;
1141       }
1142
1143       Temp src00 = bld.tmp(src0.type(), 1);
1144       Temp src01 = bld.tmp(dst.type(), 1);
1145       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1146       Temp src10 = bld.tmp(src1.type(), 1);
1147       Temp src11 = bld.tmp(dst.type(), 1);
1148       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1149       if (dst.regClass() == s2) {
1150          Temp carry = bld.tmp(s1);
1151          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1152          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1153          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1154       } else if (dst.regClass() == v2) {
1155          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1156          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1157          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1158          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1159       } else {
1160          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1161          nir_print_instr(&instr->instr, stderr);
1162          fprintf(stderr, "\n");
1163       }
1164       break;
1165    }
1166    case nir_op_isub: {
1167       if (dst.regClass() == s1) {
1168          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1169          break;
1170       }
1171
1172       Temp src0 = get_alu_src(ctx, instr->src[0]);
1173       Temp src1 = get_alu_src(ctx, instr->src[1]);
1174       if (dst.regClass() == v1) {
1175          bld.vsub32(Definition(dst), src0, src1);
1176          break;
1177       }
1178
1179       Temp src00 = bld.tmp(src0.type(), 1);
1180       Temp src01 = bld.tmp(dst.type(), 1);
1181       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1182       Temp src10 = bld.tmp(src1.type(), 1);
1183       Temp src11 = bld.tmp(dst.type(), 1);
1184       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1185       if (dst.regClass() == s2) {
1186          Temp carry = bld.tmp(s1);
1187          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1188          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1189          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1190       } else if (dst.regClass() == v2) {
1191          Temp lower = bld.tmp(v1);
1192          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1193          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1194          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1195       } else {
1196          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1197          nir_print_instr(&instr->instr, stderr);
1198          fprintf(stderr, "\n");
1199       }
1200       break;
1201    }
1202    case nir_op_usub_borrow: {
1203       Temp src0 = get_alu_src(ctx, instr->src[0]);
1204       Temp src1 = get_alu_src(ctx, instr->src[1]);
1205       if (dst.regClass() == s1) {
1206          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1207          break;
1208       } else if (dst.regClass() == v1) {
1209          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1210          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1211          break;
1212       }
1213
1214       Temp src00 = bld.tmp(src0.type(), 1);
1215       Temp src01 = bld.tmp(dst.type(), 1);
1216       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1217       Temp src10 = bld.tmp(src1.type(), 1);
1218       Temp src11 = bld.tmp(dst.type(), 1);
1219       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1220       if (dst.regClass() == s2) {
1221          Temp borrow = bld.tmp(s1);
1222          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1223          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1224          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1225       } else if (dst.regClass() == v2) {
1226          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1227          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1228          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1229          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1230       } else {
1231          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1232          nir_print_instr(&instr->instr, stderr);
1233          fprintf(stderr, "\n");
1234       }
1235       break;
1236    }
1237    case nir_op_imul: {
1238       if (dst.regClass() == v1) {
1239          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1240                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1241       } else if (dst.regClass() == s1) {
1242          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1243       } else {
1244          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1245          nir_print_instr(&instr->instr, stderr);
1246          fprintf(stderr, "\n");
1247       }
1248       break;
1249    }
1250    case nir_op_umul_high: {
1251       if (dst.regClass() == v1) {
1252          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1253       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1254          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1255       } else if (dst.regClass() == s1) {
1256          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1257                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1258          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1259       } else {
1260          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1261          nir_print_instr(&instr->instr, stderr);
1262          fprintf(stderr, "\n");
1263       }
1264       break;
1265    }
1266    case nir_op_imul_high: {
1267       if (dst.regClass() == v1) {
1268          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1269       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1270          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1271       } else if (dst.regClass() == s1) {
1272          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1273                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1274          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1275       } else {
1276          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1277          nir_print_instr(&instr->instr, stderr);
1278          fprintf(stderr, "\n");
1279       }
1280       break;
1281    }
1282    case nir_op_fmul: {
1283       if (dst.size() == 1) {
1284          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1285       } else if (dst.size() == 2) {
1286          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1287                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1288       } else {
1289          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1290          nir_print_instr(&instr->instr, stderr);
1291          fprintf(stderr, "\n");
1292       }
1293       break;
1294    }
1295    case nir_op_fadd: {
1296       if (dst.size() == 1) {
1297          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1298       } else if (dst.size() == 2) {
1299          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1300                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1301       } else {
1302          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1303          nir_print_instr(&instr->instr, stderr);
1304          fprintf(stderr, "\n");
1305       }
1306       break;
1307    }
1308    case nir_op_fsub: {
1309       Temp src0 = get_alu_src(ctx, instr->src[0]);
1310       Temp src1 = get_alu_src(ctx, instr->src[1]);
1311       if (dst.size() == 1) {
1312          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1313             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1314          else
1315             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1316       } else if (dst.size() == 2) {
1317          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1318                                      get_alu_src(ctx, instr->src[0]),
1319                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1320          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1321          sub->neg[1] = true;
1322       } else {
1323          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1324          nir_print_instr(&instr->instr, stderr);
1325          fprintf(stderr, "\n");
1326       }
1327       break;
1328    }
1329    case nir_op_fmax: {
1330       if (dst.size() == 1) {
1331          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1332       } else if (dst.size() == 2) {
1333          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1334                   get_alu_src(ctx, instr->src[0]),
1335                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1336       } else {
1337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1338          nir_print_instr(&instr->instr, stderr);
1339          fprintf(stderr, "\n");
1340       }
1341       break;
1342    }
1343    case nir_op_fmin: {
1344       if (dst.size() == 1) {
1345          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1346       } else if (dst.size() == 2) {
1347          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]),
1349                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1350       } else {
1351          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1352          nir_print_instr(&instr->instr, stderr);
1353          fprintf(stderr, "\n");
1354       }
1355       break;
1356    }
1357    case nir_op_fmax3: {
1358       if (dst.size() == 1) {
1359          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1360       } else {
1361          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1362          nir_print_instr(&instr->instr, stderr);
1363          fprintf(stderr, "\n");
1364       }
1365       break;
1366    }
1367    case nir_op_fmin3: {
1368       if (dst.size() == 1) {
1369          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1370       } else {
1371          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1372          nir_print_instr(&instr->instr, stderr);
1373          fprintf(stderr, "\n");
1374       }
1375       break;
1376    }
1377    case nir_op_fmed3: {
1378       if (dst.size() == 1) {
1379          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1380       } else {
1381          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1382          nir_print_instr(&instr->instr, stderr);
1383          fprintf(stderr, "\n");
1384       }
1385       break;
1386    }
1387    case nir_op_umax3: {
1388       if (dst.size() == 1) {
1389          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1390       } else {
1391          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1392          nir_print_instr(&instr->instr, stderr);
1393          fprintf(stderr, "\n");
1394       }
1395       break;
1396    }
1397    case nir_op_umin3: {
1398       if (dst.size() == 1) {
1399          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1400       } else {
1401          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1402          nir_print_instr(&instr->instr, stderr);
1403          fprintf(stderr, "\n");
1404       }
1405       break;
1406    }
1407    case nir_op_umed3: {
1408       if (dst.size() == 1) {
1409          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1410       } else {
1411          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1412          nir_print_instr(&instr->instr, stderr);
1413          fprintf(stderr, "\n");
1414       }
1415       break;
1416    }
1417    case nir_op_imax3: {
1418       if (dst.size() == 1) {
1419          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1420       } else {
1421          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1422          nir_print_instr(&instr->instr, stderr);
1423          fprintf(stderr, "\n");
1424       }
1425       break;
1426    }
1427    case nir_op_imin3: {
1428       if (dst.size() == 1) {
1429          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1430       } else {
1431          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1432          nir_print_instr(&instr->instr, stderr);
1433          fprintf(stderr, "\n");
1434       }
1435       break;
1436    }
1437    case nir_op_imed3: {
1438       if (dst.size() == 1) {
1439          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1440       } else {
1441          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1442          nir_print_instr(&instr->instr, stderr);
1443          fprintf(stderr, "\n");
1444       }
1445       break;
1446    }
1447    case nir_op_cube_face_coord: {
1448       Temp in = get_alu_src(ctx, instr->src[0], 3);
1449       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1450                       emit_extract_vector(ctx, in, 1, v1),
1451                       emit_extract_vector(ctx, in, 2, v1) };
1452       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1453       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1454       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1455       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1456       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1457       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1458       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1459       break;
1460    }
1461    case nir_op_cube_face_index: {
1462       Temp in = get_alu_src(ctx, instr->src[0], 3);
1463       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1464                       emit_extract_vector(ctx, in, 1, v1),
1465                       emit_extract_vector(ctx, in, 2, v1) };
1466       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1467       break;
1468    }
1469    case nir_op_bcsel: {
1470       emit_bcsel(ctx, instr, dst);
1471       break;
1472    }
1473    case nir_op_frsq: {
1474       if (dst.size() == 1) {
1475          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1476       } else if (dst.size() == 2) {
1477          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1478       } else {
1479          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1480          nir_print_instr(&instr->instr, stderr);
1481          fprintf(stderr, "\n");
1482       }
1483       break;
1484    }
1485    case nir_op_fneg: {
1486       Temp src = get_alu_src(ctx, instr->src[0]);
1487       if (dst.size() == 1) {
1488          if (ctx->block->fp_mode.must_flush_denorms32)
1489             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1490          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1491       } else if (dst.size() == 2) {
1492          if (ctx->block->fp_mode.must_flush_denorms16_64)
1493             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1494          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1495          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1496          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1497          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1498       } else {
1499          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1500          nir_print_instr(&instr->instr, stderr);
1501          fprintf(stderr, "\n");
1502       }
1503       break;
1504    }
1505    case nir_op_fabs: {
1506       Temp src = get_alu_src(ctx, instr->src[0]);
1507       if (dst.size() == 1) {
1508          if (ctx->block->fp_mode.must_flush_denorms32)
1509             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1510          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1511       } else if (dst.size() == 2) {
1512          if (ctx->block->fp_mode.must_flush_denorms16_64)
1513             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1514          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1515          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1516          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1517          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1518       } else {
1519          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1520          nir_print_instr(&instr->instr, stderr);
1521          fprintf(stderr, "\n");
1522       }
1523       break;
1524    }
1525    case nir_op_fsat: {
1526       Temp src = get_alu_src(ctx, instr->src[0]);
1527       if (dst.size() == 1) {
1528          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1529       } else if (dst.size() == 2) {
1530          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1531          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1532          vop3->clamp = true;
1533       } else {
1534          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1535          nir_print_instr(&instr->instr, stderr);
1536          fprintf(stderr, "\n");
1537       }
1538       break;
1539    }
1540    case nir_op_flog2: {
1541       if (dst.size() == 1) {
1542          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1543       } else {
1544          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1545          nir_print_instr(&instr->instr, stderr);
1546          fprintf(stderr, "\n");
1547       }
1548       break;
1549    }
1550    case nir_op_frcp: {
1551       if (dst.size() == 1) {
1552          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1553       } else if (dst.size() == 2) {
1554          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1555       } else {
1556          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1557          nir_print_instr(&instr->instr, stderr);
1558          fprintf(stderr, "\n");
1559       }
1560       break;
1561    }
1562    case nir_op_fexp2: {
1563       if (dst.size() == 1) {
1564          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1565       } else {
1566          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1567          nir_print_instr(&instr->instr, stderr);
1568          fprintf(stderr, "\n");
1569       }
1570       break;
1571    }
1572    case nir_op_fsqrt: {
1573       if (dst.size() == 1) {
1574          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1575       } else if (dst.size() == 2) {
1576          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1577       } else {
1578          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1579          nir_print_instr(&instr->instr, stderr);
1580          fprintf(stderr, "\n");
1581       }
1582       break;
1583    }
1584    case nir_op_ffract: {
1585       if (dst.size() == 1) {
1586          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1587       } else if (dst.size() == 2) {
1588          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1589       } else {
1590          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1591          nir_print_instr(&instr->instr, stderr);
1592          fprintf(stderr, "\n");
1593       }
1594       break;
1595    }
1596    case nir_op_ffloor: {
1597       if (dst.size() == 1) {
1598          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1599       } else if (dst.size() == 2) {
1600          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1601       } else {
1602          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1603          nir_print_instr(&instr->instr, stderr);
1604          fprintf(stderr, "\n");
1605       }
1606       break;
1607    }
1608    case nir_op_fceil: {
1609       if (dst.size() == 1) {
1610          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1611       } else if (dst.size() == 2) {
1612          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1613       } else {
1614          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1615          nir_print_instr(&instr->instr, stderr);
1616          fprintf(stderr, "\n");
1617       }
1618       break;
1619    }
1620    case nir_op_ftrunc: {
1621       if (dst.size() == 1) {
1622          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1623       } else if (dst.size() == 2) {
1624          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_fround_even: {
1633       if (dst.size() == 1) {
1634          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1635       } else if (dst.size() == 2) {
1636          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1637       } else {
1638          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1639          nir_print_instr(&instr->instr, stderr);
1640          fprintf(stderr, "\n");
1641       }
1642       break;
1643    }
1644    case nir_op_fsin:
1645    case nir_op_fcos: {
1646       Temp src = get_alu_src(ctx, instr->src[0]);
1647       aco_ptr<Instruction> norm;
1648       if (dst.size() == 1) {
1649          Temp tmp;
1650          Operand half_pi(0x3e22f983u);
1651          if (src.type() == RegType::sgpr)
1652             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1653          else
1654             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1655
1656          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1657          if (ctx->options->chip_class < GFX9)
1658             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1659
1660          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1661          bld.vop1(opcode, Definition(dst), tmp);
1662       } else {
1663          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1664          nir_print_instr(&instr->instr, stderr);
1665          fprintf(stderr, "\n");
1666       }
1667       break;
1668    }
1669    case nir_op_ldexp: {
1670       if (dst.size() == 1) {
1671          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1672                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1673                   get_alu_src(ctx, instr->src[1]));
1674       } else if (dst.size() == 2) {
1675          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1676                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1677                   get_alu_src(ctx, instr->src[1]));
1678       } else {
1679          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1680          nir_print_instr(&instr->instr, stderr);
1681          fprintf(stderr, "\n");
1682       }
1683       break;
1684    }
1685    case nir_op_frexp_sig: {
1686       if (dst.size() == 1) {
1687          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1688                   get_alu_src(ctx, instr->src[0]));
1689       } else if (dst.size() == 2) {
1690          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1691                   get_alu_src(ctx, instr->src[0]));
1692       } else {
1693          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1694          nir_print_instr(&instr->instr, stderr);
1695          fprintf(stderr, "\n");
1696       }
1697       break;
1698    }
1699    case nir_op_frexp_exp: {
1700       if (instr->src[0].src.ssa->bit_size == 32) {
1701          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1702                   get_alu_src(ctx, instr->src[0]));
1703       } else if (instr->src[0].src.ssa->bit_size == 64) {
1704          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1705                   get_alu_src(ctx, instr->src[0]));
1706       } else {
1707          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1708          nir_print_instr(&instr->instr, stderr);
1709          fprintf(stderr, "\n");
1710       }
1711       break;
1712    }
1713    case nir_op_fsign: {
1714       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1715       if (dst.size() == 1) {
1716          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1717          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1718          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1719          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1720       } else if (dst.size() == 2) {
1721          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1722          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1723          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1724
1725          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1726          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1727          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1728
1729          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1730       } else {
1731          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1732          nir_print_instr(&instr->instr, stderr);
1733          fprintf(stderr, "\n");
1734       }
1735       break;
1736    }
1737    case nir_op_f2f32: {
1738       if (instr->src[0].src.ssa->bit_size == 64) {
1739          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1740       } else {
1741          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1742          nir_print_instr(&instr->instr, stderr);
1743          fprintf(stderr, "\n");
1744       }
1745       break;
1746    }
1747    case nir_op_f2f64: {
1748       if (instr->src[0].src.ssa->bit_size == 32) {
1749          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1750       } else {
1751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1752          nir_print_instr(&instr->instr, stderr);
1753          fprintf(stderr, "\n");
1754       }
1755       break;
1756    }
1757    case nir_op_i2f32: {
1758       assert(dst.size() == 1);
1759       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1760       break;
1761    }
1762    case nir_op_i2f64: {
1763       if (instr->src[0].src.ssa->bit_size == 32) {
1764          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1765       } else if (instr->src[0].src.ssa->bit_size == 64) {
1766          Temp src = get_alu_src(ctx, instr->src[0]);
1767          RegClass rc = RegClass(src.type(), 1);
1768          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1769          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1770          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1771          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1772          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1773          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1774
1775       } else {
1776          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1777          nir_print_instr(&instr->instr, stderr);
1778          fprintf(stderr, "\n");
1779       }
1780       break;
1781    }
1782    case nir_op_u2f32: {
1783       assert(dst.size() == 1);
1784       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1785       break;
1786    }
1787    case nir_op_u2f64: {
1788       if (instr->src[0].src.ssa->bit_size == 32) {
1789          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1790       } else if (instr->src[0].src.ssa->bit_size == 64) {
1791          Temp src = get_alu_src(ctx, instr->src[0]);
1792          RegClass rc = RegClass(src.type(), 1);
1793          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1794          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1795          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1796          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1797          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1798          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1799       } else {
1800          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1801          nir_print_instr(&instr->instr, stderr);
1802          fprintf(stderr, "\n");
1803       }
1804       break;
1805    }
1806    case nir_op_f2i32: {
1807       Temp src = get_alu_src(ctx, instr->src[0]);
1808       if (instr->src[0].src.ssa->bit_size == 32) {
1809          if (dst.type() == RegType::vgpr)
1810             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1811          else
1812             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1813                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1814
1815       } else if (instr->src[0].src.ssa->bit_size == 64) {
1816          if (dst.type() == RegType::vgpr)
1817             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1818          else
1819             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1820                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1821
1822       } else {
1823          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1824          nir_print_instr(&instr->instr, stderr);
1825          fprintf(stderr, "\n");
1826       }
1827       break;
1828    }
1829    case nir_op_f2u32: {
1830       Temp src = get_alu_src(ctx, instr->src[0]);
1831       if (instr->src[0].src.ssa->bit_size == 32) {
1832          if (dst.type() == RegType::vgpr)
1833             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1834          else
1835             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1836                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1837
1838       } else if (instr->src[0].src.ssa->bit_size == 64) {
1839          if (dst.type() == RegType::vgpr)
1840             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1841          else
1842             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1843                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1844
1845       } else {
1846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1847          nir_print_instr(&instr->instr, stderr);
1848          fprintf(stderr, "\n");
1849       }
1850       break;
1851    }
1852    case nir_op_f2i64: {
1853       Temp src = get_alu_src(ctx, instr->src[0]);
1854       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1855          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1856          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1857          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1858          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1859          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1860          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1861          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1862          Temp new_exponent = bld.tmp(v1);
1863          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1864          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1865          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1866          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1867          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1868          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1869          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1870          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1871          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1872          Temp new_lower = bld.tmp(v1);
1873          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1874          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1875          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1876
1877       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1878          if (src.type() == RegType::vgpr)
1879             src = bld.as_uniform(src);
1880          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1881          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1882          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1883          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1884          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1885          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1886          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1887          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1888          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1889          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1890          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1891          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1892          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1893          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1894          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1895          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1896          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1897          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1898          Temp borrow = bld.tmp(s1);
1899          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1900          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1901          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1902
1903       } else if (instr->src[0].src.ssa->bit_size == 64) {
1904          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1905          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1906          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1907          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1908          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1909          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1910          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1911          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1912          if (dst.type() == RegType::sgpr) {
1913             lower = bld.as_uniform(lower);
1914             upper = bld.as_uniform(upper);
1915          }
1916          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1917
1918       } else {
1919          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1920          nir_print_instr(&instr->instr, stderr);
1921          fprintf(stderr, "\n");
1922       }
1923       break;
1924    }
1925    case nir_op_f2u64: {
1926       Temp src = get_alu_src(ctx, instr->src[0]);
1927       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1928          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1929          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1930          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1931          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1932          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1933          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1934          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1935          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1936          Temp new_exponent = bld.tmp(v1);
1937          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1938          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1939          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1940          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1941          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1942          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1943          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1944          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1945          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1946
1947       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1948          if (src.type() == RegType::vgpr)
1949             src = bld.as_uniform(src);
1950          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1951          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1952          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1953          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1954          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1955          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1956          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1957          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1958          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1959          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1960          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1961          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1962          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1963          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1964          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1965          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1966          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1967          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1968
1969       } else if (instr->src[0].src.ssa->bit_size == 64) {
1970          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1971          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1972          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1973          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1974          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1975          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1976          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1977          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1978          if (dst.type() == RegType::sgpr) {
1979             lower = bld.as_uniform(lower);
1980             upper = bld.as_uniform(upper);
1981          }
1982          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1983
1984       } else {
1985          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1986          nir_print_instr(&instr->instr, stderr);
1987          fprintf(stderr, "\n");
1988       }
1989       break;
1990    }
1991    case nir_op_b2f32: {
1992       Temp src = get_alu_src(ctx, instr->src[0]);
1993       assert(src.regClass() == bld.lm);
1994
1995       if (dst.regClass() == s1) {
1996          src = bool_to_scalar_condition(ctx, src);
1997          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1998       } else if (dst.regClass() == v1) {
1999          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2000       } else {
2001          unreachable("Wrong destination register class for nir_op_b2f32.");
2002       }
2003       break;
2004    }
2005    case nir_op_b2f64: {
2006       Temp src = get_alu_src(ctx, instr->src[0]);
2007       assert(src.regClass() == bld.lm);
2008
2009       if (dst.regClass() == s2) {
2010          src = bool_to_scalar_condition(ctx, src);
2011          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2012       } else if (dst.regClass() == v2) {
2013          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2014          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2015          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2016       } else {
2017          unreachable("Wrong destination register class for nir_op_b2f64.");
2018       }
2019       break;
2020    }
2021    case nir_op_i2i32: {
2022       Temp src = get_alu_src(ctx, instr->src[0]);
2023       if (instr->src[0].src.ssa->bit_size == 64) {
2024          /* we can actually just say dst = src, as it would map the lower register */
2025          emit_extract_vector(ctx, src, 0, dst);
2026       } else {
2027          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2028          nir_print_instr(&instr->instr, stderr);
2029          fprintf(stderr, "\n");
2030       }
2031       break;
2032    }
2033    case nir_op_u2u32: {
2034       Temp src = get_alu_src(ctx, instr->src[0]);
2035       if (instr->src[0].src.ssa->bit_size == 16) {
2036          if (dst.regClass() == s1) {
2037             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2038          } else {
2039             // TODO: do better with SDWA
2040             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2041          }
2042       } else if (instr->src[0].src.ssa->bit_size == 64) {
2043          /* we can actually just say dst = src, as it would map the lower register */
2044          emit_extract_vector(ctx, src, 0, dst);
2045       } else {
2046          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2047          nir_print_instr(&instr->instr, stderr);
2048          fprintf(stderr, "\n");
2049       }
2050       break;
2051    }
2052    case nir_op_i2i64: {
2053       Temp src = get_alu_src(ctx, instr->src[0]);
2054       if (src.regClass() == s1) {
2055          Temp high = bld.sopc(aco_opcode::s_ashr_i32, bld.def(s1, scc), src, Operand(31u));
2056          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2057       } else if (src.regClass() == v1) {
2058          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2059          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2060       } else {
2061          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2062          nir_print_instr(&instr->instr, stderr);
2063          fprintf(stderr, "\n");
2064       }
2065       break;
2066    }
2067    case nir_op_u2u64: {
2068       Temp src = get_alu_src(ctx, instr->src[0]);
2069       if (instr->src[0].src.ssa->bit_size == 32) {
2070          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2071       } else {
2072          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2073          nir_print_instr(&instr->instr, stderr);
2074          fprintf(stderr, "\n");
2075       }
2076       break;
2077    }
2078    case nir_op_b2i32: {
2079       Temp src = get_alu_src(ctx, instr->src[0]);
2080       assert(src.regClass() == bld.lm);
2081
2082       if (dst.regClass() == s1) {
2083          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2084          bool_to_scalar_condition(ctx, src, dst);
2085       } else if (dst.regClass() == v1) {
2086          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2087       } else {
2088          unreachable("Invalid register class for b2i32");
2089       }
2090       break;
2091    }
2092    case nir_op_i2b1: {
2093       Temp src = get_alu_src(ctx, instr->src[0]);
2094       assert(dst.regClass() == bld.lm);
2095
2096       if (src.type() == RegType::vgpr) {
2097          assert(src.regClass() == v1 || src.regClass() == v2);
2098          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2099                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2100       } else {
2101          assert(src.regClass() == s1 || src.regClass() == s2);
2102          Temp tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2103                              bld.scc(bld.def(s1)), Operand(0u), src);
2104          bool_to_vector_condition(ctx, tmp, dst);
2105       }
2106       break;
2107    }
2108    case nir_op_pack_64_2x32_split: {
2109       Temp src0 = get_alu_src(ctx, instr->src[0]);
2110       Temp src1 = get_alu_src(ctx, instr->src[1]);
2111
2112       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2113       break;
2114    }
2115    case nir_op_unpack_64_2x32_split_x:
2116       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2117       break;
2118    case nir_op_unpack_64_2x32_split_y:
2119       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2120       break;
2121    case nir_op_pack_half_2x16: {
2122       Temp src = get_alu_src(ctx, instr->src[0], 2);
2123
2124       if (dst.regClass() == v1) {
2125          Temp src0 = bld.tmp(v1);
2126          Temp src1 = bld.tmp(v1);
2127          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2128          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2129             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2130          else
2131             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2132                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2133                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2134       } else {
2135          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2136          nir_print_instr(&instr->instr, stderr);
2137          fprintf(stderr, "\n");
2138       }
2139       break;
2140    }
2141    case nir_op_unpack_half_2x16_split_x: {
2142       if (dst.regClass() == v1) {
2143          Builder bld(ctx->program, ctx->block);
2144          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2145       } else {
2146          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2147          nir_print_instr(&instr->instr, stderr);
2148          fprintf(stderr, "\n");
2149       }
2150       break;
2151    }
2152    case nir_op_unpack_half_2x16_split_y: {
2153       if (dst.regClass() == v1) {
2154          Builder bld(ctx->program, ctx->block);
2155          /* TODO: use SDWA here */
2156          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2157                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2158       } else {
2159          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2160          nir_print_instr(&instr->instr, stderr);
2161          fprintf(stderr, "\n");
2162       }
2163       break;
2164    }
2165    case nir_op_fquantize2f16: {
2166       Temp src = get_alu_src(ctx, instr->src[0]);
2167       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2168
2169       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2170
2171       Temp cmp_res = bld.tmp(bld.lm);
2172       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2173
2174       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2175
2176       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32) {
2177          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2178          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2179       } else {
2180          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2181       }
2182       break;
2183    }
2184    case nir_op_bfm: {
2185       Temp bits = get_alu_src(ctx, instr->src[0]);
2186       Temp offset = get_alu_src(ctx, instr->src[1]);
2187
2188       if (dst.regClass() == s1) {
2189          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2190       } else if (dst.regClass() == v1) {
2191          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2192       } else {
2193          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2194          nir_print_instr(&instr->instr, stderr);
2195          fprintf(stderr, "\n");
2196       }
2197       break;
2198    }
2199    case nir_op_bitfield_select: {
2200       /* (mask & insert) | (~mask & base) */
2201       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2202       Temp insert = get_alu_src(ctx, instr->src[1]);
2203       Temp base = get_alu_src(ctx, instr->src[2]);
2204
2205       /* dst = (insert & bitmask) | (base & ~bitmask) */
2206       if (dst.regClass() == s1) {
2207          aco_ptr<Instruction> sop2;
2208          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2209          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2210          Operand lhs;
2211          if (const_insert && const_bitmask) {
2212             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2213          } else {
2214             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2215             lhs = Operand(insert);
2216          }
2217
2218          Operand rhs;
2219          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2220          if (const_base && const_bitmask) {
2221             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2222          } else {
2223             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2224             rhs = Operand(base);
2225          }
2226
2227          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2228
2229       } else if (dst.regClass() == v1) {
2230          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2231             base = as_vgpr(ctx, base);
2232          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2233             insert = as_vgpr(ctx, insert);
2234
2235          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2236
2237       } else {
2238          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2239          nir_print_instr(&instr->instr, stderr);
2240          fprintf(stderr, "\n");
2241       }
2242       break;
2243    }
2244    case nir_op_ubfe:
2245    case nir_op_ibfe: {
2246       Temp base = get_alu_src(ctx, instr->src[0]);
2247       Temp offset = get_alu_src(ctx, instr->src[1]);
2248       Temp bits = get_alu_src(ctx, instr->src[2]);
2249
2250       if (dst.type() == RegType::sgpr) {
2251          Operand extract;
2252          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2253          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2254          if (const_offset && const_bits) {
2255             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2256             extract = Operand(const_extract);
2257          } else {
2258             Operand width;
2259             if (const_bits) {
2260                width = Operand(const_bits->u32 << 16);
2261             } else {
2262                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2263             }
2264             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2265          }
2266
2267          aco_opcode opcode;
2268          if (dst.regClass() == s1) {
2269             if (instr->op == nir_op_ubfe)
2270                opcode = aco_opcode::s_bfe_u32;
2271             else
2272                opcode = aco_opcode::s_bfe_i32;
2273          } else if (dst.regClass() == s2) {
2274             if (instr->op == nir_op_ubfe)
2275                opcode = aco_opcode::s_bfe_u64;
2276             else
2277                opcode = aco_opcode::s_bfe_i64;
2278          } else {
2279             unreachable("Unsupported BFE bit size");
2280          }
2281
2282          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2283
2284       } else {
2285          aco_opcode opcode;
2286          if (dst.regClass() == v1) {
2287             if (instr->op == nir_op_ubfe)
2288                opcode = aco_opcode::v_bfe_u32;
2289             else
2290                opcode = aco_opcode::v_bfe_i32;
2291          } else {
2292             unreachable("Unsupported BFE bit size");
2293          }
2294
2295          emit_vop3a_instruction(ctx, instr, opcode, dst);
2296       }
2297       break;
2298    }
2299    case nir_op_bit_count: {
2300       Temp src = get_alu_src(ctx, instr->src[0]);
2301       if (src.regClass() == s1) {
2302          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2303       } else if (src.regClass() == v1) {
2304          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2305       } else if (src.regClass() == v2) {
2306          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2307                   emit_extract_vector(ctx, src, 1, v1),
2308                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2309                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2310       } else if (src.regClass() == s2) {
2311          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2312       } else {
2313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2314          nir_print_instr(&instr->instr, stderr);
2315          fprintf(stderr, "\n");
2316       }
2317       break;
2318    }
2319    case nir_op_flt: {
2320       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2321       break;
2322    }
2323    case nir_op_fge: {
2324       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2325       break;
2326    }
2327    case nir_op_feq: {
2328       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2329       break;
2330    }
2331    case nir_op_fne: {
2332       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2333       break;
2334    }
2335    case nir_op_ilt: {
2336       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2337       break;
2338    }
2339    case nir_op_ige: {
2340       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2341       break;
2342    }
2343    case nir_op_ieq: {
2344       if (instr->src[0].src.ssa->bit_size == 1)
2345          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2346       else
2347          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32, aco_opcode::s_cmp_eq_u64);
2348       break;
2349    }
2350    case nir_op_ine: {
2351       if (instr->src[0].src.ssa->bit_size == 1)
2352          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2353       else
2354          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32, aco_opcode::s_cmp_lg_u64);
2355       break;
2356    }
2357    case nir_op_ult: {
2358       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2359       break;
2360    }
2361    case nir_op_uge: {
2362       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2363       break;
2364    }
2365    case nir_op_fddx:
2366    case nir_op_fddy:
2367    case nir_op_fddx_fine:
2368    case nir_op_fddy_fine:
2369    case nir_op_fddx_coarse:
2370    case nir_op_fddy_coarse: {
2371       Definition tl = bld.def(v1);
2372       uint16_t dpp_ctrl;
2373       if (instr->op == nir_op_fddx_fine) {
2374          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2375          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2376       } else if (instr->op == nir_op_fddy_fine) {
2377          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2378          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2379       } else {
2380          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2381          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2382             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2383          else
2384             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2385       }
2386
2387       Definition tmp = bld.def(v1);
2388       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2389       emit_wqm(ctx, tmp.getTemp(), dst, true);
2390       break;
2391    }
2392    default:
2393       fprintf(stderr, "Unknown NIR ALU instr: ");
2394       nir_print_instr(&instr->instr, stderr);
2395       fprintf(stderr, "\n");
2396    }
2397 }
2398
2399 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2400 {
2401    Temp dst = get_ssa_temp(ctx, &instr->def);
2402
2403    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2404    // which get truncated the lsb if double and msb if int
2405    // for now, we only use s_mov_b64 with 64bit inline constants
2406    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2407    assert(dst.type() == RegType::sgpr);
2408
2409    Builder bld(ctx->program, ctx->block);
2410
2411    if (instr->def.bit_size == 1) {
2412       assert(dst.regClass() == bld.lm);
2413       int val = instr->value[0].b ? -1 : 0;
2414       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2415       bld.sop1(Builder::s_mov, Definition(dst), op);
2416    } else if (dst.size() == 1) {
2417       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2418    } else {
2419       assert(dst.size() != 1);
2420       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2421       if (instr->def.bit_size == 64)
2422          for (unsigned i = 0; i < dst.size(); i++)
2423             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2424       else {
2425          for (unsigned i = 0; i < dst.size(); i++)
2426             vec->operands[i] = Operand{instr->value[i].u32};
2427       }
2428       vec->definitions[0] = Definition(dst);
2429       ctx->block->instructions.emplace_back(std::move(vec));
2430    }
2431 }
2432
2433 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2434 {
2435    uint32_t new_mask = 0;
2436    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2437       if (mask & (1u << i))
2438          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2439    return new_mask;
2440 }
2441
2442 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2443 {
2444    /* This wouldn't work inside control flow or with indirect offsets but
2445     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2446
2447    unsigned write_mask = nir_intrinsic_write_mask(instr);
2448    unsigned component = nir_intrinsic_component(instr);
2449    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2450    unsigned idx = nir_intrinsic_base(instr) + component;
2451
2452    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2453    if (off_instr->type != nir_instr_type_load_const) {
2454       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2455       nir_print_instr(off_instr, stderr);
2456       fprintf(stderr, "\n");
2457    }
2458    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2459
2460    if (instr->src[0].ssa->bit_size == 64)
2461       write_mask = widen_mask(write_mask, 2);
2462
2463    for (unsigned i = 0; i < 8; ++i) {
2464       if (write_mask & (1 << i)) {
2465          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2466          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2467       }
2468       idx++;
2469    }
2470 }
2471
2472 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2473 {
2474    Builder bld(ctx->program, ctx->block);
2475    unsigned write_mask = nir_intrinsic_write_mask(instr);
2476    Operand values[4];
2477    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2478    for (unsigned i = 0; i < 4; ++i) {
2479       if (write_mask & (1 << i)) {
2480          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2481          values[i] = Operand(tmp);
2482       } else {
2483          values[i] = Operand(v1);
2484       }
2485    }
2486
2487    unsigned index = nir_intrinsic_base(instr) / 4;
2488    unsigned target, col_format;
2489    unsigned enabled_channels = 0xF;
2490    aco_opcode compr_op = (aco_opcode)0;
2491
2492    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2493    assert(offset && "Non-const offsets on exports not yet supported");
2494    index += offset->u32;
2495
2496    assert(index != FRAG_RESULT_COLOR);
2497
2498    /* Unlike vertex shader exports, it's fine to use multiple exports to
2499     * export separate channels of one target. So shaders which export both
2500     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2501     * TODO: combine the exports in those cases and create better code
2502     */
2503
2504    if (index == FRAG_RESULT_SAMPLE_MASK) {
2505
2506       if (ctx->program->info->ps.writes_z) {
2507          target = V_008DFC_SQ_EXP_MRTZ;
2508          enabled_channels = 0x4;
2509          col_format = (unsigned) -1;
2510
2511          values[2] = values[0];
2512          values[0] = Operand(v1);
2513       } else {
2514          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2515                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2516          return;
2517       }
2518
2519    } else if (index == FRAG_RESULT_DEPTH) {
2520
2521       target = V_008DFC_SQ_EXP_MRTZ;
2522       enabled_channels = 0x1;
2523       col_format = (unsigned) -1;
2524
2525    } else if (index == FRAG_RESULT_STENCIL) {
2526
2527       if (ctx->program->info->ps.writes_z) {
2528          target = V_008DFC_SQ_EXP_MRTZ;
2529          enabled_channels = 0x2;
2530          col_format = (unsigned) -1;
2531
2532          values[1] = values[0];
2533          values[0] = Operand(v1);
2534       } else {
2535          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2536          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2537                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2538          return;
2539       }
2540
2541    } else {
2542       index -= FRAG_RESULT_DATA0;
2543       target = V_008DFC_SQ_EXP_MRT + index;
2544       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2545    }
2546    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2547    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2548
2549    switch (col_format)
2550    {
2551    case V_028714_SPI_SHADER_ZERO:
2552       enabled_channels = 0; /* writemask */
2553       target = V_008DFC_SQ_EXP_NULL;
2554       break;
2555
2556    case V_028714_SPI_SHADER_32_R:
2557       enabled_channels = 1;
2558       break;
2559
2560    case V_028714_SPI_SHADER_32_GR:
2561       enabled_channels = 0x3;
2562       break;
2563
2564    case V_028714_SPI_SHADER_32_AR:
2565       if (ctx->options->chip_class >= GFX10) {
2566          /* Special case: on GFX10, the outputs are different for 32_AR */
2567          enabled_channels = 0x3;
2568          values[1] = values[3];
2569          values[3] = Operand(v1);
2570       } else {
2571          enabled_channels = 0x9;
2572       }
2573       break;
2574
2575    case V_028714_SPI_SHADER_FP16_ABGR:
2576       enabled_channels = 0x5;
2577       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2578       break;
2579
2580    case V_028714_SPI_SHADER_UNORM16_ABGR:
2581       enabled_channels = 0x5;
2582       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2583       break;
2584
2585    case V_028714_SPI_SHADER_SNORM16_ABGR:
2586       enabled_channels = 0x5;
2587       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2588       break;
2589
2590    case V_028714_SPI_SHADER_UINT16_ABGR: {
2591       enabled_channels = 0x5;
2592       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2593       if (is_int8 || is_int10) {
2594          /* clamp */
2595          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2596          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2597
2598          for (unsigned i = 0; i < 4; i++) {
2599             if ((write_mask >> i) & 1) {
2600                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2601                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2602                                     values[i]);
2603             }
2604          }
2605       }
2606       break;
2607    }
2608
2609    case V_028714_SPI_SHADER_SINT16_ABGR:
2610       enabled_channels = 0x5;
2611       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2612       if (is_int8 || is_int10) {
2613          /* clamp */
2614          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2615          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2616          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2617          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2618
2619          for (unsigned i = 0; i < 4; i++) {
2620             if ((write_mask >> i) & 1) {
2621                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2622                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2623                                     values[i]);
2624                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2625                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2626                                     values[i]);
2627             }
2628          }
2629       }
2630       break;
2631
2632    case V_028714_SPI_SHADER_32_ABGR:
2633       enabled_channels = 0xF;
2634       break;
2635
2636    default:
2637       break;
2638    }
2639
2640    if (target == V_008DFC_SQ_EXP_NULL)
2641       return;
2642
2643    if ((bool) compr_op) {
2644       for (int i = 0; i < 2; i++) {
2645          /* check if at least one of the values to be compressed is enabled */
2646          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2647          if (enabled) {
2648             enabled_channels |= enabled << (i*2);
2649             values[i] = bld.vop3(compr_op, bld.def(v1),
2650                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2651                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2652          } else {
2653             values[i] = Operand(v1);
2654          }
2655       }
2656       values[2] = Operand(v1);
2657       values[3] = Operand(v1);
2658    } else {
2659       for (int i = 0; i < 4; i++)
2660          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2661    }
2662
2663    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2664            enabled_channels, target, (bool) compr_op);
2665 }
2666
2667 Operand load_lds_size_m0(isel_context *ctx)
2668 {
2669    /* TODO: m0 does not need to be initialized on GFX9+ */
2670    Builder bld(ctx->program, ctx->block);
2671    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2672 }
2673
2674 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2675               Temp address, unsigned base_offset, unsigned align)
2676 {
2677    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2678
2679    Builder bld(ctx->program, ctx->block);
2680
2681    Operand m = load_lds_size_m0(ctx);
2682
2683    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2684    unsigned bytes_read = 0;
2685    unsigned result_size = 0;
2686    unsigned total_bytes = num_components * elem_size_bytes;
2687    std::array<Temp, 4> result;
2688
2689    while (bytes_read < total_bytes) {
2690       unsigned todo = total_bytes - bytes_read;
2691       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2692       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2693
2694       aco_opcode op = aco_opcode::last_opcode;
2695       bool read2 = false;
2696       if (todo >= 16 && aligned16) {
2697          op = aco_opcode::ds_read_b128;
2698          todo = 16;
2699       } else if (todo >= 16 && aligned8) {
2700          op = aco_opcode::ds_read2_b64;
2701          read2 = true;
2702          todo = 16;
2703       } else if (todo >= 12 && aligned16) {
2704          op = aco_opcode::ds_read_b96;
2705          todo = 12;
2706       } else if (todo >= 8 && aligned8) {
2707          op = aco_opcode::ds_read_b64;
2708          todo = 8;
2709       } else if (todo >= 8) {
2710          op = aco_opcode::ds_read2_b32;
2711          read2 = true;
2712          todo = 8;
2713       } else if (todo >= 4) {
2714          op = aco_opcode::ds_read_b32;
2715          todo = 4;
2716       } else {
2717          assert(false);
2718       }
2719       assert(todo % elem_size_bytes == 0);
2720       unsigned num_elements = todo / elem_size_bytes;
2721       unsigned offset = base_offset + bytes_read;
2722       unsigned max_offset = read2 ? 1019 : 65535;
2723
2724       Temp address_offset = address;
2725       if (offset > max_offset) {
2726          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2727          offset = bytes_read;
2728       }
2729       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2730
2731       Temp res;
2732       if (num_components == 1 && dst.type() == RegType::vgpr)
2733          res = dst;
2734       else
2735          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2736
2737       if (read2)
2738          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2739       else
2740          res = bld.ds(op, Definition(res), address_offset, m, offset);
2741
2742       if (num_components == 1) {
2743          assert(todo == total_bytes);
2744          if (dst.type() == RegType::sgpr)
2745             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2746          return;
2747       }
2748
2749       if (dst.type() == RegType::sgpr)
2750          res = bld.as_uniform(res);
2751
2752       if (num_elements == 1) {
2753          result[result_size++] = res;
2754       } else {
2755          assert(res != dst && res.size() % num_elements == 0);
2756          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2757          split->operands[0] = Operand(res);
2758          for (unsigned i = 0; i < num_elements; i++)
2759             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2760          ctx->block->instructions.emplace_back(std::move(split));
2761       }
2762
2763       bytes_read += todo;
2764    }
2765
2766    assert(result_size == num_components && result_size > 1);
2767    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2768    for (unsigned i = 0; i < result_size; i++)
2769       vec->operands[i] = Operand(result[i]);
2770    vec->definitions[0] = Definition(dst);
2771    ctx->block->instructions.emplace_back(std::move(vec));
2772    ctx->allocated_vec.emplace(dst.id(), result);
2773 }
2774
2775 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2776 {
2777    if (start == 0 && size == data.size())
2778       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2779
2780    unsigned size_hint = 1;
2781    auto it = ctx->allocated_vec.find(data.id());
2782    if (it != ctx->allocated_vec.end())
2783       size_hint = it->second[0].size();
2784    if (size % size_hint || start % size_hint)
2785       size_hint = 1;
2786
2787    start /= size_hint;
2788    size /= size_hint;
2789
2790    Temp elems[size];
2791    for (unsigned i = 0; i < size; i++)
2792       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2793
2794    if (size == 1)
2795       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2796
2797    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2798    for (unsigned i = 0; i < size; i++)
2799       vec->operands[i] = Operand(elems[i]);
2800    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2801    vec->definitions[0] = Definition(res);
2802    ctx->block->instructions.emplace_back(std::move(vec));
2803    return res;
2804 }
2805
2806 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2807 {
2808    Builder bld(ctx->program, ctx->block);
2809    unsigned bytes_written = 0;
2810    while (bytes_written < total_size * 4) {
2811       unsigned todo = total_size * 4 - bytes_written;
2812       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2813       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2814
2815       aco_opcode op = aco_opcode::last_opcode;
2816       bool write2 = false;
2817       unsigned size = 0;
2818       if (todo >= 16 && aligned16) {
2819          op = aco_opcode::ds_write_b128;
2820          size = 4;
2821       } else if (todo >= 16 && aligned8) {
2822          op = aco_opcode::ds_write2_b64;
2823          write2 = true;
2824          size = 4;
2825       } else if (todo >= 12 && aligned16) {
2826          op = aco_opcode::ds_write_b96;
2827          size = 3;
2828       } else if (todo >= 8 && aligned8) {
2829          op = aco_opcode::ds_write_b64;
2830          size = 2;
2831       } else if (todo >= 8) {
2832          op = aco_opcode::ds_write2_b32;
2833          write2 = true;
2834          size = 2;
2835       } else if (todo >= 4) {
2836          op = aco_opcode::ds_write_b32;
2837          size = 1;
2838       } else {
2839          assert(false);
2840       }
2841
2842       unsigned offset = offset0 + offset1 + bytes_written;
2843       unsigned max_offset = write2 ? 1020 : 65535;
2844       Temp address_offset = address;
2845       if (offset > max_offset) {
2846          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2847          offset = offset1 + bytes_written;
2848       }
2849       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2850
2851       if (write2) {
2852          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2853          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2854          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2855       } else {
2856          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2857          bld.ds(op, address_offset, val, m, offset);
2858       }
2859
2860       bytes_written += size * 4;
2861    }
2862 }
2863
2864 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2865                Temp address, unsigned base_offset, unsigned align)
2866 {
2867    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2868
2869    Operand m = load_lds_size_m0(ctx);
2870
2871    /* we need at most two stores for 32bit variables */
2872    int start[2], count[2];
2873    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2874    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2875    assert(wrmask == 0);
2876
2877    /* one combined store is sufficient */
2878    if (count[0] == count[1]) {
2879       Builder bld(ctx->program, ctx->block);
2880
2881       Temp address_offset = address;
2882       if ((base_offset >> 2) + start[1] > 255) {
2883          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2884          base_offset = 0;
2885       }
2886
2887       assert(count[0] == 1);
2888       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2889       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2890       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2891       base_offset = base_offset / elem_size_bytes;
2892       bld.ds(op, address_offset, val0, val1, m,
2893              base_offset + start[0], base_offset + start[1]);
2894       return;
2895    }
2896
2897    for (unsigned i = 0; i < 2; i++) {
2898       if (count[i] == 0)
2899          continue;
2900
2901       unsigned elem_size_words = elem_size_bytes / 4;
2902       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2903                       base_offset, start[i] * elem_size_bytes, align);
2904    }
2905    return;
2906 }
2907
2908 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2909 {
2910    if (ctx->stage == vertex_vs) {
2911       visit_store_vs_output(ctx, instr);
2912    } else if (ctx->stage == fragment_fs) {
2913       visit_store_fs_output(ctx, instr);
2914    } else {
2915       unreachable("Shader stage not implemented");
2916    }
2917 }
2918
2919 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2920 {
2921    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2922    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2923
2924    Builder bld(ctx->program, ctx->block);
2925    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2926    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2927 }
2928
2929 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2930 {
2931    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2932    for (unsigned i = 0; i < num_components; i++)
2933       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
2934    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
2935       assert(num_components == 4);
2936       Builder bld(ctx->program, ctx->block);
2937       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
2938    }
2939
2940    for (Operand& op : vec->operands)
2941       op = op.isUndefined() ? Operand(0u) : op;
2942
2943    vec->definitions[0] = Definition(dst);
2944    ctx->block->instructions.emplace_back(std::move(vec));
2945    emit_split_vector(ctx, dst, num_components);
2946    return;
2947 }
2948
2949 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2950 {
2951    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2952    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2953    unsigned idx = nir_intrinsic_base(instr);
2954    unsigned component = nir_intrinsic_component(instr);
2955    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
2956
2957    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2958    if (offset) {
2959       assert(offset->u32 == 0);
2960    } else {
2961       /* the lower 15bit of the prim_mask contain the offset into LDS
2962        * while the upper bits contain the number of prims */
2963       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2964       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2965       Builder bld(ctx->program, ctx->block);
2966       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2967       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2968       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2969       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2970       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2971    }
2972
2973    if (instr->dest.ssa.num_components == 1) {
2974       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2975    } else {
2976       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2977       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2978       {
2979          Temp tmp = {ctx->program->allocateId(), v1};
2980          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2981          vec->operands[i] = Operand(tmp);
2982       }
2983       vec->definitions[0] = Definition(dst);
2984       ctx->block->instructions.emplace_back(std::move(vec));
2985    }
2986 }
2987
2988 unsigned get_num_channels_from_data_format(unsigned data_format)
2989 {
2990    switch (data_format) {
2991    case V_008F0C_BUF_DATA_FORMAT_8:
2992    case V_008F0C_BUF_DATA_FORMAT_16:
2993    case V_008F0C_BUF_DATA_FORMAT_32:
2994       return 1;
2995    case V_008F0C_BUF_DATA_FORMAT_8_8:
2996    case V_008F0C_BUF_DATA_FORMAT_16_16:
2997    case V_008F0C_BUF_DATA_FORMAT_32_32:
2998       return 2;
2999    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3000    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3001    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3002       return 3;
3003    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3004    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3005    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3006    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3007    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3008       return 4;
3009    default:
3010       break;
3011    }
3012
3013    return 4;
3014 }
3015
3016 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3017  * so we may need to fix it up. */
3018 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3019 {
3020    Builder bld(ctx->program, ctx->block);
3021
3022    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3023       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3024
3025    /* For the integer-like cases, do a natural sign extension.
3026     *
3027     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3028     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3029     * exponent.
3030     */
3031    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3032    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3033
3034    /* Convert back to the right type. */
3035    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3036       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3037       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3038       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3039    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3040       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3041    }
3042
3043    return alpha;
3044 }
3045
3046 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3047 {
3048    Builder bld(ctx->program, ctx->block);
3049    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3050    if (ctx->stage & sw_vs) {
3051
3052       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3053       if (off_instr->type != nir_instr_type_load_const) {
3054          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3055          nir_print_instr(off_instr, stderr);
3056          fprintf(stderr, "\n");
3057       }
3058       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3059
3060       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3061
3062       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3063       unsigned component = nir_intrinsic_component(instr);
3064       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3065       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3066       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3067       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3068
3069       unsigned dfmt = attrib_format & 0xf;
3070
3071       unsigned nfmt = (attrib_format >> 4) & 0x7;
3072       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3073       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3074       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3075       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3076       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3077       if (post_shuffle)
3078          num_channels = MAX2(num_channels, 3);
3079
3080       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3081
3082       Temp index;
3083       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3084          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3085          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3086          if (divisor) {
3087             ctx->needs_instance_id = true;
3088             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3089             if (divisor != 1) {
3090                Temp divided = bld.tmp(v1);
3091                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3092                index = bld.vadd32(bld.def(v1), start_instance, divided);
3093             } else {
3094                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3095             }
3096          } else {
3097             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3098          }
3099       } else {
3100          index = bld.vadd32(bld.def(v1),
3101                             get_arg(ctx, ctx->args->ac.base_vertex),
3102                             get_arg(ctx, ctx->args->ac.vertex_id));
3103       }
3104
3105       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3106          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3107          attrib_offset = attrib_offset % attrib_stride;
3108       }
3109
3110       Operand soffset(0u);
3111       if (attrib_offset >= 4096) {
3112          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3113          attrib_offset = 0;
3114       }
3115
3116       aco_opcode opcode;
3117       switch (num_channels) {
3118       case 1:
3119          opcode = aco_opcode::tbuffer_load_format_x;
3120          break;
3121       case 2:
3122          opcode = aco_opcode::tbuffer_load_format_xy;
3123          break;
3124       case 3:
3125          opcode = aco_opcode::tbuffer_load_format_xyz;
3126          break;
3127       case 4:
3128          opcode = aco_opcode::tbuffer_load_format_xyzw;
3129          break;
3130       default:
3131          unreachable("Unimplemented load_input vector size");
3132       }
3133
3134       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3135
3136       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3137       mubuf->operands[0] = Operand(index);
3138       mubuf->operands[1] = Operand(list);
3139       mubuf->operands[2] = soffset;
3140       mubuf->definitions[0] = Definition(tmp);
3141       mubuf->idxen = true;
3142       mubuf->can_reorder = true;
3143       mubuf->dfmt = dfmt;
3144       mubuf->nfmt = nfmt;
3145       assert(attrib_offset < 4096);
3146       mubuf->offset = attrib_offset;
3147       ctx->block->instructions.emplace_back(std::move(mubuf));
3148
3149       emit_split_vector(ctx, tmp, tmp.size());
3150
3151       if (tmp.id() != dst.id()) {
3152          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3153                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3154
3155          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3156          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3157          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3158
3159          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3160          for (unsigned i = 0; i < dst.size(); i++) {
3161             unsigned idx = i + component;
3162             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3163                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3164                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3165             } else if (idx < num_channels) {
3166                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3167             } else if (is_float && idx == 3) {
3168                vec->operands[i] = Operand(0x3f800000u);
3169             } else if (!is_float && idx == 3) {
3170                vec->operands[i] = Operand(1u);
3171             } else {
3172                vec->operands[i] = Operand(0u);
3173             }
3174          }
3175          vec->definitions[0] = Definition(dst);
3176          ctx->block->instructions.emplace_back(std::move(vec));
3177          emit_split_vector(ctx, dst, dst.size());
3178       }
3179
3180    } else if (ctx->stage == fragment_fs) {
3181       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3182       if (off_instr->type != nir_instr_type_load_const ||
3183           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3184          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3185          nir_print_instr(off_instr, stderr);
3186          fprintf(stderr, "\n");
3187       }
3188
3189       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3190       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3191       if (offset) {
3192          assert(offset->u32 == 0);
3193       } else {
3194          /* the lower 15bit of the prim_mask contain the offset into LDS
3195           * while the upper bits contain the number of prims */
3196          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3197          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3198          Builder bld(ctx->program, ctx->block);
3199          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3200          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3201          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3202          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3203          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3204       }
3205
3206       unsigned idx = nir_intrinsic_base(instr);
3207       unsigned component = nir_intrinsic_component(instr);
3208
3209       if (dst.size() == 1) {
3210          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3211       } else {
3212          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3213          for (unsigned i = 0; i < dst.size(); i++)
3214             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3215          vec->definitions[0] = Definition(dst);
3216          bld.insert(std::move(vec));
3217       }
3218
3219    } else {
3220       unreachable("Shader stage not implemented");
3221    }
3222 }
3223
3224 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3225 {
3226    if (ctx->program->info->need_indirect_descriptor_sets) {
3227       Builder bld(ctx->program, ctx->block);
3228       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3229       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3230    }
3231
3232    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3233 }
3234
3235
3236 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3237 {
3238    Builder bld(ctx->program, ctx->block);
3239    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3240    if (!ctx->divergent_vals[instr->dest.ssa.index])
3241       index = bld.as_uniform(index);
3242    unsigned desc_set = nir_intrinsic_desc_set(instr);
3243    unsigned binding = nir_intrinsic_binding(instr);
3244
3245    Temp desc_ptr;
3246    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3247    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3248    unsigned offset = layout->binding[binding].offset;
3249    unsigned stride;
3250    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3251        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3252       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3253       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3254       offset = pipeline_layout->push_constant_size + 16 * idx;
3255       stride = 16;
3256    } else {
3257       desc_ptr = load_desc_ptr(ctx, desc_set);
3258       stride = layout->binding[binding].size;
3259    }
3260
3261    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3262    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3263    if (stride != 1) {
3264       if (nir_const_index) {
3265          const_index = const_index * stride;
3266       } else if (index.type() == RegType::vgpr) {
3267          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3268          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3269       } else {
3270          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3271       }
3272    }
3273    if (offset) {
3274       if (nir_const_index) {
3275          const_index = const_index + offset;
3276       } else if (index.type() == RegType::vgpr) {
3277          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3278       } else {
3279          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3280       }
3281    }
3282
3283    if (nir_const_index && const_index == 0) {
3284       index = desc_ptr;
3285    } else if (index.type() == RegType::vgpr) {
3286       index = bld.vadd32(bld.def(v1),
3287                          nir_const_index ? Operand(const_index) : Operand(index),
3288                          Operand(desc_ptr));
3289    } else {
3290       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3291                        nir_const_index ? Operand(const_index) : Operand(index),
3292                        Operand(desc_ptr));
3293    }
3294
3295    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3296 }
3297
3298 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3299                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3300 {
3301    Builder bld(ctx->program, ctx->block);
3302
3303    unsigned num_bytes = dst.size() * 4;
3304    bool dlc = glc && ctx->options->chip_class >= GFX10;
3305
3306    aco_opcode op;
3307    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3308       if (ctx->options->chip_class < GFX8)
3309          offset = as_vgpr(ctx, offset);
3310
3311       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3312       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3313       unsigned const_offset = 0;
3314
3315       Temp lower = Temp();
3316       if (num_bytes > 16) {
3317          assert(num_components == 3 || num_components == 4);
3318          op = aco_opcode::buffer_load_dwordx4;
3319          lower = bld.tmp(v4);
3320          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3321          mubuf->definitions[0] = Definition(lower);
3322          mubuf->operands[0] = vaddr;
3323          mubuf->operands[1] = Operand(rsrc);
3324          mubuf->operands[2] = soffset;
3325          mubuf->offen = (offset.type() == RegType::vgpr);
3326          mubuf->glc = glc;
3327          mubuf->dlc = dlc;
3328          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3329          mubuf->can_reorder = readonly;
3330          bld.insert(std::move(mubuf));
3331          emit_split_vector(ctx, lower, 2);
3332          num_bytes -= 16;
3333          const_offset = 16;
3334       }
3335
3336       switch (num_bytes) {
3337          case 4:
3338             op = aco_opcode::buffer_load_dword;
3339             break;
3340          case 8:
3341             op = aco_opcode::buffer_load_dwordx2;
3342             break;
3343          case 12:
3344             op = aco_opcode::buffer_load_dwordx3;
3345             break;
3346          case 16:
3347             op = aco_opcode::buffer_load_dwordx4;
3348             break;
3349          default:
3350             unreachable("Load SSBO not implemented for this size.");
3351       }
3352       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3353       mubuf->operands[0] = vaddr;
3354       mubuf->operands[1] = Operand(rsrc);
3355       mubuf->operands[2] = soffset;
3356       mubuf->offen = (offset.type() == RegType::vgpr);
3357       mubuf->glc = glc;
3358       mubuf->dlc = dlc;
3359       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3360       mubuf->can_reorder = readonly;
3361       mubuf->offset = const_offset;
3362       aco_ptr<Instruction> instr = std::move(mubuf);
3363
3364       if (dst.size() > 4) {
3365          assert(lower != Temp());
3366          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3367          instr->definitions[0] = Definition(upper);
3368          bld.insert(std::move(instr));
3369          if (dst.size() == 8)
3370             emit_split_vector(ctx, upper, 2);
3371          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3372          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3373          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3374          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3375          if (dst.size() == 8)
3376             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3377       }
3378
3379       if (dst.type() == RegType::sgpr) {
3380          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3381          instr->definitions[0] = Definition(vec);
3382          bld.insert(std::move(instr));
3383          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3384       } else {
3385          instr->definitions[0] = Definition(dst);
3386          bld.insert(std::move(instr));
3387       }
3388    } else {
3389       switch (num_bytes) {
3390          case 4:
3391             op = aco_opcode::s_buffer_load_dword;
3392             break;
3393          case 8:
3394             op = aco_opcode::s_buffer_load_dwordx2;
3395             break;
3396          case 12:
3397          case 16:
3398             op = aco_opcode::s_buffer_load_dwordx4;
3399             break;
3400          case 24:
3401          case 32:
3402             op = aco_opcode::s_buffer_load_dwordx8;
3403             break;
3404          default:
3405             unreachable("Load SSBO not implemented for this size.");
3406       }
3407       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3408       load->operands[0] = Operand(rsrc);
3409       load->operands[1] = Operand(bld.as_uniform(offset));
3410       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3411       load->definitions[0] = Definition(dst);
3412       load->glc = glc;
3413       load->dlc = dlc;
3414       load->barrier = readonly ? barrier_none : barrier_buffer;
3415       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3416       assert(ctx->options->chip_class >= GFX8 || !glc);
3417
3418       /* trim vector */
3419       if (dst.size() == 3) {
3420          Temp vec = bld.tmp(s4);
3421          load->definitions[0] = Definition(vec);
3422          bld.insert(std::move(load));
3423          emit_split_vector(ctx, vec, 4);
3424
3425          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3426                     emit_extract_vector(ctx, vec, 0, s1),
3427                     emit_extract_vector(ctx, vec, 1, s1),
3428                     emit_extract_vector(ctx, vec, 2, s1));
3429       } else if (dst.size() == 6) {
3430          Temp vec = bld.tmp(s8);
3431          load->definitions[0] = Definition(vec);
3432          bld.insert(std::move(load));
3433          emit_split_vector(ctx, vec, 4);
3434
3435          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3436                     emit_extract_vector(ctx, vec, 0, s2),
3437                     emit_extract_vector(ctx, vec, 1, s2),
3438                     emit_extract_vector(ctx, vec, 2, s2));
3439       } else {
3440          bld.insert(std::move(load));
3441       }
3442
3443    }
3444    emit_split_vector(ctx, dst, num_components);
3445 }
3446
3447 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3448 {
3449    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3450    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3451
3452    Builder bld(ctx->program, ctx->block);
3453
3454    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3455    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3456    unsigned binding = nir_intrinsic_binding(idx_instr);
3457    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3458
3459    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3460       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3461                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3462                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3463                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3464       if (ctx->options->chip_class >= GFX10) {
3465          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3466                       S_008F0C_OOB_SELECT(3) |
3467                       S_008F0C_RESOURCE_LEVEL(1);
3468       } else {
3469          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3470                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3471       }
3472       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3473                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3474                                      Operand(0xFFFFFFFFu),
3475                                      Operand(desc_type));
3476       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3477                         rsrc, upper_dwords);
3478    } else {
3479       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3480       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3481    }
3482
3483    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3484 }
3485
3486 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3487 {
3488    Builder bld(ctx->program, ctx->block);
3489    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3490
3491    unsigned offset = nir_intrinsic_base(instr);
3492    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3493    if (index_cv && instr->dest.ssa.bit_size == 32) {
3494
3495       unsigned count = instr->dest.ssa.num_components;
3496       unsigned start = (offset + index_cv->u32) / 4u;
3497       start -= ctx->args->ac.base_inline_push_consts;
3498       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3499          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3500          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3501          for (unsigned i = 0; i < count; ++i) {
3502             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3503             vec->operands[i] = Operand{elems[i]};
3504          }
3505          vec->definitions[0] = Definition(dst);
3506          ctx->block->instructions.emplace_back(std::move(vec));
3507          ctx->allocated_vec.emplace(dst.id(), elems);
3508          return;
3509       }
3510    }
3511
3512    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3513    if (offset != 0) // TODO check if index != 0 as well
3514       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3515    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3516    Temp vec = dst;
3517    bool trim = false;
3518    aco_opcode op;
3519
3520    switch (dst.size()) {
3521    case 1:
3522       op = aco_opcode::s_load_dword;
3523       break;
3524    case 2:
3525       op = aco_opcode::s_load_dwordx2;
3526       break;
3527    case 3:
3528       vec = bld.tmp(s4);
3529       trim = true;
3530    case 4:
3531       op = aco_opcode::s_load_dwordx4;
3532       break;
3533    case 6:
3534       vec = bld.tmp(s8);
3535       trim = true;
3536    case 8:
3537       op = aco_opcode::s_load_dwordx8;
3538       break;
3539    default:
3540       unreachable("unimplemented or forbidden load_push_constant.");
3541    }
3542
3543    bld.smem(op, Definition(vec), ptr, index);
3544
3545    if (trim) {
3546       emit_split_vector(ctx, vec, 4);
3547       RegClass rc = dst.size() == 3 ? s1 : s2;
3548       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3549                  emit_extract_vector(ctx, vec, 0, rc),
3550                  emit_extract_vector(ctx, vec, 1, rc),
3551                  emit_extract_vector(ctx, vec, 2, rc));
3552
3553    }
3554    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3555 }
3556
3557 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3558 {
3559    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3560
3561    Builder bld(ctx->program, ctx->block);
3562
3563    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3564                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3565                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3566                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3567    if (ctx->options->chip_class >= GFX10) {
3568       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3569                    S_008F0C_OOB_SELECT(3) |
3570                    S_008F0C_RESOURCE_LEVEL(1);
3571    } else {
3572       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3573                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3574    }
3575
3576    unsigned base = nir_intrinsic_base(instr);
3577    unsigned range = nir_intrinsic_range(instr);
3578
3579    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3580    if (base && offset.type() == RegType::sgpr)
3581       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3582    else if (base && offset.type() == RegType::vgpr)
3583       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3584
3585    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3586                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3587                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3588                           Operand(desc_type));
3589
3590    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3591 }
3592
3593 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3594 {
3595    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3596       ctx->cf_info.exec_potentially_empty = true;
3597
3598    ctx->program->needs_exact = true;
3599
3600    // TODO: optimize uniform conditions
3601    Builder bld(ctx->program, ctx->block);
3602    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3603    assert(src.regClass() == bld.lm);
3604    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3605    bld.pseudo(aco_opcode::p_discard_if, src);
3606    ctx->block->kind |= block_kind_uses_discard_if;
3607    return;
3608 }
3609
3610 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3611 {
3612    Builder bld(ctx->program, ctx->block);
3613
3614    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3615       ctx->cf_info.exec_potentially_empty = true;
3616
3617    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3618                     ctx->cf_info.parent_loop.has_divergent_continue;
3619
3620    if (ctx->block->loop_nest_depth &&
3621        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3622       /* we handle discards the same way as jump instructions */
3623       append_logical_end(ctx->block);
3624
3625       /* in loops, discard behaves like break */
3626       Block *linear_target = ctx->cf_info.parent_loop.exit;
3627       ctx->block->kind |= block_kind_discard;
3628
3629       if (!divergent) {
3630          /* uniform discard - loop ends here */
3631          assert(nir_instr_is_last(&instr->instr));
3632          ctx->block->kind |= block_kind_uniform;
3633          ctx->cf_info.has_branch = true;
3634          bld.branch(aco_opcode::p_branch);
3635          add_linear_edge(ctx->block->index, linear_target);
3636          return;
3637       }
3638
3639       /* we add a break right behind the discard() instructions */
3640       ctx->block->kind |= block_kind_break;
3641       unsigned idx = ctx->block->index;
3642
3643       /* remove critical edges from linear CFG */
3644       bld.branch(aco_opcode::p_branch);
3645       Block* break_block = ctx->program->create_and_insert_block();
3646       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3647       break_block->kind |= block_kind_uniform;
3648       add_linear_edge(idx, break_block);
3649       add_linear_edge(break_block->index, linear_target);
3650       bld.reset(break_block);
3651       bld.branch(aco_opcode::p_branch);
3652
3653       Block* continue_block = ctx->program->create_and_insert_block();
3654       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3655       add_linear_edge(idx, continue_block);
3656       append_logical_start(continue_block);
3657       ctx->block = continue_block;
3658
3659       return;
3660    }
3661
3662    /* it can currently happen that NIR doesn't remove the unreachable code */
3663    if (!nir_instr_is_last(&instr->instr)) {
3664       ctx->program->needs_exact = true;
3665       /* save exec somewhere temporarily so that it doesn't get
3666        * overwritten before the discard from outer exec masks */
3667       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3668       bld.pseudo(aco_opcode::p_discard_if, cond);
3669       ctx->block->kind |= block_kind_uses_discard_if;
3670       return;
3671    }
3672
3673    /* This condition is incorrect for uniformly branched discards in a loop
3674     * predicated by a divergent condition, but the above code catches that case
3675     * and the discard would end up turning into a discard_if.
3676     * For example:
3677     * if (divergent) {
3678     *    while (...) {
3679     *       if (uniform) {
3680     *          discard;
3681     *       }
3682     *    }
3683     * }
3684     */
3685    if (!ctx->cf_info.parent_if.is_divergent) {
3686       /* program just ends here */
3687       ctx->block->kind |= block_kind_uniform;
3688       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3689               0 /* enabled mask */, 9 /* dest */,
3690               false /* compressed */, true/* done */, true /* valid mask */);
3691       bld.sopp(aco_opcode::s_endpgm);
3692       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3693    } else {
3694       ctx->block->kind |= block_kind_discard;
3695       /* branch and linear edge is added by visit_if() */
3696    }
3697 }
3698
3699 enum aco_descriptor_type {
3700    ACO_DESC_IMAGE,
3701    ACO_DESC_FMASK,
3702    ACO_DESC_SAMPLER,
3703    ACO_DESC_BUFFER,
3704    ACO_DESC_PLANE_0,
3705    ACO_DESC_PLANE_1,
3706    ACO_DESC_PLANE_2,
3707 };
3708
3709 static bool
3710 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3711    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3712       return false;
3713    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3714    return dim == ac_image_cube ||
3715           dim == ac_image_1darray ||
3716           dim == ac_image_2darray ||
3717           dim == ac_image_2darraymsaa;
3718 }
3719
3720 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3721                       enum aco_descriptor_type desc_type,
3722                       const nir_tex_instr *tex_instr, bool image, bool write)
3723 {
3724 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3725    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3726    if (it != ctx->tex_desc.end())
3727       return it->second;
3728 */
3729    Temp index = Temp();
3730    bool index_set = false;
3731    unsigned constant_index = 0;
3732    unsigned descriptor_set;
3733    unsigned base_index;
3734    Builder bld(ctx->program, ctx->block);
3735
3736    if (!deref_instr) {
3737       assert(tex_instr && !image);
3738       descriptor_set = 0;
3739       base_index = tex_instr->sampler_index;
3740    } else {
3741       while(deref_instr->deref_type != nir_deref_type_var) {
3742          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3743          if (!array_size)
3744             array_size = 1;
3745
3746          assert(deref_instr->deref_type == nir_deref_type_array);
3747          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3748          if (const_value) {
3749             constant_index += array_size * const_value->u32;
3750          } else {
3751             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3752             if (indirect.type() == RegType::vgpr)
3753                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3754
3755             if (array_size != 1)
3756                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3757
3758             if (!index_set) {
3759                index = indirect;
3760                index_set = true;
3761             } else {
3762                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3763             }
3764          }
3765
3766          deref_instr = nir_src_as_deref(deref_instr->parent);
3767       }
3768       descriptor_set = deref_instr->var->data.descriptor_set;
3769       base_index = deref_instr->var->data.binding;
3770    }
3771
3772    Temp list = load_desc_ptr(ctx, descriptor_set);
3773    list = convert_pointer_to_64_bit(ctx, list);
3774
3775    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3776    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3777    unsigned offset = binding->offset;
3778    unsigned stride = binding->size;
3779    aco_opcode opcode;
3780    RegClass type;
3781
3782    assert(base_index < layout->binding_count);
3783
3784    switch (desc_type) {
3785    case ACO_DESC_IMAGE:
3786       type = s8;
3787       opcode = aco_opcode::s_load_dwordx8;
3788       break;
3789    case ACO_DESC_FMASK:
3790       type = s8;
3791       opcode = aco_opcode::s_load_dwordx8;
3792       offset += 32;
3793       break;
3794    case ACO_DESC_SAMPLER:
3795       type = s4;
3796       opcode = aco_opcode::s_load_dwordx4;
3797       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3798          offset += radv_combined_image_descriptor_sampler_offset(binding);
3799       break;
3800    case ACO_DESC_BUFFER:
3801       type = s4;
3802       opcode = aco_opcode::s_load_dwordx4;
3803       break;
3804    case ACO_DESC_PLANE_0:
3805    case ACO_DESC_PLANE_1:
3806       type = s8;
3807       opcode = aco_opcode::s_load_dwordx8;
3808       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3809       break;
3810    case ACO_DESC_PLANE_2:
3811       type = s4;
3812       opcode = aco_opcode::s_load_dwordx4;
3813       offset += 64;
3814       break;
3815    default:
3816       unreachable("invalid desc_type\n");
3817    }
3818
3819    offset += constant_index * stride;
3820
3821    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3822       (!index_set || binding->immutable_samplers_equal)) {
3823       if (binding->immutable_samplers_equal)
3824          constant_index = 0;
3825
3826       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3827       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3828                         Operand(samplers[constant_index * 4 + 0]),
3829                         Operand(samplers[constant_index * 4 + 1]),
3830                         Operand(samplers[constant_index * 4 + 2]),
3831                         Operand(samplers[constant_index * 4 + 3]));
3832    }
3833
3834    Operand off;
3835    if (!index_set) {
3836       off = Operand(offset);
3837    } else {
3838       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3839                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3840    }
3841
3842    Temp res = bld.smem(opcode, bld.def(type), list, off);
3843
3844    if (desc_type == ACO_DESC_PLANE_2) {
3845       Temp components[8];
3846       for (unsigned i = 0; i < 8; i++)
3847          components[i] = bld.tmp(s1);
3848       bld.pseudo(aco_opcode::p_split_vector,
3849                  Definition(components[0]),
3850                  Definition(components[1]),
3851                  Definition(components[2]),
3852                  Definition(components[3]),
3853                  res);
3854
3855       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3856       bld.pseudo(aco_opcode::p_split_vector,
3857                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3858                  Definition(components[4]),
3859                  Definition(components[5]),
3860                  Definition(components[6]),
3861                  Definition(components[7]),
3862                  desc2);
3863
3864       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3865                        components[0], components[1], components[2], components[3],
3866                        components[4], components[5], components[6], components[7]);
3867    }
3868
3869    return res;
3870 }
3871
3872 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3873 {
3874    switch (dim) {
3875    case GLSL_SAMPLER_DIM_BUF:
3876       return 1;
3877    case GLSL_SAMPLER_DIM_1D:
3878       return array ? 2 : 1;
3879    case GLSL_SAMPLER_DIM_2D:
3880       return array ? 3 : 2;
3881    case GLSL_SAMPLER_DIM_MS:
3882       return array ? 4 : 3;
3883    case GLSL_SAMPLER_DIM_3D:
3884    case GLSL_SAMPLER_DIM_CUBE:
3885       return 3;
3886    case GLSL_SAMPLER_DIM_RECT:
3887    case GLSL_SAMPLER_DIM_SUBPASS:
3888       return 2;
3889    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3890       return 3;
3891    default:
3892       break;
3893    }
3894    return 0;
3895 }
3896
3897
3898 /* Adjust the sample index according to FMASK.
3899  *
3900  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3901  * which is the identity mapping. Each nibble says which physical sample
3902  * should be fetched to get that sample.
3903  *
3904  * For example, 0x11111100 means there are only 2 samples stored and
3905  * the second sample covers 3/4 of the pixel. When reading samples 0
3906  * and 1, return physical sample 0 (determined by the first two 0s
3907  * in FMASK), otherwise return physical sample 1.
3908  *
3909  * The sample index should be adjusted as follows:
3910  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3911  */
3912 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3913 {
3914    Builder bld(ctx->program, ctx->block);
3915    Temp fmask = bld.tmp(v1);
3916    unsigned dim = ctx->options->chip_class >= GFX10
3917                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3918                   : 0;
3919
3920    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3921    load->operands[0] = Operand(coords);
3922    load->operands[1] = Operand(fmask_desc_ptr);
3923    load->definitions[0] = Definition(fmask);
3924    load->glc = false;
3925    load->dlc = false;
3926    load->dmask = 0x1;
3927    load->unrm = true;
3928    load->da = da;
3929    load->dim = dim;
3930    load->can_reorder = true; /* fmask images shouldn't be modified */
3931    ctx->block->instructions.emplace_back(std::move(load));
3932
3933    Operand sample_index4;
3934    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3935       sample_index4 = Operand(sample_index.constantValue() << 2);
3936    } else if (sample_index.regClass() == s1) {
3937       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3938    } else {
3939       assert(sample_index.regClass() == v1);
3940       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3941    }
3942
3943    Temp final_sample;
3944    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3945       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3946    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3947       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3948    else
3949       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3950
3951    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3952     * resource descriptor is 0 (invalid),
3953     */
3954    Temp compare = bld.tmp(bld.lm);
3955    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3956                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3957
3958    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3959
3960    /* Replace the MSAA sample index. */
3961    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3962 }
3963
3964 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3965 {
3966
3967    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3968    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3969    bool is_array = glsl_sampler_type_is_array(type);
3970    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3971    assert(!add_frag_pos && "Input attachments should be lowered.");
3972    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3973    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3974    int count = image_type_to_components_count(dim, is_array);
3975    std::vector<Operand> coords(count);
3976
3977    if (is_ms) {
3978       Operand sample_index;
3979       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3980       if (sample_cv)
3981          sample_index = Operand(sample_cv->u32);
3982       else
3983          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3984
3985       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3986          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3987          for (unsigned i = 0; i < vec->operands.size(); i++)
3988             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3989          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3990          vec->definitions[0] = Definition(fmask_load_address);
3991          ctx->block->instructions.emplace_back(std::move(vec));
3992
3993          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3994          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3995       }
3996       count--;
3997       coords[count] = sample_index;
3998    }
3999
4000    if (count == 1 && !gfx9_1d)
4001       return emit_extract_vector(ctx, src0, 0, v1);
4002
4003    if (gfx9_1d) {
4004       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4005       coords.resize(coords.size() + 1);
4006       coords[1] = Operand((uint32_t) 0);
4007       if (is_array)
4008          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4009    } else {
4010       for (int i = 0; i < count; i++)
4011          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4012    }
4013
4014    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4015    for (unsigned i = 0; i < coords.size(); i++)
4016       vec->operands[i] = coords[i];
4017    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4018    vec->definitions[0] = Definition(res);
4019    ctx->block->instructions.emplace_back(std::move(vec));
4020    return res;
4021 }
4022
4023
4024 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4025 {
4026    Builder bld(ctx->program, ctx->block);
4027    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4028    const struct glsl_type *type = glsl_without_array(var->type);
4029    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4030    bool is_array = glsl_sampler_type_is_array(type);
4031    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4032
4033    if (dim == GLSL_SAMPLER_DIM_BUF) {
4034       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4035       unsigned num_channels = util_last_bit(mask);
4036       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4037       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4038
4039       aco_opcode opcode;
4040       switch (num_channels) {
4041       case 1:
4042          opcode = aco_opcode::buffer_load_format_x;
4043          break;
4044       case 2:
4045          opcode = aco_opcode::buffer_load_format_xy;
4046          break;
4047       case 3:
4048          opcode = aco_opcode::buffer_load_format_xyz;
4049          break;
4050       case 4:
4051          opcode = aco_opcode::buffer_load_format_xyzw;
4052          break;
4053       default:
4054          unreachable(">4 channel buffer image load");
4055       }
4056       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4057       load->operands[0] = Operand(vindex);
4058       load->operands[1] = Operand(rsrc);
4059       load->operands[2] = Operand((uint32_t) 0);
4060       Temp tmp;
4061       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4062          tmp = dst;
4063       else
4064          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4065       load->definitions[0] = Definition(tmp);
4066       load->idxen = true;
4067       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4068       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4069       load->barrier = barrier_image;
4070       ctx->block->instructions.emplace_back(std::move(load));
4071
4072       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4073       return;
4074    }
4075
4076    Temp coords = get_image_coords(ctx, instr, type);
4077    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4078
4079    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4080    unsigned num_components = util_bitcount(dmask);
4081    Temp tmp;
4082    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4083       tmp = dst;
4084    else
4085       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4086
4087    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4088    load->operands[0] = Operand(coords);
4089    load->operands[1] = Operand(resource);
4090    load->definitions[0] = Definition(tmp);
4091    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4092    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4093    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4094    load->dmask = dmask;
4095    load->unrm = true;
4096    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4097    load->barrier = barrier_image;
4098    ctx->block->instructions.emplace_back(std::move(load));
4099
4100    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4101    return;
4102 }
4103
4104 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4105 {
4106    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4107    const struct glsl_type *type = glsl_without_array(var->type);
4108    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4109    bool is_array = glsl_sampler_type_is_array(type);
4110    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4111
4112    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4113
4114    if (dim == GLSL_SAMPLER_DIM_BUF) {
4115       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4116       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4117       aco_opcode opcode;
4118       switch (data.size()) {
4119       case 1:
4120          opcode = aco_opcode::buffer_store_format_x;
4121          break;
4122       case 2:
4123          opcode = aco_opcode::buffer_store_format_xy;
4124          break;
4125       case 3:
4126          opcode = aco_opcode::buffer_store_format_xyz;
4127          break;
4128       case 4:
4129          opcode = aco_opcode::buffer_store_format_xyzw;
4130          break;
4131       default:
4132          unreachable(">4 channel buffer image store");
4133       }
4134       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4135       store->operands[0] = Operand(vindex);
4136       store->operands[1] = Operand(rsrc);
4137       store->operands[2] = Operand((uint32_t) 0);
4138       store->operands[3] = Operand(data);
4139       store->idxen = true;
4140       store->glc = glc;
4141       store->dlc = false;
4142       store->disable_wqm = true;
4143       store->barrier = barrier_image;
4144       ctx->program->needs_exact = true;
4145       ctx->block->instructions.emplace_back(std::move(store));
4146       return;
4147    }
4148
4149    assert(data.type() == RegType::vgpr);
4150    Temp coords = get_image_coords(ctx, instr, type);
4151    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4152
4153    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4154    store->operands[0] = Operand(coords);
4155    store->operands[1] = Operand(resource);
4156    store->operands[2] = Operand(s4);
4157    store->operands[3] = Operand(data);
4158    store->glc = glc;
4159    store->dlc = false;
4160    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4161    store->dmask = (1 << data.size()) - 1;
4162    store->unrm = true;
4163    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4164    store->disable_wqm = true;
4165    store->barrier = barrier_image;
4166    ctx->program->needs_exact = true;
4167    ctx->block->instructions.emplace_back(std::move(store));
4168    return;
4169 }
4170
4171 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4172 {
4173    /* return the previous value if dest is ever used */
4174    bool return_previous = false;
4175    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4176       return_previous = true;
4177       break;
4178    }
4179    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4180       return_previous = true;
4181       break;
4182    }
4183
4184    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4185    const struct glsl_type *type = glsl_without_array(var->type);
4186    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4187    bool is_array = glsl_sampler_type_is_array(type);
4188    Builder bld(ctx->program, ctx->block);
4189
4190    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4191    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4192
4193    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4194       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4195
4196    aco_opcode buf_op, image_op;
4197    switch (instr->intrinsic) {
4198       case nir_intrinsic_image_deref_atomic_add:
4199          buf_op = aco_opcode::buffer_atomic_add;
4200          image_op = aco_opcode::image_atomic_add;
4201          break;
4202       case nir_intrinsic_image_deref_atomic_umin:
4203          buf_op = aco_opcode::buffer_atomic_umin;
4204          image_op = aco_opcode::image_atomic_umin;
4205          break;
4206       case nir_intrinsic_image_deref_atomic_imin:
4207          buf_op = aco_opcode::buffer_atomic_smin;
4208          image_op = aco_opcode::image_atomic_smin;
4209          break;
4210       case nir_intrinsic_image_deref_atomic_umax:
4211          buf_op = aco_opcode::buffer_atomic_umax;
4212          image_op = aco_opcode::image_atomic_umax;
4213          break;
4214       case nir_intrinsic_image_deref_atomic_imax:
4215          buf_op = aco_opcode::buffer_atomic_smax;
4216          image_op = aco_opcode::image_atomic_smax;
4217          break;
4218       case nir_intrinsic_image_deref_atomic_and:
4219          buf_op = aco_opcode::buffer_atomic_and;
4220          image_op = aco_opcode::image_atomic_and;
4221          break;
4222       case nir_intrinsic_image_deref_atomic_or:
4223          buf_op = aco_opcode::buffer_atomic_or;
4224          image_op = aco_opcode::image_atomic_or;
4225          break;
4226       case nir_intrinsic_image_deref_atomic_xor:
4227          buf_op = aco_opcode::buffer_atomic_xor;
4228          image_op = aco_opcode::image_atomic_xor;
4229          break;
4230       case nir_intrinsic_image_deref_atomic_exchange:
4231          buf_op = aco_opcode::buffer_atomic_swap;
4232          image_op = aco_opcode::image_atomic_swap;
4233          break;
4234       case nir_intrinsic_image_deref_atomic_comp_swap:
4235          buf_op = aco_opcode::buffer_atomic_cmpswap;
4236          image_op = aco_opcode::image_atomic_cmpswap;
4237          break;
4238       default:
4239          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4240    }
4241
4242    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4243
4244    if (dim == GLSL_SAMPLER_DIM_BUF) {
4245       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4246       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4247       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4248       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4249       mubuf->operands[0] = Operand(vindex);
4250       mubuf->operands[1] = Operand(resource);
4251       mubuf->operands[2] = Operand((uint32_t)0);
4252       mubuf->operands[3] = Operand(data);
4253       if (return_previous)
4254          mubuf->definitions[0] = Definition(dst);
4255       mubuf->offset = 0;
4256       mubuf->idxen = true;
4257       mubuf->glc = return_previous;
4258       mubuf->dlc = false; /* Not needed for atomics */
4259       mubuf->disable_wqm = true;
4260       mubuf->barrier = barrier_image;
4261       ctx->program->needs_exact = true;
4262       ctx->block->instructions.emplace_back(std::move(mubuf));
4263       return;
4264    }
4265
4266    Temp coords = get_image_coords(ctx, instr, type);
4267    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4268    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4269    mimg->operands[0] = Operand(coords);
4270    mimg->operands[1] = Operand(resource);
4271    mimg->operands[2] = Operand(s4); /* no sampler */
4272    mimg->operands[3] = Operand(data);
4273    if (return_previous)
4274       mimg->definitions[0] = Definition(dst);
4275    mimg->glc = return_previous;
4276    mimg->dlc = false; /* Not needed for atomics */
4277    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4278    mimg->dmask = (1 << data.size()) - 1;
4279    mimg->unrm = true;
4280    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4281    mimg->disable_wqm = true;
4282    mimg->barrier = barrier_image;
4283    ctx->program->needs_exact = true;
4284    ctx->block->instructions.emplace_back(std::move(mimg));
4285    return;
4286 }
4287
4288 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4289 {
4290    if (in_elements && ctx->options->chip_class == GFX8) {
4291       Builder bld(ctx->program, ctx->block);
4292
4293       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4294       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4295       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4296       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4297
4298       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4299       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4300
4301       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4302       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4303       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4304
4305       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4306       /* idea
4307        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4308        * in case 12 (or 3?), we have to divide by 3:
4309        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4310        * use v_mul_hi_u32 with magic number to divide
4311        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4312        * disable v_skip
4313        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4314        */
4315
4316    } else {
4317       emit_extract_vector(ctx, desc, 2, dst);
4318    }
4319 }
4320
4321 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4322 {
4323    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4324    const struct glsl_type *type = glsl_without_array(var->type);
4325    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4326    bool is_array = glsl_sampler_type_is_array(type);
4327    Builder bld(ctx->program, ctx->block);
4328
4329    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4330       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4331       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4332    }
4333
4334    /* LOD */
4335    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4336
4337    /* Resource */
4338    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4339
4340    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4341
4342    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4343    mimg->operands[0] = Operand(lod);
4344    mimg->operands[1] = Operand(resource);
4345    unsigned& dmask = mimg->dmask;
4346    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4347    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4348    mimg->da = glsl_sampler_type_is_array(type);
4349    mimg->can_reorder = true;
4350    Definition& def = mimg->definitions[0];
4351    ctx->block->instructions.emplace_back(std::move(mimg));
4352
4353    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4354        glsl_sampler_type_is_array(type)) {
4355
4356       assert(instr->dest.ssa.num_components == 3);
4357       Temp tmp = {ctx->program->allocateId(), v3};
4358       def = Definition(tmp);
4359       emit_split_vector(ctx, tmp, 3);
4360
4361       /* divide 3rd value by 6 by multiplying with magic number */
4362       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4363       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4364
4365       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4366                  emit_extract_vector(ctx, tmp, 0, v1),
4367                  emit_extract_vector(ctx, tmp, 1, v1),
4368                  by_6);
4369
4370    } else if (ctx->options->chip_class == GFX9 &&
4371               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4372               glsl_sampler_type_is_array(type)) {
4373       assert(instr->dest.ssa.num_components == 2);
4374       def = Definition(dst);
4375       dmask = 0x5;
4376    } else {
4377       def = Definition(dst);
4378    }
4379
4380    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4381 }
4382
4383 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4384 {
4385    Builder bld(ctx->program, ctx->block);
4386    unsigned num_components = instr->num_components;
4387
4388    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4389    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4390    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4391
4392    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4393    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4394 }
4395
4396 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4397 {
4398    Builder bld(ctx->program, ctx->block);
4399    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4400    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4401    unsigned writemask = nir_intrinsic_write_mask(instr);
4402
4403    Temp offset;
4404    if (ctx->options->chip_class < GFX8)
4405       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4406    else
4407       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4408
4409    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4410    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4411
4412    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4413                ctx->options->chip_class >= GFX8;
4414    if (smem)
4415       offset = bld.as_uniform(offset);
4416    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4417
4418    while (writemask) {
4419       int start, count;
4420       u_bit_scan_consecutive_range(&writemask, &start, &count);
4421       if (count == 3 && smem) {
4422          writemask |= 1u << (start + 2);
4423          count = 2;
4424       }
4425       int num_bytes = count * elem_size_bytes;
4426
4427       if (num_bytes > 16) {
4428          assert(elem_size_bytes == 8);
4429          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4430          count = 2;
4431          num_bytes = 16;
4432       }
4433
4434       // TODO: check alignment of sub-dword stores
4435       // TODO: split 3 bytes. there is no store instruction for that
4436
4437       Temp write_data;
4438       if (count != instr->num_components) {
4439          emit_split_vector(ctx, data, instr->num_components);
4440          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4441          for (int i = 0; i < count; i++) {
4442             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4443             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4444          }
4445          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4446          vec->definitions[0] = Definition(write_data);
4447          ctx->block->instructions.emplace_back(std::move(vec));
4448       } else if (!smem && data.type() != RegType::vgpr) {
4449          assert(num_bytes % 4 == 0);
4450          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4451       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4452          assert(num_bytes % 4 == 0);
4453          write_data = bld.as_uniform(data);
4454       } else {
4455          write_data = data;
4456       }
4457
4458       aco_opcode vmem_op, smem_op;
4459       switch (num_bytes) {
4460          case 4:
4461             vmem_op = aco_opcode::buffer_store_dword;
4462             smem_op = aco_opcode::s_buffer_store_dword;
4463             break;
4464          case 8:
4465             vmem_op = aco_opcode::buffer_store_dwordx2;
4466             smem_op = aco_opcode::s_buffer_store_dwordx2;
4467             break;
4468          case 12:
4469             vmem_op = aco_opcode::buffer_store_dwordx3;
4470             smem_op = aco_opcode::last_opcode;
4471             assert(!smem);
4472             break;
4473          case 16:
4474             vmem_op = aco_opcode::buffer_store_dwordx4;
4475             smem_op = aco_opcode::s_buffer_store_dwordx4;
4476             break;
4477          default:
4478             unreachable("Store SSBO not implemented for this size.");
4479       }
4480       if (ctx->stage == fragment_fs)
4481          smem_op = aco_opcode::p_fs_buffer_store_smem;
4482
4483       if (smem) {
4484          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4485          store->operands[0] = Operand(rsrc);
4486          if (start) {
4487             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4488                                 offset, Operand(start * elem_size_bytes));
4489             store->operands[1] = Operand(off);
4490          } else {
4491             store->operands[1] = Operand(offset);
4492          }
4493          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4494             store->operands[1].setFixed(m0);
4495          store->operands[2] = Operand(write_data);
4496          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4497          store->dlc = false;
4498          store->disable_wqm = true;
4499          store->barrier = barrier_buffer;
4500          ctx->block->instructions.emplace_back(std::move(store));
4501          ctx->program->wb_smem_l1_on_end = true;
4502          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4503             ctx->block->kind |= block_kind_needs_lowering;
4504             ctx->program->needs_exact = true;
4505          }
4506       } else {
4507          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4508          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4509          store->operands[1] = Operand(rsrc);
4510          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4511          store->operands[3] = Operand(write_data);
4512          store->offset = start * elem_size_bytes;
4513          store->offen = (offset.type() == RegType::vgpr);
4514          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4515          store->dlc = false;
4516          store->disable_wqm = true;
4517          store->barrier = barrier_buffer;
4518          ctx->program->needs_exact = true;
4519          ctx->block->instructions.emplace_back(std::move(store));
4520       }
4521    }
4522 }
4523
4524 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4525 {
4526    /* return the previous value if dest is ever used */
4527    bool return_previous = false;
4528    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4529       return_previous = true;
4530       break;
4531    }
4532    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4533       return_previous = true;
4534       break;
4535    }
4536
4537    Builder bld(ctx->program, ctx->block);
4538    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4539
4540    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4541       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4542                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4543
4544    Temp offset;
4545    if (ctx->options->chip_class < GFX8)
4546       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4547    else
4548       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4549
4550    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4551    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4552
4553    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4554
4555    aco_opcode op32, op64;
4556    switch (instr->intrinsic) {
4557       case nir_intrinsic_ssbo_atomic_add:
4558          op32 = aco_opcode::buffer_atomic_add;
4559          op64 = aco_opcode::buffer_atomic_add_x2;
4560          break;
4561       case nir_intrinsic_ssbo_atomic_imin:
4562          op32 = aco_opcode::buffer_atomic_smin;
4563          op64 = aco_opcode::buffer_atomic_smin_x2;
4564          break;
4565       case nir_intrinsic_ssbo_atomic_umin:
4566          op32 = aco_opcode::buffer_atomic_umin;
4567          op64 = aco_opcode::buffer_atomic_umin_x2;
4568          break;
4569       case nir_intrinsic_ssbo_atomic_imax:
4570          op32 = aco_opcode::buffer_atomic_smax;
4571          op64 = aco_opcode::buffer_atomic_smax_x2;
4572          break;
4573       case nir_intrinsic_ssbo_atomic_umax:
4574          op32 = aco_opcode::buffer_atomic_umax;
4575          op64 = aco_opcode::buffer_atomic_umax_x2;
4576          break;
4577       case nir_intrinsic_ssbo_atomic_and:
4578          op32 = aco_opcode::buffer_atomic_and;
4579          op64 = aco_opcode::buffer_atomic_and_x2;
4580          break;
4581       case nir_intrinsic_ssbo_atomic_or:
4582          op32 = aco_opcode::buffer_atomic_or;
4583          op64 = aco_opcode::buffer_atomic_or_x2;
4584          break;
4585       case nir_intrinsic_ssbo_atomic_xor:
4586          op32 = aco_opcode::buffer_atomic_xor;
4587          op64 = aco_opcode::buffer_atomic_xor_x2;
4588          break;
4589       case nir_intrinsic_ssbo_atomic_exchange:
4590          op32 = aco_opcode::buffer_atomic_swap;
4591          op64 = aco_opcode::buffer_atomic_swap_x2;
4592          break;
4593       case nir_intrinsic_ssbo_atomic_comp_swap:
4594          op32 = aco_opcode::buffer_atomic_cmpswap;
4595          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4596          break;
4597       default:
4598          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4599    }
4600    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4601    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4602    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4603    mubuf->operands[1] = Operand(rsrc);
4604    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4605    mubuf->operands[3] = Operand(data);
4606    if (return_previous)
4607       mubuf->definitions[0] = Definition(dst);
4608    mubuf->offset = 0;
4609    mubuf->offen = (offset.type() == RegType::vgpr);
4610    mubuf->glc = return_previous;
4611    mubuf->dlc = false; /* Not needed for atomics */
4612    mubuf->disable_wqm = true;
4613    mubuf->barrier = barrier_buffer;
4614    ctx->program->needs_exact = true;
4615    ctx->block->instructions.emplace_back(std::move(mubuf));
4616 }
4617
4618 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4619
4620    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4621    Builder bld(ctx->program, ctx->block);
4622    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4623    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4624 }
4625
4626 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4627 {
4628    Builder bld(ctx->program, ctx->block);
4629    unsigned num_components = instr->num_components;
4630    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4631
4632    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4633    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4634
4635    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4636    bool dlc = glc && ctx->options->chip_class >= GFX10;
4637    aco_opcode op;
4638    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4639       bool global = ctx->options->chip_class >= GFX9;
4640       aco_opcode op;
4641       switch (num_bytes) {
4642       case 4:
4643          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4644          break;
4645       case 8:
4646          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4647          break;
4648       case 12:
4649          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4650          break;
4651       case 16:
4652          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4653          break;
4654       default:
4655          unreachable("load_global not implemented for this size.");
4656       }
4657       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4658       flat->operands[0] = Operand(addr);
4659       flat->operands[1] = Operand(s1);
4660       flat->glc = glc;
4661       flat->dlc = dlc;
4662       flat->barrier = barrier_buffer;
4663
4664       if (dst.type() == RegType::sgpr) {
4665          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4666          flat->definitions[0] = Definition(vec);
4667          ctx->block->instructions.emplace_back(std::move(flat));
4668          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4669       } else {
4670          flat->definitions[0] = Definition(dst);
4671          ctx->block->instructions.emplace_back(std::move(flat));
4672       }
4673       emit_split_vector(ctx, dst, num_components);
4674    } else {
4675       switch (num_bytes) {
4676          case 4:
4677             op = aco_opcode::s_load_dword;
4678             break;
4679          case 8:
4680             op = aco_opcode::s_load_dwordx2;
4681             break;
4682          case 12:
4683          case 16:
4684             op = aco_opcode::s_load_dwordx4;
4685             break;
4686          default:
4687             unreachable("load_global not implemented for this size.");
4688       }
4689       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4690       load->operands[0] = Operand(addr);
4691       load->operands[1] = Operand(0u);
4692       load->definitions[0] = Definition(dst);
4693       load->glc = glc;
4694       load->dlc = dlc;
4695       load->barrier = barrier_buffer;
4696       assert(ctx->options->chip_class >= GFX8 || !glc);
4697
4698       if (dst.size() == 3) {
4699          /* trim vector */
4700          Temp vec = bld.tmp(s4);
4701          load->definitions[0] = Definition(vec);
4702          ctx->block->instructions.emplace_back(std::move(load));
4703          emit_split_vector(ctx, vec, 4);
4704
4705          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4706                     emit_extract_vector(ctx, vec, 0, s1),
4707                     emit_extract_vector(ctx, vec, 1, s1),
4708                     emit_extract_vector(ctx, vec, 2, s1));
4709       } else {
4710          ctx->block->instructions.emplace_back(std::move(load));
4711       }
4712    }
4713 }
4714
4715 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4716 {
4717    Builder bld(ctx->program, ctx->block);
4718    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4719
4720    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4721    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4722
4723    unsigned writemask = nir_intrinsic_write_mask(instr);
4724    while (writemask) {
4725       int start, count;
4726       u_bit_scan_consecutive_range(&writemask, &start, &count);
4727       unsigned num_bytes = count * elem_size_bytes;
4728
4729       Temp write_data = data;
4730       if (count != instr->num_components) {
4731          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4732          for (int i = 0; i < count; i++)
4733             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4734          write_data = bld.tmp(RegType::vgpr, count);
4735          vec->definitions[0] = Definition(write_data);
4736          ctx->block->instructions.emplace_back(std::move(vec));
4737       }
4738
4739       unsigned offset = start * elem_size_bytes;
4740       if (offset > 0 && ctx->options->chip_class < GFX9) {
4741          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4742          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4743          Temp carry = bld.tmp(bld.lm);
4744          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4745
4746          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4747                   Operand(offset), addr0);
4748          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4749                   Operand(0u), addr1,
4750                   carry).def(1).setHint(vcc);
4751
4752          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4753
4754          offset = 0;
4755       }
4756
4757       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4758       bool global = ctx->options->chip_class >= GFX9;
4759       aco_opcode op;
4760       switch (num_bytes) {
4761       case 4:
4762          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4763          break;
4764       case 8:
4765          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4766          break;
4767       case 12:
4768          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4769          break;
4770       case 16:
4771          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4772          break;
4773       default:
4774          unreachable("store_global not implemented for this size.");
4775       }
4776       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4777       flat->operands[0] = Operand(addr);
4778       flat->operands[1] = Operand(s1);
4779       flat->operands[2] = Operand(data);
4780       flat->glc = glc;
4781       flat->dlc = false;
4782       flat->offset = offset;
4783       flat->disable_wqm = true;
4784       flat->barrier = barrier_buffer;
4785       ctx->program->needs_exact = true;
4786       ctx->block->instructions.emplace_back(std::move(flat));
4787    }
4788 }
4789
4790 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4791 {
4792    /* return the previous value if dest is ever used */
4793    bool return_previous = false;
4794    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4795       return_previous = true;
4796       break;
4797    }
4798    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4799       return_previous = true;
4800       break;
4801    }
4802
4803    Builder bld(ctx->program, ctx->block);
4804    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4805    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4806
4807    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4808       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4809                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4810
4811    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4812
4813    bool global = ctx->options->chip_class >= GFX9;
4814    aco_opcode op32, op64;
4815    switch (instr->intrinsic) {
4816       case nir_intrinsic_global_atomic_add:
4817          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4818          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4819          break;
4820       case nir_intrinsic_global_atomic_imin:
4821          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4822          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4823          break;
4824       case nir_intrinsic_global_atomic_umin:
4825          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4826          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4827          break;
4828       case nir_intrinsic_global_atomic_imax:
4829          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4830          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4831          break;
4832       case nir_intrinsic_global_atomic_umax:
4833          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4834          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4835          break;
4836       case nir_intrinsic_global_atomic_and:
4837          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4838          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4839          break;
4840       case nir_intrinsic_global_atomic_or:
4841          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4842          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4843          break;
4844       case nir_intrinsic_global_atomic_xor:
4845          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4846          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4847          break;
4848       case nir_intrinsic_global_atomic_exchange:
4849          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4850          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4851          break;
4852       case nir_intrinsic_global_atomic_comp_swap:
4853          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4854          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4855          break;
4856       default:
4857          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4858    }
4859    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4860    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4861    flat->operands[0] = Operand(addr);
4862    flat->operands[1] = Operand(s1);
4863    flat->operands[2] = Operand(data);
4864    if (return_previous)
4865       flat->definitions[0] = Definition(dst);
4866    flat->glc = return_previous;
4867    flat->dlc = false; /* Not needed for atomics */
4868    flat->offset = 0;
4869    flat->disable_wqm = true;
4870    flat->barrier = barrier_buffer;
4871    ctx->program->needs_exact = true;
4872    ctx->block->instructions.emplace_back(std::move(flat));
4873 }
4874
4875 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4876    Builder bld(ctx->program, ctx->block);
4877    switch(instr->intrinsic) {
4878       case nir_intrinsic_group_memory_barrier:
4879       case nir_intrinsic_memory_barrier:
4880          bld.barrier(aco_opcode::p_memory_barrier_all);
4881          break;
4882       case nir_intrinsic_memory_barrier_atomic_counter:
4883          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4884          break;
4885       case nir_intrinsic_memory_barrier_buffer:
4886          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4887          break;
4888       case nir_intrinsic_memory_barrier_image:
4889          bld.barrier(aco_opcode::p_memory_barrier_image);
4890          break;
4891       case nir_intrinsic_memory_barrier_shared:
4892          bld.barrier(aco_opcode::p_memory_barrier_shared);
4893          break;
4894       default:
4895          unreachable("Unimplemented memory barrier intrinsic");
4896          break;
4897    }
4898 }
4899
4900 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4901 {
4902    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4903    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4904    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4905    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4906    Builder bld(ctx->program, ctx->block);
4907
4908    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4909    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4910    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4911 }
4912
4913 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4914 {
4915    unsigned writemask = nir_intrinsic_write_mask(instr);
4916    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4917    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4918    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4919    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4920
4921    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4922    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4923 }
4924
4925 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4926 {
4927    unsigned offset = nir_intrinsic_base(instr);
4928    Operand m = load_lds_size_m0(ctx);
4929    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4930    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4931
4932    unsigned num_operands = 3;
4933    aco_opcode op32, op64, op32_rtn, op64_rtn;
4934    switch(instr->intrinsic) {
4935       case nir_intrinsic_shared_atomic_add:
4936          op32 = aco_opcode::ds_add_u32;
4937          op64 = aco_opcode::ds_add_u64;
4938          op32_rtn = aco_opcode::ds_add_rtn_u32;
4939          op64_rtn = aco_opcode::ds_add_rtn_u64;
4940          break;
4941       case nir_intrinsic_shared_atomic_imin:
4942          op32 = aco_opcode::ds_min_i32;
4943          op64 = aco_opcode::ds_min_i64;
4944          op32_rtn = aco_opcode::ds_min_rtn_i32;
4945          op64_rtn = aco_opcode::ds_min_rtn_i64;
4946          break;
4947       case nir_intrinsic_shared_atomic_umin:
4948          op32 = aco_opcode::ds_min_u32;
4949          op64 = aco_opcode::ds_min_u64;
4950          op32_rtn = aco_opcode::ds_min_rtn_u32;
4951          op64_rtn = aco_opcode::ds_min_rtn_u64;
4952          break;
4953       case nir_intrinsic_shared_atomic_imax:
4954          op32 = aco_opcode::ds_max_i32;
4955          op64 = aco_opcode::ds_max_i64;
4956          op32_rtn = aco_opcode::ds_max_rtn_i32;
4957          op64_rtn = aco_opcode::ds_max_rtn_i64;
4958          break;
4959       case nir_intrinsic_shared_atomic_umax:
4960          op32 = aco_opcode::ds_max_u32;
4961          op64 = aco_opcode::ds_max_u64;
4962          op32_rtn = aco_opcode::ds_max_rtn_u32;
4963          op64_rtn = aco_opcode::ds_max_rtn_u64;
4964          break;
4965       case nir_intrinsic_shared_atomic_and:
4966          op32 = aco_opcode::ds_and_b32;
4967          op64 = aco_opcode::ds_and_b64;
4968          op32_rtn = aco_opcode::ds_and_rtn_b32;
4969          op64_rtn = aco_opcode::ds_and_rtn_b64;
4970          break;
4971       case nir_intrinsic_shared_atomic_or:
4972          op32 = aco_opcode::ds_or_b32;
4973          op64 = aco_opcode::ds_or_b64;
4974          op32_rtn = aco_opcode::ds_or_rtn_b32;
4975          op64_rtn = aco_opcode::ds_or_rtn_b64;
4976          break;
4977       case nir_intrinsic_shared_atomic_xor:
4978          op32 = aco_opcode::ds_xor_b32;
4979          op64 = aco_opcode::ds_xor_b64;
4980          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4981          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4982          break;
4983       case nir_intrinsic_shared_atomic_exchange:
4984          op32 = aco_opcode::ds_write_b32;
4985          op64 = aco_opcode::ds_write_b64;
4986          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4987          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4988          break;
4989       case nir_intrinsic_shared_atomic_comp_swap:
4990          op32 = aco_opcode::ds_cmpst_b32;
4991          op64 = aco_opcode::ds_cmpst_b64;
4992          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4993          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4994          num_operands = 4;
4995          break;
4996       default:
4997          unreachable("Unhandled shared atomic intrinsic");
4998    }
4999
5000    /* return the previous value if dest is ever used */
5001    bool return_previous = false;
5002    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5003       return_previous = true;
5004       break;
5005    }
5006    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5007       return_previous = true;
5008       break;
5009    }
5010
5011    aco_opcode op;
5012    if (data.size() == 1) {
5013       assert(instr->dest.ssa.bit_size == 32);
5014       op = return_previous ? op32_rtn : op32;
5015    } else {
5016       assert(instr->dest.ssa.bit_size == 64);
5017       op = return_previous ? op64_rtn : op64;
5018    }
5019
5020    if (offset > 65535) {
5021       Builder bld(ctx->program, ctx->block);
5022       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5023       offset = 0;
5024    }
5025
5026    aco_ptr<DS_instruction> ds;
5027    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5028    ds->operands[0] = Operand(address);
5029    ds->operands[1] = Operand(data);
5030    if (num_operands == 4)
5031       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5032    ds->operands[num_operands - 1] = m;
5033    ds->offset0 = offset;
5034    if (return_previous)
5035       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5036    ctx->block->instructions.emplace_back(std::move(ds));
5037 }
5038
5039 Temp get_scratch_resource(isel_context *ctx)
5040 {
5041    Builder bld(ctx->program, ctx->block);
5042    Temp scratch_addr = ctx->program->private_segment_buffer;
5043    if (ctx->stage != compute_cs)
5044       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5045
5046    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5047                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5048
5049    if (ctx->program->chip_class >= GFX10) {
5050       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5051                    S_008F0C_OOB_SELECT(3) |
5052                    S_008F0C_RESOURCE_LEVEL(1);
5053    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5054       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5055                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5056    }
5057
5058    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5059    if (ctx->program->chip_class <= GFX8)
5060       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5061
5062    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5063 }
5064
5065 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5066    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5067    Builder bld(ctx->program, ctx->block);
5068    Temp rsrc = get_scratch_resource(ctx);
5069    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5070    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5071
5072    aco_opcode op;
5073    switch (dst.size()) {
5074       case 1:
5075          op = aco_opcode::buffer_load_dword;
5076          break;
5077       case 2:
5078          op = aco_opcode::buffer_load_dwordx2;
5079          break;
5080       case 3:
5081          op = aco_opcode::buffer_load_dwordx3;
5082          break;
5083       case 4:
5084          op = aco_opcode::buffer_load_dwordx4;
5085          break;
5086       case 6:
5087       case 8: {
5088          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5089          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5090                                 bld.def(v4), offset, rsrc,
5091                                 ctx->program->scratch_offset, 0, true);
5092          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5093                                                   aco_opcode::buffer_load_dwordx4,
5094                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5095                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5096          emit_split_vector(ctx, lower, 2);
5097          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5098          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5099          if (dst.size() == 8) {
5100             emit_split_vector(ctx, upper, 2);
5101             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5102             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5103          } else {
5104             elems[2] = upper;
5105          }
5106
5107          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5108                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5109          for (unsigned i = 0; i < dst.size() / 2; i++)
5110             vec->operands[i] = Operand(elems[i]);
5111          vec->definitions[0] = Definition(dst);
5112          bld.insert(std::move(vec));
5113          ctx->allocated_vec.emplace(dst.id(), elems);
5114          return;
5115       }
5116       default:
5117          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5118    }
5119
5120    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5121    emit_split_vector(ctx, dst, instr->num_components);
5122 }
5123
5124 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5125    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5126    Builder bld(ctx->program, ctx->block);
5127    Temp rsrc = get_scratch_resource(ctx);
5128    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5129    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5130
5131    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5132    unsigned writemask = nir_intrinsic_write_mask(instr);
5133
5134    while (writemask) {
5135       int start, count;
5136       u_bit_scan_consecutive_range(&writemask, &start, &count);
5137       int num_bytes = count * elem_size_bytes;
5138
5139       if (num_bytes > 16) {
5140          assert(elem_size_bytes == 8);
5141          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5142          count = 2;
5143          num_bytes = 16;
5144       }
5145
5146       // TODO: check alignment of sub-dword stores
5147       // TODO: split 3 bytes. there is no store instruction for that
5148
5149       Temp write_data;
5150       if (count != instr->num_components) {
5151          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5152          for (int i = 0; i < count; i++) {
5153             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5154             vec->operands[i] = Operand(elem);
5155          }
5156          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5157          vec->definitions[0] = Definition(write_data);
5158          ctx->block->instructions.emplace_back(std::move(vec));
5159       } else {
5160          write_data = data;
5161       }
5162
5163       aco_opcode op;
5164       switch (num_bytes) {
5165          case 4:
5166             op = aco_opcode::buffer_store_dword;
5167             break;
5168          case 8:
5169             op = aco_opcode::buffer_store_dwordx2;
5170             break;
5171          case 12:
5172             op = aco_opcode::buffer_store_dwordx3;
5173             break;
5174          case 16:
5175             op = aco_opcode::buffer_store_dwordx4;
5176             break;
5177          default:
5178             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5179       }
5180
5181       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5182    }
5183 }
5184
5185 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5186    uint8_t log2_ps_iter_samples;
5187    if (ctx->program->info->ps.force_persample) {
5188       log2_ps_iter_samples =
5189          util_logbase2(ctx->options->key.fs.num_samples);
5190    } else {
5191       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5192    }
5193
5194    /* The bit pattern matches that used by fixed function fragment
5195     * processing. */
5196    static const unsigned ps_iter_masks[] = {
5197       0xffff, /* not used */
5198       0x5555,
5199       0x1111,
5200       0x0101,
5201       0x0001,
5202    };
5203    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5204
5205    Builder bld(ctx->program, ctx->block);
5206
5207    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5208                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5209    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5210    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5211    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5212    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5213 }
5214
5215 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5216 {
5217    Builder bld(ctx->program, ctx->block);
5218
5219    if (cluster_size == 1) {
5220       return src;
5221    } if (op == nir_op_iand && cluster_size == 4) {
5222       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5223       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5224       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5225                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5226    } else if (op == nir_op_ior && cluster_size == 4) {
5227       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5228       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5229                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5230    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5231       //subgroupAnd(val) -> (exec & ~val) == 0
5232       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5233       return bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5234    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5235       //subgroupOr(val) -> (val & exec) != 0
5236       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5237       return bool_to_vector_condition(ctx, tmp);
5238    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5239       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5240       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5241       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5242       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5243       return bool_to_vector_condition(ctx, tmp);
5244    } else {
5245       //subgroupClustered{And,Or,Xor}(val, n) ->
5246       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5247       //cluster_offset = ~(n - 1) & lane_id
5248       //cluster_mask = ((1 << n) - 1)
5249       //subgroupClusteredAnd():
5250       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5251       //subgroupClusteredOr():
5252       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5253       //subgroupClusteredXor():
5254       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5255       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5256       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5257
5258       Temp tmp;
5259       if (op == nir_op_iand)
5260          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5261       else
5262          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5263
5264       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5265       if (ctx->program->wave_size == 64)
5266          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5267       else
5268          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5269       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5270       if (cluster_mask != 0xffffffff)
5271          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5272
5273       Definition cmp_def = Definition();
5274       if (op == nir_op_iand) {
5275          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5276       } else if (op == nir_op_ior) {
5277          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5278       } else if (op == nir_op_ixor) {
5279          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5280                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5281          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5282       }
5283       cmp_def.setHint(vcc);
5284       return cmp_def.getTemp();
5285    }
5286 }
5287
5288 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5289 {
5290    Builder bld(ctx->program, ctx->block);
5291
5292    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5293    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5294    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5295    Temp tmp;
5296    if (op == nir_op_iand)
5297       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5298    else
5299       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5300
5301    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5302    Temp lo = lohi.def(0).getTemp();
5303    Temp hi = lohi.def(1).getTemp();
5304    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5305
5306    Definition cmp_def = Definition();
5307    if (op == nir_op_iand)
5308       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5309    else if (op == nir_op_ior)
5310       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5311    else if (op == nir_op_ixor)
5312       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5313                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5314    cmp_def.setHint(vcc);
5315    return cmp_def.getTemp();
5316 }
5317
5318 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5319 {
5320    Builder bld(ctx->program, ctx->block);
5321
5322    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5323    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5324    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5325    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5326    if (op == nir_op_iand)
5327       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5328    else if (op == nir_op_ior)
5329       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5330    else if (op == nir_op_ixor)
5331       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5332
5333    assert(false);
5334    return Temp();
5335 }
5336
5337 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5338 {
5339    Builder bld(ctx->program, ctx->block);
5340    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5341    if (src.regClass().type() == RegType::vgpr) {
5342       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5343    } else if (src.regClass() == s1) {
5344       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5345    } else if (src.regClass() == s2) {
5346       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5347    } else {
5348       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5349       nir_print_instr(&instr->instr, stderr);
5350       fprintf(stderr, "\n");
5351    }
5352 }
5353
5354 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5355 {
5356    Builder bld(ctx->program, ctx->block);
5357    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5358    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5359    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5360
5361    /* Build DD X/Y */
5362    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5363    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5364    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5365    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5366    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5367    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5368
5369    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5370    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5371    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5372    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5373    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5374    Temp wqm1 = bld.tmp(v1);
5375    emit_wqm(ctx, tmp1, wqm1, true);
5376    Temp wqm2 = bld.tmp(v1);
5377    emit_wqm(ctx, tmp2, wqm2, true);
5378    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5379    return;
5380 }
5381
5382 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5383 {
5384    Builder bld(ctx->program, ctx->block);
5385    switch(instr->intrinsic) {
5386    case nir_intrinsic_load_barycentric_sample:
5387    case nir_intrinsic_load_barycentric_pixel:
5388    case nir_intrinsic_load_barycentric_centroid: {
5389       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5390       Temp bary = Temp(0, s2);
5391       switch (mode) {
5392       case INTERP_MODE_SMOOTH:
5393       case INTERP_MODE_NONE:
5394          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5395             bary = get_arg(ctx, ctx->args->ac.persp_center);
5396          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5397             bary = ctx->persp_centroid;
5398          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5399             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5400          break;
5401       case INTERP_MODE_NOPERSPECTIVE:
5402          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5403             bary = get_arg(ctx, ctx->args->ac.linear_center);
5404          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5405             bary = ctx->linear_centroid;
5406          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5407             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5408          break;
5409       default:
5410          break;
5411       }
5412       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5413       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5414       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5415       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5416                  Operand(p1), Operand(p2));
5417       emit_split_vector(ctx, dst, 2);
5418       break;
5419    }
5420    case nir_intrinsic_load_barycentric_at_sample: {
5421       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5422       switch (ctx->options->key.fs.num_samples) {
5423          case 2: sample_pos_offset += 1 << 3; break;
5424          case 4: sample_pos_offset += 3 << 3; break;
5425          case 8: sample_pos_offset += 7 << 3; break;
5426          default: break;
5427       }
5428       Temp sample_pos;
5429       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5430       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5431       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5432       if (addr.type() == RegType::sgpr) {
5433          Operand offset;
5434          if (const_addr) {
5435             sample_pos_offset += const_addr->u32 << 3;
5436             offset = Operand(sample_pos_offset);
5437          } else if (ctx->options->chip_class >= GFX9) {
5438             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5439          } else {
5440             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5441             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5442          }
5443          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5444
5445       } else if (ctx->options->chip_class >= GFX9) {
5446          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5447          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5448       } else {
5449          /* addr += private_segment_buffer + sample_pos_offset */
5450          Temp tmp0 = bld.tmp(s1);
5451          Temp tmp1 = bld.tmp(s1);
5452          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5453          Definition scc_tmp = bld.def(s1, scc);
5454          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5455          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5456          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5457          Temp pck0 = bld.tmp(v1);
5458          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5459          tmp1 = as_vgpr(ctx, tmp1);
5460          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5461          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5462
5463          /* sample_pos = flat_load_dwordx2 addr */
5464          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5465       }
5466
5467       /* sample_pos -= 0.5 */
5468       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5469       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5470       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5471       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5472       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5473
5474       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5475       break;
5476    }
5477    case nir_intrinsic_load_barycentric_at_offset: {
5478       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5479       RegClass rc = RegClass(offset.type(), 1);
5480       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5481       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5482       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5483       break;
5484    }
5485    case nir_intrinsic_load_front_face: {
5486       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5487                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5488       break;
5489    }
5490    case nir_intrinsic_load_view_index:
5491    case nir_intrinsic_load_layer_id: {
5492       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5493          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5494          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5495          break;
5496       }
5497
5498       unsigned idx = nir_intrinsic_base(instr);
5499       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5500                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5501       break;
5502    }
5503    case nir_intrinsic_load_frag_coord: {
5504       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5505       break;
5506    }
5507    case nir_intrinsic_load_sample_pos: {
5508       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5509       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5510       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5511                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5512                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5513       break;
5514    }
5515    case nir_intrinsic_load_interpolated_input:
5516       visit_load_interpolated_input(ctx, instr);
5517       break;
5518    case nir_intrinsic_store_output:
5519       visit_store_output(ctx, instr);
5520       break;
5521    case nir_intrinsic_load_input:
5522       visit_load_input(ctx, instr);
5523       break;
5524    case nir_intrinsic_load_ubo:
5525       visit_load_ubo(ctx, instr);
5526       break;
5527    case nir_intrinsic_load_push_constant:
5528       visit_load_push_constant(ctx, instr);
5529       break;
5530    case nir_intrinsic_load_constant:
5531       visit_load_constant(ctx, instr);
5532       break;
5533    case nir_intrinsic_vulkan_resource_index:
5534       visit_load_resource(ctx, instr);
5535       break;
5536    case nir_intrinsic_discard:
5537       visit_discard(ctx, instr);
5538       break;
5539    case nir_intrinsic_discard_if:
5540       visit_discard_if(ctx, instr);
5541       break;
5542    case nir_intrinsic_load_shared:
5543       visit_load_shared(ctx, instr);
5544       break;
5545    case nir_intrinsic_store_shared:
5546       visit_store_shared(ctx, instr);
5547       break;
5548    case nir_intrinsic_shared_atomic_add:
5549    case nir_intrinsic_shared_atomic_imin:
5550    case nir_intrinsic_shared_atomic_umin:
5551    case nir_intrinsic_shared_atomic_imax:
5552    case nir_intrinsic_shared_atomic_umax:
5553    case nir_intrinsic_shared_atomic_and:
5554    case nir_intrinsic_shared_atomic_or:
5555    case nir_intrinsic_shared_atomic_xor:
5556    case nir_intrinsic_shared_atomic_exchange:
5557    case nir_intrinsic_shared_atomic_comp_swap:
5558       visit_shared_atomic(ctx, instr);
5559       break;
5560    case nir_intrinsic_image_deref_load:
5561       visit_image_load(ctx, instr);
5562       break;
5563    case nir_intrinsic_image_deref_store:
5564       visit_image_store(ctx, instr);
5565       break;
5566    case nir_intrinsic_image_deref_atomic_add:
5567    case nir_intrinsic_image_deref_atomic_umin:
5568    case nir_intrinsic_image_deref_atomic_imin:
5569    case nir_intrinsic_image_deref_atomic_umax:
5570    case nir_intrinsic_image_deref_atomic_imax:
5571    case nir_intrinsic_image_deref_atomic_and:
5572    case nir_intrinsic_image_deref_atomic_or:
5573    case nir_intrinsic_image_deref_atomic_xor:
5574    case nir_intrinsic_image_deref_atomic_exchange:
5575    case nir_intrinsic_image_deref_atomic_comp_swap:
5576       visit_image_atomic(ctx, instr);
5577       break;
5578    case nir_intrinsic_image_deref_size:
5579       visit_image_size(ctx, instr);
5580       break;
5581    case nir_intrinsic_load_ssbo:
5582       visit_load_ssbo(ctx, instr);
5583       break;
5584    case nir_intrinsic_store_ssbo:
5585       visit_store_ssbo(ctx, instr);
5586       break;
5587    case nir_intrinsic_load_global:
5588       visit_load_global(ctx, instr);
5589       break;
5590    case nir_intrinsic_store_global:
5591       visit_store_global(ctx, instr);
5592       break;
5593    case nir_intrinsic_global_atomic_add:
5594    case nir_intrinsic_global_atomic_imin:
5595    case nir_intrinsic_global_atomic_umin:
5596    case nir_intrinsic_global_atomic_imax:
5597    case nir_intrinsic_global_atomic_umax:
5598    case nir_intrinsic_global_atomic_and:
5599    case nir_intrinsic_global_atomic_or:
5600    case nir_intrinsic_global_atomic_xor:
5601    case nir_intrinsic_global_atomic_exchange:
5602    case nir_intrinsic_global_atomic_comp_swap:
5603       visit_global_atomic(ctx, instr);
5604       break;
5605    case nir_intrinsic_ssbo_atomic_add:
5606    case nir_intrinsic_ssbo_atomic_imin:
5607    case nir_intrinsic_ssbo_atomic_umin:
5608    case nir_intrinsic_ssbo_atomic_imax:
5609    case nir_intrinsic_ssbo_atomic_umax:
5610    case nir_intrinsic_ssbo_atomic_and:
5611    case nir_intrinsic_ssbo_atomic_or:
5612    case nir_intrinsic_ssbo_atomic_xor:
5613    case nir_intrinsic_ssbo_atomic_exchange:
5614    case nir_intrinsic_ssbo_atomic_comp_swap:
5615       visit_atomic_ssbo(ctx, instr);
5616       break;
5617    case nir_intrinsic_load_scratch:
5618       visit_load_scratch(ctx, instr);
5619       break;
5620    case nir_intrinsic_store_scratch:
5621       visit_store_scratch(ctx, instr);
5622       break;
5623    case nir_intrinsic_get_buffer_size:
5624       visit_get_buffer_size(ctx, instr);
5625       break;
5626    case nir_intrinsic_barrier: {
5627       unsigned* bsize = ctx->program->info->cs.block_size;
5628       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5629       if (workgroup_size > ctx->program->wave_size)
5630          bld.sopp(aco_opcode::s_barrier);
5631       break;
5632    }
5633    case nir_intrinsic_group_memory_barrier:
5634    case nir_intrinsic_memory_barrier:
5635    case nir_intrinsic_memory_barrier_atomic_counter:
5636    case nir_intrinsic_memory_barrier_buffer:
5637    case nir_intrinsic_memory_barrier_image:
5638    case nir_intrinsic_memory_barrier_shared:
5639       emit_memory_barrier(ctx, instr);
5640       break;
5641    case nir_intrinsic_load_num_work_groups: {
5642       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5643       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5644       emit_split_vector(ctx, dst, 3);
5645       break;
5646    }
5647    case nir_intrinsic_load_local_invocation_id: {
5648       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5649       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5650       emit_split_vector(ctx, dst, 3);
5651       break;
5652    }
5653    case nir_intrinsic_load_work_group_id: {
5654       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5655       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5656       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5657                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5658                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5659                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5660       emit_split_vector(ctx, dst, 3);
5661       break;
5662    }
5663    case nir_intrinsic_load_local_invocation_index: {
5664       Temp id = emit_mbcnt(ctx, bld.def(v1));
5665
5666       /* The tg_size bits [6:11] contain the subgroup id,
5667        * we need this multiplied by the wave size, and then OR the thread id to it.
5668        */
5669       if (ctx->program->wave_size == 64) {
5670          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5671          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5672                                 get_arg(ctx, ctx->args->ac.tg_size));
5673          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5674       } else {
5675          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5676          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5677                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5678          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5679       }
5680       break;
5681    }
5682    case nir_intrinsic_load_subgroup_id: {
5683       if (ctx->stage == compute_cs) {
5684          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5685                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5686       } else {
5687          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5688       }
5689       break;
5690    }
5691    case nir_intrinsic_load_subgroup_invocation: {
5692       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5693       break;
5694    }
5695    case nir_intrinsic_load_num_subgroups: {
5696       if (ctx->stage == compute_cs)
5697          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5698                   get_arg(ctx, ctx->args->ac.tg_size));
5699       else
5700          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5701       break;
5702    }
5703    case nir_intrinsic_ballot: {
5704       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5705       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5706       Definition tmp = bld.def(dst.regClass());
5707       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5708       if (instr->src[0].ssa->bit_size == 1) {
5709          assert(src.regClass() == bld.lm);
5710          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5711       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5712          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5713       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5714          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5715       } else {
5716          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5717          nir_print_instr(&instr->instr, stderr);
5718          fprintf(stderr, "\n");
5719       }
5720       if (dst.size() != bld.lm.size()) {
5721          /* Wave32 with ballot size set to 64 */
5722          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5723       }
5724       emit_wqm(ctx, tmp.getTemp(), dst);
5725       break;
5726    }
5727    case nir_intrinsic_shuffle:
5728    case nir_intrinsic_read_invocation: {
5729       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5730       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5731          emit_uniform_subgroup(ctx, instr, src);
5732       } else {
5733          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5734          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5735             tid = bld.as_uniform(tid);
5736          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5737          if (src.regClass() == v1) {
5738             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5739          } else if (src.regClass() == v2) {
5740             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5741             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5742             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5743             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5744             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5745             emit_split_vector(ctx, dst, 2);
5746          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5747             assert(src.regClass() == bld.lm);
5748             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5749             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5750          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5751             assert(src.regClass() == bld.lm);
5752             Temp tmp;
5753             if (ctx->program->wave_size == 64)
5754                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5755             else
5756                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5757             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5758             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5759             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5760          } else {
5761             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5762             nir_print_instr(&instr->instr, stderr);
5763             fprintf(stderr, "\n");
5764          }
5765       }
5766       break;
5767    }
5768    case nir_intrinsic_load_sample_id: {
5769       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5770                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5771       break;
5772    }
5773    case nir_intrinsic_load_sample_mask_in: {
5774       visit_load_sample_mask_in(ctx, instr);
5775       break;
5776    }
5777    case nir_intrinsic_read_first_invocation: {
5778       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5779       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5780       if (src.regClass() == v1) {
5781          emit_wqm(ctx,
5782                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5783                   dst);
5784       } else if (src.regClass() == v2) {
5785          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5786          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5787          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5788          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5789          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5790          emit_split_vector(ctx, dst, 2);
5791       } else if (instr->dest.ssa.bit_size == 1) {
5792          assert(src.regClass() == bld.lm);
5793          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5794                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5795          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5796       } else if (src.regClass() == s1) {
5797          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5798       } else if (src.regClass() == s2) {
5799          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5800       } else {
5801          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5802          nir_print_instr(&instr->instr, stderr);
5803          fprintf(stderr, "\n");
5804       }
5805       break;
5806    }
5807    case nir_intrinsic_vote_all: {
5808       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5809       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5810       assert(src.regClass() == bld.lm);
5811       assert(dst.regClass() == bld.lm);
5812
5813       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5814       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5815       emit_wqm(ctx, val, dst);
5816       break;
5817    }
5818    case nir_intrinsic_vote_any: {
5819       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5820       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5821       assert(src.regClass() == bld.lm);
5822       assert(dst.regClass() == bld.lm);
5823
5824       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5825       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), Operand(0u), bld.scc(tmp));
5826       emit_wqm(ctx, val, dst);
5827       break;
5828    }
5829    case nir_intrinsic_reduce:
5830    case nir_intrinsic_inclusive_scan:
5831    case nir_intrinsic_exclusive_scan: {
5832       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5833       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5834       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5835       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5836          nir_intrinsic_cluster_size(instr) : 0;
5837       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5838
5839       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5840          emit_uniform_subgroup(ctx, instr, src);
5841       } else if (instr->dest.ssa.bit_size == 1) {
5842          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5843             op = nir_op_iand;
5844          else if (op == nir_op_iadd)
5845             op = nir_op_ixor;
5846          else if (op == nir_op_umax || op == nir_op_imax)
5847             op = nir_op_ior;
5848          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5849
5850          switch (instr->intrinsic) {
5851          case nir_intrinsic_reduce:
5852             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5853             break;
5854          case nir_intrinsic_exclusive_scan:
5855             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5856             break;
5857          case nir_intrinsic_inclusive_scan:
5858             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5859             break;
5860          default:
5861             assert(false);
5862          }
5863       } else if (cluster_size == 1) {
5864          bld.copy(Definition(dst), src);
5865       } else {
5866          src = as_vgpr(ctx, src);
5867
5868          ReduceOp reduce_op;
5869          switch (op) {
5870          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5871             CASE(iadd)
5872             CASE(imul)
5873             CASE(fadd)
5874             CASE(fmul)
5875             CASE(imin)
5876             CASE(umin)
5877             CASE(fmin)
5878             CASE(imax)
5879             CASE(umax)
5880             CASE(fmax)
5881             CASE(iand)
5882             CASE(ior)
5883             CASE(ixor)
5884             default:
5885                unreachable("unknown reduction op");
5886          #undef CASE
5887          }
5888
5889          aco_opcode aco_op;
5890          switch (instr->intrinsic) {
5891             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5892             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5893             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5894             default:
5895                unreachable("unknown reduce intrinsic");
5896          }
5897
5898          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5899          reduce->operands[0] = Operand(src);
5900          // filled in by aco_reduce_assign.cpp, used internally as part of the
5901          // reduce sequence
5902          assert(dst.size() == 1 || dst.size() == 2);
5903          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5904          reduce->operands[2] = Operand(v1.as_linear());
5905
5906          Temp tmp_dst = bld.tmp(dst.regClass());
5907          reduce->definitions[0] = Definition(tmp_dst);
5908          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
5909          reduce->definitions[2] = Definition();
5910          reduce->definitions[3] = Definition(scc, s1);
5911          reduce->definitions[4] = Definition();
5912          reduce->reduce_op = reduce_op;
5913          reduce->cluster_size = cluster_size;
5914          ctx->block->instructions.emplace_back(std::move(reduce));
5915
5916          emit_wqm(ctx, tmp_dst, dst);
5917       }
5918       break;
5919    }
5920    case nir_intrinsic_quad_broadcast: {
5921       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5922       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5923          emit_uniform_subgroup(ctx, instr, src);
5924       } else {
5925          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5926          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5927          if (instr->dest.ssa.bit_size == 1) {
5928             assert(src.regClass() == bld.lm);
5929             assert(dst.regClass() == bld.lm);
5930             uint32_t half_mask = 0x11111111u << lane;
5931             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5932             Temp tmp = bld.tmp(bld.lm);
5933             bld.sop1(Builder::s_wqm, Definition(tmp),
5934                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
5935                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
5936             emit_wqm(ctx, tmp, dst);
5937          } else if (instr->dest.ssa.bit_size == 32) {
5938             emit_wqm(ctx,
5939                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5940                                   dpp_quad_perm(lane, lane, lane, lane)),
5941                      dst);
5942          } else if (instr->dest.ssa.bit_size == 64) {
5943             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5944             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5945             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5946             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5947             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5948             emit_split_vector(ctx, dst, 2);
5949          } else {
5950             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5951             nir_print_instr(&instr->instr, stderr);
5952             fprintf(stderr, "\n");
5953          }
5954       }
5955       break;
5956    }
5957    case nir_intrinsic_quad_swap_horizontal:
5958    case nir_intrinsic_quad_swap_vertical:
5959    case nir_intrinsic_quad_swap_diagonal:
5960    case nir_intrinsic_quad_swizzle_amd: {
5961       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5962       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5963          emit_uniform_subgroup(ctx, instr, src);
5964          break;
5965       }
5966       uint16_t dpp_ctrl = 0;
5967       switch (instr->intrinsic) {
5968       case nir_intrinsic_quad_swap_horizontal:
5969          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5970          break;
5971       case nir_intrinsic_quad_swap_vertical:
5972          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5973          break;
5974       case nir_intrinsic_quad_swap_diagonal:
5975          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5976          break;
5977       case nir_intrinsic_quad_swizzle_amd: {
5978          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5979          break;
5980       }
5981       default:
5982          break;
5983       }
5984
5985       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5986       if (instr->dest.ssa.bit_size == 1) {
5987          assert(src.regClass() == bld.lm);
5988          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5989          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5990          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
5991          emit_wqm(ctx, tmp, dst);
5992       } else if (instr->dest.ssa.bit_size == 32) {
5993          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5994          emit_wqm(ctx, tmp, dst);
5995       } else if (instr->dest.ssa.bit_size == 64) {
5996          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5997          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5998          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5999          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6000          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6001          emit_split_vector(ctx, dst, 2);
6002       } else {
6003          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6004          nir_print_instr(&instr->instr, stderr);
6005          fprintf(stderr, "\n");
6006       }
6007       break;
6008    }
6009    case nir_intrinsic_masked_swizzle_amd: {
6010       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6011       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6012          emit_uniform_subgroup(ctx, instr, src);
6013          break;
6014       }
6015       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6016       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6017       if (dst.regClass() == v1) {
6018          emit_wqm(ctx,
6019                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6020                   dst);
6021       } else if (dst.regClass() == v2) {
6022          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6023          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6024          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6025          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6026          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6027          emit_split_vector(ctx, dst, 2);
6028       } else {
6029          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6030          nir_print_instr(&instr->instr, stderr);
6031          fprintf(stderr, "\n");
6032       }
6033       break;
6034    }
6035    case nir_intrinsic_write_invocation_amd: {
6036       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6037       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6038       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6039       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6040       if (dst.regClass() == v1) {
6041          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6042          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
6043       } else if (dst.regClass() == v2) {
6044          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6045          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6046          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6047          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6048          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
6049          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
6050          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6051          emit_split_vector(ctx, dst, 2);
6052       } else {
6053          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6054          nir_print_instr(&instr->instr, stderr);
6055          fprintf(stderr, "\n");
6056       }
6057       break;
6058    }
6059    case nir_intrinsic_mbcnt_amd: {
6060       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6061       RegClass rc = RegClass(src.type(), 1);
6062       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6063       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6064       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6065       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6066       emit_wqm(ctx, wqm_tmp, dst);
6067       break;
6068    }
6069    case nir_intrinsic_load_helper_invocation: {
6070       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6071       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6072       ctx->block->kind |= block_kind_needs_lowering;
6073       ctx->program->needs_exact = true;
6074       break;
6075    }
6076    case nir_intrinsic_is_helper_invocation: {
6077       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6078       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6079       ctx->block->kind |= block_kind_needs_lowering;
6080       ctx->program->needs_exact = true;
6081       break;
6082    }
6083    case nir_intrinsic_demote:
6084       bld.pseudo(aco_opcode::p_demote_to_helper);
6085       ctx->block->kind |= block_kind_uses_demote;
6086       ctx->program->needs_exact = true;
6087       break;
6088    case nir_intrinsic_demote_if: {
6089       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6090       assert(src.regClass() == bld.lm);
6091       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6092       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6093       ctx->block->kind |= block_kind_uses_demote;
6094       ctx->program->needs_exact = true;
6095       break;
6096    }
6097    case nir_intrinsic_first_invocation: {
6098       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6099                get_ssa_temp(ctx, &instr->dest.ssa));
6100       break;
6101    }
6102    case nir_intrinsic_shader_clock:
6103       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6104       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6105       break;
6106    case nir_intrinsic_load_vertex_id_zero_base: {
6107       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6108       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6109       break;
6110    }
6111    case nir_intrinsic_load_first_vertex: {
6112       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6113       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6114       break;
6115    }
6116    case nir_intrinsic_load_base_instance: {
6117       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6118       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6119       break;
6120    }
6121    case nir_intrinsic_load_instance_id: {
6122       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6123       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6124       break;
6125    }
6126    case nir_intrinsic_load_draw_id: {
6127       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6128       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6129       break;
6130    }
6131    default:
6132       fprintf(stderr, "Unimplemented intrinsic instr: ");
6133       nir_print_instr(&instr->instr, stderr);
6134       fprintf(stderr, "\n");
6135       abort();
6136
6137       break;
6138    }
6139 }
6140
6141
6142 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6143                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6144                     enum glsl_base_type *stype)
6145 {
6146    nir_deref_instr *texture_deref_instr = NULL;
6147    nir_deref_instr *sampler_deref_instr = NULL;
6148    int plane = -1;
6149
6150    for (unsigned i = 0; i < instr->num_srcs; i++) {
6151       switch (instr->src[i].src_type) {
6152       case nir_tex_src_texture_deref:
6153          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6154          break;
6155       case nir_tex_src_sampler_deref:
6156          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6157          break;
6158       case nir_tex_src_plane:
6159          plane = nir_src_as_int(instr->src[i].src);
6160          break;
6161       default:
6162          break;
6163       }
6164    }
6165
6166    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6167
6168    if (!sampler_deref_instr)
6169       sampler_deref_instr = texture_deref_instr;
6170
6171    if (plane >= 0) {
6172       assert(instr->op != nir_texop_txf_ms &&
6173              instr->op != nir_texop_samples_identical);
6174       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6175       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6176    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6177       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6178    } else {
6179       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6180    }
6181    if (samp_ptr) {
6182       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6183
6184       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6185          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6186          Builder bld(ctx->program, ctx->block);
6187
6188          /* to avoid unnecessary moves, we split and recombine sampler and image */
6189          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6190                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6191          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6192          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6193                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6194                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6195          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6196                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6197
6198          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6199          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6200                                img[0], img[1], img[2], img[3],
6201                                img[4], img[5], img[6], img[7]);
6202          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6203                                 samp[0], samp[1], samp[2], samp[3]);
6204       }
6205    }
6206    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6207                      instr->op == nir_texop_samples_identical))
6208       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6209 }
6210
6211 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6212                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6213 {
6214    Builder bld(ctx->program, ctx->block);
6215
6216    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6217    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6218    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6219
6220    Operand neg_one(0xbf800000u);
6221    Operand one(0x3f800000u);
6222    Operand two(0x40000000u);
6223    Operand four(0x40800000u);
6224
6225    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6226    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6227    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6228
6229    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6230    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6231    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6232    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6233
6234    // select sc
6235    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6236    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6237                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6238                        one, is_ma_y);
6239    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6240
6241    // select tc
6242    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6243    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6244    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6245
6246    // select ma
6247    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6248                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6249                   deriv_z, is_ma_z);
6250    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6251    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6252 }
6253
6254 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6255 {
6256    Builder bld(ctx->program, ctx->block);
6257    Temp coord_args[4], ma, tc, sc, id;
6258    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6259       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6260
6261    if (is_array) {
6262       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6263
6264       // see comment in ac_prepare_cube_coords()
6265       if (ctx->options->chip_class <= GFX8)
6266          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6267    }
6268
6269    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6270
6271    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6272    vop3a->operands[0] = Operand(ma);
6273    vop3a->abs[0] = true;
6274    Temp invma = bld.tmp(v1);
6275    vop3a->definitions[0] = Definition(invma);
6276    ctx->block->instructions.emplace_back(std::move(vop3a));
6277
6278    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6279    if (!is_deriv)
6280       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6281
6282    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6283    if (!is_deriv)
6284       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6285
6286    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6287
6288    if (is_deriv) {
6289       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6290       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6291
6292       for (unsigned i = 0; i < 2; i++) {
6293          // see comment in ac_prepare_cube_coords()
6294          Temp deriv_ma;
6295          Temp deriv_sc, deriv_tc;
6296          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6297                            &deriv_ma, &deriv_sc, &deriv_tc);
6298
6299          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6300
6301          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6302                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6303                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6304          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6305                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6306                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6307          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6308       }
6309
6310       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6311       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6312    }
6313
6314    if (is_array)
6315       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6316    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6317
6318 }
6319
6320 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6321 {
6322    Temp coord_vec[3];
6323    for (unsigned i = 0; i < coords.size(); i++)
6324       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6325
6326    Builder bld(ctx->program, ctx->block);
6327    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6328
6329    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6330    for (unsigned i = 0; i < coords.size(); i++)
6331       vec->operands[i] = Operand(coord_vec[i]);
6332    Temp res = bld.tmp(RegType::vgpr, coords.size());
6333    vec->definitions[0] = Definition(res);
6334    ctx->block->instructions.emplace_back(std::move(vec));
6335    return res;
6336 }
6337
6338 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6339 {
6340    if (vec->parent_instr->type != nir_instr_type_alu)
6341       return;
6342    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6343    if (vec_instr->op != nir_op_vec(vec->num_components))
6344       return;
6345
6346    for (unsigned i = 0; i < vec->num_components; i++) {
6347       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6348               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6349    }
6350 }
6351
6352 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6353 {
6354    Builder bld(ctx->program, ctx->block);
6355    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6356         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6357    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6358         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6359    nir_const_value *sample_index_cv = NULL;
6360    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6361    enum glsl_base_type stype;
6362    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6363
6364    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6365                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6366    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6367                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6368
6369    for (unsigned i = 0; i < instr->num_srcs; i++) {
6370       switch (instr->src[i].src_type) {
6371       case nir_tex_src_coord:
6372          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6373          break;
6374       case nir_tex_src_bias:
6375          if (instr->op == nir_texop_txb) {
6376             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6377             has_bias = true;
6378          }
6379          break;
6380       case nir_tex_src_lod: {
6381          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6382
6383          if (val && val->f32 <= 0.0) {
6384             level_zero = true;
6385          } else {
6386             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6387             has_lod = true;
6388          }
6389          break;
6390       }
6391       case nir_tex_src_comparator:
6392          if (instr->is_shadow) {
6393             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6394             has_compare = true;
6395          }
6396          break;
6397       case nir_tex_src_offset:
6398          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6399          get_const_vec(instr->src[i].src.ssa, const_offset);
6400          has_offset = true;
6401          break;
6402       case nir_tex_src_ddx:
6403          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6404          has_ddx = true;
6405          break;
6406       case nir_tex_src_ddy:
6407          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6408          has_ddy = true;
6409          break;
6410       case nir_tex_src_ms_index:
6411          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6412          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6413          has_sample_index = true;
6414          break;
6415       case nir_tex_src_texture_offset:
6416       case nir_tex_src_sampler_offset:
6417       default:
6418          break;
6419       }
6420    }
6421 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6422    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6423       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6424
6425    if (instr->op == nir_texop_texture_samples) {
6426       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6427
6428       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6429       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6430       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6431       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6432
6433       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6434                samples, Operand(1u), bld.scc(is_msaa));
6435       return;
6436    }
6437
6438    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6439       aco_ptr<Instruction> tmp_instr;
6440       Temp acc, pack = Temp();
6441
6442       uint32_t pack_const = 0;
6443       for (unsigned i = 0; i < offset.size(); i++) {
6444          if (!const_offset[i])
6445             continue;
6446          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6447       }
6448
6449       if (offset.type() == RegType::sgpr) {
6450          for (unsigned i = 0; i < offset.size(); i++) {
6451             if (const_offset[i])
6452                continue;
6453
6454             acc = emit_extract_vector(ctx, offset, i, s1);
6455             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6456
6457             if (i) {
6458                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6459             }
6460
6461             if (pack == Temp()) {
6462                pack = acc;
6463             } else {
6464                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6465             }
6466          }
6467
6468          if (pack_const && pack != Temp())
6469             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6470       } else {
6471          for (unsigned i = 0; i < offset.size(); i++) {
6472             if (const_offset[i])
6473                continue;
6474
6475             acc = emit_extract_vector(ctx, offset, i, v1);
6476             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6477
6478             if (i) {
6479                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6480             }
6481
6482             if (pack == Temp()) {
6483                pack = acc;
6484             } else {
6485                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6486             }
6487          }
6488
6489          if (pack_const && pack != Temp())
6490             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6491       }
6492       if (pack_const && pack == Temp())
6493          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6494       else if (pack == Temp())
6495          has_offset = false;
6496       else
6497          offset = pack;
6498    }
6499
6500    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6501       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6502
6503    /* pack derivatives */
6504    if (has_ddx || has_ddy) {
6505       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6506          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6507                              ddx, Operand(0u), ddy, Operand(0u));
6508       } else {
6509          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6510       }
6511       has_derivs = true;
6512    }
6513
6514    if (instr->coord_components > 1 &&
6515        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6516        instr->is_array &&
6517        instr->op != nir_texop_txf)
6518       coords = apply_round_slice(ctx, coords, 1);
6519
6520    if (instr->coord_components > 2 &&
6521       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6522        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6523        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6524        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6525        instr->is_array &&
6526        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6527       coords = apply_round_slice(ctx, coords, 2);
6528
6529    if (ctx->options->chip_class == GFX9 &&
6530        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6531        instr->op != nir_texop_lod && instr->coord_components) {
6532       assert(coords.size() > 0 && coords.size() < 3);
6533
6534       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6535       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6536       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6537       if (coords.size() > 1)
6538          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6539       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6540       vec->definitions[0] = Definition(coords);
6541       ctx->block->instructions.emplace_back(std::move(vec));
6542    }
6543
6544    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6545
6546    if (instr->op == nir_texop_samples_identical)
6547       resource = fmask_ptr;
6548
6549    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6550              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6551             instr->op != nir_texop_txs) {
6552       assert(has_sample_index);
6553       Operand op(sample_index);
6554       if (sample_index_cv)
6555          op = Operand(sample_index_cv->u32);
6556       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6557    }
6558
6559    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6560       Temp split_coords[coords.size()];
6561       emit_split_vector(ctx, coords, coords.size());
6562       for (unsigned i = 0; i < coords.size(); i++)
6563          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6564
6565       unsigned i = 0;
6566       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6567          Temp off = emit_extract_vector(ctx, offset, i, v1);
6568          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6569       }
6570
6571       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6572       for (unsigned i = 0; i < coords.size(); i++)
6573          vec->operands[i] = Operand(split_coords[i]);
6574       coords = bld.tmp(coords.regClass());
6575       vec->definitions[0] = Definition(coords);
6576       ctx->block->instructions.emplace_back(std::move(vec));
6577
6578       has_offset = false;
6579    }
6580
6581    /* Build tex instruction */
6582    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6583    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6584                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6585                   : 0;
6586    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6587    Temp tmp_dst = dst;
6588
6589    /* gather4 selects the component by dmask and always returns vec4 */
6590    if (instr->op == nir_texop_tg4) {
6591       assert(instr->dest.ssa.num_components == 4);
6592       if (instr->is_shadow)
6593          dmask = 1;
6594       else
6595          dmask = 1 << instr->component;
6596       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6597          tmp_dst = bld.tmp(v4);
6598    } else if (instr->op == nir_texop_samples_identical) {
6599       tmp_dst = bld.tmp(v1);
6600    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6601       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6602    }
6603
6604    aco_ptr<MIMG_instruction> tex;
6605    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6606       if (!has_lod)
6607          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6608
6609       bool div_by_6 = instr->op == nir_texop_txs &&
6610                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6611                       instr->is_array &&
6612                       (dmask & (1 << 2));
6613       if (tmp_dst.id() == dst.id() && div_by_6)
6614          tmp_dst = bld.tmp(tmp_dst.regClass());
6615
6616       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6617       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6618       tex->operands[1] = Operand(resource);
6619       if (ctx->options->chip_class == GFX9 &&
6620           instr->op == nir_texop_txs &&
6621           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6622           instr->is_array) {
6623          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6624       } else if (instr->op == nir_texop_query_levels) {
6625          tex->dmask = 1 << 3;
6626       } else {
6627          tex->dmask = dmask;
6628       }
6629       tex->da = da;
6630       tex->definitions[0] = Definition(tmp_dst);
6631       tex->dim = dim;
6632       tex->can_reorder = true;
6633       ctx->block->instructions.emplace_back(std::move(tex));
6634
6635       if (div_by_6) {
6636          /* divide 3rd value by 6 by multiplying with magic number */
6637          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6638          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6639          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6640          assert(instr->dest.ssa.num_components == 3);
6641          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6642          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6643                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6644                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6645                               by_6);
6646
6647       }
6648
6649       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6650       return;
6651    }
6652
6653    Temp tg4_compare_cube_wa64 = Temp();
6654
6655    if (tg4_integer_workarounds) {
6656       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6657       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6658       tex->operands[1] = Operand(resource);
6659       tex->dim = dim;
6660       tex->dmask = 0x3;
6661       tex->da = da;
6662       Temp size = bld.tmp(v2);
6663       tex->definitions[0] = Definition(size);
6664       tex->can_reorder = true;
6665       ctx->block->instructions.emplace_back(std::move(tex));
6666       emit_split_vector(ctx, size, size.size());
6667
6668       Temp half_texel[2];
6669       for (unsigned i = 0; i < 2; i++) {
6670          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6671          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6672          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6673          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6674       }
6675
6676       Temp orig_coords[2] = {
6677          emit_extract_vector(ctx, coords, 0, v1),
6678          emit_extract_vector(ctx, coords, 1, v1)};
6679       Temp new_coords[2] = {
6680          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6681          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6682       };
6683
6684       if (tg4_integer_cube_workaround) {
6685          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6686          Temp desc[resource.size()];
6687          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6688                                                                            Format::PSEUDO, 1, resource.size())};
6689          split->operands[0] = Operand(resource);
6690          for (unsigned i = 0; i < resource.size(); i++) {
6691             desc[i] = bld.tmp(s1);
6692             split->definitions[i] = Definition(desc[i]);
6693          }
6694          ctx->block->instructions.emplace_back(std::move(split));
6695
6696          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6697          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6698                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6699
6700          Temp nfmt;
6701          if (stype == GLSL_TYPE_UINT) {
6702             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6703                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6704                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6705                             bld.scc(compare_cube_wa));
6706          } else {
6707             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6708                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6709                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6710                             bld.scc(compare_cube_wa));
6711          }
6712          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6713          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6714
6715          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6716
6717          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6718                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6719          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6720
6721          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6722                                                                          Format::PSEUDO, resource.size(), 1)};
6723          for (unsigned i = 0; i < resource.size(); i++)
6724             vec->operands[i] = Operand(desc[i]);
6725          resource = bld.tmp(resource.regClass());
6726          vec->definitions[0] = Definition(resource);
6727          ctx->block->instructions.emplace_back(std::move(vec));
6728
6729          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6730                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6731          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6732                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6733       }
6734
6735       if (coords.size() == 3) {
6736          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6737                              new_coords[0], new_coords[1],
6738                              emit_extract_vector(ctx, coords, 2, v1));
6739       } else {
6740          assert(coords.size() == 2);
6741          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6742                              new_coords[0], new_coords[1]);
6743       }
6744    }
6745
6746    std::vector<Operand> args;
6747    if (has_offset)
6748       args.emplace_back(Operand(offset));
6749    if (has_bias)
6750       args.emplace_back(Operand(bias));
6751    if (has_compare)
6752       args.emplace_back(Operand(compare));
6753    if (has_derivs)
6754       args.emplace_back(Operand(derivs));
6755    args.emplace_back(Operand(coords));
6756    if (has_sample_index)
6757       args.emplace_back(Operand(sample_index));
6758    if (has_lod)
6759       args.emplace_back(lod);
6760
6761    Temp arg;
6762    if (args.size() > 1) {
6763       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6764       unsigned size = 0;
6765       for (unsigned i = 0; i < args.size(); i++) {
6766          size += args[i].size();
6767          vec->operands[i] = args[i];
6768       }
6769       RegClass rc = RegClass(RegType::vgpr, size);
6770       Temp tmp = bld.tmp(rc);
6771       vec->definitions[0] = Definition(tmp);
6772       ctx->block->instructions.emplace_back(std::move(vec));
6773       arg = tmp;
6774    } else {
6775       assert(args[0].isTemp());
6776       arg = as_vgpr(ctx, args[0].getTemp());
6777    }
6778
6779    /* we don't need the bias, sample index, compare value or offset to be
6780     * computed in WQM but if the p_create_vector copies the coordinates, then it
6781     * needs to be in WQM */
6782    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6783        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6784        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6785       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6786
6787    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6788       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6789
6790       assert(coords.size() == 1);
6791       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6792       aco_opcode op;
6793       switch (last_bit) {
6794       case 1:
6795          op = aco_opcode::buffer_load_format_x; break;
6796       case 2:
6797          op = aco_opcode::buffer_load_format_xy; break;
6798       case 3:
6799          op = aco_opcode::buffer_load_format_xyz; break;
6800       case 4:
6801          op = aco_opcode::buffer_load_format_xyzw; break;
6802       default:
6803          unreachable("Tex instruction loads more than 4 components.");
6804       }
6805
6806       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6807       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6808          tmp_dst = dst;
6809       else
6810          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6811
6812       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6813       mubuf->operands[0] = Operand(coords);
6814       mubuf->operands[1] = Operand(resource);
6815       mubuf->operands[2] = Operand((uint32_t) 0);
6816       mubuf->definitions[0] = Definition(tmp_dst);
6817       mubuf->idxen = true;
6818       mubuf->can_reorder = true;
6819       ctx->block->instructions.emplace_back(std::move(mubuf));
6820
6821       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6822       return;
6823    }
6824
6825
6826    if (instr->op == nir_texop_txf ||
6827        instr->op == nir_texop_txf_ms ||
6828        instr->op == nir_texop_samples_identical) {
6829       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6830       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6831       tex->operands[0] = Operand(arg);
6832       tex->operands[1] = Operand(resource);
6833       tex->dim = dim;
6834       tex->dmask = dmask;
6835       tex->unrm = true;
6836       tex->da = da;
6837       tex->definitions[0] = Definition(tmp_dst);
6838       tex->can_reorder = true;
6839       ctx->block->instructions.emplace_back(std::move(tex));
6840
6841       if (instr->op == nir_texop_samples_identical) {
6842          assert(dmask == 1 && dst.regClass() == v1);
6843          assert(dst.id() != tmp_dst.id());
6844
6845          Temp tmp = bld.tmp(bld.lm);
6846          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6847          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6848
6849       } else {
6850          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6851       }
6852       return;
6853    }
6854
6855    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6856    aco_opcode opcode = aco_opcode::image_sample;
6857    if (has_offset) { /* image_sample_*_o */
6858       if (has_compare) {
6859          opcode = aco_opcode::image_sample_c_o;
6860          if (has_derivs)
6861             opcode = aco_opcode::image_sample_c_d_o;
6862          if (has_bias)
6863             opcode = aco_opcode::image_sample_c_b_o;
6864          if (level_zero)
6865             opcode = aco_opcode::image_sample_c_lz_o;
6866          if (has_lod)
6867             opcode = aco_opcode::image_sample_c_l_o;
6868       } else {
6869          opcode = aco_opcode::image_sample_o;
6870          if (has_derivs)
6871             opcode = aco_opcode::image_sample_d_o;
6872          if (has_bias)
6873             opcode = aco_opcode::image_sample_b_o;
6874          if (level_zero)
6875             opcode = aco_opcode::image_sample_lz_o;
6876          if (has_lod)
6877             opcode = aco_opcode::image_sample_l_o;
6878       }
6879    } else { /* no offset */
6880       if (has_compare) {
6881          opcode = aco_opcode::image_sample_c;
6882          if (has_derivs)
6883             opcode = aco_opcode::image_sample_c_d;
6884          if (has_bias)
6885             opcode = aco_opcode::image_sample_c_b;
6886          if (level_zero)
6887             opcode = aco_opcode::image_sample_c_lz;
6888          if (has_lod)
6889             opcode = aco_opcode::image_sample_c_l;
6890       } else {
6891          opcode = aco_opcode::image_sample;
6892          if (has_derivs)
6893             opcode = aco_opcode::image_sample_d;
6894          if (has_bias)
6895             opcode = aco_opcode::image_sample_b;
6896          if (level_zero)
6897             opcode = aco_opcode::image_sample_lz;
6898          if (has_lod)
6899             opcode = aco_opcode::image_sample_l;
6900       }
6901    }
6902
6903    if (instr->op == nir_texop_tg4) {
6904       if (has_offset) {
6905          opcode = aco_opcode::image_gather4_lz_o;
6906          if (has_compare)
6907             opcode = aco_opcode::image_gather4_c_lz_o;
6908       } else {
6909          opcode = aco_opcode::image_gather4_lz;
6910          if (has_compare)
6911             opcode = aco_opcode::image_gather4_c_lz;
6912       }
6913    } else if (instr->op == nir_texop_lod) {
6914       opcode = aco_opcode::image_get_lod;
6915    }
6916
6917    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6918    tex->operands[0] = Operand(arg);
6919    tex->operands[1] = Operand(resource);
6920    tex->operands[2] = Operand(sampler);
6921    tex->dim = dim;
6922    tex->dmask = dmask;
6923    tex->da = da;
6924    tex->definitions[0] = Definition(tmp_dst);
6925    tex->can_reorder = true;
6926    ctx->block->instructions.emplace_back(std::move(tex));
6927
6928    if (tg4_integer_cube_workaround) {
6929       assert(tmp_dst.id() != dst.id());
6930       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6931
6932       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6933       Temp val[4];
6934       for (unsigned i = 0; i < dst.size(); i++) {
6935          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6936          Temp cvt_val;
6937          if (stype == GLSL_TYPE_UINT)
6938             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6939          else
6940             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6941          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6942       }
6943       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6944       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6945                            val[0], val[1], val[2], val[3]);
6946    }
6947    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6948    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6949
6950 }
6951
6952
6953 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6954 {
6955    Temp tmp = get_ssa_temp(ctx, ssa);
6956    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6957       return Operand(tmp.regClass());
6958    else
6959       return Operand(tmp);
6960 }
6961
6962 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6963 {
6964    aco_ptr<Pseudo_instruction> phi;
6965    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6966    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
6967
6968    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
6969    logical |= ctx->block->kind & block_kind_merge;
6970    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6971
6972    /* we want a sorted list of sources, since the predecessor list is also sorted */
6973    std::map<unsigned, nir_ssa_def*> phi_src;
6974    nir_foreach_phi_src(src, instr)
6975       phi_src[src->pred->index] = src->src.ssa;
6976
6977    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
6978    unsigned num_operands = 0;
6979    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
6980    unsigned num_defined = 0;
6981    unsigned cur_pred_idx = 0;
6982    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
6983       if (cur_pred_idx < preds.size()) {
6984          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
6985          unsigned block = ctx->cf_info.nir_to_aco[src.first];
6986          unsigned skipped = 0;
6987          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
6988             skipped++;
6989          if (cur_pred_idx + skipped < preds.size()) {
6990             for (unsigned i = 0; i < skipped; i++)
6991                operands[num_operands++] = Operand(dst.regClass());
6992             cur_pred_idx += skipped;
6993          } else {
6994             continue;
6995          }
6996       }
6997       cur_pred_idx++;
6998       Operand op = get_phi_operand(ctx, src.second);
6999       operands[num_operands++] = op;
7000       num_defined += !op.isUndefined();
7001    }
7002    /* handle block_kind_continue_or_break at loop exit blocks */
7003    while (cur_pred_idx++ < preds.size())
7004       operands[num_operands++] = Operand(dst.regClass());
7005
7006    if (num_defined == 0) {
7007       Builder bld(ctx->program, ctx->block);
7008       if (dst.regClass() == s1) {
7009          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7010       } else if (dst.regClass() == v1) {
7011          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7012       } else {
7013          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7014          for (unsigned i = 0; i < dst.size(); i++)
7015             vec->operands[i] = Operand(0u);
7016          vec->definitions[0] = Definition(dst);
7017          ctx->block->instructions.emplace_back(std::move(vec));
7018       }
7019       return;
7020    }
7021
7022    /* we can use a linear phi in some cases if one src is undef */
7023    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7024       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7025
7026       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7027       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7028       assert(invert->kind & block_kind_invert);
7029
7030       unsigned then_block = invert->linear_preds[0];
7031
7032       Block* insert_block = NULL;
7033       for (unsigned i = 0; i < num_operands; i++) {
7034          Operand op = operands[i];
7035          if (op.isUndefined())
7036             continue;
7037          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7038          phi->operands[0] = op;
7039          break;
7040       }
7041       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7042       phi->operands[1] = Operand(dst.regClass());
7043       phi->definitions[0] = Definition(dst);
7044       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7045       return;
7046    }
7047
7048    /* try to scalarize vector phis */
7049    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7050       // TODO: scalarize linear phis on divergent ifs
7051       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7052       std::array<Temp, 4> new_vec;
7053       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7054          Operand src = operands[i];
7055          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7056             can_scalarize = false;
7057       }
7058       if (can_scalarize) {
7059          unsigned num_components = instr->dest.ssa.num_components;
7060          assert(dst.size() % num_components == 0);
7061          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7062
7063          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7064          for (unsigned k = 0; k < num_components; k++) {
7065             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7066             for (unsigned i = 0; i < num_operands; i++) {
7067                Operand src = operands[i];
7068                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7069             }
7070             Temp phi_dst = {ctx->program->allocateId(), rc};
7071             phi->definitions[0] = Definition(phi_dst);
7072             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7073             new_vec[k] = phi_dst;
7074             vec->operands[k] = Operand(phi_dst);
7075          }
7076          vec->definitions[0] = Definition(dst);
7077          ctx->block->instructions.emplace_back(std::move(vec));
7078          ctx->allocated_vec.emplace(dst.id(), new_vec);
7079          return;
7080       }
7081    }
7082
7083    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7084    for (unsigned i = 0; i < num_operands; i++)
7085       phi->operands[i] = operands[i];
7086    phi->definitions[0] = Definition(dst);
7087    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7088 }
7089
7090
7091 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7092 {
7093    Temp dst = get_ssa_temp(ctx, &instr->def);
7094
7095    assert(dst.type() == RegType::sgpr);
7096
7097    if (dst.size() == 1) {
7098       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7099    } else {
7100       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7101       for (unsigned i = 0; i < dst.size(); i++)
7102          vec->operands[i] = Operand(0u);
7103       vec->definitions[0] = Definition(dst);
7104       ctx->block->instructions.emplace_back(std::move(vec));
7105    }
7106 }
7107
7108 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7109 {
7110    Builder bld(ctx->program, ctx->block);
7111    Block *logical_target;
7112    append_logical_end(ctx->block);
7113    unsigned idx = ctx->block->index;
7114
7115    switch (instr->type) {
7116    case nir_jump_break:
7117       logical_target = ctx->cf_info.parent_loop.exit;
7118       add_logical_edge(idx, logical_target);
7119       ctx->block->kind |= block_kind_break;
7120
7121       if (!ctx->cf_info.parent_if.is_divergent &&
7122           !ctx->cf_info.parent_loop.has_divergent_continue) {
7123          /* uniform break - directly jump out of the loop */
7124          ctx->block->kind |= block_kind_uniform;
7125          ctx->cf_info.has_branch = true;
7126          bld.branch(aco_opcode::p_branch);
7127          add_linear_edge(idx, logical_target);
7128          return;
7129       }
7130       ctx->cf_info.parent_loop.has_divergent_branch = true;
7131       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7132       break;
7133    case nir_jump_continue:
7134       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7135       add_logical_edge(idx, logical_target);
7136       ctx->block->kind |= block_kind_continue;
7137
7138       if (ctx->cf_info.parent_if.is_divergent) {
7139          /* for potential uniform breaks after this continue,
7140             we must ensure that they are handled correctly */
7141          ctx->cf_info.parent_loop.has_divergent_continue = true;
7142          ctx->cf_info.parent_loop.has_divergent_branch = true;
7143          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7144       } else {
7145          /* uniform continue - directly jump to the loop header */
7146          ctx->block->kind |= block_kind_uniform;
7147          ctx->cf_info.has_branch = true;
7148          bld.branch(aco_opcode::p_branch);
7149          add_linear_edge(idx, logical_target);
7150          return;
7151       }
7152       break;
7153    default:
7154       fprintf(stderr, "Unknown NIR jump instr: ");
7155       nir_print_instr(&instr->instr, stderr);
7156       fprintf(stderr, "\n");
7157       abort();
7158    }
7159
7160    /* remove critical edges from linear CFG */
7161    bld.branch(aco_opcode::p_branch);
7162    Block* break_block = ctx->program->create_and_insert_block();
7163    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7164    break_block->kind |= block_kind_uniform;
7165    add_linear_edge(idx, break_block);
7166    /* the loop_header pointer might be invalidated by this point */
7167    if (instr->type == nir_jump_continue)
7168       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7169    add_linear_edge(break_block->index, logical_target);
7170    bld.reset(break_block);
7171    bld.branch(aco_opcode::p_branch);
7172
7173    Block* continue_block = ctx->program->create_and_insert_block();
7174    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7175    add_linear_edge(idx, continue_block);
7176    append_logical_start(continue_block);
7177    ctx->block = continue_block;
7178    return;
7179 }
7180
7181 void visit_block(isel_context *ctx, nir_block *block)
7182 {
7183    nir_foreach_instr(instr, block) {
7184       switch (instr->type) {
7185       case nir_instr_type_alu:
7186          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7187          break;
7188       case nir_instr_type_load_const:
7189          visit_load_const(ctx, nir_instr_as_load_const(instr));
7190          break;
7191       case nir_instr_type_intrinsic:
7192          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7193          break;
7194       case nir_instr_type_tex:
7195          visit_tex(ctx, nir_instr_as_tex(instr));
7196          break;
7197       case nir_instr_type_phi:
7198          visit_phi(ctx, nir_instr_as_phi(instr));
7199          break;
7200       case nir_instr_type_ssa_undef:
7201          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7202          break;
7203       case nir_instr_type_deref:
7204          break;
7205       case nir_instr_type_jump:
7206          visit_jump(ctx, nir_instr_as_jump(instr));
7207          break;
7208       default:
7209          fprintf(stderr, "Unknown NIR instr type: ");
7210          nir_print_instr(instr, stderr);
7211          fprintf(stderr, "\n");
7212          //abort();
7213       }
7214    }
7215
7216    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7217       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7218 }
7219
7220
7221
7222 static void visit_loop(isel_context *ctx, nir_loop *loop)
7223 {
7224    append_logical_end(ctx->block);
7225    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7226    Builder bld(ctx->program, ctx->block);
7227    bld.branch(aco_opcode::p_branch);
7228    unsigned loop_preheader_idx = ctx->block->index;
7229
7230    Block loop_exit = Block();
7231    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7232    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7233
7234    Block* loop_header = ctx->program->create_and_insert_block();
7235    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7236    loop_header->kind |= block_kind_loop_header;
7237    add_edge(loop_preheader_idx, loop_header);
7238    ctx->block = loop_header;
7239
7240    /* emit loop body */
7241    unsigned loop_header_idx = loop_header->index;
7242    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7243    append_logical_start(ctx->block);
7244    visit_cf_list(ctx, &loop->body);
7245
7246    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7247    if (!ctx->cf_info.has_branch) {
7248       append_logical_end(ctx->block);
7249       if (ctx->cf_info.exec_potentially_empty) {
7250          /* Discards can result in code running with an empty exec mask.
7251           * This would result in divergent breaks not ever being taken. As a
7252           * workaround, break the loop when the loop mask is empty instead of
7253           * always continuing. */
7254          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7255          unsigned block_idx = ctx->block->index;
7256
7257          /* create helper blocks to avoid critical edges */
7258          Block *break_block = ctx->program->create_and_insert_block();
7259          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7260          break_block->kind = block_kind_uniform;
7261          bld.reset(break_block);
7262          bld.branch(aco_opcode::p_branch);
7263          add_linear_edge(block_idx, break_block);
7264          add_linear_edge(break_block->index, &loop_exit);
7265
7266          Block *continue_block = ctx->program->create_and_insert_block();
7267          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7268          continue_block->kind = block_kind_uniform;
7269          bld.reset(continue_block);
7270          bld.branch(aco_opcode::p_branch);
7271          add_linear_edge(block_idx, continue_block);
7272          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7273
7274          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7275          ctx->block = &ctx->program->blocks[block_idx];
7276       } else {
7277          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7278          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7279             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7280          else
7281             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7282       }
7283
7284       bld.reset(ctx->block);
7285       bld.branch(aco_opcode::p_branch);
7286    }
7287
7288    /* fixup phis in loop header from unreachable blocks */
7289    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7290       bool linear = ctx->cf_info.has_branch;
7291       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7292       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7293          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7294              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7295             /* the last operand should be the one that needs to be removed */
7296             instr->operands.pop_back();
7297          } else if (!is_phi(instr)) {
7298             break;
7299          }
7300       }
7301    }
7302
7303    ctx->cf_info.has_branch = false;
7304
7305    // TODO: if the loop has not a single exit, we must add one °°
7306    /* emit loop successor block */
7307    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7308    append_logical_start(ctx->block);
7309
7310    #if 0
7311    // TODO: check if it is beneficial to not branch on continues
7312    /* trim linear phis in loop header */
7313    for (auto&& instr : loop_entry->instructions) {
7314       if (instr->opcode == aco_opcode::p_linear_phi) {
7315          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7316          new_phi->definitions[0] = instr->definitions[0];
7317          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7318             new_phi->operands[i] = instr->operands[i];
7319          /* check that the remaining operands are all the same */
7320          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7321             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7322          instr.swap(new_phi);
7323       } else if (instr->opcode == aco_opcode::p_phi) {
7324          continue;
7325       } else {
7326          break;
7327       }
7328    }
7329    #endif
7330 }
7331
7332 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7333 {
7334    ic->cond = cond;
7335
7336    append_logical_end(ctx->block);
7337    ctx->block->kind |= block_kind_branch;
7338
7339    /* branch to linear then block */
7340    assert(cond.regClass() == ctx->program->lane_mask);
7341    aco_ptr<Pseudo_branch_instruction> branch;
7342    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7343    branch->operands[0] = Operand(cond);
7344    ctx->block->instructions.push_back(std::move(branch));
7345
7346    ic->BB_if_idx = ctx->block->index;
7347    ic->BB_invert = Block();
7348    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7349    /* Invert blocks are intentionally not marked as top level because they
7350     * are not part of the logical cfg. */
7351    ic->BB_invert.kind |= block_kind_invert;
7352    ic->BB_endif = Block();
7353    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7354    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7355
7356    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7357    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7358    ctx->cf_info.parent_if.is_divergent = true;
7359    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7360
7361    /** emit logical then block */
7362    Block* BB_then_logical = ctx->program->create_and_insert_block();
7363    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7364    add_edge(ic->BB_if_idx, BB_then_logical);
7365    ctx->block = BB_then_logical;
7366    append_logical_start(BB_then_logical);
7367 }
7368
7369 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7370 {
7371    Block *BB_then_logical = ctx->block;
7372    append_logical_end(BB_then_logical);
7373     /* branch from logical then block to invert block */
7374    aco_ptr<Pseudo_branch_instruction> branch;
7375    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7376    BB_then_logical->instructions.emplace_back(std::move(branch));
7377    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7378    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7379       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7380    BB_then_logical->kind |= block_kind_uniform;
7381    assert(!ctx->cf_info.has_branch);
7382    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7383    ctx->cf_info.parent_loop.has_divergent_branch = false;
7384
7385    /** emit linear then block */
7386    Block* BB_then_linear = ctx->program->create_and_insert_block();
7387    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7388    BB_then_linear->kind |= block_kind_uniform;
7389    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7390    /* branch from linear then block to invert block */
7391    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7392    BB_then_linear->instructions.emplace_back(std::move(branch));
7393    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7394
7395    /** emit invert merge block */
7396    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7397    ic->invert_idx = ctx->block->index;
7398
7399    /* branch to linear else block (skip else) */
7400    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7401    branch->operands[0] = Operand(ic->cond);
7402    ctx->block->instructions.push_back(std::move(branch));
7403
7404    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7405    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7406
7407    /** emit logical else block */
7408    Block* BB_else_logical = ctx->program->create_and_insert_block();
7409    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7410    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7411    add_linear_edge(ic->invert_idx, BB_else_logical);
7412    ctx->block = BB_else_logical;
7413    append_logical_start(BB_else_logical);
7414 }
7415
7416 static void end_divergent_if(isel_context *ctx, if_context *ic)
7417 {
7418    Block *BB_else_logical = ctx->block;
7419    append_logical_end(BB_else_logical);
7420
7421    /* branch from logical else block to endif block */
7422    aco_ptr<Pseudo_branch_instruction> branch;
7423    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7424    BB_else_logical->instructions.emplace_back(std::move(branch));
7425    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7426    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7427       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7428    BB_else_logical->kind |= block_kind_uniform;
7429
7430    assert(!ctx->cf_info.has_branch);
7431    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7432
7433
7434    /** emit linear else block */
7435    Block* BB_else_linear = ctx->program->create_and_insert_block();
7436    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7437    BB_else_linear->kind |= block_kind_uniform;
7438    add_linear_edge(ic->invert_idx, BB_else_linear);
7439
7440    /* branch from linear else block to endif block */
7441    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7442    BB_else_linear->instructions.emplace_back(std::move(branch));
7443    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7444
7445
7446    /** emit endif merge block */
7447    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7448    append_logical_start(ctx->block);
7449
7450
7451    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7452    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7453    /* uniform control flow never has an empty exec-mask */
7454    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7455       ctx->cf_info.exec_potentially_empty = false;
7456 }
7457
7458 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7459 {
7460    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7461    Builder bld(ctx->program, ctx->block);
7462    aco_ptr<Pseudo_branch_instruction> branch;
7463
7464    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7465       /**
7466        * Uniform conditionals are represented in the following way*) :
7467        *
7468        * The linear and logical CFG:
7469        *                        BB_IF
7470        *                        /    \
7471        *       BB_THEN (logical)      BB_ELSE (logical)
7472        *                        \    /
7473        *                        BB_ENDIF
7474        *
7475        * *) Exceptions may be due to break and continue statements within loops
7476        *    If a break/continue happens within uniform control flow, it branches
7477        *    to the loop exit/entry block. Otherwise, it branches to the next
7478        *    merge block.
7479        **/
7480       append_logical_end(ctx->block);
7481       ctx->block->kind |= block_kind_uniform;
7482
7483       /* emit branch */
7484       assert(cond.regClass() == bld.lm);
7485       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7486       cond = bool_to_scalar_condition(ctx, cond);
7487
7488       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7489       branch->operands[0] = Operand(cond);
7490       branch->operands[0].setFixed(scc);
7491       ctx->block->instructions.emplace_back(std::move(branch));
7492
7493       unsigned BB_if_idx = ctx->block->index;
7494       Block BB_endif = Block();
7495       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7496       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7497
7498       /** emit then block */
7499       Block* BB_then = ctx->program->create_and_insert_block();
7500       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7501       add_edge(BB_if_idx, BB_then);
7502       append_logical_start(BB_then);
7503       ctx->block = BB_then;
7504       visit_cf_list(ctx, &if_stmt->then_list);
7505       BB_then = ctx->block;
7506       bool then_branch = ctx->cf_info.has_branch;
7507       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7508
7509       if (!then_branch) {
7510          append_logical_end(BB_then);
7511          /* branch from then block to endif block */
7512          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7513          BB_then->instructions.emplace_back(std::move(branch));
7514          add_linear_edge(BB_then->index, &BB_endif);
7515          if (!then_branch_divergent)
7516             add_logical_edge(BB_then->index, &BB_endif);
7517          BB_then->kind |= block_kind_uniform;
7518       }
7519
7520       ctx->cf_info.has_branch = false;
7521       ctx->cf_info.parent_loop.has_divergent_branch = false;
7522
7523       /** emit else block */
7524       Block* BB_else = ctx->program->create_and_insert_block();
7525       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7526       add_edge(BB_if_idx, BB_else);
7527       append_logical_start(BB_else);
7528       ctx->block = BB_else;
7529       visit_cf_list(ctx, &if_stmt->else_list);
7530       BB_else = ctx->block;
7531
7532       if (!ctx->cf_info.has_branch) {
7533          append_logical_end(BB_else);
7534          /* branch from then block to endif block */
7535          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7536          BB_else->instructions.emplace_back(std::move(branch));
7537          add_linear_edge(BB_else->index, &BB_endif);
7538          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7539             add_logical_edge(BB_else->index, &BB_endif);
7540          BB_else->kind |= block_kind_uniform;
7541       }
7542
7543       ctx->cf_info.has_branch &= then_branch;
7544       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7545
7546       /** emit endif merge block */
7547       if (!ctx->cf_info.has_branch) {
7548          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7549          append_logical_start(ctx->block);
7550       }
7551    } else { /* non-uniform condition */
7552       /**
7553        * To maintain a logical and linear CFG without critical edges,
7554        * non-uniform conditionals are represented in the following way*) :
7555        *
7556        * The linear CFG:
7557        *                        BB_IF
7558        *                        /    \
7559        *       BB_THEN (logical)      BB_THEN (linear)
7560        *                        \    /
7561        *                        BB_INVERT (linear)
7562        *                        /    \
7563        *       BB_ELSE (logical)      BB_ELSE (linear)
7564        *                        \    /
7565        *                        BB_ENDIF
7566        *
7567        * The logical CFG:
7568        *                        BB_IF
7569        *                        /    \
7570        *       BB_THEN (logical)      BB_ELSE (logical)
7571        *                        \    /
7572        *                        BB_ENDIF
7573        *
7574        * *) Exceptions may be due to break and continue statements within loops
7575        **/
7576
7577       if_context ic;
7578
7579       begin_divergent_if_then(ctx, &ic, cond);
7580       visit_cf_list(ctx, &if_stmt->then_list);
7581
7582       begin_divergent_if_else(ctx, &ic);
7583       visit_cf_list(ctx, &if_stmt->else_list);
7584
7585       end_divergent_if(ctx, &ic);
7586    }
7587 }
7588
7589 static void visit_cf_list(isel_context *ctx,
7590                           struct exec_list *list)
7591 {
7592    foreach_list_typed(nir_cf_node, node, node, list) {
7593       switch (node->type) {
7594       case nir_cf_node_block:
7595          visit_block(ctx, nir_cf_node_as_block(node));
7596          break;
7597       case nir_cf_node_if:
7598          visit_if(ctx, nir_cf_node_as_if(node));
7599          break;
7600       case nir_cf_node_loop:
7601          visit_loop(ctx, nir_cf_node_as_loop(node));
7602          break;
7603       default:
7604          unreachable("unimplemented cf list type");
7605       }
7606    }
7607 }
7608
7609 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7610 {
7611    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7612    uint64_t mask = ctx->vs_output.mask[slot];
7613    if (!is_pos && !mask)
7614       return;
7615    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7616       return;
7617    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7618    exp->enabled_mask = mask;
7619    for (unsigned i = 0; i < 4; ++i) {
7620       if (mask & (1 << i))
7621          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7622       else
7623          exp->operands[i] = Operand(v1);
7624    }
7625    exp->valid_mask = false;
7626    exp->done = false;
7627    exp->compressed = false;
7628    if (is_pos)
7629       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7630    else
7631       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7632    ctx->block->instructions.emplace_back(std::move(exp));
7633 }
7634
7635 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7636 {
7637    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7638    exp->enabled_mask = 0;
7639    for (unsigned i = 0; i < 4; ++i)
7640       exp->operands[i] = Operand(v1);
7641    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7642       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7643       exp->enabled_mask |= 0x1;
7644    }
7645    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7646       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7647       exp->enabled_mask |= 0x4;
7648    }
7649    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7650       if (ctx->options->chip_class < GFX9) {
7651          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7652          exp->enabled_mask |= 0x8;
7653       } else {
7654          Builder bld(ctx->program, ctx->block);
7655
7656          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7657                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7658          if (exp->operands[2].isTemp())
7659             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7660
7661          exp->operands[2] = Operand(out);
7662          exp->enabled_mask |= 0x4;
7663       }
7664    }
7665    exp->valid_mask = false;
7666    exp->done = false;
7667    exp->compressed = false;
7668    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7669    ctx->block->instructions.emplace_back(std::move(exp));
7670 }
7671
7672 static void create_vs_exports(isel_context *ctx)
7673 {
7674    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7675
7676    if (outinfo->export_prim_id) {
7677       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7678       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7679    }
7680
7681    if (ctx->options->key.has_multiview_view_index) {
7682       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7683       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7684    }
7685
7686    /* the order these position exports are created is important */
7687    int next_pos = 0;
7688    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7689    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7690       export_vs_psiz_layer_viewport(ctx, &next_pos);
7691    }
7692    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7693       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7694    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7695       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7696
7697    if (ctx->options->key.vs_common_out.export_clip_dists) {
7698       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7699          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7700       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7701          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7702    }
7703
7704    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7705       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7706           i != VARYING_SLOT_PRIMITIVE_ID)
7707          continue;
7708
7709       export_vs_varying(ctx, i, false, NULL);
7710    }
7711 }
7712
7713 static void emit_stream_output(isel_context *ctx,
7714                                Temp const *so_buffers,
7715                                Temp const *so_write_offset,
7716                                const struct radv_stream_output *output)
7717 {
7718    unsigned num_comps = util_bitcount(output->component_mask);
7719    unsigned loc = output->location;
7720    unsigned buf = output->buffer;
7721    unsigned offset = output->offset;
7722
7723    assert(num_comps && num_comps <= 4);
7724    if (!num_comps || num_comps > 4)
7725       return;
7726
7727    unsigned start = ffs(output->component_mask) - 1;
7728
7729    Temp out[4];
7730    bool all_undef = true;
7731    assert(ctx->stage == vertex_vs);
7732    for (unsigned i = 0; i < num_comps; i++) {
7733       out[i] = ctx->vs_output.outputs[loc][start + i];
7734       all_undef = all_undef && !out[i].id();
7735    }
7736    if (all_undef)
7737       return;
7738
7739    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7741    for (unsigned i = 0; i < num_comps; ++i)
7742       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7743    vec->definitions[0] = Definition(write_data);
7744    ctx->block->instructions.emplace_back(std::move(vec));
7745
7746    aco_opcode opcode;
7747    switch (num_comps) {
7748    case 1:
7749       opcode = aco_opcode::buffer_store_dword;
7750       break;
7751    case 2:
7752       opcode = aco_opcode::buffer_store_dwordx2;
7753       break;
7754    case 3:
7755       opcode = aco_opcode::buffer_store_dwordx3;
7756       break;
7757    case 4:
7758       opcode = aco_opcode::buffer_store_dwordx4;
7759       break;
7760    }
7761
7762    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7763    store->operands[0] = Operand(so_write_offset[buf]);
7764    store->operands[1] = Operand(so_buffers[buf]);
7765    store->operands[2] = Operand((uint32_t) 0);
7766    store->operands[3] = Operand(write_data);
7767    if (offset > 4095) {
7768       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7769       Builder bld(ctx->program, ctx->block);
7770       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7771    } else {
7772       store->offset = offset;
7773    }
7774    store->offen = true;
7775    store->glc = true;
7776    store->dlc = false;
7777    store->slc = true;
7778    store->can_reorder = true;
7779    ctx->block->instructions.emplace_back(std::move(store));
7780 }
7781
7782 static void emit_streamout(isel_context *ctx, unsigned stream)
7783 {
7784    Builder bld(ctx->program, ctx->block);
7785
7786    Temp so_buffers[4];
7787    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7788    for (unsigned i = 0; i < 4; i++) {
7789       unsigned stride = ctx->program->info->so.strides[i];
7790       if (!stride)
7791          continue;
7792
7793       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7794    }
7795
7796    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7797                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7798
7799    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7800
7801    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7802
7803    if_context ic;
7804    begin_divergent_if_then(ctx, &ic, can_emit);
7805
7806    bld.reset(ctx->block);
7807
7808    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7809
7810    Temp so_write_offset[4];
7811
7812    for (unsigned i = 0; i < 4; i++) {
7813       unsigned stride = ctx->program->info->so.strides[i];
7814       if (!stride)
7815          continue;
7816
7817       if (stride == 1) {
7818          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7819                                 get_arg(ctx, ctx->args->streamout_write_idx),
7820                                 get_arg(ctx, ctx->args->streamout_offset[i]));
7821          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7822
7823          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7824       } else {
7825          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7826          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
7827                                  get_arg(ctx, ctx->args->streamout_offset[i]));
7828          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7829       }
7830    }
7831
7832    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7833       struct radv_stream_output *output =
7834          &ctx->program->info->so.outputs[i];
7835       if (stream != output->stream)
7836          continue;
7837
7838       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7839    }
7840
7841    begin_divergent_if_else(ctx, &ic);
7842    end_divergent_if(ctx, &ic);
7843 }
7844
7845 } /* end namespace */
7846
7847 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
7848 {
7849    /* Split all arguments except for the first (ring_offsets) and the last
7850     * (exec) so that the dead channels don't stay live throughout the program.
7851     */
7852    for (unsigned i = 1; i < startpgm->definitions.size() - 1; i++) {
7853       if (startpgm->definitions[i].regClass().size() > 1) {
7854          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
7855                            startpgm->definitions[i].regClass().size());
7856       }
7857    }
7858 }
7859
7860 void handle_bc_optimize(isel_context *ctx)
7861 {
7862    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7863    Builder bld(ctx->program, ctx->block);
7864    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7865    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7866    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7867    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
7868    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
7869    if (uses_center && uses_centroid) {
7870       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
7871                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
7872
7873       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7874          Temp new_coord[2];
7875          for (unsigned i = 0; i < 2; i++) {
7876             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
7877             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
7878             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7879                                     persp_centroid, persp_center, sel);
7880          }
7881          ctx->persp_centroid = bld.tmp(v2);
7882          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
7883                     Operand(new_coord[0]), Operand(new_coord[1]));
7884          emit_split_vector(ctx, ctx->persp_centroid, 2);
7885       }
7886
7887       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7888          Temp new_coord[2];
7889          for (unsigned i = 0; i < 2; i++) {
7890             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
7891             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
7892             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7893                                     linear_centroid, linear_center, sel);
7894          }
7895          ctx->linear_centroid = bld.tmp(v2);
7896          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
7897                     Operand(new_coord[0]), Operand(new_coord[1]));
7898          emit_split_vector(ctx, ctx->linear_centroid, 2);
7899       }
7900    }
7901 }
7902
7903 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
7904 {
7905    Program *program = ctx->program;
7906
7907    unsigned float_controls = shader->info.float_controls_execution_mode;
7908
7909    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
7910       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
7911    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
7912       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
7913                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
7914
7915    program->next_fp_mode.must_flush_denorms32 =
7916       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
7917    program->next_fp_mode.must_flush_denorms16_64 =
7918       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
7919                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
7920
7921    program->next_fp_mode.care_about_round32 =
7922       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
7923
7924    program->next_fp_mode.care_about_round16_64 =
7925       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
7926                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
7927
7928    /* default to preserving fp16 and fp64 denorms, since it's free */
7929    if (program->next_fp_mode.must_flush_denorms16_64)
7930       program->next_fp_mode.denorm16_64 = 0;
7931    else
7932       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
7933
7934    /* preserving fp32 denorms is expensive, so only do it if asked */
7935    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
7936       program->next_fp_mode.denorm32 = fp_denorm_keep;
7937    else
7938       program->next_fp_mode.denorm32 = 0;
7939
7940    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
7941       program->next_fp_mode.round32 = fp_round_tz;
7942    else
7943       program->next_fp_mode.round32 = fp_round_ne;
7944
7945    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
7946       program->next_fp_mode.round16_64 = fp_round_tz;
7947    else
7948       program->next_fp_mode.round16_64 = fp_round_ne;
7949
7950    ctx->block->fp_mode = program->next_fp_mode;
7951 }
7952
7953 void select_program(Program *program,
7954                     unsigned shader_count,
7955                     struct nir_shader *const *shaders,
7956                     ac_shader_config* config,
7957                     struct radv_shader_args *args)
7958 {
7959    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
7960
7961    for (unsigned i = 0; i < shader_count; i++) {
7962       nir_shader *nir = shaders[i];
7963       init_context(&ctx, nir);
7964
7965       setup_fp_mode(&ctx, nir);
7966
7967       if (!i) {
7968          /* needs to be after init_context() for FS */
7969          Pseudo_instruction *startpgm = add_startpgm(&ctx);
7970          append_logical_start(ctx.block);
7971          split_arguments(&ctx, startpgm);
7972       }
7973
7974       if_context ic;
7975       if (shader_count >= 2) {
7976          Builder bld(ctx.program, ctx.block);
7977          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7978          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
7979          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
7980
7981          begin_divergent_if_then(&ctx, &ic, cond);
7982       }
7983
7984       if (i) {
7985          Builder bld(ctx.program, ctx.block);
7986          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7987          bld.sopp(aco_opcode::s_barrier);
7988       }
7989
7990       if (ctx.stage == fragment_fs)
7991          handle_bc_optimize(&ctx);
7992
7993       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7994       visit_cf_list(&ctx, &func->body);
7995
7996       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7997          emit_streamout(&ctx, 0);
7998
7999       if (ctx.stage == vertex_vs)
8000          create_vs_exports(&ctx);
8001
8002       if (shader_count >= 2) {
8003          begin_divergent_if_else(&ctx, &ic);
8004          end_divergent_if(&ctx, &ic);
8005       }
8006
8007       ralloc_free(ctx.divergent_vals);
8008    }
8009
8010    program->config->float_mode = program->blocks[0].fp_mode.val;
8011
8012    append_logical_end(ctx.block);
8013    ctx.block->kind |= block_kind_uniform;
8014    Builder bld(ctx.program, ctx.block);
8015    if (ctx.program->wb_smem_l1_on_end)
8016       bld.smem(aco_opcode::s_dcache_wb, false);
8017    bld.sopp(aco_opcode::s_endpgm);
8018
8019    /* cleanup CFG */
8020    for (Block& BB : program->blocks) {
8021       for (unsigned idx : BB.linear_preds)
8022          program->blocks[idx].linear_succs.emplace_back(BB.index);
8023       for (unsigned idx : BB.logical_preds)
8024          program->blocks[idx].logical_succs.emplace_back(BB.index);
8025    }
8026 }
8027 }