src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "ac_shader_util.h"
  31 #include "aco_ir.h"
  32 #include "aco_builder.h"
  33 #include "aco_interface.h"
  34 #include "aco_instruction_selection_setup.cpp"
  35 #include "util/fast_idiv_by_const.h"
  36
  37 namespace aco {
  38 namespace {
  39
  40 class loop_info_RAII {
  41    isel_context* ctx;
  42    unsigned header_idx_old;
  43    Block* exit_old;
  44    bool divergent_cont_old;
  45    bool divergent_branch_old;
  46    bool divergent_if_old;
  47
  48 public:
  49    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  50       : ctx(ctx),
  51         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  52         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  53         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  54         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  55    {
  56       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  57       ctx->cf_info.parent_loop.exit = loop_exit;
  58       ctx->cf_info.parent_loop.has_divergent_continue = false;
  59       ctx->cf_info.parent_loop.has_divergent_branch = false;
  60       ctx->cf_info.parent_if.is_divergent = false;
  61       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  62    }
  63
  64    ~loop_info_RAII()
  65    {
  66       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  67       ctx->cf_info.parent_loop.exit = exit_old;
  68       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  69       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  70       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  71       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  72       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  73          ctx->cf_info.exec_potentially_empty = false;
  74    }
  75 };
  76
  77 struct if_context {
  78    Temp cond;
  79
  80    bool divergent_old;
  81    bool exec_potentially_empty_old;
  82
  83    unsigned BB_if_idx;
  84    unsigned invert_idx;
  85    bool then_branch_divergent;
  86    Block BB_invert;
  87    Block BB_endif;
  88 };
  89
  90 static void visit_cf_list(struct isel_context *ctx,
  91                           struct exec_list *list);
  92
  93 static void add_logical_edge(unsigned pred_idx, Block *succ)
  94 {
  95    succ->logical_preds.emplace_back(pred_idx);
  96 }
  97
  98
  99 static void add_linear_edge(unsigned pred_idx, Block *succ)
 100 {
 101    succ->linear_preds.emplace_back(pred_idx);
 102 }
 103
 104 static void add_edge(unsigned pred_idx, Block *succ)
 105 {
 106    add_logical_edge(pred_idx, succ);
 107    add_linear_edge(pred_idx, succ);
 108 }
 109
 110 static void append_logical_start(Block *b)
 111 {
 112    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 113 }
 114
 115 static void append_logical_end(Block *b)
 116 {
 117    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 118 }
 119
 120 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 121 {
 122    assert(ctx->allocated[def->index].id());
 123    return ctx->allocated[def->index];
 124 }
 125
 126 Temp emit_mbcnt(isel_context *ctx, Definition dst,
 127                 Operand mask_lo = Operand((uint32_t) -1), Operand mask_hi = Operand((uint32_t) -1))
 128 {
 129    Builder bld(ctx->program, ctx->block);
 130    Definition lo_def = ctx->program->wave_size == 32 ? dst : bld.def(v1);
 131    Temp thread_id_lo = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, lo_def, mask_lo, Operand(0u));
 132
 133    if (ctx->program->wave_size == 32) {
 134       return thread_id_lo;
 135    } else {
 136       Temp thread_id_hi = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, dst, mask_hi, thread_id_lo);
 137       return thread_id_hi;
 138    }
 139 }
 140
 141 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 142 {
 143    Builder bld(ctx->program, ctx->block);
 144
 145    if (!dst.id())
 146       dst = bld.tmp(src.regClass());
 147
 148    assert(src.size() == dst.size());
 149
 150    if (ctx->stage != fragment_fs) {
 151       if (!dst.id())
 152          return src;
 153
 154       bld.copy(Definition(dst), src);
 155       return dst;
 156    }
 157
 158    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 159    ctx->program->needs_wqm |= program_needs_wqm;
 160    return dst;
 161 }
 162
 163 static Temp emit_bpermute(isel_context *ctx, Builder &bld, Temp index, Temp data)
 164 {
 165    if (index.regClass() == s1)
 166       return bld.readlane(bld.def(s1), data, index);
 167
 168    Temp index_x4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), index);
 169
 170    /* Currently not implemented on GFX6-7 */
 171    assert(ctx->options->chip_class >= GFX8);
 172
 173    if (ctx->options->chip_class <= GFX9 || ctx->program->wave_size == 32) {
 174       return bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), index_x4, data);
 175    }
 176
 177    /* GFX10, wave64 mode:
 178     * The bpermute instruction is limited to half-wave operation, which means that it can't
 179     * properly support subgroup shuffle like older generations (or wave32 mode), so we
 180     * emulate it here.
 181     */
 182    if (!ctx->has_gfx10_wave64_bpermute) {
 183       ctx->has_gfx10_wave64_bpermute = true;
 184       ctx->program->config->num_shared_vgprs = 8; /* Shared VGPRs are allocated in groups of 8 */
 185       ctx->program->vgpr_limit -= 4; /* We allocate 8 shared VGPRs, so we'll have 4 fewer normal VGPRs */
 186    }
 187
 188    Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
 189    Temp lane_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), lane_id);
 190    Temp index_is_hi = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x20u), index);
 191    Temp cmp = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2, vcc), lane_is_hi, index_is_hi);
 192
 193    return bld.reduction(aco_opcode::p_wave64_bpermute, bld.def(v1), bld.def(s2), bld.def(s1, scc),
 194                         bld.vcc(cmp), Operand(v2.as_linear()), index_x4, data, gfx10_wave64_bpermute);
 195 }
 196
 197 Temp as_vgpr(isel_context *ctx, Temp val)
 198 {
 199    if (val.type() == RegType::sgpr) {
 200       Builder bld(ctx->program, ctx->block);
 201       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 202    }
 203    assert(val.type() == RegType::vgpr);
 204    return val;
 205 }
 206
 207 //assumes a != 0xffffffff
 208 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 209 {
 210    assert(b != 0);
 211    Builder bld(ctx->program, ctx->block);
 212
 213    if (util_is_power_of_two_or_zero(b)) {
 214       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 215       return;
 216    }
 217
 218    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 219
 220    assert(info.multiplier <= 0xffffffff);
 221
 222    bool pre_shift = info.pre_shift != 0;
 223    bool increment = info.increment != 0;
 224    bool multiply = true;
 225    bool post_shift = info.post_shift != 0;
 226
 227    if (!pre_shift && !increment && !multiply && !post_shift) {
 228       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 229       return;
 230    }
 231
 232    Temp pre_shift_dst = a;
 233    if (pre_shift) {
 234       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 235       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 236    }
 237
 238    Temp increment_dst = pre_shift_dst;
 239    if (increment) {
 240       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 241       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 242    }
 243
 244    Temp multiply_dst = increment_dst;
 245    if (multiply) {
 246       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 247       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 248                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 249    }
 250
 251    if (post_shift) {
 252       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 253    }
 254 }
 255
 256 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 257 {
 258    Builder bld(ctx->program, ctx->block);
 259    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 260 }
 261
 262
 263 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 264 {
 265    /* no need to extract the whole vector */
 266    if (src.regClass() == dst_rc) {
 267       assert(idx == 0);
 268       return src;
 269    }
 270    assert(src.size() > idx);
 271    Builder bld(ctx->program, ctx->block);
 272    auto it = ctx->allocated_vec.find(src.id());
 273    /* the size check needs to be early because elements other than 0 may be garbage */
 274    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 275       if (it->second[idx].regClass() == dst_rc) {
 276          return it->second[idx];
 277       } else {
 278          assert(dst_rc.size() == it->second[idx].regClass().size());
 279          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 280          return bld.copy(bld.def(dst_rc), it->second[idx]);
 281       }
 282    }
 283
 284    if (src.size() == dst_rc.size()) {
 285       assert(idx == 0);
 286       return bld.copy(bld.def(dst_rc), src);
 287    } else {
 288       Temp dst = bld.tmp(dst_rc);
 289       emit_extract_vector(ctx, src, idx, dst);
 290       return dst;
 291    }
 292 }
 293
 294 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 295 {
 296    if (num_components == 1)
 297       return;
 298    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 299       return;
 300    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 301    split->operands[0] = Operand(vec_src);
 302    std::array<Temp,4> elems;
 303    for (unsigned i = 0; i < num_components; i++) {
 304       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 305       split->definitions[i] = Definition(elems[i]);
 306    }
 307    ctx->block->instructions.emplace_back(std::move(split));
 308    ctx->allocated_vec.emplace(vec_src.id(), elems);
 309 }
 310
 311 /* This vector expansion uses a mask to determine which elements in the new vector
 312  * come from the original vector. The other elements are undefined. */
 313 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 314 {
 315    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 316
 317    if (vec_src == dst)
 318       return;
 319
 320    Builder bld(ctx->program, ctx->block);
 321    if (num_components == 1) {
 322       if (dst.type() == RegType::sgpr)
 323          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 324       else
 325          bld.copy(Definition(dst), vec_src);
 326       return;
 327    }
 328
 329    unsigned component_size = dst.size() / num_components;
 330    std::array<Temp,4> elems;
 331
 332    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 333    vec->definitions[0] = Definition(dst);
 334    unsigned k = 0;
 335    for (unsigned i = 0; i < num_components; i++) {
 336       if (mask & (1 << i)) {
 337          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 338          if (dst.type() == RegType::sgpr)
 339             src = bld.as_uniform(src);
 340          vec->operands[i] = Operand(src);
 341       } else {
 342          vec->operands[i] = Operand(0u);
 343       }
 344       elems[i] = vec->operands[i].getTemp();
 345    }
 346    ctx->block->instructions.emplace_back(std::move(vec));
 347    ctx->allocated_vec.emplace(dst.id(), elems);
 348 }
 349
 350 Temp bool_to_vector_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s2))
 351 {
 352    Builder bld(ctx->program, ctx->block);
 353    if (!dst.id())
 354       dst = bld.tmp(bld.lm);
 355
 356    assert(val.regClass() == s1);
 357    assert(dst.regClass() == bld.lm);
 358
 359    return bld.sop2(Builder::s_cselect, bld.hint_vcc(Definition(dst)), Operand((uint32_t) -1), Operand(0u), bld.scc(val));
 360 }
 361
 362 Temp bool_to_scalar_condition(isel_context *ctx, Temp val, Temp dst = Temp(0, s1))
 363 {
 364    Builder bld(ctx->program, ctx->block);
 365    if (!dst.id())
 366       dst = bld.tmp(s1);
 367
 368    assert(val.regClass() == bld.lm);
 369    assert(dst.regClass() == s1);
 370
 371    /* if we're currently in WQM mode, ensure that the source is also computed in WQM */
 372    Temp tmp = bld.tmp(s1);
 373    bld.sop2(Builder::s_and, bld.def(bld.lm), bld.scc(Definition(tmp)), val, Operand(exec, bld.lm));
 374    return emit_wqm(ctx, tmp, dst);
 375 }
 376
 377 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 378 {
 379    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 380       return get_ssa_temp(ctx, src.src.ssa);
 381
 382    if (src.src.ssa->num_components == size) {
 383       bool identity_swizzle = true;
 384       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 385          if (src.swizzle[i] != i)
 386             identity_swizzle = false;
 387       }
 388       if (identity_swizzle)
 389          return get_ssa_temp(ctx, src.src.ssa);
 390    }
 391
 392    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 393    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 394    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 395    assert(vec.size() % elem_size == 0);
 396
 397    RegClass elem_rc = RegClass(vec.type(), elem_size);
 398    if (size == 1) {
 399       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 400    } else {
 401       assert(size <= 4);
 402       std::array<Temp,4> elems;
 403       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 404       for (unsigned i = 0; i < size; ++i) {
 405          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 406          vec_instr->operands[i] = Operand{elems[i]};
 407       }
 408       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 409       vec_instr->definitions[0] = Definition(dst);
 410       ctx->block->instructions.emplace_back(std::move(vec_instr));
 411       ctx->allocated_vec.emplace(dst.id(), elems);
 412       return dst;
 413    }
 414 }
 415
 416 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 417 {
 418    if (ptr.size() == 2)
 419       return ptr;
 420    Builder bld(ctx->program, ctx->block);
 421    if (ptr.type() == RegType::vgpr)
 422       ptr = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), ptr);
 423    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 424                      ptr, Operand((unsigned)ctx->options->address32_hi));
 425 }
 426
 427 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 428 {
 429    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 430    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 431    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 432    sop2->definitions[0] = Definition(dst);
 433    if (writes_scc)
 434       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 435    ctx->block->instructions.emplace_back(std::move(sop2));
 436 }
 437
 438 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 439 {
 440    Builder bld(ctx->program, ctx->block);
 441    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 442    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 443    if (src1.type() == RegType::sgpr) {
 444       if (commutative && src0.type() == RegType::vgpr) {
 445          Temp t = src0;
 446          src0 = src1;
 447          src1 = t;
 448       } else if (src0.type() == RegType::vgpr &&
 449                  op != aco_opcode::v_madmk_f32 &&
 450                  op != aco_opcode::v_madak_f32 &&
 451                  op != aco_opcode::v_madmk_f16 &&
 452                  op != aco_opcode::v_madak_f16) {
 453          /* If the instruction is not commutative, we emit a VOP3A instruction */
 454          bld.vop2_e64(op, Definition(dst), src0, src1);
 455          return;
 456       } else {
 457          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 458       }
 459    }
 460    bld.vop2(op, Definition(dst), src0, src1);
 461 }
 462
 463 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 464 {
 465    Temp src0 = get_alu_src(ctx, instr->src[0]);
 466    Temp src1 = get_alu_src(ctx, instr->src[1]);
 467    Temp src2 = get_alu_src(ctx, instr->src[2]);
 468
 469    /* ensure that the instruction has at most 1 sgpr operand
 470     * The optimizer will inline constants for us */
 471    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 472       src0 = as_vgpr(ctx, src0);
 473    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 474       src1 = as_vgpr(ctx, src1);
 475    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 476       src2 = as_vgpr(ctx, src2);
 477
 478    Builder bld(ctx->program, ctx->block);
 479    bld.vop3(op, Definition(dst), src0, src1, src2);
 480 }
 481
 482 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 483 {
 484    Builder bld(ctx->program, ctx->block);
 485    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 486 }
 487
 488 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 489 {
 490    Temp src0 = get_alu_src(ctx, instr->src[0]);
 491    Temp src1 = get_alu_src(ctx, instr->src[1]);
 492    assert(src0.size() == src1.size());
 493
 494    aco_ptr<Instruction> vopc;
 495    if (src1.type() == RegType::sgpr) {
 496       if (src0.type() == RegType::vgpr) {
 497          /* to swap the operands, we might also have to change the opcode */
 498          switch (op) {
 499             case aco_opcode::v_cmp_lt_f32:
 500                op = aco_opcode::v_cmp_gt_f32;
 501                break;
 502             case aco_opcode::v_cmp_ge_f32:
 503                op = aco_opcode::v_cmp_le_f32;
 504                break;
 505             case aco_opcode::v_cmp_lt_i32:
 506                op = aco_opcode::v_cmp_gt_i32;
 507                break;
 508             case aco_opcode::v_cmp_ge_i32:
 509                op = aco_opcode::v_cmp_le_i32;
 510                break;
 511             case aco_opcode::v_cmp_lt_u32:
 512                op = aco_opcode::v_cmp_gt_u32;
 513                break;
 514             case aco_opcode::v_cmp_ge_u32:
 515                op = aco_opcode::v_cmp_le_u32;
 516                break;
 517             case aco_opcode::v_cmp_lt_f64:
 518                op = aco_opcode::v_cmp_gt_f64;
 519                break;
 520             case aco_opcode::v_cmp_ge_f64:
 521                op = aco_opcode::v_cmp_le_f64;
 522                break;
 523             case aco_opcode::v_cmp_lt_i64:
 524                op = aco_opcode::v_cmp_gt_i64;
 525                break;
 526             case aco_opcode::v_cmp_ge_i64:
 527                op = aco_opcode::v_cmp_le_i64;
 528                break;
 529             case aco_opcode::v_cmp_lt_u64:
 530                op = aco_opcode::v_cmp_gt_u64;
 531                break;
 532             case aco_opcode::v_cmp_ge_u64:
 533                op = aco_opcode::v_cmp_le_u64;
 534                break;
 535             default: /* eq and ne are commutative */
 536                break;
 537          }
 538          Temp t = src0;
 539          src0 = src1;
 540          src1 = t;
 541       } else {
 542          src1 = as_vgpr(ctx, src1);
 543       }
 544    }
 545
 546    Builder bld(ctx->program, ctx->block);
 547    bld.vopc(op, bld.hint_vcc(Definition(dst)), src0, src1);
 548 }
 549
 550 void emit_sopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 551 {
 552    Temp src0 = get_alu_src(ctx, instr->src[0]);
 553    Temp src1 = get_alu_src(ctx, instr->src[1]);
 554    Builder bld(ctx->program, ctx->block);
 555
 556    assert(dst.regClass() == bld.lm);
 557    assert(src0.type() == RegType::sgpr);
 558    assert(src1.type() == RegType::sgpr);
 559    assert(src0.regClass() == src1.regClass());
 560
 561    /* Emit the SALU comparison instruction */
 562    Temp cmp = bld.sopc(op, bld.scc(bld.def(s1)), src0, src1);
 563    /* Turn the result into a per-lane bool */
 564    bool_to_vector_condition(ctx, cmp, dst);
 565 }
 566
 567 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, Temp dst,
 568                      aco_opcode v32_op, aco_opcode v64_op, aco_opcode s32_op = aco_opcode::num_opcodes, aco_opcode s64_op = aco_opcode::num_opcodes)
 569 {
 570    aco_opcode s_op = instr->src[0].src.ssa->bit_size == 64 ? s64_op : s32_op;
 571    aco_opcode v_op = instr->src[0].src.ssa->bit_size == 64 ? v64_op : v32_op;
 572    bool divergent_vals = ctx->divergent_vals[instr->dest.dest.ssa.index];
 573    bool use_valu = s_op == aco_opcode::num_opcodes ||
 574                    divergent_vals ||
 575                    ctx->allocated[instr->src[0].src.ssa->index].type() == RegType::vgpr ||
 576                    ctx->allocated[instr->src[1].src.ssa->index].type() == RegType::vgpr;
 577    aco_opcode op = use_valu ? v_op : s_op;
 578    assert(op != aco_opcode::num_opcodes);
 579
 580    if (use_valu)
 581       emit_vopc_instruction(ctx, instr, op, dst);
 582    else
 583       emit_sopc_instruction(ctx, instr, op, dst);
 584 }
 585
 586 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, Builder::WaveSpecificOpcode op, Temp dst)
 587 {
 588    Builder bld(ctx->program, ctx->block);
 589    Temp src0 = get_alu_src(ctx, instr->src[0]);
 590    Temp src1 = get_alu_src(ctx, instr->src[1]);
 591
 592    assert(dst.regClass() == bld.lm);
 593    assert(src0.regClass() == bld.lm);
 594    assert(src1.regClass() == bld.lm);
 595
 596    bld.sop2(op, Definition(dst), bld.def(s1, scc), src0, src1);
 597 }
 598
 599 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 600 {
 601    Builder bld(ctx->program, ctx->block);
 602    Temp cond = get_alu_src(ctx, instr->src[0]);
 603    Temp then = get_alu_src(ctx, instr->src[1]);
 604    Temp els = get_alu_src(ctx, instr->src[2]);
 605
 606    assert(cond.regClass() == bld.lm);
 607
 608    if (dst.type() == RegType::vgpr) {
 609       aco_ptr<Instruction> bcsel;
 610       if (dst.size() == 1) {
 611          then = as_vgpr(ctx, then);
 612          els = as_vgpr(ctx, els);
 613
 614          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 615       } else if (dst.size() == 2) {
 616          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 617          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 618          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 619          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 620
 621          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 622          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 623
 624          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 625       } else {
 626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 627          nir_print_instr(&instr->instr, stderr);
 628          fprintf(stderr, "\n");
 629       }
 630       return;
 631    }
 632
 633    if (instr->dest.dest.ssa.bit_size == 1) {
 634       assert(dst.regClass() == bld.lm);
 635       assert(then.regClass() == bld.lm);
 636       assert(els.regClass() == bld.lm);
 637    }
 638
 639    if (!ctx->divergent_vals[instr->src[0].src.ssa->index]) { /* uniform condition and values in sgpr */
 640       if (dst.regClass() == s1 || dst.regClass() == s2) {
 641          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 642          assert(dst.size() == then.size());
 643          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 644          bld.sop2(op, Definition(dst), then, els, bld.scc(bool_to_scalar_condition(ctx, cond)));
 645       } else {
 646          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 647          nir_print_instr(&instr->instr, stderr);
 648          fprintf(stderr, "\n");
 649       }
 650       return;
 651    }
 652
 653    /* divergent boolean bcsel
 654     * this implements bcsel on bools: dst = s0 ? s1 : s2
 655     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 656    assert(instr->dest.dest.ssa.bit_size == 1);
 657
 658    if (cond.id() != then.id())
 659       then = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), cond, then);
 660
 661    if (cond.id() == els.id())
 662       bld.sop1(Builder::s_mov, Definition(dst), then);
 663    else
 664       bld.sop2(Builder::s_or, Definition(dst), bld.def(s1, scc), then,
 665                bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), els, cond));
 666 }
 667
 668 void emit_scaled_op(isel_context *ctx, Builder& bld, Definition dst, Temp val,
 669                     aco_opcode op, uint32_t undo)
 670 {
 671    /* multiply by 16777216 to handle denormals */
 672    Temp is_denormal = bld.vopc(aco_opcode::v_cmp_class_f32, bld.hint_vcc(bld.def(bld.lm)),
 673                                as_vgpr(ctx, val), bld.copy(bld.def(v1), Operand((1u << 7) | (1u << 4))));
 674    Temp scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x4b800000u), val);
 675    scaled = bld.vop1(op, bld.def(v1), scaled);
 676    scaled = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(undo), scaled);
 677
 678    Temp not_scaled = bld.vop1(op, bld.def(v1), val);
 679
 680    bld.vop2(aco_opcode::v_cndmask_b32, dst, not_scaled, scaled, is_denormal);
 681 }
 682
 683 void emit_rcp(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 684 {
 685    if (ctx->block->fp_mode.denorm32 == 0) {
 686       bld.vop1(aco_opcode::v_rcp_f32, dst, val);
 687       return;
 688    }
 689
 690    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rcp_f32, 0x4b800000u);
 691 }
 692
 693 void emit_rsq(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 694 {
 695    if (ctx->block->fp_mode.denorm32 == 0) {
 696       bld.vop1(aco_opcode::v_rsq_f32, dst, val);
 697       return;
 698    }
 699
 700    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_rsq_f32, 0x45800000u);
 701 }
 702
 703 void emit_sqrt(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 704 {
 705    if (ctx->block->fp_mode.denorm32 == 0) {
 706       bld.vop1(aco_opcode::v_sqrt_f32, dst, val);
 707       return;
 708    }
 709
 710    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_sqrt_f32, 0x39800000u);
 711 }
 712
 713 void emit_log2(isel_context *ctx, Builder& bld, Definition dst, Temp val)
 714 {
 715    if (ctx->block->fp_mode.denorm32 == 0) {
 716       bld.vop1(aco_opcode::v_log_f32, dst, val);
 717       return;
 718    }
 719
 720    emit_scaled_op(ctx, bld, dst, val, aco_opcode::v_log_f32, 0xc1c00000u);
 721 }
 722
 723 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 724 {
 725    if (!instr->dest.dest.is_ssa) {
 726       fprintf(stderr, "nir alu dst not in ssa: ");
 727       nir_print_instr(&instr->instr, stderr);
 728       fprintf(stderr, "\n");
 729       abort();
 730    }
 731    Builder bld(ctx->program, ctx->block);
 732    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 733    switch(instr->op) {
 734    case nir_op_vec2:
 735    case nir_op_vec3:
 736    case nir_op_vec4: {
 737       std::array<Temp,4> elems;
 738       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 739       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 740          elems[i] = get_alu_src(ctx, instr->src[i]);
 741          vec->operands[i] = Operand{elems[i]};
 742       }
 743       vec->definitions[0] = Definition(dst);
 744       ctx->block->instructions.emplace_back(std::move(vec));
 745       ctx->allocated_vec.emplace(dst.id(), elems);
 746       break;
 747    }
 748    case nir_op_mov: {
 749       Temp src = get_alu_src(ctx, instr->src[0]);
 750       aco_ptr<Instruction> mov;
 751       if (dst.type() == RegType::sgpr) {
 752          if (src.type() == RegType::vgpr)
 753             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 754          else if (src.regClass() == s1)
 755             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 756          else if (src.regClass() == s2)
 757             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 758          else
 759             unreachable("wrong src register class for nir_op_imov");
 760       } else if (dst.regClass() == v1) {
 761          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 762       } else if (dst.regClass() == v2) {
 763          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 764       } else {
 765          nir_print_instr(&instr->instr, stderr);
 766          unreachable("Should have been lowered to scalar.");
 767       }
 768       break;
 769    }
 770    case nir_op_inot: {
 771       Temp src = get_alu_src(ctx, instr->src[0]);
 772       if (instr->dest.dest.ssa.bit_size == 1) {
 773          assert(src.regClass() == bld.lm);
 774          assert(dst.regClass() == bld.lm);
 775          bld.sop2(Builder::s_andn2, Definition(dst), bld.def(s1, scc), Operand(exec, bld.lm), src);
 776       } else if (dst.regClass() == v1) {
 777          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 778       } else if (dst.type() == RegType::sgpr) {
 779          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 780          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 781       } else {
 782          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 783          nir_print_instr(&instr->instr, stderr);
 784          fprintf(stderr, "\n");
 785       }
 786       break;
 787    }
 788    case nir_op_ineg: {
 789       Temp src = get_alu_src(ctx, instr->src[0]);
 790       if (dst.regClass() == v1) {
 791          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 792       } else if (dst.regClass() == s1) {
 793          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 794       } else if (dst.size() == 2) {
 795          Temp src0 = bld.tmp(dst.type(), 1);
 796          Temp src1 = bld.tmp(dst.type(), 1);
 797          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 798
 799          if (dst.regClass() == s2) {
 800             Temp carry = bld.tmp(s1);
 801             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 802             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 803             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 804          } else {
 805             Temp lower = bld.tmp(v1);
 806             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 807             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 808             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 809          }
 810       } else {
 811          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 812          nir_print_instr(&instr->instr, stderr);
 813          fprintf(stderr, "\n");
 814       }
 815       break;
 816    }
 817    case nir_op_iabs: {
 818       if (dst.regClass() == s1) {
 819          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 820       } else if (dst.regClass() == v1) {
 821          Temp src = get_alu_src(ctx, instr->src[0]);
 822          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 823       } else {
 824          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 825          nir_print_instr(&instr->instr, stderr);
 826          fprintf(stderr, "\n");
 827       }
 828       break;
 829    }
 830    case nir_op_isign: {
 831       Temp src = get_alu_src(ctx, instr->src[0]);
 832       if (dst.regClass() == s1) {
 833          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 834          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 835          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 836       } else if (dst.regClass() == s2) {
 837          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 838          Temp neqz;
 839          if (ctx->program->chip_class >= GFX8)
 840             neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 841          else
 842             neqz = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), src, Operand(0u)).def(1).getTemp();
 843          /* SCC gets zero-extended to 64 bit */
 844          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, bld.scc(neqz));
 845       } else if (dst.regClass() == v1) {
 846          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 847          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 848          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 849       } else if (dst.regClass() == v2) {
 850          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 851          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 852          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
 853          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 854          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 855          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 856       } else {
 857          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 858          nir_print_instr(&instr->instr, stderr);
 859          fprintf(stderr, "\n");
 860       }
 861       break;
 862    }
 863    case nir_op_imax: {
 864       if (dst.regClass() == v1) {
 865          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 866       } else if (dst.regClass() == s1) {
 867          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 868       } else {
 869          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 870          nir_print_instr(&instr->instr, stderr);
 871          fprintf(stderr, "\n");
 872       }
 873       break;
 874    }
 875    case nir_op_umax: {
 876       if (dst.regClass() == v1) {
 877          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 878       } else if (dst.regClass() == s1) {
 879          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 880       } else {
 881          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 882          nir_print_instr(&instr->instr, stderr);
 883          fprintf(stderr, "\n");
 884       }
 885       break;
 886    }
 887    case nir_op_imin: {
 888       if (dst.regClass() == v1) {
 889          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 890       } else if (dst.regClass() == s1) {
 891          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 892       } else {
 893          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 894          nir_print_instr(&instr->instr, stderr);
 895          fprintf(stderr, "\n");
 896       }
 897       break;
 898    }
 899    case nir_op_umin: {
 900       if (dst.regClass() == v1) {
 901          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 902       } else if (dst.regClass() == s1) {
 903          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 904       } else {
 905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 906          nir_print_instr(&instr->instr, stderr);
 907          fprintf(stderr, "\n");
 908       }
 909       break;
 910    }
 911    case nir_op_ior: {
 912       if (instr->dest.dest.ssa.bit_size == 1) {
 913          emit_boolean_logic(ctx, instr, Builder::s_or, dst);
 914       } else if (dst.regClass() == v1) {
 915          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 916       } else if (dst.regClass() == s1) {
 917          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 918       } else if (dst.regClass() == s2) {
 919          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 920       } else {
 921          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 922          nir_print_instr(&instr->instr, stderr);
 923          fprintf(stderr, "\n");
 924       }
 925       break;
 926    }
 927    case nir_op_iand: {
 928       if (instr->dest.dest.ssa.bit_size == 1) {
 929          emit_boolean_logic(ctx, instr, Builder::s_and, dst);
 930       } else if (dst.regClass() == v1) {
 931          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 932       } else if (dst.regClass() == s1) {
 933          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 934       } else if (dst.regClass() == s2) {
 935          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 936       } else {
 937          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 938          nir_print_instr(&instr->instr, stderr);
 939          fprintf(stderr, "\n");
 940       }
 941       break;
 942    }
 943    case nir_op_ixor: {
 944       if (instr->dest.dest.ssa.bit_size == 1) {
 945          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
 946       } else if (dst.regClass() == v1) {
 947          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 948       } else if (dst.regClass() == s1) {
 949          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 950       } else if (dst.regClass() == s2) {
 951          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_ushr: {
 960       if (dst.regClass() == v1) {
 961          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 962       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 963          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 964                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 965       } else if (dst.regClass() == v2) {
 966          bld.vop3(aco_opcode::v_lshr_b64, Definition(dst),
 967                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
 968       } else if (dst.regClass() == s2) {
 969          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 970       } else if (dst.regClass() == s1) {
 971          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 972       } else {
 973          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 974          nir_print_instr(&instr->instr, stderr);
 975          fprintf(stderr, "\n");
 976       }
 977       break;
 978    }
 979    case nir_op_ishl: {
 980       if (dst.regClass() == v1) {
 981          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 982       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
 983          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 984                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 985       } else if (dst.regClass() == v2) {
 986          bld.vop3(aco_opcode::v_lshl_b64, Definition(dst),
 987                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
 988       } else if (dst.regClass() == s1) {
 989          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 990       } else if (dst.regClass() == s2) {
 991          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 992       } else {
 993          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 994          nir_print_instr(&instr->instr, stderr);
 995          fprintf(stderr, "\n");
 996       }
 997       break;
 998    }
 999    case nir_op_ishr: {
1000       if (dst.regClass() == v1) {
1001          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
1002       } else if (dst.regClass() == v2 && ctx->program->chip_class >= GFX8) {
1003          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
1004                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
1005       } else if (dst.regClass() == v2) {
1006          bld.vop3(aco_opcode::v_ashr_i64, Definition(dst),
1007                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1008       } else if (dst.regClass() == s1) {
1009          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
1010       } else if (dst.regClass() == s2) {
1011          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
1012       } else {
1013          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1014          nir_print_instr(&instr->instr, stderr);
1015          fprintf(stderr, "\n");
1016       }
1017       break;
1018    }
1019    case nir_op_find_lsb: {
1020       Temp src = get_alu_src(ctx, instr->src[0]);
1021       if (src.regClass() == s1) {
1022          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
1023       } else if (src.regClass() == v1) {
1024          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
1025       } else if (src.regClass() == s2) {
1026          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
1027       } else {
1028          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1029          nir_print_instr(&instr->instr, stderr);
1030          fprintf(stderr, "\n");
1031       }
1032       break;
1033    }
1034    case nir_op_ufind_msb:
1035    case nir_op_ifind_msb: {
1036       Temp src = get_alu_src(ctx, instr->src[0]);
1037       if (src.regClass() == s1 || src.regClass() == s2) {
1038          aco_opcode op = src.regClass() == s2 ?
1039                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
1040                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
1041          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
1042
1043          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
1044                                         Operand(src.size() * 32u - 1u), msb_rev);
1045          Temp msb = sub.def(0).getTemp();
1046          Temp carry = sub.def(1).getTemp();
1047
1048          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
1049       } else if (src.regClass() == v1) {
1050          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
1051          Temp msb_rev = bld.tmp(v1);
1052          emit_vop1_instruction(ctx, instr, op, msb_rev);
1053          Temp msb = bld.tmp(v1);
1054          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
1055          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
1056       } else {
1057          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1058          nir_print_instr(&instr->instr, stderr);
1059          fprintf(stderr, "\n");
1060       }
1061       break;
1062    }
1063    case nir_op_bitfield_reverse: {
1064       if (dst.regClass() == s1) {
1065          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1066       } else if (dst.regClass() == v1) {
1067          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
1068       } else {
1069          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1070          nir_print_instr(&instr->instr, stderr);
1071          fprintf(stderr, "\n");
1072       }
1073       break;
1074    }
1075    case nir_op_iadd: {
1076       if (dst.regClass() == s1) {
1077          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
1078          break;
1079       }
1080
1081       Temp src0 = get_alu_src(ctx, instr->src[0]);
1082       Temp src1 = get_alu_src(ctx, instr->src[1]);
1083       if (dst.regClass() == v1) {
1084          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
1085          break;
1086       }
1087
1088       assert(src0.size() == 2 && src1.size() == 2);
1089       Temp src00 = bld.tmp(src0.type(), 1);
1090       Temp src01 = bld.tmp(dst.type(), 1);
1091       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1092       Temp src10 = bld.tmp(src1.type(), 1);
1093       Temp src11 = bld.tmp(dst.type(), 1);
1094       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1095
1096       if (dst.regClass() == s2) {
1097          Temp carry = bld.tmp(s1);
1098          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1099          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
1100          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1101       } else if (dst.regClass() == v2) {
1102          Temp dst0 = bld.tmp(v1);
1103          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
1104          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
1105          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1106       } else {
1107          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1108          nir_print_instr(&instr->instr, stderr);
1109          fprintf(stderr, "\n");
1110       }
1111       break;
1112    }
1113    case nir_op_uadd_sat: {
1114       Temp src0 = get_alu_src(ctx, instr->src[0]);
1115       Temp src1 = get_alu_src(ctx, instr->src[1]);
1116       if (dst.regClass() == s1) {
1117          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1118          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1119                   src0, src1);
1120          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1121       } else if (dst.regClass() == v1) {
1122          if (ctx->options->chip_class >= GFX9) {
1123             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1124             add->operands[0] = Operand(src0);
1125             add->operands[1] = Operand(src1);
1126             add->definitions[0] = Definition(dst);
1127             add->clamp = 1;
1128             ctx->block->instructions.emplace_back(std::move(add));
1129          } else {
1130             if (src1.regClass() != v1)
1131                std::swap(src0, src1);
1132             assert(src1.regClass() == v1);
1133             Temp tmp = bld.tmp(v1);
1134             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1135             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1136          }
1137       } else {
1138          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1139          nir_print_instr(&instr->instr, stderr);
1140          fprintf(stderr, "\n");
1141       }
1142       break;
1143    }
1144    case nir_op_uadd_carry: {
1145       Temp src0 = get_alu_src(ctx, instr->src[0]);
1146       Temp src1 = get_alu_src(ctx, instr->src[1]);
1147       if (dst.regClass() == s1) {
1148          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1149          break;
1150       }
1151       if (dst.regClass() == v1) {
1152          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1153          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1154          break;
1155       }
1156
1157       Temp src00 = bld.tmp(src0.type(), 1);
1158       Temp src01 = bld.tmp(dst.type(), 1);
1159       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1160       Temp src10 = bld.tmp(src1.type(), 1);
1161       Temp src11 = bld.tmp(dst.type(), 1);
1162       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1163       if (dst.regClass() == s2) {
1164          Temp carry = bld.tmp(s1);
1165          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1166          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1167          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1168       } else if (dst.regClass() == v2) {
1169          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1170          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1171          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1172          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1173       } else {
1174          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1175          nir_print_instr(&instr->instr, stderr);
1176          fprintf(stderr, "\n");
1177       }
1178       break;
1179    }
1180    case nir_op_isub: {
1181       if (dst.regClass() == s1) {
1182          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1183          break;
1184       }
1185
1186       Temp src0 = get_alu_src(ctx, instr->src[0]);
1187       Temp src1 = get_alu_src(ctx, instr->src[1]);
1188       if (dst.regClass() == v1) {
1189          bld.vsub32(Definition(dst), src0, src1);
1190          break;
1191       }
1192
1193       Temp src00 = bld.tmp(src0.type(), 1);
1194       Temp src01 = bld.tmp(dst.type(), 1);
1195       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1196       Temp src10 = bld.tmp(src1.type(), 1);
1197       Temp src11 = bld.tmp(dst.type(), 1);
1198       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1199       if (dst.regClass() == s2) {
1200          Temp carry = bld.tmp(s1);
1201          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1202          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1203          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1204       } else if (dst.regClass() == v2) {
1205          Temp lower = bld.tmp(v1);
1206          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1207          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1208          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1209       } else {
1210          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1211          nir_print_instr(&instr->instr, stderr);
1212          fprintf(stderr, "\n");
1213       }
1214       break;
1215    }
1216    case nir_op_usub_borrow: {
1217       Temp src0 = get_alu_src(ctx, instr->src[0]);
1218       Temp src1 = get_alu_src(ctx, instr->src[1]);
1219       if (dst.regClass() == s1) {
1220          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1221          break;
1222       } else if (dst.regClass() == v1) {
1223          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1224          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1225          break;
1226       }
1227
1228       Temp src00 = bld.tmp(src0.type(), 1);
1229       Temp src01 = bld.tmp(dst.type(), 1);
1230       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1231       Temp src10 = bld.tmp(src1.type(), 1);
1232       Temp src11 = bld.tmp(dst.type(), 1);
1233       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1234       if (dst.regClass() == s2) {
1235          Temp borrow = bld.tmp(s1);
1236          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1237          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1238          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1239       } else if (dst.regClass() == v2) {
1240          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1241          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1242          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1243          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1244       } else {
1245          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1246          nir_print_instr(&instr->instr, stderr);
1247          fprintf(stderr, "\n");
1248       }
1249       break;
1250    }
1251    case nir_op_imul: {
1252       if (dst.regClass() == v1) {
1253          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1254                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1255       } else if (dst.regClass() == s1) {
1256          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1257       } else {
1258          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1259          nir_print_instr(&instr->instr, stderr);
1260          fprintf(stderr, "\n");
1261       }
1262       break;
1263    }
1264    case nir_op_umul_high: {
1265       if (dst.regClass() == v1) {
1266          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1267       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1268          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1269       } else if (dst.regClass() == s1) {
1270          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1271                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1272          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1273       } else {
1274          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1275          nir_print_instr(&instr->instr, stderr);
1276          fprintf(stderr, "\n");
1277       }
1278       break;
1279    }
1280    case nir_op_imul_high: {
1281       if (dst.regClass() == v1) {
1282          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1283       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1284          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1285       } else if (dst.regClass() == s1) {
1286          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1287                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1288          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1289       } else {
1290          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1291          nir_print_instr(&instr->instr, stderr);
1292          fprintf(stderr, "\n");
1293       }
1294       break;
1295    }
1296    case nir_op_fmul: {
1297       if (dst.size() == 1) {
1298          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1299       } else if (dst.size() == 2) {
1300          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1301                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1302       } else {
1303          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1304          nir_print_instr(&instr->instr, stderr);
1305          fprintf(stderr, "\n");
1306       }
1307       break;
1308    }
1309    case nir_op_fadd: {
1310       if (dst.size() == 1) {
1311          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1312       } else if (dst.size() == 2) {
1313          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1314                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1315       } else {
1316          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1317          nir_print_instr(&instr->instr, stderr);
1318          fprintf(stderr, "\n");
1319       }
1320       break;
1321    }
1322    case nir_op_fsub: {
1323       Temp src0 = get_alu_src(ctx, instr->src[0]);
1324       Temp src1 = get_alu_src(ctx, instr->src[1]);
1325       if (dst.size() == 1) {
1326          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1327             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1328          else
1329             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1330       } else if (dst.size() == 2) {
1331          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1332                                      get_alu_src(ctx, instr->src[0]),
1333                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1334          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1335          sub->neg[1] = true;
1336       } else {
1337          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1338          nir_print_instr(&instr->instr, stderr);
1339          fprintf(stderr, "\n");
1340       }
1341       break;
1342    }
1343    case nir_op_fmax: {
1344       if (dst.size() == 1) {
1345          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1346       } else if (dst.size() == 2) {
1347          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1348                   get_alu_src(ctx, instr->src[0]),
1349                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1350       } else {
1351          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1352          nir_print_instr(&instr->instr, stderr);
1353          fprintf(stderr, "\n");
1354       }
1355       break;
1356    }
1357    case nir_op_fmin: {
1358       if (dst.size() == 1) {
1359          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1360       } else if (dst.size() == 2) {
1361          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1362                   get_alu_src(ctx, instr->src[0]),
1363                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1364       } else {
1365          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1366          nir_print_instr(&instr->instr, stderr);
1367          fprintf(stderr, "\n");
1368       }
1369       break;
1370    }
1371    case nir_op_fmax3: {
1372       if (dst.size() == 1) {
1373          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1374       } else {
1375          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1376          nir_print_instr(&instr->instr, stderr);
1377          fprintf(stderr, "\n");
1378       }
1379       break;
1380    }
1381    case nir_op_fmin3: {
1382       if (dst.size() == 1) {
1383          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1384       } else {
1385          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1386          nir_print_instr(&instr->instr, stderr);
1387          fprintf(stderr, "\n");
1388       }
1389       break;
1390    }
1391    case nir_op_fmed3: {
1392       if (dst.size() == 1) {
1393          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1394       } else {
1395          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1396          nir_print_instr(&instr->instr, stderr);
1397          fprintf(stderr, "\n");
1398       }
1399       break;
1400    }
1401    case nir_op_umax3: {
1402       if (dst.size() == 1) {
1403          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1404       } else {
1405          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1406          nir_print_instr(&instr->instr, stderr);
1407          fprintf(stderr, "\n");
1408       }
1409       break;
1410    }
1411    case nir_op_umin3: {
1412       if (dst.size() == 1) {
1413          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1414       } else {
1415          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1416          nir_print_instr(&instr->instr, stderr);
1417          fprintf(stderr, "\n");
1418       }
1419       break;
1420    }
1421    case nir_op_umed3: {
1422       if (dst.size() == 1) {
1423          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1424       } else {
1425          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1426          nir_print_instr(&instr->instr, stderr);
1427          fprintf(stderr, "\n");
1428       }
1429       break;
1430    }
1431    case nir_op_imax3: {
1432       if (dst.size() == 1) {
1433          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1434       } else {
1435          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1436          nir_print_instr(&instr->instr, stderr);
1437          fprintf(stderr, "\n");
1438       }
1439       break;
1440    }
1441    case nir_op_imin3: {
1442       if (dst.size() == 1) {
1443          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1444       } else {
1445          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1446          nir_print_instr(&instr->instr, stderr);
1447          fprintf(stderr, "\n");
1448       }
1449       break;
1450    }
1451    case nir_op_imed3: {
1452       if (dst.size() == 1) {
1453          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1454       } else {
1455          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1456          nir_print_instr(&instr->instr, stderr);
1457          fprintf(stderr, "\n");
1458       }
1459       break;
1460    }
1461    case nir_op_cube_face_coord: {
1462       Temp in = get_alu_src(ctx, instr->src[0], 3);
1463       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1464                       emit_extract_vector(ctx, in, 1, v1),
1465                       emit_extract_vector(ctx, in, 2, v1) };
1466       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1467       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1468       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1469       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1470       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1471       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1472       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1473       break;
1474    }
1475    case nir_op_cube_face_index: {
1476       Temp in = get_alu_src(ctx, instr->src[0], 3);
1477       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1478                       emit_extract_vector(ctx, in, 1, v1),
1479                       emit_extract_vector(ctx, in, 2, v1) };
1480       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1481       break;
1482    }
1483    case nir_op_bcsel: {
1484       emit_bcsel(ctx, instr, dst);
1485       break;
1486    }
1487    case nir_op_frsq: {
1488       if (dst.size() == 1) {
1489          emit_rsq(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1490       } else if (dst.size() == 2) {
1491          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1492       } else {
1493          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1494          nir_print_instr(&instr->instr, stderr);
1495          fprintf(stderr, "\n");
1496       }
1497       break;
1498    }
1499    case nir_op_fneg: {
1500       Temp src = get_alu_src(ctx, instr->src[0]);
1501       if (dst.size() == 1) {
1502          if (ctx->block->fp_mode.must_flush_denorms32)
1503             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1504          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1505       } else if (dst.size() == 2) {
1506          if (ctx->block->fp_mode.must_flush_denorms16_64)
1507             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1508          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1509          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1510          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1511          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1512       } else {
1513          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1514          nir_print_instr(&instr->instr, stderr);
1515          fprintf(stderr, "\n");
1516       }
1517       break;
1518    }
1519    case nir_op_fabs: {
1520       Temp src = get_alu_src(ctx, instr->src[0]);
1521       if (dst.size() == 1) {
1522          if (ctx->block->fp_mode.must_flush_denorms32)
1523             src = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0x3f800000u), as_vgpr(ctx, src));
1524          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1525       } else if (dst.size() == 2) {
1526          if (ctx->block->fp_mode.must_flush_denorms16_64)
1527             src = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), Operand(0x3FF0000000000000lu), as_vgpr(ctx, src));
1528          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1529          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1530          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1531          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_fsat: {
1540       Temp src = get_alu_src(ctx, instr->src[0]);
1541       if (dst.size() == 1) {
1542          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1543       } else if (dst.size() == 2) {
1544          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1545          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1546          vop3->clamp = true;
1547       } else {
1548          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1549          nir_print_instr(&instr->instr, stderr);
1550          fprintf(stderr, "\n");
1551       }
1552       break;
1553    }
1554    case nir_op_flog2: {
1555       if (dst.size() == 1) {
1556          emit_log2(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1557       } else {
1558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1559          nir_print_instr(&instr->instr, stderr);
1560          fprintf(stderr, "\n");
1561       }
1562       break;
1563    }
1564    case nir_op_frcp: {
1565       if (dst.size() == 1) {
1566          emit_rcp(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1567       } else if (dst.size() == 2) {
1568          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1569       } else {
1570          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1571          nir_print_instr(&instr->instr, stderr);
1572          fprintf(stderr, "\n");
1573       }
1574       break;
1575    }
1576    case nir_op_fexp2: {
1577       if (dst.size() == 1) {
1578          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1579       } else {
1580          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1581          nir_print_instr(&instr->instr, stderr);
1582          fprintf(stderr, "\n");
1583       }
1584       break;
1585    }
1586    case nir_op_fsqrt: {
1587       if (dst.size() == 1) {
1588          emit_sqrt(ctx, bld, Definition(dst), get_alu_src(ctx, instr->src[0]));
1589       } else if (dst.size() == 2) {
1590          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1591       } else {
1592          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1593          nir_print_instr(&instr->instr, stderr);
1594          fprintf(stderr, "\n");
1595       }
1596       break;
1597    }
1598    case nir_op_ffract: {
1599       if (dst.size() == 1) {
1600          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1601       } else if (dst.size() == 2) {
1602          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1603       } else {
1604          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1605          nir_print_instr(&instr->instr, stderr);
1606          fprintf(stderr, "\n");
1607       }
1608       break;
1609    }
1610    case nir_op_ffloor: {
1611       if (dst.size() == 1) {
1612          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1613       } else if (dst.size() == 2) {
1614          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1615       } else {
1616          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1617          nir_print_instr(&instr->instr, stderr);
1618          fprintf(stderr, "\n");
1619       }
1620       break;
1621    }
1622    case nir_op_fceil: {
1623       if (dst.size() == 1) {
1624          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1625       } else if (dst.size() == 2) {
1626          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1627       } else {
1628          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1629          nir_print_instr(&instr->instr, stderr);
1630          fprintf(stderr, "\n");
1631       }
1632       break;
1633    }
1634    case nir_op_ftrunc: {
1635       if (dst.size() == 1) {
1636          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1637       } else if (dst.size() == 2) {
1638          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1639       } else {
1640          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1641          nir_print_instr(&instr->instr, stderr);
1642          fprintf(stderr, "\n");
1643       }
1644       break;
1645    }
1646    case nir_op_fround_even: {
1647       if (dst.size() == 1) {
1648          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1649       } else if (dst.size() == 2) {
1650          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1651       } else {
1652          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1653          nir_print_instr(&instr->instr, stderr);
1654          fprintf(stderr, "\n");
1655       }
1656       break;
1657    }
1658    case nir_op_fsin:
1659    case nir_op_fcos: {
1660       Temp src = get_alu_src(ctx, instr->src[0]);
1661       aco_ptr<Instruction> norm;
1662       if (dst.size() == 1) {
1663          Temp half_pi = bld.copy(bld.def(s1), Operand(0x3e22f983u));
1664          Temp tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, as_vgpr(ctx, src));
1665
1666          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1667          if (ctx->options->chip_class < GFX9)
1668             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1669
1670          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1671          bld.vop1(opcode, Definition(dst), tmp);
1672       } else {
1673          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1674          nir_print_instr(&instr->instr, stderr);
1675          fprintf(stderr, "\n");
1676       }
1677       break;
1678    }
1679    case nir_op_ldexp: {
1680       if (dst.size() == 1) {
1681          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1682                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1683                   get_alu_src(ctx, instr->src[1]));
1684       } else if (dst.size() == 2) {
1685          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1686                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1687                   get_alu_src(ctx, instr->src[1]));
1688       } else {
1689          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1690          nir_print_instr(&instr->instr, stderr);
1691          fprintf(stderr, "\n");
1692       }
1693       break;
1694    }
1695    case nir_op_frexp_sig: {
1696       if (dst.size() == 1) {
1697          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1698                   get_alu_src(ctx, instr->src[0]));
1699       } else if (dst.size() == 2) {
1700          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1701                   get_alu_src(ctx, instr->src[0]));
1702       } else {
1703          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1704          nir_print_instr(&instr->instr, stderr);
1705          fprintf(stderr, "\n");
1706       }
1707       break;
1708    }
1709    case nir_op_frexp_exp: {
1710       if (instr->src[0].src.ssa->bit_size == 32) {
1711          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1712                   get_alu_src(ctx, instr->src[0]));
1713       } else if (instr->src[0].src.ssa->bit_size == 64) {
1714          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1715                   get_alu_src(ctx, instr->src[0]));
1716       } else {
1717          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1718          nir_print_instr(&instr->instr, stderr);
1719          fprintf(stderr, "\n");
1720       }
1721       break;
1722    }
1723    case nir_op_fsign: {
1724       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1725       if (dst.size() == 1) {
1726          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1727          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1728          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1729          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1730       } else if (dst.size() == 2) {
1731          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1732          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1733          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, emit_extract_vector(ctx, src, 1, v1), cond);
1734
1735          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), src);
1736          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1737          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1738
1739          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1740       } else {
1741          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1742          nir_print_instr(&instr->instr, stderr);
1743          fprintf(stderr, "\n");
1744       }
1745       break;
1746    }
1747    case nir_op_f2f32: {
1748       if (instr->src[0].src.ssa->bit_size == 64) {
1749          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1750       } else {
1751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1752          nir_print_instr(&instr->instr, stderr);
1753          fprintf(stderr, "\n");
1754       }
1755       break;
1756    }
1757    case nir_op_f2f64: {
1758       if (instr->src[0].src.ssa->bit_size == 32) {
1759          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1760       } else {
1761          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1762          nir_print_instr(&instr->instr, stderr);
1763          fprintf(stderr, "\n");
1764       }
1765       break;
1766    }
1767    case nir_op_i2f32: {
1768       assert(dst.size() == 1);
1769       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1770       break;
1771    }
1772    case nir_op_i2f64: {
1773       if (instr->src[0].src.ssa->bit_size == 32) {
1774          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1775       } else if (instr->src[0].src.ssa->bit_size == 64) {
1776          Temp src = get_alu_src(ctx, instr->src[0]);
1777          RegClass rc = RegClass(src.type(), 1);
1778          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1779          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1780          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1781          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1782          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1783          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1784
1785       } else {
1786          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1787          nir_print_instr(&instr->instr, stderr);
1788          fprintf(stderr, "\n");
1789       }
1790       break;
1791    }
1792    case nir_op_u2f32: {
1793       assert(dst.size() == 1);
1794       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1795       break;
1796    }
1797    case nir_op_u2f64: {
1798       if (instr->src[0].src.ssa->bit_size == 32) {
1799          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1800       } else if (instr->src[0].src.ssa->bit_size == 64) {
1801          Temp src = get_alu_src(ctx, instr->src[0]);
1802          RegClass rc = RegClass(src.type(), 1);
1803          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1804          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1805          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1806          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1807          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1808          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1809       } else {
1810          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1811          nir_print_instr(&instr->instr, stderr);
1812          fprintf(stderr, "\n");
1813       }
1814       break;
1815    }
1816    case nir_op_f2i32: {
1817       Temp src = get_alu_src(ctx, instr->src[0]);
1818       if (instr->src[0].src.ssa->bit_size == 32) {
1819          if (dst.type() == RegType::vgpr)
1820             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1821          else
1822             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1823                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1824
1825       } else if (instr->src[0].src.ssa->bit_size == 64) {
1826          if (dst.type() == RegType::vgpr)
1827             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1828          else
1829             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1830                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1831
1832       } else {
1833          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1834          nir_print_instr(&instr->instr, stderr);
1835          fprintf(stderr, "\n");
1836       }
1837       break;
1838    }
1839    case nir_op_f2u32: {
1840       Temp src = get_alu_src(ctx, instr->src[0]);
1841       if (instr->src[0].src.ssa->bit_size == 32) {
1842          if (dst.type() == RegType::vgpr)
1843             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1844          else
1845             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1846                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1847
1848       } else if (instr->src[0].src.ssa->bit_size == 64) {
1849          if (dst.type() == RegType::vgpr)
1850             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1851          else
1852             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1853                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1854
1855       } else {
1856          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1857          nir_print_instr(&instr->instr, stderr);
1858          fprintf(stderr, "\n");
1859       }
1860       break;
1861    }
1862    case nir_op_f2i64: {
1863       Temp src = get_alu_src(ctx, instr->src[0]);
1864       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1865          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1866          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1867          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1868          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1869          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1870          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1871          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1872          Temp new_exponent = bld.tmp(v1);
1873          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1874          if (ctx->program->chip_class >= GFX8)
1875             mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1876          else
1877             mantissa = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), mantissa, new_exponent);
1878          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1879          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1880          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1881          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1882          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1883          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1884          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1885          Temp new_lower = bld.tmp(v1);
1886          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1887          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1888          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1889
1890       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1891          if (src.type() == RegType::vgpr)
1892             src = bld.as_uniform(src);
1893          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1894          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1895          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1896          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1897          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1898          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1899          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1900          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1901          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1902          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1903          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1904          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1905          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1906          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1907          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1908          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1909          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1910          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1911          Temp borrow = bld.tmp(s1);
1912          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1913          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1914          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1915
1916       } else if (instr->src[0].src.ssa->bit_size == 64) {
1917          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1918          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1919          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1920          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1921          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1922          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1923          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1924          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1925          if (dst.type() == RegType::sgpr) {
1926             lower = bld.as_uniform(lower);
1927             upper = bld.as_uniform(upper);
1928          }
1929          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1930
1931       } else {
1932          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1933          nir_print_instr(&instr->instr, stderr);
1934          fprintf(stderr, "\n");
1935       }
1936       break;
1937    }
1938    case nir_op_f2u64: {
1939       Temp src = get_alu_src(ctx, instr->src[0]);
1940       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1941          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1942          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(bld.lm)), Operand(64u), exponent);
1943          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1944          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1945          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1946          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1947          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1948          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1949          Temp new_exponent = bld.tmp(v1);
1950          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1951          if (ctx->program->chip_class >= GFX8)
1952             mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1953          else
1954             mantissa = bld.vop3(aco_opcode::v_lshl_b64, bld.def(v2), mantissa, new_exponent);
1955          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1956          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1957          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1958          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1959          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1960          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1961          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1962
1963       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1964          if (src.type() == RegType::vgpr)
1965             src = bld.as_uniform(src);
1966          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1967          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1968          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1969          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1970          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1971          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1972          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1973          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1974          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1975          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1976          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1977          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1978          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1979          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1980          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1981          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1982          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1983          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1984
1985       } else if (instr->src[0].src.ssa->bit_size == 64) {
1986          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1987          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1988          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1989          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1990          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1991          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1992          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1993          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1994          if (dst.type() == RegType::sgpr) {
1995             lower = bld.as_uniform(lower);
1996             upper = bld.as_uniform(upper);
1997          }
1998          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1999
2000       } else {
2001          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2002          nir_print_instr(&instr->instr, stderr);
2003          fprintf(stderr, "\n");
2004       }
2005       break;
2006    }
2007    case nir_op_b2f32: {
2008       Temp src = get_alu_src(ctx, instr->src[0]);
2009       assert(src.regClass() == bld.lm);
2010
2011       if (dst.regClass() == s1) {
2012          src = bool_to_scalar_condition(ctx, src);
2013          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
2014       } else if (dst.regClass() == v1) {
2015          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
2016       } else {
2017          unreachable("Wrong destination register class for nir_op_b2f32.");
2018       }
2019       break;
2020    }
2021    case nir_op_b2f64: {
2022       Temp src = get_alu_src(ctx, instr->src[0]);
2023       assert(src.regClass() == bld.lm);
2024
2025       if (dst.regClass() == s2) {
2026          src = bool_to_scalar_condition(ctx, src);
2027          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
2028       } else if (dst.regClass() == v2) {
2029          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
2030          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one, src);
2031          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
2032       } else {
2033          unreachable("Wrong destination register class for nir_op_b2f64.");
2034       }
2035       break;
2036    }
2037    case nir_op_i2i32: {
2038       Temp src = get_alu_src(ctx, instr->src[0]);
2039       if (instr->src[0].src.ssa->bit_size == 64) {
2040          /* we can actually just say dst = src, as it would map the lower register */
2041          emit_extract_vector(ctx, src, 0, dst);
2042       } else {
2043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2044          nir_print_instr(&instr->instr, stderr);
2045          fprintf(stderr, "\n");
2046       }
2047       break;
2048    }
2049    case nir_op_u2u32: {
2050       Temp src = get_alu_src(ctx, instr->src[0]);
2051       if (instr->src[0].src.ssa->bit_size == 16) {
2052          if (dst.regClass() == s1) {
2053             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
2054          } else {
2055             // TODO: do better with SDWA
2056             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
2057          }
2058       } else if (instr->src[0].src.ssa->bit_size == 64) {
2059          /* we can actually just say dst = src, as it would map the lower register */
2060          emit_extract_vector(ctx, src, 0, dst);
2061       } else {
2062          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2063          nir_print_instr(&instr->instr, stderr);
2064          fprintf(stderr, "\n");
2065       }
2066       break;
2067    }
2068    case nir_op_i2i64: {
2069       Temp src = get_alu_src(ctx, instr->src[0]);
2070       if (src.regClass() == s1) {
2071          Temp high = bld.sopc(aco_opcode::s_ashr_i32, bld.def(s1, scc), src, Operand(31u));
2072          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2073       } else if (src.regClass() == v1) {
2074          Temp high = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
2075          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, high);
2076       } else {
2077          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2078          nir_print_instr(&instr->instr, stderr);
2079          fprintf(stderr, "\n");
2080       }
2081       break;
2082    }
2083    case nir_op_u2u64: {
2084       Temp src = get_alu_src(ctx, instr->src[0]);
2085       if (instr->src[0].src.ssa->bit_size == 32) {
2086          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
2087       } else {
2088          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2089          nir_print_instr(&instr->instr, stderr);
2090          fprintf(stderr, "\n");
2091       }
2092       break;
2093    }
2094    case nir_op_b2i32: {
2095       Temp src = get_alu_src(ctx, instr->src[0]);
2096       assert(src.regClass() == bld.lm);
2097
2098       if (dst.regClass() == s1) {
2099          // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
2100          bool_to_scalar_condition(ctx, src, dst);
2101       } else if (dst.regClass() == v1) {
2102          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2103       } else {
2104          unreachable("Invalid register class for b2i32");
2105       }
2106       break;
2107    }
2108    case nir_op_i2b1: {
2109       Temp src = get_alu_src(ctx, instr->src[0]);
2110       assert(dst.regClass() == bld.lm);
2111
2112       if (src.type() == RegType::vgpr) {
2113          assert(src.regClass() == v1 || src.regClass() == v2);
2114          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2115                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2116       } else {
2117          assert(src.regClass() == s1 || src.regClass() == s2);
2118          Temp tmp;
2119          if (src.regClass() == s2 && ctx->program->chip_class <= GFX7) {
2120             tmp = bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), Operand(0u), src).def(1).getTemp();
2121          } else {
2122             tmp = bld.sopc(src.size() == 2 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::s_cmp_lg_u32,
2123                            bld.scc(bld.def(s1)), Operand(0u), src);
2124          }
2125          bool_to_vector_condition(ctx, tmp, dst);
2126       }
2127       break;
2128    }
2129    case nir_op_pack_64_2x32_split: {
2130       Temp src0 = get_alu_src(ctx, instr->src[0]);
2131       Temp src1 = get_alu_src(ctx, instr->src[1]);
2132
2133       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2134       break;
2135    }
2136    case nir_op_unpack_64_2x32_split_x:
2137       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2138       break;
2139    case nir_op_unpack_64_2x32_split_y:
2140       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2141       break;
2142    case nir_op_pack_half_2x16: {
2143       Temp src = get_alu_src(ctx, instr->src[0], 2);
2144
2145       if (dst.regClass() == v1) {
2146          Temp src0 = bld.tmp(v1);
2147          Temp src1 = bld.tmp(v1);
2148          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2149          if (!ctx->block->fp_mode.care_about_round32 || ctx->block->fp_mode.round32 == fp_round_tz)
2150             bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2151          else
2152             bld.vop3(aco_opcode::v_cvt_pk_u16_u32, Definition(dst),
2153                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src0),
2154                      bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), src1));
2155       } else {
2156          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2157          nir_print_instr(&instr->instr, stderr);
2158          fprintf(stderr, "\n");
2159       }
2160       break;
2161    }
2162    case nir_op_unpack_half_2x16_split_x: {
2163       if (dst.regClass() == v1) {
2164          Builder bld(ctx->program, ctx->block);
2165          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2166       } else {
2167          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2168          nir_print_instr(&instr->instr, stderr);
2169          fprintf(stderr, "\n");
2170       }
2171       break;
2172    }
2173    case nir_op_unpack_half_2x16_split_y: {
2174       if (dst.regClass() == v1) {
2175          Builder bld(ctx->program, ctx->block);
2176          /* TODO: use SDWA here */
2177          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2178                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2179       } else {
2180          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2181          nir_print_instr(&instr->instr, stderr);
2182          fprintf(stderr, "\n");
2183       }
2184       break;
2185    }
2186    case nir_op_fquantize2f16: {
2187       Temp src = get_alu_src(ctx, instr->src[0]);
2188       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), src);
2189       Temp f32, cmp_res;
2190
2191       if (ctx->program->chip_class >= GFX8) {
2192          Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2193          cmp_res = bld.vopc_e64(aco_opcode::v_cmp_class_f16, bld.hint_vcc(bld.def(bld.lm)), f16, mask);
2194          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2195       } else {
2196          /* 0x38800000 is smallest half float value (2^-14) in 32-bit float,
2197           * so compare the result and flush to 0 if it's smaller.
2198           */
2199          f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2200          Temp smallest = bld.copy(bld.def(s1), Operand(0x38800000u));
2201          Instruction* vop3 = bld.vopc_e64(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), f32, smallest);
2202          static_cast<VOP3A_instruction*>(vop3)->abs[0] = true;
2203          cmp_res = vop3->definitions[0].getTemp();
2204       }
2205
2206       if (ctx->block->fp_mode.preserve_signed_zero_inf_nan32 || ctx->program->chip_class < GFX8) {
2207          Temp copysign_0 = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0u), as_vgpr(ctx, src));
2208          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), copysign_0, f32, cmp_res);
2209       } else {
2210          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2211       }
2212       break;
2213    }
2214    case nir_op_bfm: {
2215       Temp bits = get_alu_src(ctx, instr->src[0]);
2216       Temp offset = get_alu_src(ctx, instr->src[1]);
2217
2218       if (dst.regClass() == s1) {
2219          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2220       } else if (dst.regClass() == v1) {
2221          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2222       } else {
2223          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2224          nir_print_instr(&instr->instr, stderr);
2225          fprintf(stderr, "\n");
2226       }
2227       break;
2228    }
2229    case nir_op_bitfield_select: {
2230       /* (mask & insert) | (~mask & base) */
2231       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2232       Temp insert = get_alu_src(ctx, instr->src[1]);
2233       Temp base = get_alu_src(ctx, instr->src[2]);
2234
2235       /* dst = (insert & bitmask) | (base & ~bitmask) */
2236       if (dst.regClass() == s1) {
2237          aco_ptr<Instruction> sop2;
2238          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2239          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2240          Operand lhs;
2241          if (const_insert && const_bitmask) {
2242             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2243          } else {
2244             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2245             lhs = Operand(insert);
2246          }
2247
2248          Operand rhs;
2249          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2250          if (const_base && const_bitmask) {
2251             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2252          } else {
2253             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2254             rhs = Operand(base);
2255          }
2256
2257          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2258
2259       } else if (dst.regClass() == v1) {
2260          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2261             base = as_vgpr(ctx, base);
2262          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2263             insert = as_vgpr(ctx, insert);
2264
2265          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2266
2267       } else {
2268          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2269          nir_print_instr(&instr->instr, stderr);
2270          fprintf(stderr, "\n");
2271       }
2272       break;
2273    }
2274    case nir_op_ubfe:
2275    case nir_op_ibfe: {
2276       Temp base = get_alu_src(ctx, instr->src[0]);
2277       Temp offset = get_alu_src(ctx, instr->src[1]);
2278       Temp bits = get_alu_src(ctx, instr->src[2]);
2279
2280       if (dst.type() == RegType::sgpr) {
2281          Operand extract;
2282          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2283          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2284          if (const_offset && const_bits) {
2285             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2286             extract = Operand(const_extract);
2287          } else {
2288             Operand width;
2289             if (const_bits) {
2290                width = Operand(const_bits->u32 << 16);
2291             } else {
2292                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2293             }
2294             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2295          }
2296
2297          aco_opcode opcode;
2298          if (dst.regClass() == s1) {
2299             if (instr->op == nir_op_ubfe)
2300                opcode = aco_opcode::s_bfe_u32;
2301             else
2302                opcode = aco_opcode::s_bfe_i32;
2303          } else if (dst.regClass() == s2) {
2304             if (instr->op == nir_op_ubfe)
2305                opcode = aco_opcode::s_bfe_u64;
2306             else
2307                opcode = aco_opcode::s_bfe_i64;
2308          } else {
2309             unreachable("Unsupported BFE bit size");
2310          }
2311
2312          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2313
2314       } else {
2315          aco_opcode opcode;
2316          if (dst.regClass() == v1) {
2317             if (instr->op == nir_op_ubfe)
2318                opcode = aco_opcode::v_bfe_u32;
2319             else
2320                opcode = aco_opcode::v_bfe_i32;
2321          } else {
2322             unreachable("Unsupported BFE bit size");
2323          }
2324
2325          emit_vop3a_instruction(ctx, instr, opcode, dst);
2326       }
2327       break;
2328    }
2329    case nir_op_bit_count: {
2330       Temp src = get_alu_src(ctx, instr->src[0]);
2331       if (src.regClass() == s1) {
2332          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2333       } else if (src.regClass() == v1) {
2334          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2335       } else if (src.regClass() == v2) {
2336          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2337                   emit_extract_vector(ctx, src, 1, v1),
2338                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2339                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2340       } else if (src.regClass() == s2) {
2341          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2342       } else {
2343          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2344          nir_print_instr(&instr->instr, stderr);
2345          fprintf(stderr, "\n");
2346       }
2347       break;
2348    }
2349    case nir_op_flt: {
2350       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_f32, aco_opcode::v_cmp_lt_f64);
2351       break;
2352    }
2353    case nir_op_fge: {
2354       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_f32, aco_opcode::v_cmp_ge_f64);
2355       break;
2356    }
2357    case nir_op_feq: {
2358       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_f32, aco_opcode::v_cmp_eq_f64);
2359       break;
2360    }
2361    case nir_op_fne: {
2362       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_neq_f32, aco_opcode::v_cmp_neq_f64);
2363       break;
2364    }
2365    case nir_op_ilt: {
2366       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_i32, aco_opcode::v_cmp_lt_i64, aco_opcode::s_cmp_lt_i32);
2367       break;
2368    }
2369    case nir_op_ige: {
2370       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_i32, aco_opcode::v_cmp_ge_i64, aco_opcode::s_cmp_ge_i32);
2371       break;
2372    }
2373    case nir_op_ieq: {
2374       if (instr->src[0].src.ssa->bit_size == 1)
2375          emit_boolean_logic(ctx, instr, Builder::s_xnor, dst);
2376       else
2377          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_eq_i32, aco_opcode::v_cmp_eq_i64, aco_opcode::s_cmp_eq_i32,
2378                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_eq_u64 : aco_opcode::num_opcodes);
2379       break;
2380    }
2381    case nir_op_ine: {
2382       if (instr->src[0].src.ssa->bit_size == 1)
2383          emit_boolean_logic(ctx, instr, Builder::s_xor, dst);
2384       else
2385          emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lg_i32, aco_opcode::v_cmp_lg_i64, aco_opcode::s_cmp_lg_i32,
2386                          ctx->program->chip_class >= GFX8 ? aco_opcode::s_cmp_lg_u64 : aco_opcode::num_opcodes);
2387       break;
2388    }
2389    case nir_op_ult: {
2390       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_lt_u32, aco_opcode::v_cmp_lt_u64, aco_opcode::s_cmp_lt_u32);
2391       break;
2392    }
2393    case nir_op_uge: {
2394       emit_comparison(ctx, instr, dst, aco_opcode::v_cmp_ge_u32, aco_opcode::v_cmp_ge_u64, aco_opcode::s_cmp_ge_u32);
2395       break;
2396    }
2397    case nir_op_fddx:
2398    case nir_op_fddy:
2399    case nir_op_fddx_fine:
2400    case nir_op_fddy_fine:
2401    case nir_op_fddx_coarse:
2402    case nir_op_fddy_coarse: {
2403       Temp src = get_alu_src(ctx, instr->src[0]);
2404       uint16_t dpp_ctrl1, dpp_ctrl2;
2405       if (instr->op == nir_op_fddx_fine) {
2406          dpp_ctrl1 = dpp_quad_perm(0, 0, 2, 2);
2407          dpp_ctrl2 = dpp_quad_perm(1, 1, 3, 3);
2408       } else if (instr->op == nir_op_fddy_fine) {
2409          dpp_ctrl1 = dpp_quad_perm(0, 1, 0, 1);
2410          dpp_ctrl2 = dpp_quad_perm(2, 3, 2, 3);
2411       } else {
2412          dpp_ctrl1 = dpp_quad_perm(0, 0, 0, 0);
2413          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2414             dpp_ctrl2 = dpp_quad_perm(1, 1, 1, 1);
2415          else
2416             dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
2417       }
2418
2419       Temp tmp;
2420       if (ctx->program->chip_class >= GFX8) {
2421          Temp tl = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl1);
2422          tmp = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), src, tl, dpp_ctrl2);
2423       } else {
2424          Temp tl = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl1);
2425          Temp tr = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl2);
2426          tmp = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), tr, tl);
2427       }
2428       emit_wqm(ctx, tmp, dst, true);
2429       break;
2430    }
2431    default:
2432       fprintf(stderr, "Unknown NIR ALU instr: ");
2433       nir_print_instr(&instr->instr, stderr);
2434       fprintf(stderr, "\n");
2435    }
2436 }
2437
2438 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2439 {
2440    Temp dst = get_ssa_temp(ctx, &instr->def);
2441
2442    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2443    // which get truncated the lsb if double and msb if int
2444    // for now, we only use s_mov_b64 with 64bit inline constants
2445    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2446    assert(dst.type() == RegType::sgpr);
2447
2448    Builder bld(ctx->program, ctx->block);
2449
2450    if (instr->def.bit_size == 1) {
2451       assert(dst.regClass() == bld.lm);
2452       int val = instr->value[0].b ? -1 : 0;
2453       Operand op = bld.lm.size() == 1 ? Operand((uint32_t) val) : Operand((uint64_t) val);
2454       bld.sop1(Builder::s_mov, Definition(dst), op);
2455    } else if (dst.size() == 1) {
2456       bld.copy(Definition(dst), Operand(instr->value[0].u32));
2457    } else {
2458       assert(dst.size() != 1);
2459       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2460       if (instr->def.bit_size == 64)
2461          for (unsigned i = 0; i < dst.size(); i++)
2462             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2463       else {
2464          for (unsigned i = 0; i < dst.size(); i++)
2465             vec->operands[i] = Operand{instr->value[i].u32};
2466       }
2467       vec->definitions[0] = Definition(dst);
2468       ctx->block->instructions.emplace_back(std::move(vec));
2469    }
2470 }
2471
2472 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2473 {
2474    uint32_t new_mask = 0;
2475    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2476       if (mask & (1u << i))
2477          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2478    return new_mask;
2479 }
2480
2481 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2482 {
2483    /* This wouldn't work inside control flow or with indirect offsets but
2484     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2485
2486    unsigned write_mask = nir_intrinsic_write_mask(instr);
2487    unsigned component = nir_intrinsic_component(instr);
2488    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2489    unsigned idx = nir_intrinsic_base(instr) + component;
2490
2491    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2492    if (off_instr->type != nir_instr_type_load_const) {
2493       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2494       nir_print_instr(off_instr, stderr);
2495       fprintf(stderr, "\n");
2496    }
2497    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2498
2499    if (instr->src[0].ssa->bit_size == 64)
2500       write_mask = widen_mask(write_mask, 2);
2501
2502    for (unsigned i = 0; i < 8; ++i) {
2503       if (write_mask & (1 << i)) {
2504          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2505          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2506       }
2507       idx++;
2508    }
2509 }
2510
2511 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2512 {
2513    Builder bld(ctx->program, ctx->block);
2514    unsigned write_mask = nir_intrinsic_write_mask(instr);
2515    Operand values[4];
2516    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2517    for (unsigned i = 0; i < 4; ++i) {
2518       if (write_mask & (1 << i)) {
2519          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2520          values[i] = Operand(tmp);
2521       } else {
2522          values[i] = Operand(v1);
2523       }
2524    }
2525
2526    unsigned index = nir_intrinsic_base(instr) / 4;
2527    unsigned target, col_format;
2528    unsigned enabled_channels = 0xF;
2529    aco_opcode compr_op = (aco_opcode)0;
2530
2531    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2532    assert(offset && "Non-const offsets on exports not yet supported");
2533    index += offset->u32;
2534
2535    assert(index != FRAG_RESULT_COLOR);
2536
2537    /* Unlike vertex shader exports, it's fine to use multiple exports to
2538     * export separate channels of one target. So shaders which export both
2539     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2540     * TODO: combine the exports in those cases and create better code
2541     */
2542
2543    if (index == FRAG_RESULT_SAMPLE_MASK) {
2544
2545       if (ctx->program->info->ps.writes_z) {
2546          target = V_008DFC_SQ_EXP_MRTZ;
2547          enabled_channels = 0x4;
2548          col_format = (unsigned) -1;
2549
2550          values[2] = values[0];
2551          values[0] = Operand(v1);
2552       } else {
2553          bld.exp(aco_opcode::exp, Operand(v1), Operand(values[0]), Operand(v1), Operand(v1),
2554                  0xc, V_008DFC_SQ_EXP_MRTZ, true);
2555          return;
2556       }
2557
2558    } else if (index == FRAG_RESULT_DEPTH) {
2559
2560       target = V_008DFC_SQ_EXP_MRTZ;
2561       enabled_channels = 0x1;
2562       col_format = (unsigned) -1;
2563
2564    } else if (index == FRAG_RESULT_STENCIL) {
2565
2566       if (ctx->program->info->ps.writes_z) {
2567          target = V_008DFC_SQ_EXP_MRTZ;
2568          enabled_channels = 0x2;
2569          col_format = (unsigned) -1;
2570
2571          values[1] = values[0];
2572          values[0] = Operand(v1);
2573       } else {
2574          values[0] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u), values[0]);
2575          bld.exp(aco_opcode::exp, values[0], Operand(v1), Operand(v1), Operand(v1),
2576                  0x3, V_008DFC_SQ_EXP_MRTZ, true);
2577          return;
2578       }
2579
2580    } else {
2581       index -= FRAG_RESULT_DATA0;
2582       target = V_008DFC_SQ_EXP_MRT + index;
2583       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2584    }
2585    bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2586    bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2587
2588    switch (col_format)
2589    {
2590    case V_028714_SPI_SHADER_ZERO:
2591       enabled_channels = 0; /* writemask */
2592       target = V_008DFC_SQ_EXP_NULL;
2593       break;
2594
2595    case V_028714_SPI_SHADER_32_R:
2596       enabled_channels = 1;
2597       break;
2598
2599    case V_028714_SPI_SHADER_32_GR:
2600       enabled_channels = 0x3;
2601       break;
2602
2603    case V_028714_SPI_SHADER_32_AR:
2604       if (ctx->options->chip_class >= GFX10) {
2605          /* Special case: on GFX10, the outputs are different for 32_AR */
2606          enabled_channels = 0x3;
2607          values[1] = values[3];
2608          values[3] = Operand(v1);
2609       } else {
2610          enabled_channels = 0x9;
2611       }
2612       break;
2613
2614    case V_028714_SPI_SHADER_FP16_ABGR:
2615       enabled_channels = 0x5;
2616       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2617       break;
2618
2619    case V_028714_SPI_SHADER_UNORM16_ABGR:
2620       enabled_channels = 0x5;
2621       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2622       break;
2623
2624    case V_028714_SPI_SHADER_SNORM16_ABGR:
2625       enabled_channels = 0x5;
2626       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2627       break;
2628
2629    case V_028714_SPI_SHADER_UINT16_ABGR: {
2630       enabled_channels = 0x5;
2631       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2632       if (is_int8 || is_int10) {
2633          /* clamp */
2634          uint32_t max_rgb = is_int8 ? 255 : is_int10 ? 1023 : 0;
2635          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2636
2637          for (unsigned i = 0; i < 4; i++) {
2638             if ((write_mask >> i) & 1) {
2639                values[i] = bld.vop2(aco_opcode::v_min_u32, bld.def(v1),
2640                                     i == 3 && is_int10 ? Operand(3u) : Operand(max_rgb_val),
2641                                     values[i]);
2642             }
2643          }
2644       }
2645       break;
2646    }
2647
2648    case V_028714_SPI_SHADER_SINT16_ABGR:
2649       enabled_channels = 0x5;
2650       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2651       if (is_int8 || is_int10) {
2652          /* clamp */
2653          uint32_t max_rgb = is_int8 ? 127 : is_int10 ? 511 : 0;
2654          uint32_t min_rgb = is_int8 ? -128 :is_int10 ? -512 : 0;
2655          Temp max_rgb_val = bld.copy(bld.def(s1), Operand(max_rgb));
2656          Temp min_rgb_val = bld.copy(bld.def(s1), Operand(min_rgb));
2657
2658          for (unsigned i = 0; i < 4; i++) {
2659             if ((write_mask >> i) & 1) {
2660                values[i] = bld.vop2(aco_opcode::v_min_i32, bld.def(v1),
2661                                     i == 3 && is_int10 ? Operand(1u) : Operand(max_rgb_val),
2662                                     values[i]);
2663                values[i] = bld.vop2(aco_opcode::v_max_i32, bld.def(v1),
2664                                     i == 3 && is_int10 ? Operand(-2u) : Operand(min_rgb_val),
2665                                     values[i]);
2666             }
2667          }
2668       }
2669       break;
2670
2671    case V_028714_SPI_SHADER_32_ABGR:
2672       enabled_channels = 0xF;
2673       break;
2674
2675    default:
2676       break;
2677    }
2678
2679    if (target == V_008DFC_SQ_EXP_NULL)
2680       return;
2681
2682    if ((bool) compr_op) {
2683       for (int i = 0; i < 2; i++) {
2684          /* check if at least one of the values to be compressed is enabled */
2685          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2686          if (enabled) {
2687             enabled_channels |= enabled << (i*2);
2688             values[i] = bld.vop3(compr_op, bld.def(v1),
2689                                  values[i*2].isUndefined() ? Operand(0u) : values[i*2],
2690                                  values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1]);
2691          } else {
2692             values[i] = Operand(v1);
2693          }
2694       }
2695       values[2] = Operand(v1);
2696       values[3] = Operand(v1);
2697    } else {
2698       for (int i = 0; i < 4; i++)
2699          values[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2700    }
2701
2702    bld.exp(aco_opcode::exp, values[0], values[1], values[2], values[3],
2703            enabled_channels, target, (bool) compr_op);
2704 }
2705
2706 Operand load_lds_size_m0(isel_context *ctx)
2707 {
2708    /* TODO: m0 does not need to be initialized on GFX9+ */
2709    Builder bld(ctx->program, ctx->block);
2710    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
2711 }
2712
2713 void load_lds(isel_context *ctx, unsigned elem_size_bytes, Temp dst,
2714               Temp address, unsigned base_offset, unsigned align)
2715 {
2716    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2717
2718    Builder bld(ctx->program, ctx->block);
2719
2720    Operand m = load_lds_size_m0(ctx);
2721
2722    unsigned num_components = dst.size() * 4u / elem_size_bytes;
2723    unsigned bytes_read = 0;
2724    unsigned result_size = 0;
2725    unsigned total_bytes = num_components * elem_size_bytes;
2726    std::array<Temp, 4> result;
2727
2728    while (bytes_read < total_bytes) {
2729       unsigned todo = total_bytes - bytes_read;
2730       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
2731       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
2732
2733       aco_opcode op = aco_opcode::last_opcode;
2734       bool read2 = false;
2735       if (todo >= 16 && aligned16) {
2736          op = aco_opcode::ds_read_b128;
2737          todo = 16;
2738       } else if (todo >= 16 && aligned8) {
2739          op = aco_opcode::ds_read2_b64;
2740          read2 = true;
2741          todo = 16;
2742       } else if (todo >= 12 && aligned16) {
2743          op = aco_opcode::ds_read_b96;
2744          todo = 12;
2745       } else if (todo >= 8 && aligned8) {
2746          op = aco_opcode::ds_read_b64;
2747          todo = 8;
2748       } else if (todo >= 8) {
2749          op = aco_opcode::ds_read2_b32;
2750          read2 = true;
2751          todo = 8;
2752       } else if (todo >= 4) {
2753          op = aco_opcode::ds_read_b32;
2754          todo = 4;
2755       } else {
2756          assert(false);
2757       }
2758       assert(todo % elem_size_bytes == 0);
2759       unsigned num_elements = todo / elem_size_bytes;
2760       unsigned offset = base_offset + bytes_read;
2761       unsigned max_offset = read2 ? 1019 : 65535;
2762
2763       Temp address_offset = address;
2764       if (offset > max_offset) {
2765          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2766          offset = bytes_read;
2767       }
2768       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
2769
2770       Temp res;
2771       if (num_components == 1 && dst.type() == RegType::vgpr)
2772          res = dst;
2773       else
2774          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
2775
2776       if (read2)
2777          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
2778       else
2779          res = bld.ds(op, Definition(res), address_offset, m, offset);
2780
2781       if (num_components == 1) {
2782          assert(todo == total_bytes);
2783          if (dst.type() == RegType::sgpr)
2784             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
2785          return;
2786       }
2787
2788       if (dst.type() == RegType::sgpr)
2789          res = bld.as_uniform(res);
2790
2791       if (num_elements == 1) {
2792          result[result_size++] = res;
2793       } else {
2794          assert(res != dst && res.size() % num_elements == 0);
2795          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
2796          split->operands[0] = Operand(res);
2797          for (unsigned i = 0; i < num_elements; i++)
2798             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
2799          ctx->block->instructions.emplace_back(std::move(split));
2800       }
2801
2802       bytes_read += todo;
2803    }
2804
2805    assert(result_size == num_components && result_size > 1);
2806    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
2807    for (unsigned i = 0; i < result_size; i++)
2808       vec->operands[i] = Operand(result[i]);
2809    vec->definitions[0] = Definition(dst);
2810    ctx->block->instructions.emplace_back(std::move(vec));
2811    ctx->allocated_vec.emplace(dst.id(), result);
2812 }
2813
2814 Temp extract_subvector(isel_context *ctx, Temp data, unsigned start, unsigned size, RegType type)
2815 {
2816    if (start == 0 && size == data.size())
2817       return type == RegType::vgpr ? as_vgpr(ctx, data) : data;
2818
2819    unsigned size_hint = 1;
2820    auto it = ctx->allocated_vec.find(data.id());
2821    if (it != ctx->allocated_vec.end())
2822       size_hint = it->second[0].size();
2823    if (size % size_hint || start % size_hint)
2824       size_hint = 1;
2825
2826    start /= size_hint;
2827    size /= size_hint;
2828
2829    Temp elems[size];
2830    for (unsigned i = 0; i < size; i++)
2831       elems[i] = emit_extract_vector(ctx, data, start + i, RegClass(type, size_hint));
2832
2833    if (size == 1)
2834       return type == RegType::vgpr ? as_vgpr(ctx, elems[0]) : elems[0];
2835
2836    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
2837    for (unsigned i = 0; i < size; i++)
2838       vec->operands[i] = Operand(elems[i]);
2839    Temp res = {ctx->program->allocateId(), RegClass(type, size * size_hint)};
2840    vec->definitions[0] = Definition(res);
2841    ctx->block->instructions.emplace_back(std::move(vec));
2842    return res;
2843 }
2844
2845 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned data_start, unsigned total_size, unsigned offset0, unsigned offset1, unsigned align)
2846 {
2847    Builder bld(ctx->program, ctx->block);
2848    unsigned bytes_written = 0;
2849    while (bytes_written < total_size * 4) {
2850       unsigned todo = total_size * 4 - bytes_written;
2851       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
2852       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
2853
2854       aco_opcode op = aco_opcode::last_opcode;
2855       bool write2 = false;
2856       unsigned size = 0;
2857       if (todo >= 16 && aligned16) {
2858          op = aco_opcode::ds_write_b128;
2859          size = 4;
2860       } else if (todo >= 16 && aligned8) {
2861          op = aco_opcode::ds_write2_b64;
2862          write2 = true;
2863          size = 4;
2864       } else if (todo >= 12 && aligned16) {
2865          op = aco_opcode::ds_write_b96;
2866          size = 3;
2867       } else if (todo >= 8 && aligned8) {
2868          op = aco_opcode::ds_write_b64;
2869          size = 2;
2870       } else if (todo >= 8) {
2871          op = aco_opcode::ds_write2_b32;
2872          write2 = true;
2873          size = 2;
2874       } else if (todo >= 4) {
2875          op = aco_opcode::ds_write_b32;
2876          size = 1;
2877       } else {
2878          assert(false);
2879       }
2880
2881       unsigned offset = offset0 + offset1 + bytes_written;
2882       unsigned max_offset = write2 ? 1020 : 65535;
2883       Temp address_offset = address;
2884       if (offset > max_offset) {
2885          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
2886          offset = offset1 + bytes_written;
2887       }
2888       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
2889
2890       if (write2) {
2891          Temp val0 = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size / 2, RegType::vgpr);
2892          Temp val1 = extract_subvector(ctx, data, data_start + (bytes_written >> 2) + 1, size / 2, RegType::vgpr);
2893          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
2894       } else {
2895          Temp val = extract_subvector(ctx, data, data_start + (bytes_written >> 2), size, RegType::vgpr);
2896          bld.ds(op, address_offset, val, m, offset);
2897       }
2898
2899       bytes_written += size * 4;
2900    }
2901 }
2902
2903 void store_lds(isel_context *ctx, unsigned elem_size_bytes, Temp data, uint32_t wrmask,
2904                Temp address, unsigned base_offset, unsigned align)
2905 {
2906    assert(util_is_power_of_two_nonzero(align) && align >= 4);
2907
2908    Operand m = load_lds_size_m0(ctx);
2909
2910    /* we need at most two stores for 32bit variables */
2911    int start[2], count[2];
2912    u_bit_scan_consecutive_range(&wrmask, &start[0], &count[0]);
2913    u_bit_scan_consecutive_range(&wrmask, &start[1], &count[1]);
2914    assert(wrmask == 0);
2915
2916    /* one combined store is sufficient */
2917    if (count[0] == count[1]) {
2918       Builder bld(ctx->program, ctx->block);
2919
2920       Temp address_offset = address;
2921       if ((base_offset >> 2) + start[1] > 255) {
2922          address_offset = bld.vadd32(bld.def(v1), Operand(base_offset), address_offset);
2923          base_offset = 0;
2924       }
2925
2926       assert(count[0] == 1);
2927       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
2928       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
2929       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
2930       base_offset = base_offset / elem_size_bytes;
2931       bld.ds(op, address_offset, val0, val1, m,
2932              base_offset + start[0], base_offset + start[1]);
2933       return;
2934    }
2935
2936    for (unsigned i = 0; i < 2; i++) {
2937       if (count[i] == 0)
2938          continue;
2939
2940       unsigned elem_size_words = elem_size_bytes / 4;
2941       ds_write_helper(ctx, m, address, data, start[i] * elem_size_words, count[i] * elem_size_words,
2942                       base_offset, start[i] * elem_size_bytes, align);
2943    }
2944    return;
2945 }
2946
2947 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2948 {
2949    if (ctx->stage == vertex_vs) {
2950       visit_store_vs_output(ctx, instr);
2951    } else if (ctx->stage == fragment_fs) {
2952       visit_store_fs_output(ctx, instr);
2953    } else {
2954       unreachable("Shader stage not implemented");
2955    }
2956 }
2957
2958 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2959 {
2960    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2961    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2962
2963    Builder bld(ctx->program, ctx->block);
2964    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2965    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2966 }
2967
2968 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2969 {
2970    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2971    for (unsigned i = 0; i < num_components; i++)
2972       vec->operands[i] = Operand(get_arg(ctx, ctx->args->ac.frag_pos[i]));
2973    if (G_0286CC_POS_W_FLOAT_ENA(ctx->program->config->spi_ps_input_ena)) {
2974       assert(num_components == 4);
2975       Builder bld(ctx->program, ctx->block);
2976       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_arg(ctx, ctx->args->ac.frag_pos[3]));
2977    }
2978
2979    for (Operand& op : vec->operands)
2980       op = op.isUndefined() ? Operand(0u) : op;
2981
2982    vec->definitions[0] = Definition(dst);
2983    ctx->block->instructions.emplace_back(std::move(vec));
2984    emit_split_vector(ctx, dst, num_components);
2985    return;
2986 }
2987
2988 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2989 {
2990    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2991    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2992    unsigned idx = nir_intrinsic_base(instr);
2993    unsigned component = nir_intrinsic_component(instr);
2994    Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
2995
2996    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2997    if (offset) {
2998       assert(offset->u32 == 0);
2999    } else {
3000       /* the lower 15bit of the prim_mask contain the offset into LDS
3001        * while the upper bits contain the number of prims */
3002       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
3003       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3004       Builder bld(ctx->program, ctx->block);
3005       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3006       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3007       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3008       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3009       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3010    }
3011
3012    if (instr->dest.ssa.num_components == 1) {
3013       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
3014    } else {
3015       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
3016       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
3017       {
3018          Temp tmp = {ctx->program->allocateId(), v1};
3019          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
3020          vec->operands[i] = Operand(tmp);
3021       }
3022       vec->definitions[0] = Definition(dst);
3023       ctx->block->instructions.emplace_back(std::move(vec));
3024    }
3025 }
3026
3027 unsigned get_num_channels_from_data_format(unsigned data_format)
3028 {
3029    switch (data_format) {
3030    case V_008F0C_BUF_DATA_FORMAT_8:
3031    case V_008F0C_BUF_DATA_FORMAT_16:
3032    case V_008F0C_BUF_DATA_FORMAT_32:
3033       return 1;
3034    case V_008F0C_BUF_DATA_FORMAT_8_8:
3035    case V_008F0C_BUF_DATA_FORMAT_16_16:
3036    case V_008F0C_BUF_DATA_FORMAT_32_32:
3037       return 2;
3038    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
3039    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
3040    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
3041       return 3;
3042    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
3043    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
3044    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
3045    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
3046    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
3047       return 4;
3048    default:
3049       break;
3050    }
3051
3052    return 4;
3053 }
3054
3055 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
3056  * so we may need to fix it up. */
3057 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
3058 {
3059    Builder bld(ctx->program, ctx->block);
3060
3061    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
3062       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
3063
3064    /* For the integer-like cases, do a natural sign extension.
3065     *
3066     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
3067     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
3068     * exponent.
3069     */
3070    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
3071    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
3072
3073    /* Convert back to the right type. */
3074    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
3075       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3076       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0xbf800000u), alpha);
3077       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
3078    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
3079       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
3080    }
3081
3082    return alpha;
3083 }
3084
3085 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
3086 {
3087    Builder bld(ctx->program, ctx->block);
3088    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3089    if (ctx->stage & sw_vs) {
3090
3091       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3092       if (off_instr->type != nir_instr_type_load_const) {
3093          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3094          nir_print_instr(off_instr, stderr);
3095          fprintf(stderr, "\n");
3096       }
3097       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
3098
3099       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->vertex_buffers));
3100
3101       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
3102       unsigned component = nir_intrinsic_component(instr);
3103       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
3104       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
3105       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
3106       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
3107
3108       unsigned dfmt = attrib_format & 0xf;
3109
3110       unsigned nfmt = (attrib_format >> 4) & 0x7;
3111       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
3112       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
3113       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
3114       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
3115       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
3116       if (post_shuffle)
3117          num_channels = MAX2(num_channels, 3);
3118
3119       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
3120
3121       Temp index;
3122       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
3123          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
3124          Temp start_instance = get_arg(ctx, ctx->args->ac.start_instance);
3125          if (divisor) {
3126             ctx->needs_instance_id = true;
3127             Temp instance_id = get_arg(ctx, ctx->args->ac.instance_id);
3128             if (divisor != 1) {
3129                Temp divided = bld.tmp(v1);
3130                emit_v_div_u32(ctx, divided, as_vgpr(ctx, instance_id), divisor);
3131                index = bld.vadd32(bld.def(v1), start_instance, divided);
3132             } else {
3133                index = bld.vadd32(bld.def(v1), start_instance, instance_id);
3134             }
3135          } else {
3136             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), start_instance);
3137          }
3138       } else {
3139          index = bld.vadd32(bld.def(v1),
3140                             get_arg(ctx, ctx->args->ac.base_vertex),
3141                             get_arg(ctx, ctx->args->ac.vertex_id));
3142       }
3143
3144       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
3145          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
3146          attrib_offset = attrib_offset % attrib_stride;
3147       }
3148
3149       Operand soffset(0u);
3150       if (attrib_offset >= 4096) {
3151          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
3152          attrib_offset = 0;
3153       }
3154
3155       aco_opcode opcode;
3156       switch (num_channels) {
3157       case 1:
3158          opcode = aco_opcode::tbuffer_load_format_x;
3159          break;
3160       case 2:
3161          opcode = aco_opcode::tbuffer_load_format_xy;
3162          break;
3163       case 3:
3164          opcode = aco_opcode::tbuffer_load_format_xyz;
3165          break;
3166       case 4:
3167          opcode = aco_opcode::tbuffer_load_format_xyzw;
3168          break;
3169       default:
3170          unreachable("Unimplemented load_input vector size");
3171       }
3172
3173       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
3174
3175       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
3176       mubuf->operands[0] = Operand(index);
3177       mubuf->operands[1] = Operand(list);
3178       mubuf->operands[2] = soffset;
3179       mubuf->definitions[0] = Definition(tmp);
3180       mubuf->idxen = true;
3181       mubuf->can_reorder = true;
3182       mubuf->dfmt = dfmt;
3183       mubuf->nfmt = nfmt;
3184       assert(attrib_offset < 4096);
3185       mubuf->offset = attrib_offset;
3186       ctx->block->instructions.emplace_back(std::move(mubuf));
3187
3188       emit_split_vector(ctx, tmp, tmp.size());
3189
3190       if (tmp.id() != dst.id()) {
3191          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
3192                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
3193
3194          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
3195          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
3196          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
3197
3198          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3199          for (unsigned i = 0; i < dst.size(); i++) {
3200             unsigned idx = i + component;
3201             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
3202                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
3203                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
3204             } else if (idx < num_channels) {
3205                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
3206             } else if (is_float && idx == 3) {
3207                vec->operands[i] = Operand(0x3f800000u);
3208             } else if (!is_float && idx == 3) {
3209                vec->operands[i] = Operand(1u);
3210             } else {
3211                vec->operands[i] = Operand(0u);
3212             }
3213          }
3214          vec->definitions[0] = Definition(dst);
3215          ctx->block->instructions.emplace_back(std::move(vec));
3216          emit_split_vector(ctx, dst, dst.size());
3217       }
3218
3219    } else if (ctx->stage == fragment_fs) {
3220       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
3221       if (off_instr->type != nir_instr_type_load_const ||
3222           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
3223          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
3224          nir_print_instr(off_instr, stderr);
3225          fprintf(stderr, "\n");
3226       }
3227
3228       Temp prim_mask = get_arg(ctx, ctx->args->ac.prim_mask);
3229       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
3230       if (offset) {
3231          assert(offset->u32 == 0);
3232       } else {
3233          /* the lower 15bit of the prim_mask contain the offset into LDS
3234           * while the upper bits contain the number of prims */
3235          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
3236          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
3237          Builder bld(ctx->program, ctx->block);
3238          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
3239          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
3240          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
3241          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
3242          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
3243       }
3244
3245       unsigned idx = nir_intrinsic_base(instr);
3246       unsigned component = nir_intrinsic_component(instr);
3247
3248       if (dst.size() == 1) {
3249          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
3250       } else {
3251          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
3252          for (unsigned i = 0; i < dst.size(); i++)
3253             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
3254          vec->definitions[0] = Definition(dst);
3255          bld.insert(std::move(vec));
3256       }
3257
3258    } else {
3259       unreachable("Shader stage not implemented");
3260    }
3261 }
3262
3263 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
3264 {
3265    if (ctx->program->info->need_indirect_descriptor_sets) {
3266       Builder bld(ctx->program, ctx->block);
3267       Temp ptr64 = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->descriptor_sets[0]));
3268       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
3269    }
3270
3271    return get_arg(ctx, ctx->args->descriptor_sets[desc_set]);
3272 }
3273
3274
3275 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
3276 {
3277    Builder bld(ctx->program, ctx->block);
3278    Temp index = get_ssa_temp(ctx, instr->src[0].ssa);
3279    if (!ctx->divergent_vals[instr->dest.ssa.index])
3280       index = bld.as_uniform(index);
3281    unsigned desc_set = nir_intrinsic_desc_set(instr);
3282    unsigned binding = nir_intrinsic_binding(instr);
3283
3284    Temp desc_ptr;
3285    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
3286    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
3287    unsigned offset = layout->binding[binding].offset;
3288    unsigned stride;
3289    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
3290        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
3291       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
3292       desc_ptr = get_arg(ctx, ctx->args->ac.push_constants);
3293       offset = pipeline_layout->push_constant_size + 16 * idx;
3294       stride = 16;
3295    } else {
3296       desc_ptr = load_desc_ptr(ctx, desc_set);
3297       stride = layout->binding[binding].size;
3298    }
3299
3300    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
3301    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
3302    if (stride != 1) {
3303       if (nir_const_index) {
3304          const_index = const_index * stride;
3305       } else if (index.type() == RegType::vgpr) {
3306          bool index24bit = layout->binding[binding].array_size <= 0x1000000;
3307          index = bld.v_mul_imm(bld.def(v1), index, stride, index24bit);
3308       } else {
3309          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
3310       }
3311    }
3312    if (offset) {
3313       if (nir_const_index) {
3314          const_index = const_index + offset;
3315       } else if (index.type() == RegType::vgpr) {
3316          index = bld.vadd32(bld.def(v1), Operand(offset), index);
3317       } else {
3318          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
3319       }
3320    }
3321
3322    if (nir_const_index && const_index == 0) {
3323       index = desc_ptr;
3324    } else if (index.type() == RegType::vgpr) {
3325       index = bld.vadd32(bld.def(v1),
3326                          nir_const_index ? Operand(const_index) : Operand(index),
3327                          Operand(desc_ptr));
3328    } else {
3329       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3330                        nir_const_index ? Operand(const_index) : Operand(index),
3331                        Operand(desc_ptr));
3332    }
3333
3334    bld.copy(Definition(get_ssa_temp(ctx, &instr->dest.ssa)), index);
3335 }
3336
3337 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst,
3338                  Temp rsrc, Temp offset, bool glc=false, bool readonly=true)
3339 {
3340    Builder bld(ctx->program, ctx->block);
3341
3342    unsigned num_bytes = dst.size() * 4;
3343    bool dlc = glc && ctx->options->chip_class >= GFX10;
3344
3345    aco_opcode op;
3346    if (dst.type() == RegType::vgpr || (ctx->options->chip_class < GFX8 && !readonly)) {
3347       if (ctx->options->chip_class < GFX8)
3348          offset = as_vgpr(ctx, offset);
3349
3350       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3351       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3352       unsigned const_offset = 0;
3353
3354       Temp lower = Temp();
3355       if (num_bytes > 16) {
3356          assert(num_components == 3 || num_components == 4);
3357          op = aco_opcode::buffer_load_dwordx4;
3358          lower = bld.tmp(v4);
3359          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3360          mubuf->definitions[0] = Definition(lower);
3361          mubuf->operands[0] = vaddr;
3362          mubuf->operands[1] = Operand(rsrc);
3363          mubuf->operands[2] = soffset;
3364          mubuf->offen = (offset.type() == RegType::vgpr);
3365          mubuf->glc = glc;
3366          mubuf->dlc = dlc;
3367          mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3368          mubuf->can_reorder = readonly;
3369          bld.insert(std::move(mubuf));
3370          emit_split_vector(ctx, lower, 2);
3371          num_bytes -= 16;
3372          const_offset = 16;
3373       }
3374
3375       switch (num_bytes) {
3376          case 4:
3377             op = aco_opcode::buffer_load_dword;
3378             break;
3379          case 8:
3380             op = aco_opcode::buffer_load_dwordx2;
3381             break;
3382          case 12:
3383             op = aco_opcode::buffer_load_dwordx3;
3384             break;
3385          case 16:
3386             op = aco_opcode::buffer_load_dwordx4;
3387             break;
3388          default:
3389             unreachable("Load SSBO not implemented for this size.");
3390       }
3391       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3392       mubuf->operands[0] = vaddr;
3393       mubuf->operands[1] = Operand(rsrc);
3394       mubuf->operands[2] = soffset;
3395       mubuf->offen = (offset.type() == RegType::vgpr);
3396       mubuf->glc = glc;
3397       mubuf->dlc = dlc;
3398       mubuf->barrier = readonly ? barrier_none : barrier_buffer;
3399       mubuf->can_reorder = readonly;
3400       mubuf->offset = const_offset;
3401       aco_ptr<Instruction> instr = std::move(mubuf);
3402
3403       if (dst.size() > 4) {
3404          assert(lower != Temp());
3405          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3406          instr->definitions[0] = Definition(upper);
3407          bld.insert(std::move(instr));
3408          if (dst.size() == 8)
3409             emit_split_vector(ctx, upper, 2);
3410          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3411          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3412          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3413          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3414          if (dst.size() == 8)
3415             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3416       }
3417
3418       if (dst.type() == RegType::sgpr) {
3419          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3420          instr->definitions[0] = Definition(vec);
3421          bld.insert(std::move(instr));
3422          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3423       } else {
3424          instr->definitions[0] = Definition(dst);
3425          bld.insert(std::move(instr));
3426       }
3427    } else {
3428       switch (num_bytes) {
3429          case 4:
3430             op = aco_opcode::s_buffer_load_dword;
3431             break;
3432          case 8:
3433             op = aco_opcode::s_buffer_load_dwordx2;
3434             break;
3435          case 12:
3436          case 16:
3437             op = aco_opcode::s_buffer_load_dwordx4;
3438             break;
3439          case 24:
3440          case 32:
3441             op = aco_opcode::s_buffer_load_dwordx8;
3442             break;
3443          default:
3444             unreachable("Load SSBO not implemented for this size.");
3445       }
3446       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3447       load->operands[0] = Operand(rsrc);
3448       load->operands[1] = Operand(bld.as_uniform(offset));
3449       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3450       load->definitions[0] = Definition(dst);
3451       load->glc = glc;
3452       load->dlc = dlc;
3453       load->barrier = readonly ? barrier_none : barrier_buffer;
3454       load->can_reorder = false; // FIXME: currently, it doesn't seem beneficial due to how our scheduler works
3455       assert(ctx->options->chip_class >= GFX8 || !glc);
3456
3457       /* trim vector */
3458       if (dst.size() == 3) {
3459          Temp vec = bld.tmp(s4);
3460          load->definitions[0] = Definition(vec);
3461          bld.insert(std::move(load));
3462          emit_split_vector(ctx, vec, 4);
3463
3464          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3465                     emit_extract_vector(ctx, vec, 0, s1),
3466                     emit_extract_vector(ctx, vec, 1, s1),
3467                     emit_extract_vector(ctx, vec, 2, s1));
3468       } else if (dst.size() == 6) {
3469          Temp vec = bld.tmp(s8);
3470          load->definitions[0] = Definition(vec);
3471          bld.insert(std::move(load));
3472          emit_split_vector(ctx, vec, 4);
3473
3474          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3475                     emit_extract_vector(ctx, vec, 0, s2),
3476                     emit_extract_vector(ctx, vec, 1, s2),
3477                     emit_extract_vector(ctx, vec, 2, s2));
3478       } else {
3479          bld.insert(std::move(load));
3480       }
3481
3482    }
3483    emit_split_vector(ctx, dst, num_components);
3484 }
3485
3486 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3487 {
3488    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3489    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3490
3491    Builder bld(ctx->program, ctx->block);
3492
3493    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3494    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3495    unsigned binding = nir_intrinsic_binding(idx_instr);
3496    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3497
3498    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3499       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3500                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3501                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3502                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3503       if (ctx->options->chip_class >= GFX10) {
3504          desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3505                       S_008F0C_OOB_SELECT(3) |
3506                       S_008F0C_RESOURCE_LEVEL(1);
3507       } else {
3508          desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3509                       S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3510       }
3511       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3512                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3513                                      Operand(0xFFFFFFFFu),
3514                                      Operand(desc_type));
3515       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3516                         rsrc, upper_dwords);
3517    } else {
3518       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3519       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3520    }
3521
3522    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3523 }
3524
3525 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3526 {
3527    Builder bld(ctx->program, ctx->block);
3528    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3529
3530    unsigned offset = nir_intrinsic_base(instr);
3531    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3532    if (index_cv && instr->dest.ssa.bit_size == 32) {
3533
3534       unsigned count = instr->dest.ssa.num_components;
3535       unsigned start = (offset + index_cv->u32) / 4u;
3536       start -= ctx->args->ac.base_inline_push_consts;
3537       if (start + count <= ctx->args->ac.num_inline_push_consts) {
3538          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3539          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3540          for (unsigned i = 0; i < count; ++i) {
3541             elems[i] = get_arg(ctx, ctx->args->ac.inline_push_consts[start + i]);
3542             vec->operands[i] = Operand{elems[i]};
3543          }
3544          vec->definitions[0] = Definition(dst);
3545          ctx->block->instructions.emplace_back(std::move(vec));
3546          ctx->allocated_vec.emplace(dst.id(), elems);
3547          return;
3548       }
3549    }
3550
3551    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3552    if (offset != 0) // TODO check if index != 0 as well
3553       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3554    Temp ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->ac.push_constants));
3555    Temp vec = dst;
3556    bool trim = false;
3557    aco_opcode op;
3558
3559    switch (dst.size()) {
3560    case 1:
3561       op = aco_opcode::s_load_dword;
3562       break;
3563    case 2:
3564       op = aco_opcode::s_load_dwordx2;
3565       break;
3566    case 3:
3567       vec = bld.tmp(s4);
3568       trim = true;
3569    case 4:
3570       op = aco_opcode::s_load_dwordx4;
3571       break;
3572    case 6:
3573       vec = bld.tmp(s8);
3574       trim = true;
3575    case 8:
3576       op = aco_opcode::s_load_dwordx8;
3577       break;
3578    default:
3579       unreachable("unimplemented or forbidden load_push_constant.");
3580    }
3581
3582    bld.smem(op, Definition(vec), ptr, index);
3583
3584    if (trim) {
3585       emit_split_vector(ctx, vec, 4);
3586       RegClass rc = dst.size() == 3 ? s1 : s2;
3587       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3588                  emit_extract_vector(ctx, vec, 0, rc),
3589                  emit_extract_vector(ctx, vec, 1, rc),
3590                  emit_extract_vector(ctx, vec, 2, rc));
3591
3592    }
3593    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3594 }
3595
3596 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3597 {
3598    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3599
3600    Builder bld(ctx->program, ctx->block);
3601
3602    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3603                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3604                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3605                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3606    if (ctx->options->chip_class >= GFX10) {
3607       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3608                    S_008F0C_OOB_SELECT(3) |
3609                    S_008F0C_RESOURCE_LEVEL(1);
3610    } else {
3611       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3612                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3613    }
3614
3615    unsigned base = nir_intrinsic_base(instr);
3616    unsigned range = nir_intrinsic_range(instr);
3617
3618    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3619    if (base && offset.type() == RegType::sgpr)
3620       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3621    else if (base && offset.type() == RegType::vgpr)
3622       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3623
3624    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3625                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(ctx->constant_data_offset)),
3626                           Operand(MIN2(base + range, ctx->shader->constant_data_size)),
3627                           Operand(desc_type));
3628
3629    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3630 }
3631
3632 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3633 {
3634    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3635       ctx->cf_info.exec_potentially_empty = true;
3636
3637    ctx->program->needs_exact = true;
3638
3639    // TODO: optimize uniform conditions
3640    Builder bld(ctx->program, ctx->block);
3641    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
3642    assert(src.regClass() == bld.lm);
3643    src = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
3644    bld.pseudo(aco_opcode::p_discard_if, src);
3645    ctx->block->kind |= block_kind_uses_discard_if;
3646    return;
3647 }
3648
3649 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3650 {
3651    Builder bld(ctx->program, ctx->block);
3652
3653    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3654       ctx->cf_info.exec_potentially_empty = true;
3655
3656    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3657                     ctx->cf_info.parent_loop.has_divergent_continue;
3658
3659    if (ctx->block->loop_nest_depth &&
3660        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3661       /* we handle discards the same way as jump instructions */
3662       append_logical_end(ctx->block);
3663
3664       /* in loops, discard behaves like break */
3665       Block *linear_target = ctx->cf_info.parent_loop.exit;
3666       ctx->block->kind |= block_kind_discard;
3667
3668       if (!divergent) {
3669          /* uniform discard - loop ends here */
3670          assert(nir_instr_is_last(&instr->instr));
3671          ctx->block->kind |= block_kind_uniform;
3672          ctx->cf_info.has_branch = true;
3673          bld.branch(aco_opcode::p_branch);
3674          add_linear_edge(ctx->block->index, linear_target);
3675          return;
3676       }
3677
3678       /* we add a break right behind the discard() instructions */
3679       ctx->block->kind |= block_kind_break;
3680       unsigned idx = ctx->block->index;
3681
3682       /* remove critical edges from linear CFG */
3683       bld.branch(aco_opcode::p_branch);
3684       Block* break_block = ctx->program->create_and_insert_block();
3685       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3686       break_block->kind |= block_kind_uniform;
3687       add_linear_edge(idx, break_block);
3688       add_linear_edge(break_block->index, linear_target);
3689       bld.reset(break_block);
3690       bld.branch(aco_opcode::p_branch);
3691
3692       Block* continue_block = ctx->program->create_and_insert_block();
3693       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3694       add_linear_edge(idx, continue_block);
3695       append_logical_start(continue_block);
3696       ctx->block = continue_block;
3697
3698       return;
3699    }
3700
3701    /* it can currently happen that NIR doesn't remove the unreachable code */
3702    if (!nir_instr_is_last(&instr->instr)) {
3703       ctx->program->needs_exact = true;
3704       /* save exec somewhere temporarily so that it doesn't get
3705        * overwritten before the discard from outer exec masks */
3706       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, bld.lm));
3707       bld.pseudo(aco_opcode::p_discard_if, cond);
3708       ctx->block->kind |= block_kind_uses_discard_if;
3709       return;
3710    }
3711
3712    /* This condition is incorrect for uniformly branched discards in a loop
3713     * predicated by a divergent condition, but the above code catches that case
3714     * and the discard would end up turning into a discard_if.
3715     * For example:
3716     * if (divergent) {
3717     *    while (...) {
3718     *       if (uniform) {
3719     *          discard;
3720     *       }
3721     *    }
3722     * }
3723     */
3724    if (!ctx->cf_info.parent_if.is_divergent) {
3725       /* program just ends here */
3726       ctx->block->kind |= block_kind_uniform;
3727       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3728               0 /* enabled mask */, 9 /* dest */,
3729               false /* compressed */, true/* done */, true /* valid mask */);
3730       bld.sopp(aco_opcode::s_endpgm);
3731       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3732    } else {
3733       ctx->block->kind |= block_kind_discard;
3734       /* branch and linear edge is added by visit_if() */
3735    }
3736 }
3737
3738 enum aco_descriptor_type {
3739    ACO_DESC_IMAGE,
3740    ACO_DESC_FMASK,
3741    ACO_DESC_SAMPLER,
3742    ACO_DESC_BUFFER,
3743    ACO_DESC_PLANE_0,
3744    ACO_DESC_PLANE_1,
3745    ACO_DESC_PLANE_2,
3746 };
3747
3748 static bool
3749 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3750    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3751       return false;
3752    ac_image_dim dim = ac_get_sampler_dim(ctx->options->chip_class, sampler_dim, is_array);
3753    return dim == ac_image_cube ||
3754           dim == ac_image_1darray ||
3755           dim == ac_image_2darray ||
3756           dim == ac_image_2darraymsaa;
3757 }
3758
3759 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3760                       enum aco_descriptor_type desc_type,
3761                       const nir_tex_instr *tex_instr, bool image, bool write)
3762 {
3763 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3764    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3765    if (it != ctx->tex_desc.end())
3766       return it->second;
3767 */
3768    Temp index = Temp();
3769    bool index_set = false;
3770    unsigned constant_index = 0;
3771    unsigned descriptor_set;
3772    unsigned base_index;
3773    Builder bld(ctx->program, ctx->block);
3774
3775    if (!deref_instr) {
3776       assert(tex_instr && !image);
3777       descriptor_set = 0;
3778       base_index = tex_instr->sampler_index;
3779    } else {
3780       while(deref_instr->deref_type != nir_deref_type_var) {
3781          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3782          if (!array_size)
3783             array_size = 1;
3784
3785          assert(deref_instr->deref_type == nir_deref_type_array);
3786          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3787          if (const_value) {
3788             constant_index += array_size * const_value->u32;
3789          } else {
3790             Temp indirect = get_ssa_temp(ctx, deref_instr->arr.index.ssa);
3791             if (indirect.type() == RegType::vgpr)
3792                indirect = bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), indirect);
3793
3794             if (array_size != 1)
3795                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3796
3797             if (!index_set) {
3798                index = indirect;
3799                index_set = true;
3800             } else {
3801                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3802             }
3803          }
3804
3805          deref_instr = nir_src_as_deref(deref_instr->parent);
3806       }
3807       descriptor_set = deref_instr->var->data.descriptor_set;
3808       base_index = deref_instr->var->data.binding;
3809    }
3810
3811    Temp list = load_desc_ptr(ctx, descriptor_set);
3812    list = convert_pointer_to_64_bit(ctx, list);
3813
3814    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3815    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3816    unsigned offset = binding->offset;
3817    unsigned stride = binding->size;
3818    aco_opcode opcode;
3819    RegClass type;
3820
3821    assert(base_index < layout->binding_count);
3822
3823    switch (desc_type) {
3824    case ACO_DESC_IMAGE:
3825       type = s8;
3826       opcode = aco_opcode::s_load_dwordx8;
3827       break;
3828    case ACO_DESC_FMASK:
3829       type = s8;
3830       opcode = aco_opcode::s_load_dwordx8;
3831       offset += 32;
3832       break;
3833    case ACO_DESC_SAMPLER:
3834       type = s4;
3835       opcode = aco_opcode::s_load_dwordx4;
3836       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3837          offset += radv_combined_image_descriptor_sampler_offset(binding);
3838       break;
3839    case ACO_DESC_BUFFER:
3840       type = s4;
3841       opcode = aco_opcode::s_load_dwordx4;
3842       break;
3843    case ACO_DESC_PLANE_0:
3844    case ACO_DESC_PLANE_1:
3845       type = s8;
3846       opcode = aco_opcode::s_load_dwordx8;
3847       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3848       break;
3849    case ACO_DESC_PLANE_2:
3850       type = s4;
3851       opcode = aco_opcode::s_load_dwordx4;
3852       offset += 64;
3853       break;
3854    default:
3855       unreachable("invalid desc_type\n");
3856    }
3857
3858    offset += constant_index * stride;
3859
3860    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3861       (!index_set || binding->immutable_samplers_equal)) {
3862       if (binding->immutable_samplers_equal)
3863          constant_index = 0;
3864
3865       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3866       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3867                         Operand(samplers[constant_index * 4 + 0]),
3868                         Operand(samplers[constant_index * 4 + 1]),
3869                         Operand(samplers[constant_index * 4 + 2]),
3870                         Operand(samplers[constant_index * 4 + 3]));
3871    }
3872
3873    Operand off;
3874    if (!index_set) {
3875       off = Operand(offset);
3876    } else {
3877       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3878                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3879    }
3880
3881    Temp res = bld.smem(opcode, bld.def(type), list, off);
3882
3883    if (desc_type == ACO_DESC_PLANE_2) {
3884       Temp components[8];
3885       for (unsigned i = 0; i < 8; i++)
3886          components[i] = bld.tmp(s1);
3887       bld.pseudo(aco_opcode::p_split_vector,
3888                  Definition(components[0]),
3889                  Definition(components[1]),
3890                  Definition(components[2]),
3891                  Definition(components[3]),
3892                  res);
3893
3894       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3895       bld.pseudo(aco_opcode::p_split_vector,
3896                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3897                  Definition(components[4]),
3898                  Definition(components[5]),
3899                  Definition(components[6]),
3900                  Definition(components[7]),
3901                  desc2);
3902
3903       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3904                        components[0], components[1], components[2], components[3],
3905                        components[4], components[5], components[6], components[7]);
3906    }
3907
3908    return res;
3909 }
3910
3911 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3912 {
3913    switch (dim) {
3914    case GLSL_SAMPLER_DIM_BUF:
3915       return 1;
3916    case GLSL_SAMPLER_DIM_1D:
3917       return array ? 2 : 1;
3918    case GLSL_SAMPLER_DIM_2D:
3919       return array ? 3 : 2;
3920    case GLSL_SAMPLER_DIM_MS:
3921       return array ? 4 : 3;
3922    case GLSL_SAMPLER_DIM_3D:
3923    case GLSL_SAMPLER_DIM_CUBE:
3924       return 3;
3925    case GLSL_SAMPLER_DIM_RECT:
3926    case GLSL_SAMPLER_DIM_SUBPASS:
3927       return 2;
3928    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3929       return 3;
3930    default:
3931       break;
3932    }
3933    return 0;
3934 }
3935
3936
3937 /* Adjust the sample index according to FMASK.
3938  *
3939  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3940  * which is the identity mapping. Each nibble says which physical sample
3941  * should be fetched to get that sample.
3942  *
3943  * For example, 0x11111100 means there are only 2 samples stored and
3944  * the second sample covers 3/4 of the pixel. When reading samples 0
3945  * and 1, return physical sample 0 (determined by the first two 0s
3946  * in FMASK), otherwise return physical sample 1.
3947  *
3948  * The sample index should be adjusted as follows:
3949  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3950  */
3951 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3952 {
3953    Builder bld(ctx->program, ctx->block);
3954    Temp fmask = bld.tmp(v1);
3955    unsigned dim = ctx->options->chip_class >= GFX10
3956                   ? ac_get_sampler_dim(ctx->options->chip_class, GLSL_SAMPLER_DIM_2D, da)
3957                   : 0;
3958
3959    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3960    load->operands[0] = Operand(coords);
3961    load->operands[1] = Operand(fmask_desc_ptr);
3962    load->definitions[0] = Definition(fmask);
3963    load->glc = false;
3964    load->dlc = false;
3965    load->dmask = 0x1;
3966    load->unrm = true;
3967    load->da = da;
3968    load->dim = dim;
3969    load->can_reorder = true; /* fmask images shouldn't be modified */
3970    ctx->block->instructions.emplace_back(std::move(load));
3971
3972    Operand sample_index4;
3973    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3974       sample_index4 = Operand(sample_index.constantValue() << 2);
3975    } else if (sample_index.regClass() == s1) {
3976       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3977    } else {
3978       assert(sample_index.regClass() == v1);
3979       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3980    }
3981
3982    Temp final_sample;
3983    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3984       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3985    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3986       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3987    else
3988       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3989
3990    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3991     * resource descriptor is 0 (invalid),
3992     */
3993    Temp compare = bld.tmp(bld.lm);
3994    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3995                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3996
3997    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3998
3999    /* Replace the MSAA sample index. */
4000    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
4001 }
4002
4003 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
4004 {
4005
4006    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
4007    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4008    bool is_array = glsl_sampler_type_is_array(type);
4009    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4010    assert(!add_frag_pos && "Input attachments should be lowered.");
4011    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
4012    bool gfx9_1d = ctx->options->chip_class == GFX9 && dim == GLSL_SAMPLER_DIM_1D;
4013    int count = image_type_to_components_count(dim, is_array);
4014    std::vector<Operand> coords(count);
4015
4016    if (is_ms) {
4017       Operand sample_index;
4018       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
4019       if (sample_cv)
4020          sample_index = Operand(sample_cv->u32);
4021       else
4022          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
4023
4024       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
4025          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
4026          for (unsigned i = 0; i < vec->operands.size(); i++)
4027             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4028          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
4029          vec->definitions[0] = Definition(fmask_load_address);
4030          ctx->block->instructions.emplace_back(std::move(vec));
4031
4032          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
4033          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
4034       }
4035       count--;
4036       coords[count] = sample_index;
4037    }
4038
4039    if (count == 1 && !gfx9_1d)
4040       return emit_extract_vector(ctx, src0, 0, v1);
4041
4042    if (gfx9_1d) {
4043       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
4044       coords.resize(coords.size() + 1);
4045       coords[1] = Operand((uint32_t) 0);
4046       if (is_array)
4047          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
4048    } else {
4049       for (int i = 0; i < count; i++)
4050          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
4051    }
4052
4053    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
4054    for (unsigned i = 0; i < coords.size(); i++)
4055       vec->operands[i] = coords[i];
4056    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
4057    vec->definitions[0] = Definition(res);
4058    ctx->block->instructions.emplace_back(std::move(vec));
4059    return res;
4060 }
4061
4062
4063 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
4064 {
4065    Builder bld(ctx->program, ctx->block);
4066    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4067    const struct glsl_type *type = glsl_without_array(var->type);
4068    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4069    bool is_array = glsl_sampler_type_is_array(type);
4070    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4071
4072    if (dim == GLSL_SAMPLER_DIM_BUF) {
4073       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
4074       unsigned num_channels = util_last_bit(mask);
4075       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4076       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4077
4078       aco_opcode opcode;
4079       switch (num_channels) {
4080       case 1:
4081          opcode = aco_opcode::buffer_load_format_x;
4082          break;
4083       case 2:
4084          opcode = aco_opcode::buffer_load_format_xy;
4085          break;
4086       case 3:
4087          opcode = aco_opcode::buffer_load_format_xyz;
4088          break;
4089       case 4:
4090          opcode = aco_opcode::buffer_load_format_xyzw;
4091          break;
4092       default:
4093          unreachable(">4 channel buffer image load");
4094       }
4095       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
4096       load->operands[0] = Operand(vindex);
4097       load->operands[1] = Operand(rsrc);
4098       load->operands[2] = Operand((uint32_t) 0);
4099       Temp tmp;
4100       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4101          tmp = dst;
4102       else
4103          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
4104       load->definitions[0] = Definition(tmp);
4105       load->idxen = true;
4106       load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT);
4107       load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4108       load->barrier = barrier_image;
4109       ctx->block->instructions.emplace_back(std::move(load));
4110
4111       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
4112       return;
4113    }
4114
4115    Temp coords = get_image_coords(ctx, instr, type);
4116    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4117
4118    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
4119    unsigned num_components = util_bitcount(dmask);
4120    Temp tmp;
4121    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
4122       tmp = dst;
4123    else
4124       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
4125
4126    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
4127    load->operands[0] = Operand(coords);
4128    load->operands[1] = Operand(resource);
4129    load->definitions[0] = Definition(tmp);
4130    load->glc = var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
4131    load->dlc = load->glc && ctx->options->chip_class >= GFX10;
4132    load->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4133    load->dmask = dmask;
4134    load->unrm = true;
4135    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4136    load->barrier = barrier_image;
4137    ctx->block->instructions.emplace_back(std::move(load));
4138
4139    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
4140    return;
4141 }
4142
4143 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
4144 {
4145    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4146    const struct glsl_type *type = glsl_without_array(var->type);
4147    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4148    bool is_array = glsl_sampler_type_is_array(type);
4149    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4150
4151    bool glc = ctx->options->chip_class == GFX6 || var->data.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
4152
4153    if (dim == GLSL_SAMPLER_DIM_BUF) {
4154       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4155       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4156       aco_opcode opcode;
4157       switch (data.size()) {
4158       case 1:
4159          opcode = aco_opcode::buffer_store_format_x;
4160          break;
4161       case 2:
4162          opcode = aco_opcode::buffer_store_format_xy;
4163          break;
4164       case 3:
4165          opcode = aco_opcode::buffer_store_format_xyz;
4166          break;
4167       case 4:
4168          opcode = aco_opcode::buffer_store_format_xyzw;
4169          break;
4170       default:
4171          unreachable(">4 channel buffer image store");
4172       }
4173       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
4174       store->operands[0] = Operand(vindex);
4175       store->operands[1] = Operand(rsrc);
4176       store->operands[2] = Operand((uint32_t) 0);
4177       store->operands[3] = Operand(data);
4178       store->idxen = true;
4179       store->glc = glc;
4180       store->dlc = false;
4181       store->disable_wqm = true;
4182       store->barrier = barrier_image;
4183       ctx->program->needs_exact = true;
4184       ctx->block->instructions.emplace_back(std::move(store));
4185       return;
4186    }
4187
4188    assert(data.type() == RegType::vgpr);
4189    Temp coords = get_image_coords(ctx, instr, type);
4190    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4191
4192    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
4193    store->operands[0] = Operand(coords);
4194    store->operands[1] = Operand(resource);
4195    store->operands[2] = Operand(s4);
4196    store->operands[3] = Operand(data);
4197    store->glc = glc;
4198    store->dlc = false;
4199    store->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4200    store->dmask = (1 << data.size()) - 1;
4201    store->unrm = true;
4202    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4203    store->disable_wqm = true;
4204    store->barrier = barrier_image;
4205    ctx->program->needs_exact = true;
4206    ctx->block->instructions.emplace_back(std::move(store));
4207    return;
4208 }
4209
4210 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4211 {
4212    /* return the previous value if dest is ever used */
4213    bool return_previous = false;
4214    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4215       return_previous = true;
4216       break;
4217    }
4218    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4219       return_previous = true;
4220       break;
4221    }
4222
4223    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4224    const struct glsl_type *type = glsl_without_array(var->type);
4225    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4226    bool is_array = glsl_sampler_type_is_array(type);
4227    Builder bld(ctx->program, ctx->block);
4228
4229    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
4230    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
4231
4232    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
4233       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
4234
4235    aco_opcode buf_op, image_op;
4236    switch (instr->intrinsic) {
4237       case nir_intrinsic_image_deref_atomic_add:
4238          buf_op = aco_opcode::buffer_atomic_add;
4239          image_op = aco_opcode::image_atomic_add;
4240          break;
4241       case nir_intrinsic_image_deref_atomic_umin:
4242          buf_op = aco_opcode::buffer_atomic_umin;
4243          image_op = aco_opcode::image_atomic_umin;
4244          break;
4245       case nir_intrinsic_image_deref_atomic_imin:
4246          buf_op = aco_opcode::buffer_atomic_smin;
4247          image_op = aco_opcode::image_atomic_smin;
4248          break;
4249       case nir_intrinsic_image_deref_atomic_umax:
4250          buf_op = aco_opcode::buffer_atomic_umax;
4251          image_op = aco_opcode::image_atomic_umax;
4252          break;
4253       case nir_intrinsic_image_deref_atomic_imax:
4254          buf_op = aco_opcode::buffer_atomic_smax;
4255          image_op = aco_opcode::image_atomic_smax;
4256          break;
4257       case nir_intrinsic_image_deref_atomic_and:
4258          buf_op = aco_opcode::buffer_atomic_and;
4259          image_op = aco_opcode::image_atomic_and;
4260          break;
4261       case nir_intrinsic_image_deref_atomic_or:
4262          buf_op = aco_opcode::buffer_atomic_or;
4263          image_op = aco_opcode::image_atomic_or;
4264          break;
4265       case nir_intrinsic_image_deref_atomic_xor:
4266          buf_op = aco_opcode::buffer_atomic_xor;
4267          image_op = aco_opcode::image_atomic_xor;
4268          break;
4269       case nir_intrinsic_image_deref_atomic_exchange:
4270          buf_op = aco_opcode::buffer_atomic_swap;
4271          image_op = aco_opcode::image_atomic_swap;
4272          break;
4273       case nir_intrinsic_image_deref_atomic_comp_swap:
4274          buf_op = aco_opcode::buffer_atomic_cmpswap;
4275          image_op = aco_opcode::image_atomic_cmpswap;
4276          break;
4277       default:
4278          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
4279    }
4280
4281    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4282
4283    if (dim == GLSL_SAMPLER_DIM_BUF) {
4284       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
4285       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
4286       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
4287       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4288       mubuf->operands[0] = Operand(vindex);
4289       mubuf->operands[1] = Operand(resource);
4290       mubuf->operands[2] = Operand((uint32_t)0);
4291       mubuf->operands[3] = Operand(data);
4292       if (return_previous)
4293          mubuf->definitions[0] = Definition(dst);
4294       mubuf->offset = 0;
4295       mubuf->idxen = true;
4296       mubuf->glc = return_previous;
4297       mubuf->dlc = false; /* Not needed for atomics */
4298       mubuf->disable_wqm = true;
4299       mubuf->barrier = barrier_image;
4300       ctx->program->needs_exact = true;
4301       ctx->block->instructions.emplace_back(std::move(mubuf));
4302       return;
4303    }
4304
4305    Temp coords = get_image_coords(ctx, instr, type);
4306    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
4307    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
4308    mimg->operands[0] = Operand(coords);
4309    mimg->operands[1] = Operand(resource);
4310    mimg->operands[2] = Operand(s4); /* no sampler */
4311    mimg->operands[3] = Operand(data);
4312    if (return_previous)
4313       mimg->definitions[0] = Definition(dst);
4314    mimg->glc = return_previous;
4315    mimg->dlc = false; /* Not needed for atomics */
4316    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4317    mimg->dmask = (1 << data.size()) - 1;
4318    mimg->unrm = true;
4319    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
4320    mimg->disable_wqm = true;
4321    mimg->barrier = barrier_image;
4322    ctx->program->needs_exact = true;
4323    ctx->block->instructions.emplace_back(std::move(mimg));
4324    return;
4325 }
4326
4327 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4328 {
4329    if (in_elements && ctx->options->chip_class == GFX8) {
4330       Builder bld(ctx->program, ctx->block);
4331
4332       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4333       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4334       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4335       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4336
4337       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4338       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4339
4340       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4341       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4342       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4343
4344       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4345       /* idea
4346        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4347        * in case 12 (or 3?), we have to divide by 3:
4348        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4349        * use v_mul_hi_u32 with magic number to divide
4350        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4351        * disable v_skip
4352        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4353        */
4354
4355    } else {
4356       emit_extract_vector(ctx, desc, 2, dst);
4357    }
4358 }
4359
4360 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4361 {
4362    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4363    const struct glsl_type *type = glsl_without_array(var->type);
4364    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
4365    bool is_array = glsl_sampler_type_is_array(type);
4366    Builder bld(ctx->program, ctx->block);
4367
4368    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4369       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4370       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4371    }
4372
4373    /* LOD */
4374    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4375
4376    /* Resource */
4377    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4378
4379    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4380
4381    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4382    mimg->operands[0] = Operand(lod);
4383    mimg->operands[1] = Operand(resource);
4384    unsigned& dmask = mimg->dmask;
4385    mimg->dim = ac_get_image_dim(ctx->options->chip_class, dim, is_array);
4386    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4387    mimg->da = glsl_sampler_type_is_array(type);
4388    mimg->can_reorder = true;
4389    Definition& def = mimg->definitions[0];
4390    ctx->block->instructions.emplace_back(std::move(mimg));
4391
4392    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4393        glsl_sampler_type_is_array(type)) {
4394
4395       assert(instr->dest.ssa.num_components == 3);
4396       Temp tmp = {ctx->program->allocateId(), v3};
4397       def = Definition(tmp);
4398       emit_split_vector(ctx, tmp, 3);
4399
4400       /* divide 3rd value by 6 by multiplying with magic number */
4401       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4402       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4403
4404       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4405                  emit_extract_vector(ctx, tmp, 0, v1),
4406                  emit_extract_vector(ctx, tmp, 1, v1),
4407                  by_6);
4408
4409    } else if (ctx->options->chip_class == GFX9 &&
4410               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4411               glsl_sampler_type_is_array(type)) {
4412       assert(instr->dest.ssa.num_components == 2);
4413       def = Definition(dst);
4414       dmask = 0x5;
4415    } else {
4416       def = Definition(dst);
4417    }
4418
4419    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4420 }
4421
4422 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4423 {
4424    Builder bld(ctx->program, ctx->block);
4425    unsigned num_components = instr->num_components;
4426
4427    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4428    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4429    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4430
4431    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4432    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc, false);
4433 }
4434
4435 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4436 {
4437    Builder bld(ctx->program, ctx->block);
4438    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4439    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4440    unsigned writemask = nir_intrinsic_write_mask(instr);
4441
4442    Temp offset;
4443    if (ctx->options->chip_class < GFX8)
4444       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4445    else
4446       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4447
4448    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4449    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4450
4451    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4452                ctx->options->chip_class >= GFX8;
4453    if (smem)
4454       offset = bld.as_uniform(offset);
4455    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4456
4457    while (writemask) {
4458       int start, count;
4459       u_bit_scan_consecutive_range(&writemask, &start, &count);
4460       if (count == 3 && smem) {
4461          writemask |= 1u << (start + 2);
4462          count = 2;
4463       }
4464       int num_bytes = count * elem_size_bytes;
4465
4466       if (num_bytes > 16) {
4467          assert(elem_size_bytes == 8);
4468          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4469          count = 2;
4470          num_bytes = 16;
4471       }
4472
4473       // TODO: check alignment of sub-dword stores
4474       // TODO: split 3 bytes. there is no store instruction for that
4475
4476       Temp write_data;
4477       if (count != instr->num_components) {
4478          emit_split_vector(ctx, data, instr->num_components);
4479          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4480          for (int i = 0; i < count; i++) {
4481             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4482             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4483          }
4484          write_data = bld.tmp(!smem ? RegType::vgpr : smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4485          vec->definitions[0] = Definition(write_data);
4486          ctx->block->instructions.emplace_back(std::move(vec));
4487       } else if (!smem && data.type() != RegType::vgpr) {
4488          assert(num_bytes % 4 == 0);
4489          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4490       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4491          assert(num_bytes % 4 == 0);
4492          write_data = bld.as_uniform(data);
4493       } else {
4494          write_data = data;
4495       }
4496
4497       aco_opcode vmem_op, smem_op;
4498       switch (num_bytes) {
4499          case 4:
4500             vmem_op = aco_opcode::buffer_store_dword;
4501             smem_op = aco_opcode::s_buffer_store_dword;
4502             break;
4503          case 8:
4504             vmem_op = aco_opcode::buffer_store_dwordx2;
4505             smem_op = aco_opcode::s_buffer_store_dwordx2;
4506             break;
4507          case 12:
4508             vmem_op = aco_opcode::buffer_store_dwordx3;
4509             smem_op = aco_opcode::last_opcode;
4510             assert(!smem);
4511             break;
4512          case 16:
4513             vmem_op = aco_opcode::buffer_store_dwordx4;
4514             smem_op = aco_opcode::s_buffer_store_dwordx4;
4515             break;
4516          default:
4517             unreachable("Store SSBO not implemented for this size.");
4518       }
4519       if (ctx->stage == fragment_fs)
4520          smem_op = aco_opcode::p_fs_buffer_store_smem;
4521
4522       if (smem) {
4523          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4524          store->operands[0] = Operand(rsrc);
4525          if (start) {
4526             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4527                                 offset, Operand(start * elem_size_bytes));
4528             store->operands[1] = Operand(off);
4529          } else {
4530             store->operands[1] = Operand(offset);
4531          }
4532          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4533             store->operands[1].setFixed(m0);
4534          store->operands[2] = Operand(write_data);
4535          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4536          store->dlc = false;
4537          store->disable_wqm = true;
4538          store->barrier = barrier_buffer;
4539          ctx->block->instructions.emplace_back(std::move(store));
4540          ctx->program->wb_smem_l1_on_end = true;
4541          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4542             ctx->block->kind |= block_kind_needs_lowering;
4543             ctx->program->needs_exact = true;
4544          }
4545       } else {
4546          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4547          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4548          store->operands[1] = Operand(rsrc);
4549          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4550          store->operands[3] = Operand(write_data);
4551          store->offset = start * elem_size_bytes;
4552          store->offen = (offset.type() == RegType::vgpr);
4553          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4554          store->dlc = false;
4555          store->disable_wqm = true;
4556          store->barrier = barrier_buffer;
4557          ctx->program->needs_exact = true;
4558          ctx->block->instructions.emplace_back(std::move(store));
4559       }
4560    }
4561 }
4562
4563 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4564 {
4565    /* return the previous value if dest is ever used */
4566    bool return_previous = false;
4567    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4568       return_previous = true;
4569       break;
4570    }
4571    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4572       return_previous = true;
4573       break;
4574    }
4575
4576    Builder bld(ctx->program, ctx->block);
4577    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4578
4579    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4580       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4581                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4582
4583    Temp offset;
4584    if (ctx->options->chip_class < GFX8)
4585       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4586    else
4587       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4588
4589    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4590    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4591
4592    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4593
4594    aco_opcode op32, op64;
4595    switch (instr->intrinsic) {
4596       case nir_intrinsic_ssbo_atomic_add:
4597          op32 = aco_opcode::buffer_atomic_add;
4598          op64 = aco_opcode::buffer_atomic_add_x2;
4599          break;
4600       case nir_intrinsic_ssbo_atomic_imin:
4601          op32 = aco_opcode::buffer_atomic_smin;
4602          op64 = aco_opcode::buffer_atomic_smin_x2;
4603          break;
4604       case nir_intrinsic_ssbo_atomic_umin:
4605          op32 = aco_opcode::buffer_atomic_umin;
4606          op64 = aco_opcode::buffer_atomic_umin_x2;
4607          break;
4608       case nir_intrinsic_ssbo_atomic_imax:
4609          op32 = aco_opcode::buffer_atomic_smax;
4610          op64 = aco_opcode::buffer_atomic_smax_x2;
4611          break;
4612       case nir_intrinsic_ssbo_atomic_umax:
4613          op32 = aco_opcode::buffer_atomic_umax;
4614          op64 = aco_opcode::buffer_atomic_umax_x2;
4615          break;
4616       case nir_intrinsic_ssbo_atomic_and:
4617          op32 = aco_opcode::buffer_atomic_and;
4618          op64 = aco_opcode::buffer_atomic_and_x2;
4619          break;
4620       case nir_intrinsic_ssbo_atomic_or:
4621          op32 = aco_opcode::buffer_atomic_or;
4622          op64 = aco_opcode::buffer_atomic_or_x2;
4623          break;
4624       case nir_intrinsic_ssbo_atomic_xor:
4625          op32 = aco_opcode::buffer_atomic_xor;
4626          op64 = aco_opcode::buffer_atomic_xor_x2;
4627          break;
4628       case nir_intrinsic_ssbo_atomic_exchange:
4629          op32 = aco_opcode::buffer_atomic_swap;
4630          op64 = aco_opcode::buffer_atomic_swap_x2;
4631          break;
4632       case nir_intrinsic_ssbo_atomic_comp_swap:
4633          op32 = aco_opcode::buffer_atomic_cmpswap;
4634          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4635          break;
4636       default:
4637          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4638    }
4639    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4640    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4641    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4642    mubuf->operands[1] = Operand(rsrc);
4643    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4644    mubuf->operands[3] = Operand(data);
4645    if (return_previous)
4646       mubuf->definitions[0] = Definition(dst);
4647    mubuf->offset = 0;
4648    mubuf->offen = (offset.type() == RegType::vgpr);
4649    mubuf->glc = return_previous;
4650    mubuf->dlc = false; /* Not needed for atomics */
4651    mubuf->disable_wqm = true;
4652    mubuf->barrier = barrier_buffer;
4653    ctx->program->needs_exact = true;
4654    ctx->block->instructions.emplace_back(std::move(mubuf));
4655 }
4656
4657 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4658
4659    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4660    Builder bld(ctx->program, ctx->block);
4661    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4662    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4663 }
4664
4665 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4666 {
4667    Builder bld(ctx->program, ctx->block);
4668    unsigned num_components = instr->num_components;
4669    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4670
4671    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4672    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4673
4674    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4675    bool dlc = glc && ctx->options->chip_class >= GFX10;
4676    aco_opcode op;
4677    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4678       bool global = ctx->options->chip_class >= GFX9;
4679       aco_opcode op;
4680       switch (num_bytes) {
4681       case 4:
4682          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4683          break;
4684       case 8:
4685          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4686          break;
4687       case 12:
4688          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4689          break;
4690       case 16:
4691          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4692          break;
4693       default:
4694          unreachable("load_global not implemented for this size.");
4695       }
4696       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4697       flat->operands[0] = Operand(addr);
4698       flat->operands[1] = Operand(s1);
4699       flat->glc = glc;
4700       flat->dlc = dlc;
4701       flat->barrier = barrier_buffer;
4702
4703       if (dst.type() == RegType::sgpr) {
4704          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4705          flat->definitions[0] = Definition(vec);
4706          ctx->block->instructions.emplace_back(std::move(flat));
4707          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4708       } else {
4709          flat->definitions[0] = Definition(dst);
4710          ctx->block->instructions.emplace_back(std::move(flat));
4711       }
4712       emit_split_vector(ctx, dst, num_components);
4713    } else {
4714       switch (num_bytes) {
4715          case 4:
4716             op = aco_opcode::s_load_dword;
4717             break;
4718          case 8:
4719             op = aco_opcode::s_load_dwordx2;
4720             break;
4721          case 12:
4722          case 16:
4723             op = aco_opcode::s_load_dwordx4;
4724             break;
4725          default:
4726             unreachable("load_global not implemented for this size.");
4727       }
4728       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4729       load->operands[0] = Operand(addr);
4730       load->operands[1] = Operand(0u);
4731       load->definitions[0] = Definition(dst);
4732       load->glc = glc;
4733       load->dlc = dlc;
4734       load->barrier = barrier_buffer;
4735       assert(ctx->options->chip_class >= GFX8 || !glc);
4736
4737       if (dst.size() == 3) {
4738          /* trim vector */
4739          Temp vec = bld.tmp(s4);
4740          load->definitions[0] = Definition(vec);
4741          ctx->block->instructions.emplace_back(std::move(load));
4742          emit_split_vector(ctx, vec, 4);
4743
4744          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4745                     emit_extract_vector(ctx, vec, 0, s1),
4746                     emit_extract_vector(ctx, vec, 1, s1),
4747                     emit_extract_vector(ctx, vec, 2, s1));
4748       } else {
4749          ctx->block->instructions.emplace_back(std::move(load));
4750       }
4751    }
4752 }
4753
4754 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4755 {
4756    Builder bld(ctx->program, ctx->block);
4757    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4758
4759    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4760    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4761
4762    unsigned writemask = nir_intrinsic_write_mask(instr);
4763    while (writemask) {
4764       int start, count;
4765       u_bit_scan_consecutive_range(&writemask, &start, &count);
4766       unsigned num_bytes = count * elem_size_bytes;
4767
4768       Temp write_data = data;
4769       if (count != instr->num_components) {
4770          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4771          for (int i = 0; i < count; i++)
4772             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4773          write_data = bld.tmp(RegType::vgpr, count);
4774          vec->definitions[0] = Definition(write_data);
4775          ctx->block->instructions.emplace_back(std::move(vec));
4776       }
4777
4778       unsigned offset = start * elem_size_bytes;
4779       if (offset > 0 && ctx->options->chip_class < GFX9) {
4780          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4781          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4782          Temp carry = bld.tmp(bld.lm);
4783          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4784
4785          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4786                   Operand(offset), addr0);
4787          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(bld.lm),
4788                   Operand(0u), addr1,
4789                   carry).def(1).setHint(vcc);
4790
4791          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4792
4793          offset = 0;
4794       }
4795
4796       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4797       bool global = ctx->options->chip_class >= GFX9;
4798       aco_opcode op;
4799       switch (num_bytes) {
4800       case 4:
4801          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4802          break;
4803       case 8:
4804          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4805          break;
4806       case 12:
4807          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4808          break;
4809       case 16:
4810          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4811          break;
4812       default:
4813          unreachable("store_global not implemented for this size.");
4814       }
4815       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4816       flat->operands[0] = Operand(addr);
4817       flat->operands[1] = Operand(s1);
4818       flat->operands[2] = Operand(data);
4819       flat->glc = glc;
4820       flat->dlc = false;
4821       flat->offset = offset;
4822       flat->disable_wqm = true;
4823       flat->barrier = barrier_buffer;
4824       ctx->program->needs_exact = true;
4825       ctx->block->instructions.emplace_back(std::move(flat));
4826    }
4827 }
4828
4829 void visit_global_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4830 {
4831    /* return the previous value if dest is ever used */
4832    bool return_previous = false;
4833    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4834       return_previous = true;
4835       break;
4836    }
4837    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4838       return_previous = true;
4839       break;
4840    }
4841
4842    Builder bld(ctx->program, ctx->block);
4843    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4844    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4845
4846    if (instr->intrinsic == nir_intrinsic_global_atomic_comp_swap)
4847       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4848                         get_ssa_temp(ctx, instr->src[2].ssa), data);
4849
4850    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4851
4852    bool global = ctx->options->chip_class >= GFX9;
4853    aco_opcode op32, op64;
4854    switch (instr->intrinsic) {
4855       case nir_intrinsic_global_atomic_add:
4856          op32 = global ? aco_opcode::global_atomic_add : aco_opcode::flat_atomic_add;
4857          op64 = global ? aco_opcode::global_atomic_add_x2 : aco_opcode::flat_atomic_add_x2;
4858          break;
4859       case nir_intrinsic_global_atomic_imin:
4860          op32 = global ? aco_opcode::global_atomic_smin : aco_opcode::flat_atomic_smin;
4861          op64 = global ? aco_opcode::global_atomic_smin_x2 : aco_opcode::flat_atomic_smin_x2;
4862          break;
4863       case nir_intrinsic_global_atomic_umin:
4864          op32 = global ? aco_opcode::global_atomic_umin : aco_opcode::flat_atomic_umin;
4865          op64 = global ? aco_opcode::global_atomic_umin_x2 : aco_opcode::flat_atomic_umin_x2;
4866          break;
4867       case nir_intrinsic_global_atomic_imax:
4868          op32 = global ? aco_opcode::global_atomic_smax : aco_opcode::flat_atomic_smax;
4869          op64 = global ? aco_opcode::global_atomic_smax_x2 : aco_opcode::flat_atomic_smax_x2;
4870          break;
4871       case nir_intrinsic_global_atomic_umax:
4872          op32 = global ? aco_opcode::global_atomic_umax : aco_opcode::flat_atomic_umax;
4873          op64 = global ? aco_opcode::global_atomic_umax_x2 : aco_opcode::flat_atomic_umax_x2;
4874          break;
4875       case nir_intrinsic_global_atomic_and:
4876          op32 = global ? aco_opcode::global_atomic_and : aco_opcode::flat_atomic_and;
4877          op64 = global ? aco_opcode::global_atomic_and_x2 : aco_opcode::flat_atomic_and_x2;
4878          break;
4879       case nir_intrinsic_global_atomic_or:
4880          op32 = global ? aco_opcode::global_atomic_or : aco_opcode::flat_atomic_or;
4881          op64 = global ? aco_opcode::global_atomic_or_x2 : aco_opcode::flat_atomic_or_x2;
4882          break;
4883       case nir_intrinsic_global_atomic_xor:
4884          op32 = global ? aco_opcode::global_atomic_xor : aco_opcode::flat_atomic_xor;
4885          op64 = global ? aco_opcode::global_atomic_xor_x2 : aco_opcode::flat_atomic_xor_x2;
4886          break;
4887       case nir_intrinsic_global_atomic_exchange:
4888          op32 = global ? aco_opcode::global_atomic_swap : aco_opcode::flat_atomic_swap;
4889          op64 = global ? aco_opcode::global_atomic_swap_x2 : aco_opcode::flat_atomic_swap_x2;
4890          break;
4891       case nir_intrinsic_global_atomic_comp_swap:
4892          op32 = global ? aco_opcode::global_atomic_cmpswap : aco_opcode::flat_atomic_cmpswap;
4893          op64 = global ? aco_opcode::global_atomic_cmpswap_x2 : aco_opcode::flat_atomic_cmpswap_x2;
4894          break;
4895       default:
4896          unreachable("visit_atomic_global should only be called with nir_intrinsic_global_atomic_* instructions.");
4897    }
4898    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4899    aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, return_previous ? 1 : 0)};
4900    flat->operands[0] = Operand(addr);
4901    flat->operands[1] = Operand(s1);
4902    flat->operands[2] = Operand(data);
4903    if (return_previous)
4904       flat->definitions[0] = Definition(dst);
4905    flat->glc = return_previous;
4906    flat->dlc = false; /* Not needed for atomics */
4907    flat->offset = 0;
4908    flat->disable_wqm = true;
4909    flat->barrier = barrier_buffer;
4910    ctx->program->needs_exact = true;
4911    ctx->block->instructions.emplace_back(std::move(flat));
4912 }
4913
4914 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4915    Builder bld(ctx->program, ctx->block);
4916    switch(instr->intrinsic) {
4917       case nir_intrinsic_group_memory_barrier:
4918       case nir_intrinsic_memory_barrier:
4919          bld.barrier(aco_opcode::p_memory_barrier_all);
4920          break;
4921       case nir_intrinsic_memory_barrier_atomic_counter:
4922          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4923          break;
4924       case nir_intrinsic_memory_barrier_buffer:
4925          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4926          break;
4927       case nir_intrinsic_memory_barrier_image:
4928          bld.barrier(aco_opcode::p_memory_barrier_image);
4929          break;
4930       case nir_intrinsic_memory_barrier_shared:
4931          bld.barrier(aco_opcode::p_memory_barrier_shared);
4932          break;
4933       default:
4934          unreachable("Unimplemented memory barrier intrinsic");
4935          break;
4936    }
4937 }
4938
4939 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4940 {
4941    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4942    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4943    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4944    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4945    Builder bld(ctx->program, ctx->block);
4946
4947    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4948    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4949    load_lds(ctx, elem_size_bytes, dst, address, nir_intrinsic_base(instr), align);
4950 }
4951
4952 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4953 {
4954    unsigned writemask = nir_intrinsic_write_mask(instr);
4955    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4956    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4957    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4958    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4959
4960    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4961    store_lds(ctx, elem_size_bytes, data, writemask, address, nir_intrinsic_base(instr), align);
4962 }
4963
4964 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4965 {
4966    unsigned offset = nir_intrinsic_base(instr);
4967    Operand m = load_lds_size_m0(ctx);
4968    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4969    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4970
4971    unsigned num_operands = 3;
4972    aco_opcode op32, op64, op32_rtn, op64_rtn;
4973    switch(instr->intrinsic) {
4974       case nir_intrinsic_shared_atomic_add:
4975          op32 = aco_opcode::ds_add_u32;
4976          op64 = aco_opcode::ds_add_u64;
4977          op32_rtn = aco_opcode::ds_add_rtn_u32;
4978          op64_rtn = aco_opcode::ds_add_rtn_u64;
4979          break;
4980       case nir_intrinsic_shared_atomic_imin:
4981          op32 = aco_opcode::ds_min_i32;
4982          op64 = aco_opcode::ds_min_i64;
4983          op32_rtn = aco_opcode::ds_min_rtn_i32;
4984          op64_rtn = aco_opcode::ds_min_rtn_i64;
4985          break;
4986       case nir_intrinsic_shared_atomic_umin:
4987          op32 = aco_opcode::ds_min_u32;
4988          op64 = aco_opcode::ds_min_u64;
4989          op32_rtn = aco_opcode::ds_min_rtn_u32;
4990          op64_rtn = aco_opcode::ds_min_rtn_u64;
4991          break;
4992       case nir_intrinsic_shared_atomic_imax:
4993          op32 = aco_opcode::ds_max_i32;
4994          op64 = aco_opcode::ds_max_i64;
4995          op32_rtn = aco_opcode::ds_max_rtn_i32;
4996          op64_rtn = aco_opcode::ds_max_rtn_i64;
4997          break;
4998       case nir_intrinsic_shared_atomic_umax:
4999          op32 = aco_opcode::ds_max_u32;
5000          op64 = aco_opcode::ds_max_u64;
5001          op32_rtn = aco_opcode::ds_max_rtn_u32;
5002          op64_rtn = aco_opcode::ds_max_rtn_u64;
5003          break;
5004       case nir_intrinsic_shared_atomic_and:
5005          op32 = aco_opcode::ds_and_b32;
5006          op64 = aco_opcode::ds_and_b64;
5007          op32_rtn = aco_opcode::ds_and_rtn_b32;
5008          op64_rtn = aco_opcode::ds_and_rtn_b64;
5009          break;
5010       case nir_intrinsic_shared_atomic_or:
5011          op32 = aco_opcode::ds_or_b32;
5012          op64 = aco_opcode::ds_or_b64;
5013          op32_rtn = aco_opcode::ds_or_rtn_b32;
5014          op64_rtn = aco_opcode::ds_or_rtn_b64;
5015          break;
5016       case nir_intrinsic_shared_atomic_xor:
5017          op32 = aco_opcode::ds_xor_b32;
5018          op64 = aco_opcode::ds_xor_b64;
5019          op32_rtn = aco_opcode::ds_xor_rtn_b32;
5020          op64_rtn = aco_opcode::ds_xor_rtn_b64;
5021          break;
5022       case nir_intrinsic_shared_atomic_exchange:
5023          op32 = aco_opcode::ds_write_b32;
5024          op64 = aco_opcode::ds_write_b64;
5025          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
5026          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
5027          break;
5028       case nir_intrinsic_shared_atomic_comp_swap:
5029          op32 = aco_opcode::ds_cmpst_b32;
5030          op64 = aco_opcode::ds_cmpst_b64;
5031          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
5032          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
5033          num_operands = 4;
5034          break;
5035       default:
5036          unreachable("Unhandled shared atomic intrinsic");
5037    }
5038
5039    /* return the previous value if dest is ever used */
5040    bool return_previous = false;
5041    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
5042       return_previous = true;
5043       break;
5044    }
5045    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
5046       return_previous = true;
5047       break;
5048    }
5049
5050    aco_opcode op;
5051    if (data.size() == 1) {
5052       assert(instr->dest.ssa.bit_size == 32);
5053       op = return_previous ? op32_rtn : op32;
5054    } else {
5055       assert(instr->dest.ssa.bit_size == 64);
5056       op = return_previous ? op64_rtn : op64;
5057    }
5058
5059    if (offset > 65535) {
5060       Builder bld(ctx->program, ctx->block);
5061       address = bld.vadd32(bld.def(v1), Operand(offset), address);
5062       offset = 0;
5063    }
5064
5065    aco_ptr<DS_instruction> ds;
5066    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
5067    ds->operands[0] = Operand(address);
5068    ds->operands[1] = Operand(data);
5069    if (num_operands == 4)
5070       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
5071    ds->operands[num_operands - 1] = m;
5072    ds->offset0 = offset;
5073    if (return_previous)
5074       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
5075    ctx->block->instructions.emplace_back(std::move(ds));
5076 }
5077
5078 Temp get_scratch_resource(isel_context *ctx)
5079 {
5080    Builder bld(ctx->program, ctx->block);
5081    Temp scratch_addr = ctx->program->private_segment_buffer;
5082    if (ctx->stage != compute_cs)
5083       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), scratch_addr, Operand(0u));
5084
5085    uint32_t rsrc_conf = S_008F0C_ADD_TID_ENABLE(1) |
5086                         S_008F0C_INDEX_STRIDE(ctx->program->wave_size == 64 ? 3 : 2);;
5087
5088    if (ctx->program->chip_class >= GFX10) {
5089       rsrc_conf |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
5090                    S_008F0C_OOB_SELECT(3) |
5091                    S_008F0C_RESOURCE_LEVEL(1);
5092    } else if (ctx->program->chip_class <= GFX7) { /* dfmt modifies stride on GFX8/GFX9 when ADD_TID_EN=1 */
5093       rsrc_conf |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
5094                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
5095    }
5096
5097    /* older generations need element size = 16 bytes. element size removed in GFX9 */
5098    if (ctx->program->chip_class <= GFX8)
5099       rsrc_conf |= S_008F0C_ELEMENT_SIZE(3);
5100
5101    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
5102 }
5103
5104 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5105    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
5106    Builder bld(ctx->program, ctx->block);
5107    Temp rsrc = get_scratch_resource(ctx);
5108    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5109    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5110
5111    aco_opcode op;
5112    switch (dst.size()) {
5113       case 1:
5114          op = aco_opcode::buffer_load_dword;
5115          break;
5116       case 2:
5117          op = aco_opcode::buffer_load_dwordx2;
5118          break;
5119       case 3:
5120          op = aco_opcode::buffer_load_dwordx3;
5121          break;
5122       case 4:
5123          op = aco_opcode::buffer_load_dwordx4;
5124          break;
5125       case 6:
5126       case 8: {
5127          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
5128          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
5129                                 bld.def(v4), offset, rsrc,
5130                                 ctx->program->scratch_offset, 0, true);
5131          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
5132                                                   aco_opcode::buffer_load_dwordx4,
5133                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
5134                                 offset, rsrc, ctx->program->scratch_offset, 16, true);
5135          emit_split_vector(ctx, lower, 2);
5136          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
5137          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
5138          if (dst.size() == 8) {
5139             emit_split_vector(ctx, upper, 2);
5140             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
5141             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
5142          } else {
5143             elems[2] = upper;
5144          }
5145
5146          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
5147                                                                          Format::PSEUDO, dst.size() / 2, 1)};
5148          for (unsigned i = 0; i < dst.size() / 2; i++)
5149             vec->operands[i] = Operand(elems[i]);
5150          vec->definitions[0] = Definition(dst);
5151          bld.insert(std::move(vec));
5152          ctx->allocated_vec.emplace(dst.id(), elems);
5153          return;
5154       }
5155       default:
5156          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
5157    }
5158
5159    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->program->scratch_offset, 0, true);
5160    emit_split_vector(ctx, dst, instr->num_components);
5161 }
5162
5163 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
5164    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
5165    Builder bld(ctx->program, ctx->block);
5166    Temp rsrc = get_scratch_resource(ctx);
5167    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5168    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
5169
5170    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
5171    unsigned writemask = nir_intrinsic_write_mask(instr);
5172
5173    while (writemask) {
5174       int start, count;
5175       u_bit_scan_consecutive_range(&writemask, &start, &count);
5176       int num_bytes = count * elem_size_bytes;
5177
5178       if (num_bytes > 16) {
5179          assert(elem_size_bytes == 8);
5180          writemask |= (((count - 2) << 1) - 1) << (start + 2);
5181          count = 2;
5182          num_bytes = 16;
5183       }
5184
5185       // TODO: check alignment of sub-dword stores
5186       // TODO: split 3 bytes. there is no store instruction for that
5187
5188       Temp write_data;
5189       if (count != instr->num_components) {
5190          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
5191          for (int i = 0; i < count; i++) {
5192             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
5193             vec->operands[i] = Operand(elem);
5194          }
5195          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
5196          vec->definitions[0] = Definition(write_data);
5197          ctx->block->instructions.emplace_back(std::move(vec));
5198       } else {
5199          write_data = data;
5200       }
5201
5202       aco_opcode op;
5203       switch (num_bytes) {
5204          case 4:
5205             op = aco_opcode::buffer_store_dword;
5206             break;
5207          case 8:
5208             op = aco_opcode::buffer_store_dwordx2;
5209             break;
5210          case 12:
5211             op = aco_opcode::buffer_store_dwordx3;
5212             break;
5213          case 16:
5214             op = aco_opcode::buffer_store_dwordx4;
5215             break;
5216          default:
5217             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
5218       }
5219
5220       bld.mubuf(op, offset, rsrc, ctx->program->scratch_offset, write_data, start * elem_size_bytes, true);
5221    }
5222 }
5223
5224 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
5225    uint8_t log2_ps_iter_samples;
5226    if (ctx->program->info->ps.force_persample) {
5227       log2_ps_iter_samples =
5228          util_logbase2(ctx->options->key.fs.num_samples);
5229    } else {
5230       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
5231    }
5232
5233    /* The bit pattern matches that used by fixed function fragment
5234     * processing. */
5235    static const unsigned ps_iter_masks[] = {
5236       0xffff, /* not used */
5237       0x5555,
5238       0x1111,
5239       0x0101,
5240       0x0001,
5241    };
5242    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
5243
5244    Builder bld(ctx->program, ctx->block);
5245
5246    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1),
5247                              get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5248    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
5249    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
5250    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5251    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, get_arg(ctx, ctx->args->ac.sample_coverage));
5252 }
5253
5254 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
5255 {
5256    Builder bld(ctx->program, ctx->block);
5257
5258    if (cluster_size == 1) {
5259       return src;
5260    } if (op == nir_op_iand && cluster_size == 4) {
5261       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5262       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5263       return bld.sop1(Builder::s_not, bld.def(bld.lm), bld.def(s1, scc),
5264                       bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc), tmp));
5265    } else if (op == nir_op_ior && cluster_size == 4) {
5266       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5267       return bld.sop1(Builder::s_wqm, bld.def(bld.lm), bld.def(s1, scc),
5268                       bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)));
5269    } else if (op == nir_op_iand && cluster_size == ctx->program->wave_size) {
5270       //subgroupAnd(val) -> (exec & ~val) == 0
5271       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5272       return bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5273    } else if (op == nir_op_ior && cluster_size == ctx->program->wave_size) {
5274       //subgroupOr(val) -> (val & exec) != 0
5275       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm)).def(1).getTemp();
5276       return bool_to_vector_condition(ctx, tmp);
5277    } else if (op == nir_op_ixor && cluster_size == ctx->program->wave_size) {
5278       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5279       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5280       tmp = bld.sop1(Builder::s_bcnt1_i32, bld.def(s1), bld.def(s1, scc), tmp);
5281       tmp = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5282       return bool_to_vector_condition(ctx, tmp);
5283    } else {
5284       //subgroupClustered{And,Or,Xor}(val, n) ->
5285       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0)) ;  just v_mbcnt_lo_u32_b32 on wave32
5286       //cluster_offset = ~(n - 1) & lane_id
5287       //cluster_mask = ((1 << n) - 1)
5288       //subgroupClusteredAnd():
5289       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5290       //subgroupClusteredOr():
5291       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5292       //subgroupClusteredXor():
5293       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5294       Temp lane_id = emit_mbcnt(ctx, bld.def(v1));
5295       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5296
5297       Temp tmp;
5298       if (op == nir_op_iand)
5299          tmp = bld.sop2(Builder::s_orn2, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5300       else
5301          tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
5302
5303       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5304
5305       if (ctx->program->chip_class <= GFX7)
5306          tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), tmp, cluster_offset);
5307       else if (ctx->program->wave_size == 64)
5308          tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5309       else
5310          tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), cluster_offset, tmp);
5311       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5312       if (cluster_mask != 0xffffffff)
5313          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5314
5315       Definition cmp_def = Definition();
5316       if (op == nir_op_iand) {
5317          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(cluster_mask), tmp).def(0);
5318       } else if (op == nir_op_ior) {
5319          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5320       } else if (op == nir_op_ixor) {
5321          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5322                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5323          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp).def(0);
5324       }
5325       cmp_def.setHint(vcc);
5326       return cmp_def.getTemp();
5327    }
5328 }
5329
5330 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5331 {
5332    Builder bld(ctx->program, ctx->block);
5333
5334    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5335    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5336    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5337    Temp tmp;
5338    if (op == nir_op_iand)
5339       tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src);
5340    else
5341       tmp = bld.sop2(Builder::s_and, bld.def(s2), bld.def(s1, scc), src, Operand(exec, bld.lm));
5342
5343    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5344    Temp lo = lohi.def(0).getTemp();
5345    Temp hi = lohi.def(1).getTemp();
5346    Temp mbcnt = emit_mbcnt(ctx, bld.def(v1), Operand(lo), Operand(hi));
5347
5348    Definition cmp_def = Definition();
5349    if (op == nir_op_iand)
5350       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5351    else if (op == nir_op_ior)
5352       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), mbcnt).def(0);
5353    else if (op == nir_op_ixor)
5354       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u),
5355                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5356    cmp_def.setHint(vcc);
5357    return cmp_def.getTemp();
5358 }
5359
5360 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5361 {
5362    Builder bld(ctx->program, ctx->block);
5363
5364    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5365    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5366    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5367    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5368    if (op == nir_op_iand)
5369       return bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5370    else if (op == nir_op_ior)
5371       return bld.sop2(Builder::s_or, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5372    else if (op == nir_op_ixor)
5373       return bld.sop2(Builder::s_xor, bld.def(bld.lm), bld.def(s1, scc), tmp, src);
5374
5375    assert(false);
5376    return Temp();
5377 }
5378
5379 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5380 {
5381    Builder bld(ctx->program, ctx->block);
5382    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5383    if (src.regClass().type() == RegType::vgpr) {
5384       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5385    } else if (src.regClass() == s1) {
5386       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5387    } else if (src.regClass() == s2) {
5388       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5389    } else {
5390       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5391       nir_print_instr(&instr->instr, stderr);
5392       fprintf(stderr, "\n");
5393    }
5394 }
5395
5396 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5397 {
5398    Builder bld(ctx->program, ctx->block);
5399    Temp persp_center = get_arg(ctx, ctx->args->ac.persp_center);
5400    Temp p1 = emit_extract_vector(ctx, persp_center, 0, v1);
5401    Temp p2 = emit_extract_vector(ctx, persp_center, 1, v1);
5402
5403    Temp ddx_1, ddx_2, ddy_1, ddy_2;
5404    uint32_t dpp_ctrl0 = dpp_quad_perm(0, 0, 0, 0);
5405    uint32_t dpp_ctrl1 = dpp_quad_perm(1, 1, 1, 1);
5406    uint32_t dpp_ctrl2 = dpp_quad_perm(2, 2, 2, 2);
5407
5408    /* Build DD X/Y */
5409    if (ctx->program->chip_class >= GFX8) {
5410       Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_ctrl0);
5411       ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl1);
5412       ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_ctrl2);
5413       Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_ctrl0);
5414       ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl1);
5415       ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_ctrl2);
5416    } else {
5417       Temp tl_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl0);
5418       ddx_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl1);
5419       ddx_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_1, tl_1);
5420       ddx_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p1, (1 << 15) | dpp_ctrl2);
5421       ddx_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddx_2, tl_1);
5422       Temp tl_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl0);
5423       ddy_1 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl1);
5424       ddy_1 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_1, tl_2);
5425       ddy_2 = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), p2, (1 << 15) | dpp_ctrl2);
5426       ddy_2 = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), ddy_2, tl_2);
5427    }
5428
5429    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5430    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5431    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5432    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5433    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5434    Temp wqm1 = bld.tmp(v1);
5435    emit_wqm(ctx, tmp1, wqm1, true);
5436    Temp wqm2 = bld.tmp(v1);
5437    emit_wqm(ctx, tmp2, wqm2, true);
5438    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5439    return;
5440 }
5441
5442 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5443 {
5444    Builder bld(ctx->program, ctx->block);
5445    switch(instr->intrinsic) {
5446    case nir_intrinsic_load_barycentric_sample:
5447    case nir_intrinsic_load_barycentric_pixel:
5448    case nir_intrinsic_load_barycentric_centroid: {
5449       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5450       Temp bary = Temp(0, s2);
5451       switch (mode) {
5452       case INTERP_MODE_SMOOTH:
5453       case INTERP_MODE_NONE:
5454          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5455             bary = get_arg(ctx, ctx->args->ac.persp_center);
5456          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5457             bary = ctx->persp_centroid;
5458          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5459             bary = get_arg(ctx, ctx->args->ac.persp_sample);
5460          break;
5461       case INTERP_MODE_NOPERSPECTIVE:
5462          if (instr->intrinsic == nir_intrinsic_load_barycentric_pixel)
5463             bary = get_arg(ctx, ctx->args->ac.linear_center);
5464          else if (instr->intrinsic == nir_intrinsic_load_barycentric_centroid)
5465             bary = ctx->linear_centroid;
5466          else if (instr->intrinsic == nir_intrinsic_load_barycentric_sample)
5467             bary = get_arg(ctx, ctx->args->ac.linear_sample);
5468          break;
5469       default:
5470          break;
5471       }
5472       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5473       Temp p1 = emit_extract_vector(ctx, bary, 0, v1);
5474       Temp p2 = emit_extract_vector(ctx, bary, 1, v1);
5475       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5476                  Operand(p1), Operand(p2));
5477       emit_split_vector(ctx, dst, 2);
5478       break;
5479    }
5480    case nir_intrinsic_load_barycentric_at_sample: {
5481       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5482       switch (ctx->options->key.fs.num_samples) {
5483          case 2: sample_pos_offset += 1 << 3; break;
5484          case 4: sample_pos_offset += 3 << 3; break;
5485          case 8: sample_pos_offset += 7 << 3; break;
5486          default: break;
5487       }
5488       Temp sample_pos;
5489       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5490       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5491       Temp private_segment_buffer = ctx->program->private_segment_buffer;
5492       if (addr.type() == RegType::sgpr) {
5493          Operand offset;
5494          if (const_addr) {
5495             sample_pos_offset += const_addr->u32 << 3;
5496             offset = Operand(sample_pos_offset);
5497          } else if (ctx->options->chip_class >= GFX9) {
5498             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5499          } else {
5500             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5501             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5502          }
5503          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), private_segment_buffer, Operand(offset));
5504
5505       } else if (ctx->options->chip_class >= GFX9) {
5506          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5507          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, private_segment_buffer, sample_pos_offset);
5508       } else {
5509          /* addr += private_segment_buffer + sample_pos_offset */
5510          Temp tmp0 = bld.tmp(s1);
5511          Temp tmp1 = bld.tmp(s1);
5512          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), private_segment_buffer);
5513          Definition scc_tmp = bld.def(s1, scc);
5514          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5515          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), bld.scc(scc_tmp.getTemp()));
5516          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5517          Temp pck0 = bld.tmp(v1);
5518          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5519          tmp1 = as_vgpr(ctx, tmp1);
5520          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(bld.lm)), tmp1, Operand(0u), carry);
5521          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5522
5523          /* sample_pos = flat_load_dwordx2 addr */
5524          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5525       }
5526
5527       /* sample_pos -= 0.5 */
5528       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5529       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5530       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5531       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5532       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5533
5534       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5535       break;
5536    }
5537    case nir_intrinsic_load_barycentric_at_offset: {
5538       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5539       RegClass rc = RegClass(offset.type(), 1);
5540       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5541       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5542       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5543       break;
5544    }
5545    case nir_intrinsic_load_front_face: {
5546       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5547                Operand(0u), get_arg(ctx, ctx->args->ac.front_face)).def(0).setHint(vcc);
5548       break;
5549    }
5550    case nir_intrinsic_load_view_index:
5551    case nir_intrinsic_load_layer_id: {
5552       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5553          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5554          bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.view_index)));
5555          break;
5556       }
5557
5558       unsigned idx = nir_intrinsic_base(instr);
5559       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5560                  Operand(2u), bld.m0(get_arg(ctx, ctx->args->ac.prim_mask)), idx, 0);
5561       break;
5562    }
5563    case nir_intrinsic_load_frag_coord: {
5564       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5565       break;
5566    }
5567    case nir_intrinsic_load_sample_pos: {
5568       Temp posx = get_arg(ctx, ctx->args->ac.frag_pos[0]);
5569       Temp posy = get_arg(ctx, ctx->args->ac.frag_pos[1]);
5570       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5571                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5572                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5573       break;
5574    }
5575    case nir_intrinsic_load_interpolated_input:
5576       visit_load_interpolated_input(ctx, instr);
5577       break;
5578    case nir_intrinsic_store_output:
5579       visit_store_output(ctx, instr);
5580       break;
5581    case nir_intrinsic_load_input:
5582       visit_load_input(ctx, instr);
5583       break;
5584    case nir_intrinsic_load_ubo:
5585       visit_load_ubo(ctx, instr);
5586       break;
5587    case nir_intrinsic_load_push_constant:
5588       visit_load_push_constant(ctx, instr);
5589       break;
5590    case nir_intrinsic_load_constant:
5591       visit_load_constant(ctx, instr);
5592       break;
5593    case nir_intrinsic_vulkan_resource_index:
5594       visit_load_resource(ctx, instr);
5595       break;
5596    case nir_intrinsic_discard:
5597       visit_discard(ctx, instr);
5598       break;
5599    case nir_intrinsic_discard_if:
5600       visit_discard_if(ctx, instr);
5601       break;
5602    case nir_intrinsic_load_shared:
5603       visit_load_shared(ctx, instr);
5604       break;
5605    case nir_intrinsic_store_shared:
5606       visit_store_shared(ctx, instr);
5607       break;
5608    case nir_intrinsic_shared_atomic_add:
5609    case nir_intrinsic_shared_atomic_imin:
5610    case nir_intrinsic_shared_atomic_umin:
5611    case nir_intrinsic_shared_atomic_imax:
5612    case nir_intrinsic_shared_atomic_umax:
5613    case nir_intrinsic_shared_atomic_and:
5614    case nir_intrinsic_shared_atomic_or:
5615    case nir_intrinsic_shared_atomic_xor:
5616    case nir_intrinsic_shared_atomic_exchange:
5617    case nir_intrinsic_shared_atomic_comp_swap:
5618       visit_shared_atomic(ctx, instr);
5619       break;
5620    case nir_intrinsic_image_deref_load:
5621       visit_image_load(ctx, instr);
5622       break;
5623    case nir_intrinsic_image_deref_store:
5624       visit_image_store(ctx, instr);
5625       break;
5626    case nir_intrinsic_image_deref_atomic_add:
5627    case nir_intrinsic_image_deref_atomic_umin:
5628    case nir_intrinsic_image_deref_atomic_imin:
5629    case nir_intrinsic_image_deref_atomic_umax:
5630    case nir_intrinsic_image_deref_atomic_imax:
5631    case nir_intrinsic_image_deref_atomic_and:
5632    case nir_intrinsic_image_deref_atomic_or:
5633    case nir_intrinsic_image_deref_atomic_xor:
5634    case nir_intrinsic_image_deref_atomic_exchange:
5635    case nir_intrinsic_image_deref_atomic_comp_swap:
5636       visit_image_atomic(ctx, instr);
5637       break;
5638    case nir_intrinsic_image_deref_size:
5639       visit_image_size(ctx, instr);
5640       break;
5641    case nir_intrinsic_load_ssbo:
5642       visit_load_ssbo(ctx, instr);
5643       break;
5644    case nir_intrinsic_store_ssbo:
5645       visit_store_ssbo(ctx, instr);
5646       break;
5647    case nir_intrinsic_load_global:
5648       visit_load_global(ctx, instr);
5649       break;
5650    case nir_intrinsic_store_global:
5651       visit_store_global(ctx, instr);
5652       break;
5653    case nir_intrinsic_global_atomic_add:
5654    case nir_intrinsic_global_atomic_imin:
5655    case nir_intrinsic_global_atomic_umin:
5656    case nir_intrinsic_global_atomic_imax:
5657    case nir_intrinsic_global_atomic_umax:
5658    case nir_intrinsic_global_atomic_and:
5659    case nir_intrinsic_global_atomic_or:
5660    case nir_intrinsic_global_atomic_xor:
5661    case nir_intrinsic_global_atomic_exchange:
5662    case nir_intrinsic_global_atomic_comp_swap:
5663       visit_global_atomic(ctx, instr);
5664       break;
5665    case nir_intrinsic_ssbo_atomic_add:
5666    case nir_intrinsic_ssbo_atomic_imin:
5667    case nir_intrinsic_ssbo_atomic_umin:
5668    case nir_intrinsic_ssbo_atomic_imax:
5669    case nir_intrinsic_ssbo_atomic_umax:
5670    case nir_intrinsic_ssbo_atomic_and:
5671    case nir_intrinsic_ssbo_atomic_or:
5672    case nir_intrinsic_ssbo_atomic_xor:
5673    case nir_intrinsic_ssbo_atomic_exchange:
5674    case nir_intrinsic_ssbo_atomic_comp_swap:
5675       visit_atomic_ssbo(ctx, instr);
5676       break;
5677    case nir_intrinsic_load_scratch:
5678       visit_load_scratch(ctx, instr);
5679       break;
5680    case nir_intrinsic_store_scratch:
5681       visit_store_scratch(ctx, instr);
5682       break;
5683    case nir_intrinsic_get_buffer_size:
5684       visit_get_buffer_size(ctx, instr);
5685       break;
5686    case nir_intrinsic_barrier: {
5687       unsigned* bsize = ctx->program->info->cs.block_size;
5688       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5689       if (workgroup_size > ctx->program->wave_size)
5690          bld.sopp(aco_opcode::s_barrier);
5691       break;
5692    }
5693    case nir_intrinsic_group_memory_barrier:
5694    case nir_intrinsic_memory_barrier:
5695    case nir_intrinsic_memory_barrier_atomic_counter:
5696    case nir_intrinsic_memory_barrier_buffer:
5697    case nir_intrinsic_memory_barrier_image:
5698    case nir_intrinsic_memory_barrier_shared:
5699       emit_memory_barrier(ctx, instr);
5700       break;
5701    case nir_intrinsic_load_num_work_groups: {
5702       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5703       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.num_work_groups)));
5704       emit_split_vector(ctx, dst, 3);
5705       break;
5706    }
5707    case nir_intrinsic_load_local_invocation_id: {
5708       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5709       bld.copy(Definition(dst), Operand(get_arg(ctx, ctx->args->ac.local_invocation_ids)));
5710       emit_split_vector(ctx, dst, 3);
5711       break;
5712    }
5713    case nir_intrinsic_load_work_group_id: {
5714       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5715       struct ac_arg *args = ctx->args->ac.workgroup_ids;
5716       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5717                  args[0].used ? Operand(get_arg(ctx, args[0])) : Operand(0u),
5718                  args[1].used ? Operand(get_arg(ctx, args[1])) : Operand(0u),
5719                  args[2].used ? Operand(get_arg(ctx, args[2])) : Operand(0u));
5720       emit_split_vector(ctx, dst, 3);
5721       break;
5722    }
5723    case nir_intrinsic_load_local_invocation_index: {
5724       Temp id = emit_mbcnt(ctx, bld.def(v1));
5725
5726       /* The tg_size bits [6:11] contain the subgroup id,
5727        * we need this multiplied by the wave size, and then OR the thread id to it.
5728        */
5729       if (ctx->program->wave_size == 64) {
5730          /* After the s_and the bits are already multiplied by 64 (left shifted by 6) so we can just feed that to v_or */
5731          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u),
5732                                 get_arg(ctx, ctx->args->ac.tg_size));
5733          bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5734       } else {
5735          /* Extract the bit field and multiply the result by 32 (left shift by 5), then do the OR  */
5736          Temp tg_num = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
5737                                 get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5738          bld.vop3(aco_opcode::v_lshl_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, Operand(0x5u), id);
5739       }
5740       break;
5741    }
5742    case nir_intrinsic_load_subgroup_id: {
5743       if (ctx->stage == compute_cs) {
5744          bld.sop2(aco_opcode::s_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc),
5745                   get_arg(ctx, ctx->args->ac.tg_size), Operand(0x6u | (0x6u << 16)));
5746       } else {
5747          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5748       }
5749       break;
5750    }
5751    case nir_intrinsic_load_subgroup_invocation: {
5752       emit_mbcnt(ctx, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5753       break;
5754    }
5755    case nir_intrinsic_load_num_subgroups: {
5756       if (ctx->stage == compute_cs)
5757          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu),
5758                   get_arg(ctx, ctx->args->ac.tg_size));
5759       else
5760          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5761       break;
5762    }
5763    case nir_intrinsic_ballot: {
5764       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5765       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5766       Definition tmp = bld.def(dst.regClass());
5767       Definition lanemask_tmp = dst.size() == bld.lm.size() ? tmp : bld.def(src.regClass());
5768       if (instr->src[0].ssa->bit_size == 1) {
5769          assert(src.regClass() == bld.lm);
5770          bld.sop2(Builder::s_and, lanemask_tmp, bld.def(s1, scc), Operand(exec, bld.lm), src);
5771       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5772          bld.vopc(aco_opcode::v_cmp_lg_u32, lanemask_tmp, Operand(0u), src);
5773       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5774          bld.vopc(aco_opcode::v_cmp_lg_u64, lanemask_tmp, Operand(0u), src);
5775       } else {
5776          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5777          nir_print_instr(&instr->instr, stderr);
5778          fprintf(stderr, "\n");
5779       }
5780       if (dst.size() != bld.lm.size()) {
5781          /* Wave32 with ballot size set to 64 */
5782          bld.pseudo(aco_opcode::p_create_vector, Definition(tmp), lanemask_tmp.getTemp(), Operand(0u));
5783       }
5784       emit_wqm(ctx, tmp.getTemp(), dst);
5785       break;
5786    }
5787    case nir_intrinsic_shuffle:
5788    case nir_intrinsic_read_invocation: {
5789       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5790       if (!ctx->divergent_vals[instr->src[0].ssa->index]) {
5791          emit_uniform_subgroup(ctx, instr, src);
5792       } else {
5793          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5794          if (instr->intrinsic == nir_intrinsic_read_invocation || !ctx->divergent_vals[instr->src[1].ssa->index])
5795             tid = bld.as_uniform(tid);
5796          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5797          if (src.regClass() == v1) {
5798             emit_wqm(ctx, emit_bpermute(ctx, bld, tid, src), dst);
5799          } else if (src.regClass() == v2) {
5800             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5801             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5802             lo = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, lo));
5803             hi = emit_wqm(ctx, emit_bpermute(ctx, bld, tid, hi));
5804             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5805             emit_split_vector(ctx, dst, 2);
5806          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == s1) {
5807             assert(src.regClass() == bld.lm);
5808             Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src, tid);
5809             bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5810          } else if (instr->dest.ssa.bit_size == 1 && tid.regClass() == v1) {
5811             assert(src.regClass() == bld.lm);
5812             Temp tmp;
5813             if (ctx->program->chip_class <= GFX7)
5814                tmp = bld.vop3(aco_opcode::v_lshr_b64, bld.def(v2), src, tid);
5815             else if (ctx->program->wave_size == 64)
5816                tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5817             else
5818                tmp = bld.vop2_e64(aco_opcode::v_lshrrev_b32, bld.def(v1), tid, src);
5819             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5820             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5821             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), tmp), dst);
5822          } else {
5823             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5824             nir_print_instr(&instr->instr, stderr);
5825             fprintf(stderr, "\n");
5826          }
5827       }
5828       break;
5829    }
5830    case nir_intrinsic_load_sample_id: {
5831       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5832                get_arg(ctx, ctx->args->ac.ancillary), Operand(8u), Operand(4u));
5833       break;
5834    }
5835    case nir_intrinsic_load_sample_mask_in: {
5836       visit_load_sample_mask_in(ctx, instr);
5837       break;
5838    }
5839    case nir_intrinsic_read_first_invocation: {
5840       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5841       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5842       if (src.regClass() == v1) {
5843          emit_wqm(ctx,
5844                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5845                   dst);
5846       } else if (src.regClass() == v2) {
5847          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5848          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5849          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5850          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5851          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5852          emit_split_vector(ctx, dst, 2);
5853       } else if (instr->dest.ssa.bit_size == 1) {
5854          assert(src.regClass() == bld.lm);
5855          Temp tmp = bld.sopc(Builder::s_bitcmp1, bld.def(s1, scc), src,
5856                              bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)));
5857          bool_to_vector_condition(ctx, emit_wqm(ctx, tmp), dst);
5858       } else if (src.regClass() == s1) {
5859          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5860       } else if (src.regClass() == s2) {
5861          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5862       } else {
5863          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5864          nir_print_instr(&instr->instr, stderr);
5865          fprintf(stderr, "\n");
5866       }
5867       break;
5868    }
5869    case nir_intrinsic_vote_all: {
5870       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5871       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5872       assert(src.regClass() == bld.lm);
5873       assert(dst.regClass() == bld.lm);
5874
5875       Temp tmp = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5876       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(0u), Operand(-1u), bld.scc(tmp));
5877       emit_wqm(ctx, val, dst);
5878       break;
5879    }
5880    case nir_intrinsic_vote_any: {
5881       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5882       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5883       assert(src.regClass() == bld.lm);
5884       assert(dst.regClass() == bld.lm);
5885
5886       Temp tmp = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), Operand(exec, bld.lm), src).def(1).getTemp();
5887       Temp val = bld.sop2(Builder::s_cselect, bld.def(bld.lm), Operand(-1u), Operand(0u), bld.scc(tmp));
5888       emit_wqm(ctx, val, dst);
5889       break;
5890    }
5891    case nir_intrinsic_reduce:
5892    case nir_intrinsic_inclusive_scan:
5893    case nir_intrinsic_exclusive_scan: {
5894       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5895       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5896       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5897       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5898          nir_intrinsic_cluster_size(instr) : 0;
5899       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : ctx->program->wave_size, ctx->program->wave_size));
5900
5901       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5902          emit_uniform_subgroup(ctx, instr, src);
5903       } else if (instr->dest.ssa.bit_size == 1) {
5904          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5905             op = nir_op_iand;
5906          else if (op == nir_op_iadd)
5907             op = nir_op_ixor;
5908          else if (op == nir_op_umax || op == nir_op_imax)
5909             op = nir_op_ior;
5910          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5911
5912          switch (instr->intrinsic) {
5913          case nir_intrinsic_reduce:
5914             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5915             break;
5916          case nir_intrinsic_exclusive_scan:
5917             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5918             break;
5919          case nir_intrinsic_inclusive_scan:
5920             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5921             break;
5922          default:
5923             assert(false);
5924          }
5925       } else if (cluster_size == 1) {
5926          bld.copy(Definition(dst), src);
5927       } else {
5928          src = as_vgpr(ctx, src);
5929
5930          ReduceOp reduce_op;
5931          switch (op) {
5932          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5933             CASE(iadd)
5934             CASE(imul)
5935             CASE(fadd)
5936             CASE(fmul)
5937             CASE(imin)
5938             CASE(umin)
5939             CASE(fmin)
5940             CASE(imax)
5941             CASE(umax)
5942             CASE(fmax)
5943             CASE(iand)
5944             CASE(ior)
5945             CASE(ixor)
5946             default:
5947                unreachable("unknown reduction op");
5948          #undef CASE
5949          }
5950
5951          aco_opcode aco_op;
5952          switch (instr->intrinsic) {
5953             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5954             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5955             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5956             default:
5957                unreachable("unknown reduce intrinsic");
5958          }
5959
5960          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5961          reduce->operands[0] = Operand(src);
5962          // filled in by aco_reduce_assign.cpp, used internally as part of the
5963          // reduce sequence
5964          assert(dst.size() == 1 || dst.size() == 2);
5965          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5966          reduce->operands[2] = Operand(v1.as_linear());
5967
5968          Temp tmp_dst = bld.tmp(dst.regClass());
5969          reduce->definitions[0] = Definition(tmp_dst);
5970          reduce->definitions[1] = bld.def(ctx->program->lane_mask); // used internally
5971          reduce->definitions[2] = Definition();
5972          reduce->definitions[3] = Definition(scc, s1);
5973          reduce->definitions[4] = Definition();
5974          reduce->reduce_op = reduce_op;
5975          reduce->cluster_size = cluster_size;
5976          ctx->block->instructions.emplace_back(std::move(reduce));
5977
5978          emit_wqm(ctx, tmp_dst, dst);
5979       }
5980       break;
5981    }
5982    case nir_intrinsic_quad_broadcast: {
5983       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5984       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5985          emit_uniform_subgroup(ctx, instr, src);
5986       } else {
5987          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5988          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5989          uint32_t dpp_ctrl = dpp_quad_perm(lane, lane, lane, lane);
5990
5991          if (instr->dest.ssa.bit_size == 1) {
5992             assert(src.regClass() == bld.lm);
5993             assert(dst.regClass() == bld.lm);
5994             uint32_t half_mask = 0x11111111u << lane;
5995             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5996             Temp tmp = bld.tmp(bld.lm);
5997             bld.sop1(Builder::s_wqm, Definition(tmp),
5998                      bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), mask_tmp,
5999                               bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm))));
6000             emit_wqm(ctx, tmp, dst);
6001          } else if (instr->dest.ssa.bit_size == 32) {
6002             if (ctx->program->chip_class >= GFX8)
6003                emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl), dst);
6004             else
6005                emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, (1 << 15) | dpp_ctrl), dst);
6006          } else if (instr->dest.ssa.bit_size == 64) {
6007             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6008             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6009             if (ctx->program->chip_class >= GFX8) {
6010                lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6011                hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6012             } else {
6013                lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, (1 << 15) | dpp_ctrl));
6014                hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, (1 << 15) | dpp_ctrl));
6015             }
6016             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6017             emit_split_vector(ctx, dst, 2);
6018          } else {
6019             fprintf(stderr, "Unimplemented NIR instr bit size: ");
6020             nir_print_instr(&instr->instr, stderr);
6021             fprintf(stderr, "\n");
6022          }
6023       }
6024       break;
6025    }
6026    case nir_intrinsic_quad_swap_horizontal:
6027    case nir_intrinsic_quad_swap_vertical:
6028    case nir_intrinsic_quad_swap_diagonal:
6029    case nir_intrinsic_quad_swizzle_amd: {
6030       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6031       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6032          emit_uniform_subgroup(ctx, instr, src);
6033          break;
6034       }
6035       uint16_t dpp_ctrl = 0;
6036       switch (instr->intrinsic) {
6037       case nir_intrinsic_quad_swap_horizontal:
6038          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
6039          break;
6040       case nir_intrinsic_quad_swap_vertical:
6041          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
6042          break;
6043       case nir_intrinsic_quad_swap_diagonal:
6044          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
6045          break;
6046       case nir_intrinsic_quad_swizzle_amd:
6047          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
6048          break;
6049       default:
6050          break;
6051       }
6052       if (ctx->program->chip_class < GFX8)
6053          dpp_ctrl |= (1 << 15);
6054
6055       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6056       if (instr->dest.ssa.bit_size == 1) {
6057          assert(src.regClass() == bld.lm);
6058          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
6059          if (ctx->program->chip_class >= GFX8)
6060             src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6061          else
6062             src = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6063          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(bld.lm), Operand(0u), src);
6064          emit_wqm(ctx, tmp, dst);
6065       } else if (instr->dest.ssa.bit_size == 32) {
6066          Temp tmp;
6067          if (ctx->program->chip_class >= GFX8)
6068             tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
6069          else
6070             tmp = bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, dpp_ctrl);
6071          emit_wqm(ctx, tmp, dst);
6072       } else if (instr->dest.ssa.bit_size == 64) {
6073          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6074          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6075          if (ctx->program->chip_class >= GFX8) {
6076             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
6077             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
6078          } else {
6079             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, dpp_ctrl));
6080             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, dpp_ctrl));
6081          }
6082          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6083          emit_split_vector(ctx, dst, 2);
6084       } else {
6085          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6086          nir_print_instr(&instr->instr, stderr);
6087          fprintf(stderr, "\n");
6088       }
6089       break;
6090    }
6091    case nir_intrinsic_masked_swizzle_amd: {
6092       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6093       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
6094          emit_uniform_subgroup(ctx, instr, src);
6095          break;
6096       }
6097       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6098       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
6099       if (dst.regClass() == v1) {
6100          emit_wqm(ctx,
6101                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
6102                   dst);
6103       } else if (dst.regClass() == v2) {
6104          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
6105          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
6106          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
6107          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
6108          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6109          emit_split_vector(ctx, dst, 2);
6110       } else {
6111          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6112          nir_print_instr(&instr->instr, stderr);
6113          fprintf(stderr, "\n");
6114       }
6115       break;
6116    }
6117    case nir_intrinsic_write_invocation_amd: {
6118       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
6119       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
6120       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
6121       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6122       if (dst.regClass() == v1) {
6123          /* src2 is ignored for writelane. RA assigns the same reg for dst */
6124          emit_wqm(ctx, bld.writelane(bld.def(v1), val, lane, src), dst);
6125       } else if (dst.regClass() == v2) {
6126          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
6127          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
6128          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
6129          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
6130          Temp lo = emit_wqm(ctx, bld.writelane(bld.def(v1), val_lo, lane, src_hi));
6131          Temp hi = emit_wqm(ctx, bld.writelane(bld.def(v1), val_hi, lane, src_hi));
6132          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
6133          emit_split_vector(ctx, dst, 2);
6134       } else {
6135          fprintf(stderr, "Unimplemented NIR instr bit size: ");
6136          nir_print_instr(&instr->instr, stderr);
6137          fprintf(stderr, "\n");
6138       }
6139       break;
6140    }
6141    case nir_intrinsic_mbcnt_amd: {
6142       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6143       RegClass rc = RegClass(src.type(), 1);
6144       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
6145       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
6146       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6147       Temp wqm_tmp = emit_mbcnt(ctx, bld.def(v1), Operand(mask_lo), Operand(mask_hi));
6148       emit_wqm(ctx, wqm_tmp, dst);
6149       break;
6150    }
6151    case nir_intrinsic_load_helper_invocation: {
6152       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6153       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
6154       ctx->block->kind |= block_kind_needs_lowering;
6155       ctx->program->needs_exact = true;
6156       break;
6157    }
6158    case nir_intrinsic_is_helper_invocation: {
6159       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6160       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
6161       ctx->block->kind |= block_kind_needs_lowering;
6162       ctx->program->needs_exact = true;
6163       break;
6164    }
6165    case nir_intrinsic_demote:
6166       bld.pseudo(aco_opcode::p_demote_to_helper);
6167       ctx->block->kind |= block_kind_uses_demote;
6168       ctx->program->needs_exact = true;
6169       break;
6170    case nir_intrinsic_demote_if: {
6171       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
6172       assert(src.regClass() == bld.lm);
6173       Temp cond = bld.sop2(Builder::s_and, bld.def(bld.lm), bld.def(s1, scc), src, Operand(exec, bld.lm));
6174       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
6175       ctx->block->kind |= block_kind_uses_demote;
6176       ctx->program->needs_exact = true;
6177       break;
6178    }
6179    case nir_intrinsic_first_invocation: {
6180       emit_wqm(ctx, bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm)),
6181                get_ssa_temp(ctx, &instr->dest.ssa));
6182       break;
6183    }
6184    case nir_intrinsic_shader_clock:
6185       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), false);
6186       emit_split_vector(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 2);
6187       break;
6188    case nir_intrinsic_load_vertex_id_zero_base: {
6189       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6190       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.vertex_id));
6191       break;
6192    }
6193    case nir_intrinsic_load_first_vertex: {
6194       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6195       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.base_vertex));
6196       break;
6197    }
6198    case nir_intrinsic_load_base_instance: {
6199       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6200       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.start_instance));
6201       break;
6202    }
6203    case nir_intrinsic_load_instance_id: {
6204       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6205       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.instance_id));
6206       break;
6207    }
6208    case nir_intrinsic_load_draw_id: {
6209       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6210       bld.copy(Definition(dst), get_arg(ctx, ctx->args->ac.draw_id));
6211       break;
6212    }
6213    default:
6214       fprintf(stderr, "Unimplemented intrinsic instr: ");
6215       nir_print_instr(&instr->instr, stderr);
6216       fprintf(stderr, "\n");
6217       abort();
6218
6219       break;
6220    }
6221 }
6222
6223
6224 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
6225                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
6226                     enum glsl_base_type *stype)
6227 {
6228    nir_deref_instr *texture_deref_instr = NULL;
6229    nir_deref_instr *sampler_deref_instr = NULL;
6230    int plane = -1;
6231
6232    for (unsigned i = 0; i < instr->num_srcs; i++) {
6233       switch (instr->src[i].src_type) {
6234       case nir_tex_src_texture_deref:
6235          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
6236          break;
6237       case nir_tex_src_sampler_deref:
6238          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
6239          break;
6240       case nir_tex_src_plane:
6241          plane = nir_src_as_int(instr->src[i].src);
6242          break;
6243       default:
6244          break;
6245       }
6246    }
6247
6248    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
6249
6250    if (!sampler_deref_instr)
6251       sampler_deref_instr = texture_deref_instr;
6252
6253    if (plane >= 0) {
6254       assert(instr->op != nir_texop_txf_ms &&
6255              instr->op != nir_texop_samples_identical);
6256       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
6257       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
6258    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
6259       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
6260    } else {
6261       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
6262    }
6263    if (samp_ptr) {
6264       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
6265
6266       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
6267          /* fix sampler aniso on SI/CI: samp[0] = samp[0] & img[7] */
6268          Builder bld(ctx->program, ctx->block);
6269
6270          /* to avoid unnecessary moves, we split and recombine sampler and image */
6271          Temp img[8] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1),
6272                         bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6273          Temp samp[4] = {bld.tmp(s1), bld.tmp(s1), bld.tmp(s1), bld.tmp(s1)};
6274          bld.pseudo(aco_opcode::p_split_vector, Definition(img[0]), Definition(img[1]),
6275                     Definition(img[2]), Definition(img[3]), Definition(img[4]),
6276                     Definition(img[5]), Definition(img[6]), Definition(img[7]), *res_ptr);
6277          bld.pseudo(aco_opcode::p_split_vector, Definition(samp[0]), Definition(samp[1]),
6278                     Definition(samp[2]), Definition(samp[3]), *samp_ptr);
6279
6280          samp[0] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), samp[0], img[7]);
6281          *res_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
6282                                img[0], img[1], img[2], img[3],
6283                                img[4], img[5], img[6], img[7]);
6284          *samp_ptr = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
6285                                 samp[0], samp[1], samp[2], samp[3]);
6286       }
6287    }
6288    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
6289                      instr->op == nir_texop_samples_identical))
6290       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
6291 }
6292
6293 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
6294                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
6295 {
6296    Builder bld(ctx->program, ctx->block);
6297
6298    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
6299    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
6300    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
6301
6302    Operand neg_one(0xbf800000u);
6303    Operand one(0x3f800000u);
6304    Operand two(0x40000000u);
6305    Operand four(0x40800000u);
6306
6307    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), Operand(0u), ma);
6308    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
6309    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
6310
6311    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(bld.lm)), four, id);
6312    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
6313    is_ma_y = bld.sop2(Builder::s_andn2, bld.hint_vcc(bld.def(bld.lm)), is_ma_y, is_ma_z);
6314    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(bld.lm)), bld.def(s1, scc), is_ma_z, is_ma_y);
6315
6316    // select sc
6317    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
6318    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
6319                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
6320                        one, is_ma_y);
6321    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6322
6323    // select tc
6324    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
6325    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
6326    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
6327
6328    // select ma
6329    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6330                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
6331                   deriv_z, is_ma_z);
6332    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
6333    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
6334 }
6335
6336 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
6337 {
6338    Builder bld(ctx->program, ctx->block);
6339    Temp coord_args[4], ma, tc, sc, id;
6340    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
6341       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
6342
6343    if (is_array) {
6344       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
6345
6346       // see comment in ac_prepare_cube_coords()
6347       if (ctx->options->chip_class <= GFX8)
6348          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6349    }
6350
6351    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6352
6353    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6354    vop3a->operands[0] = Operand(ma);
6355    vop3a->abs[0] = true;
6356    Temp invma = bld.tmp(v1);
6357    vop3a->definitions[0] = Definition(invma);
6358    ctx->block->instructions.emplace_back(std::move(vop3a));
6359
6360    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6361    if (!is_deriv)
6362       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6363
6364    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6365    if (!is_deriv)
6366       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6367
6368    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6369
6370    if (is_deriv) {
6371       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6372       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6373
6374       for (unsigned i = 0; i < 2; i++) {
6375          // see comment in ac_prepare_cube_coords()
6376          Temp deriv_ma;
6377          Temp deriv_sc, deriv_tc;
6378          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6379                            &deriv_ma, &deriv_sc, &deriv_tc);
6380
6381          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6382
6383          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6384                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6385                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6386          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6387                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6388                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6389          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6390       }
6391
6392       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6393       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6394    }
6395
6396    if (is_array)
6397       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6398    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6399
6400 }
6401
6402 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6403 {
6404    Temp coord_vec[3];
6405    for (unsigned i = 0; i < coords.size(); i++)
6406       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6407
6408    Builder bld(ctx->program, ctx->block);
6409    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6410
6411    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6412    for (unsigned i = 0; i < coords.size(); i++)
6413       vec->operands[i] = Operand(coord_vec[i]);
6414    Temp res = bld.tmp(RegType::vgpr, coords.size());
6415    vec->definitions[0] = Definition(res);
6416    ctx->block->instructions.emplace_back(std::move(vec));
6417    return res;
6418 }
6419
6420 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6421 {
6422    if (vec->parent_instr->type != nir_instr_type_alu)
6423       return;
6424    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6425    if (vec_instr->op != nir_op_vec(vec->num_components))
6426       return;
6427
6428    for (unsigned i = 0; i < vec->num_components; i++) {
6429       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6430               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6431    }
6432 }
6433
6434 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6435 {
6436    Builder bld(ctx->program, ctx->block);
6437    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6438         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6439    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6440         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6441    nir_const_value *sample_index_cv = NULL;
6442    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6443    enum glsl_base_type stype;
6444    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6445
6446    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6447                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6448    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6449                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6450
6451    for (unsigned i = 0; i < instr->num_srcs; i++) {
6452       switch (instr->src[i].src_type) {
6453       case nir_tex_src_coord:
6454          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6455          break;
6456       case nir_tex_src_bias:
6457          if (instr->op == nir_texop_txb) {
6458             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6459             has_bias = true;
6460          }
6461          break;
6462       case nir_tex_src_lod: {
6463          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6464
6465          if (val && val->f32 <= 0.0) {
6466             level_zero = true;
6467          } else {
6468             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6469             has_lod = true;
6470          }
6471          break;
6472       }
6473       case nir_tex_src_comparator:
6474          if (instr->is_shadow) {
6475             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6476             has_compare = true;
6477          }
6478          break;
6479       case nir_tex_src_offset:
6480          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6481          get_const_vec(instr->src[i].src.ssa, const_offset);
6482          has_offset = true;
6483          break;
6484       case nir_tex_src_ddx:
6485          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6486          has_ddx = true;
6487          break;
6488       case nir_tex_src_ddy:
6489          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6490          has_ddy = true;
6491          break;
6492       case nir_tex_src_ms_index:
6493          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6494          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6495          has_sample_index = true;
6496          break;
6497       case nir_tex_src_texture_offset:
6498       case nir_tex_src_sampler_offset:
6499       default:
6500          break;
6501       }
6502    }
6503 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6504    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6505       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6506
6507    if (instr->op == nir_texop_texture_samples) {
6508       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6509
6510       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6511       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6512       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6513       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6514
6515       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6516                samples, Operand(1u), bld.scc(is_msaa));
6517       return;
6518    }
6519
6520    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6521       aco_ptr<Instruction> tmp_instr;
6522       Temp acc, pack = Temp();
6523
6524       uint32_t pack_const = 0;
6525       for (unsigned i = 0; i < offset.size(); i++) {
6526          if (!const_offset[i])
6527             continue;
6528          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6529       }
6530
6531       if (offset.type() == RegType::sgpr) {
6532          for (unsigned i = 0; i < offset.size(); i++) {
6533             if (const_offset[i])
6534                continue;
6535
6536             acc = emit_extract_vector(ctx, offset, i, s1);
6537             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6538
6539             if (i) {
6540                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6541             }
6542
6543             if (pack == Temp()) {
6544                pack = acc;
6545             } else {
6546                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6547             }
6548          }
6549
6550          if (pack_const && pack != Temp())
6551             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6552       } else {
6553          for (unsigned i = 0; i < offset.size(); i++) {
6554             if (const_offset[i])
6555                continue;
6556
6557             acc = emit_extract_vector(ctx, offset, i, v1);
6558             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6559
6560             if (i) {
6561                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6562             }
6563
6564             if (pack == Temp()) {
6565                pack = acc;
6566             } else {
6567                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6568             }
6569          }
6570
6571          if (pack_const && pack != Temp())
6572             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6573       }
6574       if (pack_const && pack == Temp())
6575          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6576       else if (pack == Temp())
6577          has_offset = false;
6578       else
6579          offset = pack;
6580    }
6581
6582    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6583       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6584
6585    /* pack derivatives */
6586    if (has_ddx || has_ddy) {
6587       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class == GFX9) {
6588          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6589                              ddx, Operand(0u), ddy, Operand(0u));
6590       } else {
6591          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6592       }
6593       has_derivs = true;
6594    }
6595
6596    if (instr->coord_components > 1 &&
6597        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6598        instr->is_array &&
6599        instr->op != nir_texop_txf)
6600       coords = apply_round_slice(ctx, coords, 1);
6601
6602    if (instr->coord_components > 2 &&
6603       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6604        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6605        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6606        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6607        instr->is_array &&
6608        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6609       coords = apply_round_slice(ctx, coords, 2);
6610
6611    if (ctx->options->chip_class == GFX9 &&
6612        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6613        instr->op != nir_texop_lod && instr->coord_components) {
6614       assert(coords.size() > 0 && coords.size() < 3);
6615
6616       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6617       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6618       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6619       if (coords.size() > 1)
6620          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6621       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6622       vec->definitions[0] = Definition(coords);
6623       ctx->block->instructions.emplace_back(std::move(vec));
6624    }
6625
6626    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6627
6628    if (instr->op == nir_texop_samples_identical)
6629       resource = fmask_ptr;
6630
6631    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6632              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6633             instr->op != nir_texop_txs) {
6634       assert(has_sample_index);
6635       Operand op(sample_index);
6636       if (sample_index_cv)
6637          op = Operand(sample_index_cv->u32);
6638       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6639    }
6640
6641    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6642       Temp split_coords[coords.size()];
6643       emit_split_vector(ctx, coords, coords.size());
6644       for (unsigned i = 0; i < coords.size(); i++)
6645          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6646
6647       unsigned i = 0;
6648       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6649          Temp off = emit_extract_vector(ctx, offset, i, v1);
6650          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6651       }
6652
6653       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6654       for (unsigned i = 0; i < coords.size(); i++)
6655          vec->operands[i] = Operand(split_coords[i]);
6656       coords = bld.tmp(coords.regClass());
6657       vec->definitions[0] = Definition(coords);
6658       ctx->block->instructions.emplace_back(std::move(vec));
6659
6660       has_offset = false;
6661    }
6662
6663    /* Build tex instruction */
6664    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6665    unsigned dim = ctx->options->chip_class >= GFX10 && instr->sampler_dim != GLSL_SAMPLER_DIM_BUF
6666                   ? ac_get_sampler_dim(ctx->options->chip_class, instr->sampler_dim, instr->is_array)
6667                   : 0;
6668    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6669    Temp tmp_dst = dst;
6670
6671    /* gather4 selects the component by dmask and always returns vec4 */
6672    if (instr->op == nir_texop_tg4) {
6673       assert(instr->dest.ssa.num_components == 4);
6674       if (instr->is_shadow)
6675          dmask = 1;
6676       else
6677          dmask = 1 << instr->component;
6678       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6679          tmp_dst = bld.tmp(v4);
6680    } else if (instr->op == nir_texop_samples_identical) {
6681       tmp_dst = bld.tmp(v1);
6682    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6683       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6684    }
6685
6686    aco_ptr<MIMG_instruction> tex;
6687    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6688       if (!has_lod)
6689          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6690
6691       bool div_by_6 = instr->op == nir_texop_txs &&
6692                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6693                       instr->is_array &&
6694                       (dmask & (1 << 2));
6695       if (tmp_dst.id() == dst.id() && div_by_6)
6696          tmp_dst = bld.tmp(tmp_dst.regClass());
6697
6698       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6699       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6700       tex->operands[1] = Operand(resource);
6701       if (ctx->options->chip_class == GFX9 &&
6702           instr->op == nir_texop_txs &&
6703           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6704           instr->is_array) {
6705          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6706       } else if (instr->op == nir_texop_query_levels) {
6707          tex->dmask = 1 << 3;
6708       } else {
6709          tex->dmask = dmask;
6710       }
6711       tex->da = da;
6712       tex->definitions[0] = Definition(tmp_dst);
6713       tex->dim = dim;
6714       tex->can_reorder = true;
6715       ctx->block->instructions.emplace_back(std::move(tex));
6716
6717       if (div_by_6) {
6718          /* divide 3rd value by 6 by multiplying with magic number */
6719          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6720          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6721          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6722          assert(instr->dest.ssa.num_components == 3);
6723          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6724          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6725                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6726                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6727                               by_6);
6728
6729       }
6730
6731       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6732       return;
6733    }
6734
6735    Temp tg4_compare_cube_wa64 = Temp();
6736
6737    if (tg4_integer_workarounds) {
6738       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6739       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6740       tex->operands[1] = Operand(resource);
6741       tex->dim = dim;
6742       tex->dmask = 0x3;
6743       tex->da = da;
6744       Temp size = bld.tmp(v2);
6745       tex->definitions[0] = Definition(size);
6746       tex->can_reorder = true;
6747       ctx->block->instructions.emplace_back(std::move(tex));
6748       emit_split_vector(ctx, size, size.size());
6749
6750       Temp half_texel[2];
6751       for (unsigned i = 0; i < 2; i++) {
6752          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6753          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6754          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6755          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6756       }
6757
6758       Temp orig_coords[2] = {
6759          emit_extract_vector(ctx, coords, 0, v1),
6760          emit_extract_vector(ctx, coords, 1, v1)};
6761       Temp new_coords[2] = {
6762          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6763          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6764       };
6765
6766       if (tg4_integer_cube_workaround) {
6767          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6768          Temp desc[resource.size()];
6769          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6770                                                                            Format::PSEUDO, 1, resource.size())};
6771          split->operands[0] = Operand(resource);
6772          for (unsigned i = 0; i < resource.size(); i++) {
6773             desc[i] = bld.tmp(s1);
6774             split->definitions[i] = Definition(desc[i]);
6775          }
6776          ctx->block->instructions.emplace_back(std::move(split));
6777
6778          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6779          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6780                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6781
6782          Temp nfmt;
6783          if (stype == GLSL_TYPE_UINT) {
6784             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6785                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6786                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6787                             bld.scc(compare_cube_wa));
6788          } else {
6789             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6790                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6791                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6792                             bld.scc(compare_cube_wa));
6793          }
6794          tg4_compare_cube_wa64 = bld.tmp(bld.lm);
6795          bool_to_vector_condition(ctx, compare_cube_wa, tg4_compare_cube_wa64);
6796
6797          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6798
6799          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6800                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6801          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6802
6803          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6804                                                                          Format::PSEUDO, resource.size(), 1)};
6805          for (unsigned i = 0; i < resource.size(); i++)
6806             vec->operands[i] = Operand(desc[i]);
6807          resource = bld.tmp(resource.regClass());
6808          vec->definitions[0] = Definition(resource);
6809          ctx->block->instructions.emplace_back(std::move(vec));
6810
6811          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6812                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6813          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6814                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6815       }
6816
6817       if (coords.size() == 3) {
6818          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6819                              new_coords[0], new_coords[1],
6820                              emit_extract_vector(ctx, coords, 2, v1));
6821       } else {
6822          assert(coords.size() == 2);
6823          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6824                              new_coords[0], new_coords[1]);
6825       }
6826    }
6827
6828    std::vector<Operand> args;
6829    if (has_offset)
6830       args.emplace_back(Operand(offset));
6831    if (has_bias)
6832       args.emplace_back(Operand(bias));
6833    if (has_compare)
6834       args.emplace_back(Operand(compare));
6835    if (has_derivs)
6836       args.emplace_back(Operand(derivs));
6837    args.emplace_back(Operand(coords));
6838    if (has_sample_index)
6839       args.emplace_back(Operand(sample_index));
6840    if (has_lod)
6841       args.emplace_back(lod);
6842
6843    Temp arg;
6844    if (args.size() > 1) {
6845       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6846       unsigned size = 0;
6847       for (unsigned i = 0; i < args.size(); i++) {
6848          size += args[i].size();
6849          vec->operands[i] = args[i];
6850       }
6851       RegClass rc = RegClass(RegType::vgpr, size);
6852       Temp tmp = bld.tmp(rc);
6853       vec->definitions[0] = Definition(tmp);
6854       ctx->block->instructions.emplace_back(std::move(vec));
6855       arg = tmp;
6856    } else {
6857       assert(args[0].isTemp());
6858       arg = as_vgpr(ctx, args[0].getTemp());
6859    }
6860
6861    /* we don't need the bias, sample index, compare value or offset to be
6862     * computed in WQM but if the p_create_vector copies the coordinates, then it
6863     * needs to be in WQM */
6864    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6865        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6866        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6867       arg = emit_wqm(ctx, arg, bld.tmp(arg.regClass()), true);
6868
6869    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6870       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6871
6872       assert(coords.size() == 1);
6873       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6874       aco_opcode op;
6875       switch (last_bit) {
6876       case 1:
6877          op = aco_opcode::buffer_load_format_x; break;
6878       case 2:
6879          op = aco_opcode::buffer_load_format_xy; break;
6880       case 3:
6881          op = aco_opcode::buffer_load_format_xyz; break;
6882       case 4:
6883          op = aco_opcode::buffer_load_format_xyzw; break;
6884       default:
6885          unreachable("Tex instruction loads more than 4 components.");
6886       }
6887
6888       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6889       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6890          tmp_dst = dst;
6891       else
6892          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6893
6894       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6895       mubuf->operands[0] = Operand(coords);
6896       mubuf->operands[1] = Operand(resource);
6897       mubuf->operands[2] = Operand((uint32_t) 0);
6898       mubuf->definitions[0] = Definition(tmp_dst);
6899       mubuf->idxen = true;
6900       mubuf->can_reorder = true;
6901       ctx->block->instructions.emplace_back(std::move(mubuf));
6902
6903       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6904       return;
6905    }
6906
6907
6908    if (instr->op == nir_texop_txf ||
6909        instr->op == nir_texop_txf_ms ||
6910        instr->op == nir_texop_samples_identical) {
6911       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6912       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6913       tex->operands[0] = Operand(arg);
6914       tex->operands[1] = Operand(resource);
6915       tex->dim = dim;
6916       tex->dmask = dmask;
6917       tex->unrm = true;
6918       tex->da = da;
6919       tex->definitions[0] = Definition(tmp_dst);
6920       tex->can_reorder = true;
6921       ctx->block->instructions.emplace_back(std::move(tex));
6922
6923       if (instr->op == nir_texop_samples_identical) {
6924          assert(dmask == 1 && dst.regClass() == v1);
6925          assert(dst.id() != tmp_dst.id());
6926
6927          Temp tmp = bld.tmp(bld.lm);
6928          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6929          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6930
6931       } else {
6932          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6933       }
6934       return;
6935    }
6936
6937    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6938    aco_opcode opcode = aco_opcode::image_sample;
6939    if (has_offset) { /* image_sample_*_o */
6940       if (has_compare) {
6941          opcode = aco_opcode::image_sample_c_o;
6942          if (has_derivs)
6943             opcode = aco_opcode::image_sample_c_d_o;
6944          if (has_bias)
6945             opcode = aco_opcode::image_sample_c_b_o;
6946          if (level_zero)
6947             opcode = aco_opcode::image_sample_c_lz_o;
6948          if (has_lod)
6949             opcode = aco_opcode::image_sample_c_l_o;
6950       } else {
6951          opcode = aco_opcode::image_sample_o;
6952          if (has_derivs)
6953             opcode = aco_opcode::image_sample_d_o;
6954          if (has_bias)
6955             opcode = aco_opcode::image_sample_b_o;
6956          if (level_zero)
6957             opcode = aco_opcode::image_sample_lz_o;
6958          if (has_lod)
6959             opcode = aco_opcode::image_sample_l_o;
6960       }
6961    } else { /* no offset */
6962       if (has_compare) {
6963          opcode = aco_opcode::image_sample_c;
6964          if (has_derivs)
6965             opcode = aco_opcode::image_sample_c_d;
6966          if (has_bias)
6967             opcode = aco_opcode::image_sample_c_b;
6968          if (level_zero)
6969             opcode = aco_opcode::image_sample_c_lz;
6970          if (has_lod)
6971             opcode = aco_opcode::image_sample_c_l;
6972       } else {
6973          opcode = aco_opcode::image_sample;
6974          if (has_derivs)
6975             opcode = aco_opcode::image_sample_d;
6976          if (has_bias)
6977             opcode = aco_opcode::image_sample_b;
6978          if (level_zero)
6979             opcode = aco_opcode::image_sample_lz;
6980          if (has_lod)
6981             opcode = aco_opcode::image_sample_l;
6982       }
6983    }
6984
6985    if (instr->op == nir_texop_tg4) {
6986       if (has_offset) {
6987          opcode = aco_opcode::image_gather4_lz_o;
6988          if (has_compare)
6989             opcode = aco_opcode::image_gather4_c_lz_o;
6990       } else {
6991          opcode = aco_opcode::image_gather4_lz;
6992          if (has_compare)
6993             opcode = aco_opcode::image_gather4_c_lz;
6994       }
6995    } else if (instr->op == nir_texop_lod) {
6996       opcode = aco_opcode::image_get_lod;
6997    }
6998
6999    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
7000    tex->operands[0] = Operand(arg);
7001    tex->operands[1] = Operand(resource);
7002    tex->operands[2] = Operand(sampler);
7003    tex->dim = dim;
7004    tex->dmask = dmask;
7005    tex->da = da;
7006    tex->definitions[0] = Definition(tmp_dst);
7007    tex->can_reorder = true;
7008    ctx->block->instructions.emplace_back(std::move(tex));
7009
7010    if (tg4_integer_cube_workaround) {
7011       assert(tmp_dst.id() != dst.id());
7012       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
7013
7014       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
7015       Temp val[4];
7016       for (unsigned i = 0; i < dst.size(); i++) {
7017          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
7018          Temp cvt_val;
7019          if (stype == GLSL_TYPE_UINT)
7020             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
7021          else
7022             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
7023          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
7024       }
7025       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
7026       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
7027                            val[0], val[1], val[2], val[3]);
7028    }
7029    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
7030    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
7031
7032 }
7033
7034
7035 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
7036 {
7037    Temp tmp = get_ssa_temp(ctx, ssa);
7038    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
7039       return Operand(tmp.regClass());
7040    else
7041       return Operand(tmp);
7042 }
7043
7044 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
7045 {
7046    aco_ptr<Pseudo_instruction> phi;
7047    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
7048    assert(instr->dest.ssa.bit_size != 1 || dst.regClass() == ctx->program->lane_mask);
7049
7050    bool logical = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index];
7051    logical |= ctx->block->kind & block_kind_merge;
7052    aco_opcode opcode = logical ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
7053
7054    /* we want a sorted list of sources, since the predecessor list is also sorted */
7055    std::map<unsigned, nir_ssa_def*> phi_src;
7056    nir_foreach_phi_src(src, instr)
7057       phi_src[src->pred->index] = src->src.ssa;
7058
7059    std::vector<unsigned>& preds = logical ? ctx->block->logical_preds : ctx->block->linear_preds;
7060    unsigned num_operands = 0;
7061    Operand operands[std::max(exec_list_length(&instr->srcs), (unsigned)preds.size())];
7062    unsigned num_defined = 0;
7063    unsigned cur_pred_idx = 0;
7064    for (std::pair<unsigned, nir_ssa_def *> src : phi_src) {
7065       if (cur_pred_idx < preds.size()) {
7066          /* handle missing preds (IF merges with discard/break) and extra preds (loop exit with discard) */
7067          unsigned block = ctx->cf_info.nir_to_aco[src.first];
7068          unsigned skipped = 0;
7069          while (cur_pred_idx + skipped < preds.size() && preds[cur_pred_idx + skipped] != block)
7070             skipped++;
7071          if (cur_pred_idx + skipped < preds.size()) {
7072             for (unsigned i = 0; i < skipped; i++)
7073                operands[num_operands++] = Operand(dst.regClass());
7074             cur_pred_idx += skipped;
7075          } else {
7076             continue;
7077          }
7078       }
7079       cur_pred_idx++;
7080       Operand op = get_phi_operand(ctx, src.second);
7081       operands[num_operands++] = op;
7082       num_defined += !op.isUndefined();
7083    }
7084    /* handle block_kind_continue_or_break at loop exit blocks */
7085    while (cur_pred_idx++ < preds.size())
7086       operands[num_operands++] = Operand(dst.regClass());
7087
7088    if (num_defined == 0) {
7089       Builder bld(ctx->program, ctx->block);
7090       if (dst.regClass() == s1) {
7091          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
7092       } else if (dst.regClass() == v1) {
7093          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
7094       } else {
7095          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7096          for (unsigned i = 0; i < dst.size(); i++)
7097             vec->operands[i] = Operand(0u);
7098          vec->definitions[0] = Definition(dst);
7099          ctx->block->instructions.emplace_back(std::move(vec));
7100       }
7101       return;
7102    }
7103
7104    /* we can use a linear phi in some cases if one src is undef */
7105    if (dst.is_linear() && ctx->block->kind & block_kind_merge && num_defined == 1) {
7106       phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, num_operands, 1));
7107
7108       Block *linear_else = &ctx->program->blocks[ctx->block->linear_preds[1]];
7109       Block *invert = &ctx->program->blocks[linear_else->linear_preds[0]];
7110       assert(invert->kind & block_kind_invert);
7111
7112       unsigned then_block = invert->linear_preds[0];
7113
7114       Block* insert_block = NULL;
7115       for (unsigned i = 0; i < num_operands; i++) {
7116          Operand op = operands[i];
7117          if (op.isUndefined())
7118             continue;
7119          insert_block = ctx->block->logical_preds[i] == then_block ? invert : ctx->block;
7120          phi->operands[0] = op;
7121          break;
7122       }
7123       assert(insert_block); /* should be handled by the "num_defined == 0" case above */
7124       phi->operands[1] = Operand(dst.regClass());
7125       phi->definitions[0] = Definition(dst);
7126       insert_block->instructions.emplace(insert_block->instructions.begin(), std::move(phi));
7127       return;
7128    }
7129
7130    /* try to scalarize vector phis */
7131    if (instr->dest.ssa.bit_size != 1 && dst.size() > 1) {
7132       // TODO: scalarize linear phis on divergent ifs
7133       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
7134       std::array<Temp, 4> new_vec;
7135       for (unsigned i = 0; can_scalarize && (i < num_operands); i++) {
7136          Operand src = operands[i];
7137          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end())
7138             can_scalarize = false;
7139       }
7140       if (can_scalarize) {
7141          unsigned num_components = instr->dest.ssa.num_components;
7142          assert(dst.size() % num_components == 0);
7143          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
7144
7145          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
7146          for (unsigned k = 0; k < num_components; k++) {
7147             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7148             for (unsigned i = 0; i < num_operands; i++) {
7149                Operand src = operands[i];
7150                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
7151             }
7152             Temp phi_dst = {ctx->program->allocateId(), rc};
7153             phi->definitions[0] = Definition(phi_dst);
7154             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7155             new_vec[k] = phi_dst;
7156             vec->operands[k] = Operand(phi_dst);
7157          }
7158          vec->definitions[0] = Definition(dst);
7159          ctx->block->instructions.emplace_back(std::move(vec));
7160          ctx->allocated_vec.emplace(dst.id(), new_vec);
7161          return;
7162       }
7163    }
7164
7165    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_operands, 1));
7166    for (unsigned i = 0; i < num_operands; i++)
7167       phi->operands[i] = operands[i];
7168    phi->definitions[0] = Definition(dst);
7169    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
7170 }
7171
7172
7173 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
7174 {
7175    Temp dst = get_ssa_temp(ctx, &instr->def);
7176
7177    assert(dst.type() == RegType::sgpr);
7178
7179    if (dst.size() == 1) {
7180       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
7181    } else {
7182       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
7183       for (unsigned i = 0; i < dst.size(); i++)
7184          vec->operands[i] = Operand(0u);
7185       vec->definitions[0] = Definition(dst);
7186       ctx->block->instructions.emplace_back(std::move(vec));
7187    }
7188 }
7189
7190 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
7191 {
7192    Builder bld(ctx->program, ctx->block);
7193    Block *logical_target;
7194    append_logical_end(ctx->block);
7195    unsigned idx = ctx->block->index;
7196
7197    switch (instr->type) {
7198    case nir_jump_break:
7199       logical_target = ctx->cf_info.parent_loop.exit;
7200       add_logical_edge(idx, logical_target);
7201       ctx->block->kind |= block_kind_break;
7202
7203       if (!ctx->cf_info.parent_if.is_divergent &&
7204           !ctx->cf_info.parent_loop.has_divergent_continue) {
7205          /* uniform break - directly jump out of the loop */
7206          ctx->block->kind |= block_kind_uniform;
7207          ctx->cf_info.has_branch = true;
7208          bld.branch(aco_opcode::p_branch);
7209          add_linear_edge(idx, logical_target);
7210          return;
7211       }
7212       ctx->cf_info.parent_loop.has_divergent_branch = true;
7213       ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7214       break;
7215    case nir_jump_continue:
7216       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7217       add_logical_edge(idx, logical_target);
7218       ctx->block->kind |= block_kind_continue;
7219
7220       if (ctx->cf_info.parent_if.is_divergent) {
7221          /* for potential uniform breaks after this continue,
7222             we must ensure that they are handled correctly */
7223          ctx->cf_info.parent_loop.has_divergent_continue = true;
7224          ctx->cf_info.parent_loop.has_divergent_branch = true;
7225          ctx->cf_info.nir_to_aco[instr->instr.block->index] = ctx->block->index;
7226       } else {
7227          /* uniform continue - directly jump to the loop header */
7228          ctx->block->kind |= block_kind_uniform;
7229          ctx->cf_info.has_branch = true;
7230          bld.branch(aco_opcode::p_branch);
7231          add_linear_edge(idx, logical_target);
7232          return;
7233       }
7234       break;
7235    default:
7236       fprintf(stderr, "Unknown NIR jump instr: ");
7237       nir_print_instr(&instr->instr, stderr);
7238       fprintf(stderr, "\n");
7239       abort();
7240    }
7241
7242    /* remove critical edges from linear CFG */
7243    bld.branch(aco_opcode::p_branch);
7244    Block* break_block = ctx->program->create_and_insert_block();
7245    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7246    break_block->kind |= block_kind_uniform;
7247    add_linear_edge(idx, break_block);
7248    /* the loop_header pointer might be invalidated by this point */
7249    if (instr->type == nir_jump_continue)
7250       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
7251    add_linear_edge(break_block->index, logical_target);
7252    bld.reset(break_block);
7253    bld.branch(aco_opcode::p_branch);
7254
7255    Block* continue_block = ctx->program->create_and_insert_block();
7256    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7257    add_linear_edge(idx, continue_block);
7258    append_logical_start(continue_block);
7259    ctx->block = continue_block;
7260    return;
7261 }
7262
7263 void visit_block(isel_context *ctx, nir_block *block)
7264 {
7265    nir_foreach_instr(instr, block) {
7266       switch (instr->type) {
7267       case nir_instr_type_alu:
7268          visit_alu_instr(ctx, nir_instr_as_alu(instr));
7269          break;
7270       case nir_instr_type_load_const:
7271          visit_load_const(ctx, nir_instr_as_load_const(instr));
7272          break;
7273       case nir_instr_type_intrinsic:
7274          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
7275          break;
7276       case nir_instr_type_tex:
7277          visit_tex(ctx, nir_instr_as_tex(instr));
7278          break;
7279       case nir_instr_type_phi:
7280          visit_phi(ctx, nir_instr_as_phi(instr));
7281          break;
7282       case nir_instr_type_ssa_undef:
7283          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
7284          break;
7285       case nir_instr_type_deref:
7286          break;
7287       case nir_instr_type_jump:
7288          visit_jump(ctx, nir_instr_as_jump(instr));
7289          break;
7290       default:
7291          fprintf(stderr, "Unknown NIR instr type: ");
7292          nir_print_instr(instr, stderr);
7293          fprintf(stderr, "\n");
7294          //abort();
7295       }
7296    }
7297
7298    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7299       ctx->cf_info.nir_to_aco[block->index] = ctx->block->index;
7300 }
7301
7302
7303
7304 static void visit_loop(isel_context *ctx, nir_loop *loop)
7305 {
7306    append_logical_end(ctx->block);
7307    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
7308    Builder bld(ctx->program, ctx->block);
7309    bld.branch(aco_opcode::p_branch);
7310    unsigned loop_preheader_idx = ctx->block->index;
7311
7312    Block loop_exit = Block();
7313    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7314    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
7315
7316    Block* loop_header = ctx->program->create_and_insert_block();
7317    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
7318    loop_header->kind |= block_kind_loop_header;
7319    add_edge(loop_preheader_idx, loop_header);
7320    ctx->block = loop_header;
7321
7322    /* emit loop body */
7323    unsigned loop_header_idx = loop_header->index;
7324    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
7325    append_logical_start(ctx->block);
7326    visit_cf_list(ctx, &loop->body);
7327
7328    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
7329    if (!ctx->cf_info.has_branch) {
7330       append_logical_end(ctx->block);
7331       if (ctx->cf_info.exec_potentially_empty) {
7332          /* Discards can result in code running with an empty exec mask.
7333           * This would result in divergent breaks not ever being taken. As a
7334           * workaround, break the loop when the loop mask is empty instead of
7335           * always continuing. */
7336          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
7337          unsigned block_idx = ctx->block->index;
7338
7339          /* create helper blocks to avoid critical edges */
7340          Block *break_block = ctx->program->create_and_insert_block();
7341          break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7342          break_block->kind = block_kind_uniform;
7343          bld.reset(break_block);
7344          bld.branch(aco_opcode::p_branch);
7345          add_linear_edge(block_idx, break_block);
7346          add_linear_edge(break_block->index, &loop_exit);
7347
7348          Block *continue_block = ctx->program->create_and_insert_block();
7349          continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7350          continue_block->kind = block_kind_uniform;
7351          bld.reset(continue_block);
7352          bld.branch(aco_opcode::p_branch);
7353          add_linear_edge(block_idx, continue_block);
7354          add_linear_edge(continue_block->index, &ctx->program->blocks[loop_header_idx]);
7355
7356          add_logical_edge(block_idx, &ctx->program->blocks[loop_header_idx]);
7357          ctx->block = &ctx->program->blocks[block_idx];
7358       } else {
7359          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
7360          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7361             add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7362          else
7363             add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
7364       }
7365
7366       bld.reset(ctx->block);
7367       bld.branch(aco_opcode::p_branch);
7368    }
7369
7370    /* fixup phis in loop header from unreachable blocks */
7371    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
7372       bool linear = ctx->cf_info.has_branch;
7373       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
7374       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
7375          if ((logical && instr->opcode == aco_opcode::p_phi) ||
7376              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
7377             /* the last operand should be the one that needs to be removed */
7378             instr->operands.pop_back();
7379          } else if (!is_phi(instr)) {
7380             break;
7381          }
7382       }
7383    }
7384
7385    ctx->cf_info.has_branch = false;
7386
7387    // TODO: if the loop has not a single exit, we must add one °°
7388    /* emit loop successor block */
7389    ctx->block = ctx->program->insert_block(std::move(loop_exit));
7390    append_logical_start(ctx->block);
7391
7392    #if 0
7393    // TODO: check if it is beneficial to not branch on continues
7394    /* trim linear phis in loop header */
7395    for (auto&& instr : loop_entry->instructions) {
7396       if (instr->opcode == aco_opcode::p_linear_phi) {
7397          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7398          new_phi->definitions[0] = instr->definitions[0];
7399          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7400             new_phi->operands[i] = instr->operands[i];
7401          /* check that the remaining operands are all the same */
7402          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7403             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7404          instr.swap(new_phi);
7405       } else if (instr->opcode == aco_opcode::p_phi) {
7406          continue;
7407       } else {
7408          break;
7409       }
7410    }
7411    #endif
7412 }
7413
7414 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7415 {
7416    ic->cond = cond;
7417
7418    append_logical_end(ctx->block);
7419    ctx->block->kind |= block_kind_branch;
7420
7421    /* branch to linear then block */
7422    assert(cond.regClass() == ctx->program->lane_mask);
7423    aco_ptr<Pseudo_branch_instruction> branch;
7424    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7425    branch->operands[0] = Operand(cond);
7426    ctx->block->instructions.push_back(std::move(branch));
7427
7428    ic->BB_if_idx = ctx->block->index;
7429    ic->BB_invert = Block();
7430    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7431    /* Invert blocks are intentionally not marked as top level because they
7432     * are not part of the logical cfg. */
7433    ic->BB_invert.kind |= block_kind_invert;
7434    ic->BB_endif = Block();
7435    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7436    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7437
7438    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7439    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7440    ctx->cf_info.parent_if.is_divergent = true;
7441    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7442
7443    /** emit logical then block */
7444    Block* BB_then_logical = ctx->program->create_and_insert_block();
7445    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7446    add_edge(ic->BB_if_idx, BB_then_logical);
7447    ctx->block = BB_then_logical;
7448    append_logical_start(BB_then_logical);
7449 }
7450
7451 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7452 {
7453    Block *BB_then_logical = ctx->block;
7454    append_logical_end(BB_then_logical);
7455     /* branch from logical then block to invert block */
7456    aco_ptr<Pseudo_branch_instruction> branch;
7457    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7458    BB_then_logical->instructions.emplace_back(std::move(branch));
7459    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7460    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7461       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7462    BB_then_logical->kind |= block_kind_uniform;
7463    assert(!ctx->cf_info.has_branch);
7464    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7465    ctx->cf_info.parent_loop.has_divergent_branch = false;
7466
7467    /** emit linear then block */
7468    Block* BB_then_linear = ctx->program->create_and_insert_block();
7469    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7470    BB_then_linear->kind |= block_kind_uniform;
7471    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7472    /* branch from linear then block to invert block */
7473    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7474    BB_then_linear->instructions.emplace_back(std::move(branch));
7475    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7476
7477    /** emit invert merge block */
7478    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7479    ic->invert_idx = ctx->block->index;
7480
7481    /* branch to linear else block (skip else) */
7482    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7483    branch->operands[0] = Operand(ic->cond);
7484    ctx->block->instructions.push_back(std::move(branch));
7485
7486    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7487    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7488
7489    /** emit logical else block */
7490    Block* BB_else_logical = ctx->program->create_and_insert_block();
7491    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7492    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7493    add_linear_edge(ic->invert_idx, BB_else_logical);
7494    ctx->block = BB_else_logical;
7495    append_logical_start(BB_else_logical);
7496 }
7497
7498 static void end_divergent_if(isel_context *ctx, if_context *ic)
7499 {
7500    Block *BB_else_logical = ctx->block;
7501    append_logical_end(BB_else_logical);
7502
7503    /* branch from logical else block to endif block */
7504    aco_ptr<Pseudo_branch_instruction> branch;
7505    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7506    BB_else_logical->instructions.emplace_back(std::move(branch));
7507    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7508    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7509       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7510    BB_else_logical->kind |= block_kind_uniform;
7511
7512    assert(!ctx->cf_info.has_branch);
7513    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7514
7515
7516    /** emit linear else block */
7517    Block* BB_else_linear = ctx->program->create_and_insert_block();
7518    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7519    BB_else_linear->kind |= block_kind_uniform;
7520    add_linear_edge(ic->invert_idx, BB_else_linear);
7521
7522    /* branch from linear else block to endif block */
7523    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7524    BB_else_linear->instructions.emplace_back(std::move(branch));
7525    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7526
7527
7528    /** emit endif merge block */
7529    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7530    append_logical_start(ctx->block);
7531
7532
7533    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7534    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7535    /* uniform control flow never has an empty exec-mask */
7536    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7537       ctx->cf_info.exec_potentially_empty = false;
7538 }
7539
7540 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7541 {
7542    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7543    Builder bld(ctx->program, ctx->block);
7544    aco_ptr<Pseudo_branch_instruction> branch;
7545
7546    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7547       /**
7548        * Uniform conditionals are represented in the following way*) :
7549        *
7550        * The linear and logical CFG:
7551        *                        BB_IF
7552        *                        /    \
7553        *       BB_THEN (logical)      BB_ELSE (logical)
7554        *                        \    /
7555        *                        BB_ENDIF
7556        *
7557        * *) Exceptions may be due to break and continue statements within loops
7558        *    If a break/continue happens within uniform control flow, it branches
7559        *    to the loop exit/entry block. Otherwise, it branches to the next
7560        *    merge block.
7561        **/
7562       append_logical_end(ctx->block);
7563       ctx->block->kind |= block_kind_uniform;
7564
7565       /* emit branch */
7566       assert(cond.regClass() == bld.lm);
7567       // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7568       cond = bool_to_scalar_condition(ctx, cond);
7569
7570       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7571       branch->operands[0] = Operand(cond);
7572       branch->operands[0].setFixed(scc);
7573       ctx->block->instructions.emplace_back(std::move(branch));
7574
7575       unsigned BB_if_idx = ctx->block->index;
7576       Block BB_endif = Block();
7577       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7578       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7579
7580       /** emit then block */
7581       Block* BB_then = ctx->program->create_and_insert_block();
7582       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7583       add_edge(BB_if_idx, BB_then);
7584       append_logical_start(BB_then);
7585       ctx->block = BB_then;
7586       visit_cf_list(ctx, &if_stmt->then_list);
7587       BB_then = ctx->block;
7588       bool then_branch = ctx->cf_info.has_branch;
7589       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7590
7591       if (!then_branch) {
7592          append_logical_end(BB_then);
7593          /* branch from then block to endif block */
7594          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7595          BB_then->instructions.emplace_back(std::move(branch));
7596          add_linear_edge(BB_then->index, &BB_endif);
7597          if (!then_branch_divergent)
7598             add_logical_edge(BB_then->index, &BB_endif);
7599          BB_then->kind |= block_kind_uniform;
7600       }
7601
7602       ctx->cf_info.has_branch = false;
7603       ctx->cf_info.parent_loop.has_divergent_branch = false;
7604
7605       /** emit else block */
7606       Block* BB_else = ctx->program->create_and_insert_block();
7607       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7608       add_edge(BB_if_idx, BB_else);
7609       append_logical_start(BB_else);
7610       ctx->block = BB_else;
7611       visit_cf_list(ctx, &if_stmt->else_list);
7612       BB_else = ctx->block;
7613
7614       if (!ctx->cf_info.has_branch) {
7615          append_logical_end(BB_else);
7616          /* branch from then block to endif block */
7617          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7618          BB_else->instructions.emplace_back(std::move(branch));
7619          add_linear_edge(BB_else->index, &BB_endif);
7620          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7621             add_logical_edge(BB_else->index, &BB_endif);
7622          BB_else->kind |= block_kind_uniform;
7623       }
7624
7625       ctx->cf_info.has_branch &= then_branch;
7626       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7627
7628       /** emit endif merge block */
7629       if (!ctx->cf_info.has_branch) {
7630          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7631          append_logical_start(ctx->block);
7632       }
7633    } else { /* non-uniform condition */
7634       /**
7635        * To maintain a logical and linear CFG without critical edges,
7636        * non-uniform conditionals are represented in the following way*) :
7637        *
7638        * The linear CFG:
7639        *                        BB_IF
7640        *                        /    \
7641        *       BB_THEN (logical)      BB_THEN (linear)
7642        *                        \    /
7643        *                        BB_INVERT (linear)
7644        *                        /    \
7645        *       BB_ELSE (logical)      BB_ELSE (linear)
7646        *                        \    /
7647        *                        BB_ENDIF
7648        *
7649        * The logical CFG:
7650        *                        BB_IF
7651        *                        /    \
7652        *       BB_THEN (logical)      BB_ELSE (logical)
7653        *                        \    /
7654        *                        BB_ENDIF
7655        *
7656        * *) Exceptions may be due to break and continue statements within loops
7657        **/
7658
7659       if_context ic;
7660
7661       begin_divergent_if_then(ctx, &ic, cond);
7662       visit_cf_list(ctx, &if_stmt->then_list);
7663
7664       begin_divergent_if_else(ctx, &ic);
7665       visit_cf_list(ctx, &if_stmt->else_list);
7666
7667       end_divergent_if(ctx, &ic);
7668    }
7669 }
7670
7671 static void visit_cf_list(isel_context *ctx,
7672                           struct exec_list *list)
7673 {
7674    foreach_list_typed(nir_cf_node, node, node, list) {
7675       switch (node->type) {
7676       case nir_cf_node_block:
7677          visit_block(ctx, nir_cf_node_as_block(node));
7678          break;
7679       case nir_cf_node_if:
7680          visit_if(ctx, nir_cf_node_as_if(node));
7681          break;
7682       case nir_cf_node_loop:
7683          visit_loop(ctx, nir_cf_node_as_loop(node));
7684          break;
7685       default:
7686          unreachable("unimplemented cf list type");
7687       }
7688    }
7689 }
7690
7691 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7692 {
7693    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7694    uint64_t mask = ctx->vs_output.mask[slot];
7695    if (!is_pos && !mask)
7696       return;
7697    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7698       return;
7699    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7700    exp->enabled_mask = mask;
7701    for (unsigned i = 0; i < 4; ++i) {
7702       if (mask & (1 << i))
7703          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7704       else
7705          exp->operands[i] = Operand(v1);
7706    }
7707    exp->valid_mask = false;
7708    exp->done = false;
7709    exp->compressed = false;
7710    if (is_pos)
7711       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7712    else
7713       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7714    ctx->block->instructions.emplace_back(std::move(exp));
7715 }
7716
7717 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7718 {
7719    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7720    exp->enabled_mask = 0;
7721    for (unsigned i = 0; i < 4; ++i)
7722       exp->operands[i] = Operand(v1);
7723    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7724       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7725       exp->enabled_mask |= 0x1;
7726    }
7727    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7728       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7729       exp->enabled_mask |= 0x4;
7730    }
7731    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7732       if (ctx->options->chip_class < GFX9) {
7733          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7734          exp->enabled_mask |= 0x8;
7735       } else {
7736          Builder bld(ctx->program, ctx->block);
7737
7738          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7739                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7740          if (exp->operands[2].isTemp())
7741             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7742
7743          exp->operands[2] = Operand(out);
7744          exp->enabled_mask |= 0x4;
7745       }
7746    }
7747    exp->valid_mask = false;
7748    exp->done = false;
7749    exp->compressed = false;
7750    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7751    ctx->block->instructions.emplace_back(std::move(exp));
7752 }
7753
7754 static void create_vs_exports(isel_context *ctx)
7755 {
7756    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7757
7758    if (outinfo->export_prim_id) {
7759       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7760       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = get_arg(ctx, ctx->args->vs_prim_id);
7761    }
7762
7763    if (ctx->options->key.has_multiview_view_index) {
7764       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7765       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, get_arg(ctx, ctx->args->ac.view_index));
7766    }
7767
7768    /* the order these position exports are created is important */
7769    int next_pos = 0;
7770    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7771    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7772       export_vs_psiz_layer_viewport(ctx, &next_pos);
7773    }
7774    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7775       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7776    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7777       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7778
7779    if (ctx->options->key.vs_common_out.export_clip_dists) {
7780       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7781          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7782       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7783          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7784    }
7785
7786    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7787       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7788           i != VARYING_SLOT_PRIMITIVE_ID)
7789          continue;
7790
7791       export_vs_varying(ctx, i, false, NULL);
7792    }
7793 }
7794
7795 static void emit_stream_output(isel_context *ctx,
7796                                Temp const *so_buffers,
7797                                Temp const *so_write_offset,
7798                                const struct radv_stream_output *output)
7799 {
7800    unsigned num_comps = util_bitcount(output->component_mask);
7801    unsigned loc = output->location;
7802    unsigned buf = output->buffer;
7803    unsigned offset = output->offset;
7804
7805    assert(num_comps && num_comps <= 4);
7806    if (!num_comps || num_comps > 4)
7807       return;
7808
7809    unsigned start = ffs(output->component_mask) - 1;
7810
7811    Temp out[4];
7812    bool all_undef = true;
7813    assert(ctx->stage == vertex_vs);
7814    for (unsigned i = 0; i < num_comps; i++) {
7815       out[i] = ctx->vs_output.outputs[loc][start + i];
7816       all_undef = all_undef && !out[i].id();
7817    }
7818    if (all_undef)
7819       return;
7820
7821    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7822    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7823    for (unsigned i = 0; i < num_comps; ++i)
7824       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7825    vec->definitions[0] = Definition(write_data);
7826    ctx->block->instructions.emplace_back(std::move(vec));
7827
7828    aco_opcode opcode;
7829    switch (num_comps) {
7830    case 1:
7831       opcode = aco_opcode::buffer_store_dword;
7832       break;
7833    case 2:
7834       opcode = aco_opcode::buffer_store_dwordx2;
7835       break;
7836    case 3:
7837       opcode = aco_opcode::buffer_store_dwordx3;
7838       break;
7839    case 4:
7840       opcode = aco_opcode::buffer_store_dwordx4;
7841       break;
7842    }
7843
7844    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7845    store->operands[0] = Operand(so_write_offset[buf]);
7846    store->operands[1] = Operand(so_buffers[buf]);
7847    store->operands[2] = Operand((uint32_t) 0);
7848    store->operands[3] = Operand(write_data);
7849    if (offset > 4095) {
7850       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7851       Builder bld(ctx->program, ctx->block);
7852       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7853    } else {
7854       store->offset = offset;
7855    }
7856    store->offen = true;
7857    store->glc = true;
7858    store->dlc = false;
7859    store->slc = true;
7860    store->can_reorder = true;
7861    ctx->block->instructions.emplace_back(std::move(store));
7862 }
7863
7864 static void emit_streamout(isel_context *ctx, unsigned stream)
7865 {
7866    Builder bld(ctx->program, ctx->block);
7867
7868    Temp so_buffers[4];
7869    Temp buf_ptr = convert_pointer_to_64_bit(ctx, get_arg(ctx, ctx->args->streamout_buffers));
7870    for (unsigned i = 0; i < 4; i++) {
7871       unsigned stride = ctx->program->info->so.strides[i];
7872       if (!stride)
7873          continue;
7874
7875       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7876    }
7877
7878    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7879                                 get_arg(ctx, ctx->args->streamout_config), Operand(0x70010u));
7880
7881    Temp tid = emit_mbcnt(ctx, bld.def(v1));
7882
7883    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7884
7885    if_context ic;
7886    begin_divergent_if_then(ctx, &ic, can_emit);
7887
7888    bld.reset(ctx->block);
7889
7890    Temp so_write_index = bld.vadd32(bld.def(v1), get_arg(ctx, ctx->args->streamout_write_idx), tid);
7891
7892    Temp so_write_offset[4];
7893
7894    for (unsigned i = 0; i < 4; i++) {
7895       unsigned stride = ctx->program->info->so.strides[i];
7896       if (!stride)
7897          continue;
7898
7899       if (stride == 1) {
7900          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7901                                 get_arg(ctx, ctx->args->streamout_write_idx),
7902                                 get_arg(ctx, ctx->args->streamout_offset[i]));
7903          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7904
7905          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7906       } else {
7907          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7908          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u),
7909                                  get_arg(ctx, ctx->args->streamout_offset[i]));
7910          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7911       }
7912    }
7913
7914    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7915       struct radv_stream_output *output =
7916          &ctx->program->info->so.outputs[i];
7917       if (stream != output->stream)
7918          continue;
7919
7920       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7921    }
7922
7923    begin_divergent_if_else(ctx, &ic);
7924    end_divergent_if(ctx, &ic);
7925 }
7926
7927 } /* end namespace */
7928
7929 void split_arguments(isel_context *ctx, Pseudo_instruction *startpgm)
7930 {
7931    /* Split all arguments except for the first (ring_offsets) and the last
7932     * (exec) so that the dead channels don't stay live throughout the program.
7933     */
7934    for (unsigned i = 1; i < startpgm->definitions.size() - 1; i++) {
7935       if (startpgm->definitions[i].regClass().size() > 1) {
7936          emit_split_vector(ctx, startpgm->definitions[i].getTemp(),
7937                            startpgm->definitions[i].regClass().size());
7938       }
7939    }
7940 }
7941
7942 void handle_bc_optimize(isel_context *ctx)
7943 {
7944    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7945    Builder bld(ctx->program, ctx->block);
7946    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7947    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7948    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7949    ctx->persp_centroid = get_arg(ctx, ctx->args->ac.persp_centroid);
7950    ctx->linear_centroid = get_arg(ctx, ctx->args->ac.linear_centroid);
7951    if (uses_center && uses_centroid) {
7952       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(bld.lm)),
7953                               get_arg(ctx, ctx->args->ac.prim_mask), Operand(0u));
7954
7955       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7956          Temp new_coord[2];
7957          for (unsigned i = 0; i < 2; i++) {
7958             Temp persp_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_centroid), i, v1);
7959             Temp persp_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.persp_center), i, v1);
7960             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7961                                     persp_centroid, persp_center, sel);
7962          }
7963          ctx->persp_centroid = bld.tmp(v2);
7964          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->persp_centroid),
7965                     Operand(new_coord[0]), Operand(new_coord[1]));
7966          emit_split_vector(ctx, ctx->persp_centroid, 2);
7967       }
7968
7969       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7970          Temp new_coord[2];
7971          for (unsigned i = 0; i < 2; i++) {
7972             Temp linear_centroid = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_centroid), i, v1);
7973             Temp linear_center = emit_extract_vector(ctx, get_arg(ctx, ctx->args->ac.linear_center), i, v1);
7974             new_coord[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7975                                     linear_centroid, linear_center, sel);
7976          }
7977          ctx->linear_centroid = bld.tmp(v2);
7978          bld.pseudo(aco_opcode::p_create_vector, Definition(ctx->linear_centroid),
7979                     Operand(new_coord[0]), Operand(new_coord[1]));
7980          emit_split_vector(ctx, ctx->linear_centroid, 2);
7981       }
7982    }
7983 }
7984
7985 void setup_fp_mode(isel_context *ctx, nir_shader *shader)
7986 {
7987    Program *program = ctx->program;
7988
7989    unsigned float_controls = shader->info.float_controls_execution_mode;
7990
7991    program->next_fp_mode.preserve_signed_zero_inf_nan32 =
7992       float_controls & FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP32;
7993    program->next_fp_mode.preserve_signed_zero_inf_nan16_64 =
7994       float_controls & (FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP16 |
7995                         FLOAT_CONTROLS_SIGNED_ZERO_INF_NAN_PRESERVE_FP64);
7996
7997    program->next_fp_mode.must_flush_denorms32 =
7998       float_controls & FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP32;
7999    program->next_fp_mode.must_flush_denorms16_64 =
8000       float_controls & (FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP16 |
8001                         FLOAT_CONTROLS_DENORM_FLUSH_TO_ZERO_FP64);
8002
8003    program->next_fp_mode.care_about_round32 =
8004       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP32);
8005
8006    program->next_fp_mode.care_about_round16_64 =
8007       float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64 |
8008                         FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTE_FP64);
8009
8010    /* default to preserving fp16 and fp64 denorms, since it's free */
8011    if (program->next_fp_mode.must_flush_denorms16_64)
8012       program->next_fp_mode.denorm16_64 = 0;
8013    else
8014       program->next_fp_mode.denorm16_64 = fp_denorm_keep;
8015
8016    /* preserving fp32 denorms is expensive, so only do it if asked */
8017    if (float_controls & FLOAT_CONTROLS_DENORM_PRESERVE_FP32)
8018       program->next_fp_mode.denorm32 = fp_denorm_keep;
8019    else
8020       program->next_fp_mode.denorm32 = 0;
8021
8022    if (float_controls & FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP32)
8023       program->next_fp_mode.round32 = fp_round_tz;
8024    else
8025       program->next_fp_mode.round32 = fp_round_ne;
8026
8027    if (float_controls & (FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP16 | FLOAT_CONTROLS_ROUNDING_MODE_RTZ_FP64))
8028       program->next_fp_mode.round16_64 = fp_round_tz;
8029    else
8030       program->next_fp_mode.round16_64 = fp_round_ne;
8031
8032    ctx->block->fp_mode = program->next_fp_mode;
8033 }
8034
8035 void select_program(Program *program,
8036                     unsigned shader_count,
8037                     struct nir_shader *const *shaders,
8038                     ac_shader_config* config,
8039                     struct radv_shader_args *args)
8040 {
8041    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, args);
8042
8043    for (unsigned i = 0; i < shader_count; i++) {
8044       nir_shader *nir = shaders[i];
8045       init_context(&ctx, nir);
8046
8047       setup_fp_mode(&ctx, nir);
8048
8049       if (!i) {
8050          /* needs to be after init_context() for FS */
8051          Pseudo_instruction *startpgm = add_startpgm(&ctx);
8052          append_logical_start(ctx.block);
8053          split_arguments(&ctx, startpgm);
8054       }
8055
8056       if_context ic;
8057       if (shader_count >= 2) {
8058          Builder bld(ctx.program, ctx.block);
8059          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
8060          Temp thread_id = emit_mbcnt(&ctx, bld.def(v1));
8061          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(bld.lm)), count, thread_id);
8062
8063          begin_divergent_if_then(&ctx, &ic, cond);
8064       }
8065
8066       if (i) {
8067          Builder bld(ctx.program, ctx.block);
8068          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
8069          bld.sopp(aco_opcode::s_barrier);
8070       }
8071
8072       if (ctx.stage == fragment_fs)
8073          handle_bc_optimize(&ctx);
8074
8075       nir_function_impl *func = nir_shader_get_entrypoint(nir);
8076       visit_cf_list(&ctx, &func->body);
8077
8078       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
8079          emit_streamout(&ctx, 0);
8080
8081       if (ctx.stage == vertex_vs)
8082          create_vs_exports(&ctx);
8083
8084       if (shader_count >= 2) {
8085          begin_divergent_if_else(&ctx, &ic);
8086          end_divergent_if(&ctx, &ic);
8087       }
8088
8089       ralloc_free(ctx.divergent_vals);
8090    }
8091
8092    program->config->float_mode = program->blocks[0].fp_mode.val;
8093
8094    append_logical_end(ctx.block);
8095    ctx.block->kind |= block_kind_uniform;
8096    Builder bld(ctx.program, ctx.block);
8097    if (ctx.program->wb_smem_l1_on_end)
8098       bld.smem(aco_opcode::s_dcache_wb, false);
8099    bld.sopp(aco_opcode::s_endpgm);
8100
8101    /* cleanup CFG */
8102    for (Block& BB : program->blocks) {
8103       for (unsigned idx : BB.linear_preds)
8104          program->blocks[idx].linear_succs.emplace_back(BB.index);
8105       for (unsigned idx : BB.logical_preds)
8106          program->blocks[idx].logical_succs.emplace_back(BB.index);
8107    }
8108 }
8109 }