src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <map>
  28
  29 #include "aco_ir.h"
  30 #include "aco_builder.h"
  31 #include "aco_interface.h"
  32 #include "aco_instruction_selection_setup.cpp"
  33 #include "util/fast_idiv_by_const.h"
  34
  35 namespace aco {
  36 namespace {
  37
  38 class loop_info_RAII {
  39    isel_context* ctx;
  40    unsigned header_idx_old;
  41    Block* exit_old;
  42    bool divergent_cont_old;
  43    bool divergent_branch_old;
  44    bool divergent_if_old;
  45
  46 public:
  47    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  48       : ctx(ctx),
  49         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  50         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  51         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  52         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  53    {
  54       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  55       ctx->cf_info.parent_loop.exit = loop_exit;
  56       ctx->cf_info.parent_loop.has_divergent_continue = false;
  57       ctx->cf_info.parent_loop.has_divergent_branch = false;
  58       ctx->cf_info.parent_if.is_divergent = false;
  59       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  60    }
  61
  62    ~loop_info_RAII()
  63    {
  64       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  65       ctx->cf_info.parent_loop.exit = exit_old;
  66       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  67       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  68       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  69       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  70       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  71          ctx->cf_info.exec_potentially_empty = false;
  72    }
  73 };
  74
  75 struct if_context {
  76    Temp cond;
  77
  78    bool divergent_old;
  79    bool exec_potentially_empty_old;
  80
  81    unsigned BB_if_idx;
  82    unsigned invert_idx;
  83    bool then_branch_divergent;
  84    Block BB_invert;
  85    Block BB_endif;
  86 };
  87
  88 static void visit_cf_list(struct isel_context *ctx,
  89                           struct exec_list *list);
  90
  91 static void add_logical_edge(unsigned pred_idx, Block *succ)
  92 {
  93    succ->logical_preds.emplace_back(pred_idx);
  94 }
  95
  96
  97 static void add_linear_edge(unsigned pred_idx, Block *succ)
  98 {
  99    succ->linear_preds.emplace_back(pred_idx);
 100 }
 101
 102 static void add_edge(unsigned pred_idx, Block *succ)
 103 {
 104    add_logical_edge(pred_idx, succ);
 105    add_linear_edge(pred_idx, succ);
 106 }
 107
 108 static void append_logical_start(Block *b)
 109 {
 110    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 111 }
 112
 113 static void append_logical_end(Block *b)
 114 {
 115    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 116 }
 117
 118 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 119 {
 120    assert(ctx->allocated[def->index].id());
 121    return ctx->allocated[def->index];
 122 }
 123
 124 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 125 {
 126    Builder bld(ctx->program, ctx->block);
 127
 128    if (!dst.id())
 129       dst = bld.tmp(src.regClass());
 130
 131    if (ctx->stage != fragment_fs) {
 132       if (!dst.id())
 133          return src;
 134
 135       if (src.type() == RegType::vgpr || src.size() > 1)
 136          bld.copy(Definition(dst), src);
 137       else
 138          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 139       return dst;
 140    }
 141
 142    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 143    ctx->program->needs_wqm |= program_needs_wqm;
 144    return dst;
 145 }
 146
 147 Temp as_vgpr(isel_context *ctx, Temp val)
 148 {
 149    if (val.type() == RegType::sgpr) {
 150       Builder bld(ctx->program, ctx->block);
 151       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 152    }
 153    assert(val.type() == RegType::vgpr);
 154    return val;
 155 }
 156
 157 //assumes a != 0xffffffff
 158 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 159 {
 160    assert(b != 0);
 161    Builder bld(ctx->program, ctx->block);
 162
 163    if (util_is_power_of_two_or_zero(b)) {
 164       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 165       return;
 166    }
 167
 168    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 169
 170    assert(info.multiplier <= 0xffffffff);
 171
 172    bool pre_shift = info.pre_shift != 0;
 173    bool increment = info.increment != 0;
 174    bool multiply = true;
 175    bool post_shift = info.post_shift != 0;
 176
 177    if (!pre_shift && !increment && !multiply && !post_shift) {
 178       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 179       return;
 180    }
 181
 182    Temp pre_shift_dst = a;
 183    if (pre_shift) {
 184       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 185       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 186    }
 187
 188    Temp increment_dst = pre_shift_dst;
 189    if (increment) {
 190       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 191       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 192    }
 193
 194    Temp multiply_dst = increment_dst;
 195    if (multiply) {
 196       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 197       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 198                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 199    }
 200
 201    if (post_shift) {
 202       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 203    }
 204 }
 205
 206 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 207 {
 208    Builder bld(ctx->program, ctx->block);
 209    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 210 }
 211
 212
 213 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 214 {
 215    /* no need to extract the whole vector */
 216    if (src.regClass() == dst_rc) {
 217       assert(idx == 0);
 218       return src;
 219    }
 220    assert(src.size() > idx);
 221    Builder bld(ctx->program, ctx->block);
 222    auto it = ctx->allocated_vec.find(src.id());
 223    /* the size check needs to be early because elements other than 0 may be garbage */
 224    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 225       if (it->second[idx].regClass() == dst_rc) {
 226          return it->second[idx];
 227       } else {
 228          assert(dst_rc.size() == it->second[idx].regClass().size());
 229          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 230          return bld.copy(bld.def(dst_rc), it->second[idx]);
 231       }
 232    }
 233
 234    if (src.size() == dst_rc.size()) {
 235       assert(idx == 0);
 236       return bld.copy(bld.def(dst_rc), src);
 237    } else {
 238       Temp dst = bld.tmp(dst_rc);
 239       emit_extract_vector(ctx, src, idx, dst);
 240       return dst;
 241    }
 242 }
 243
 244 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 245 {
 246    if (num_components == 1)
 247       return;
 248    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 249       return;
 250    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 251    split->operands[0] = Operand(vec_src);
 252    std::array<Temp,4> elems;
 253    for (unsigned i = 0; i < num_components; i++) {
 254       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 255       split->definitions[i] = Definition(elems[i]);
 256    }
 257    ctx->block->instructions.emplace_back(std::move(split));
 258    ctx->allocated_vec.emplace(vec_src.id(), elems);
 259 }
 260
 261 /* This vector expansion uses a mask to determine which elements in the new vector
 262  * come from the original vector. The other elements are undefined. */
 263 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 264 {
 265    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 266
 267    if (vec_src == dst)
 268       return;
 269
 270    Builder bld(ctx->program, ctx->block);
 271    if (num_components == 1) {
 272       if (dst.type() == RegType::sgpr)
 273          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 274       else
 275          bld.copy(Definition(dst), vec_src);
 276       return;
 277    }
 278
 279    unsigned component_size = dst.size() / num_components;
 280    std::array<Temp,4> elems;
 281
 282    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 283    vec->definitions[0] = Definition(dst);
 284    unsigned k = 0;
 285    for (unsigned i = 0; i < num_components; i++) {
 286       if (mask & (1 << i)) {
 287          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 288          if (dst.type() == RegType::sgpr)
 289             src = bld.as_uniform(src);
 290          vec->operands[i] = Operand(src);
 291       } else {
 292          vec->operands[i] = Operand(0u);
 293       }
 294       elems[i] = vec->operands[i].getTemp();
 295    }
 296    ctx->block->instructions.emplace_back(std::move(vec));
 297    ctx->allocated_vec.emplace(dst.id(), elems);
 298 }
 299
 300 Temp as_divergent_bool(isel_context *ctx, Temp val, bool vcc_hint)
 301 {
 302    if (val.regClass() == s2) {
 303       return val;
 304    } else {
 305       assert(val.regClass() == s1);
 306       Builder bld(ctx->program, ctx->block);
 307       Definition& def = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2),
 308                                  Operand((uint32_t) -1), Operand(0u), bld.scc(val)).def(0);
 309       if (vcc_hint)
 310          def.setHint(vcc);
 311       return def.getTemp();
 312    }
 313 }
 314
 315 Temp as_uniform_bool(isel_context *ctx, Temp val)
 316 {
 317    if (val.regClass() == s1) {
 318       return val;
 319    } else {
 320       assert(val.regClass() == s2);
 321       Builder bld(ctx->program, ctx->block);
 322       return bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), Operand(0u), Operand(val));
 323    }
 324 }
 325
 326 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 327 {
 328    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 329       return get_ssa_temp(ctx, src.src.ssa);
 330
 331    if (src.src.ssa->num_components == size) {
 332       bool identity_swizzle = true;
 333       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 334          if (src.swizzle[i] != i)
 335             identity_swizzle = false;
 336       }
 337       if (identity_swizzle)
 338          return get_ssa_temp(ctx, src.src.ssa);
 339    }
 340
 341    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 342    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 343    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 344    assert(vec.size() % elem_size == 0);
 345
 346    RegClass elem_rc = RegClass(vec.type(), elem_size);
 347    if (size == 1) {
 348       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 349    } else {
 350       assert(size <= 4);
 351       std::array<Temp,4> elems;
 352       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 353       for (unsigned i = 0; i < size; ++i) {
 354          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 355          vec_instr->operands[i] = Operand{elems[i]};
 356       }
 357       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 358       vec_instr->definitions[0] = Definition(dst);
 359       ctx->block->instructions.emplace_back(std::move(vec_instr));
 360       ctx->allocated_vec.emplace(dst.id(), elems);
 361       return dst;
 362    }
 363 }
 364
 365 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 366 {
 367    if (ptr.size() == 2)
 368       return ptr;
 369    Builder bld(ctx->program, ctx->block);
 370    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 371                      ptr, Operand((unsigned)ctx->options->address32_hi));
 372 }
 373
 374 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 375 {
 376    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 377    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 378    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 379    sop2->definitions[0] = Definition(dst);
 380    if (writes_scc)
 381       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 382    ctx->block->instructions.emplace_back(std::move(sop2));
 383 }
 384
 385 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 386 {
 387    Builder bld(ctx->program, ctx->block);
 388    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 389    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 390    if (src1.type() == RegType::sgpr) {
 391       if (commutative && src0.type() == RegType::vgpr) {
 392          Temp t = src0;
 393          src0 = src1;
 394          src1 = t;
 395       } else if (src0.type() == RegType::vgpr &&
 396                  op != aco_opcode::v_madmk_f32 &&
 397                  op != aco_opcode::v_madak_f32 &&
 398                  op != aco_opcode::v_madmk_f16 &&
 399                  op != aco_opcode::v_madak_f16) {
 400          /* If the instruction is not commutative, we emit a VOP3A instruction */
 401          bld.vop2_e64(op, Definition(dst), src0, src1);
 402          return;
 403       } else {
 404          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 405       }
 406    }
 407    bld.vop2(op, Definition(dst), src0, src1);
 408 }
 409
 410 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 411 {
 412    Temp src0 = get_alu_src(ctx, instr->src[0]);
 413    Temp src1 = get_alu_src(ctx, instr->src[1]);
 414    Temp src2 = get_alu_src(ctx, instr->src[2]);
 415
 416    /* ensure that the instruction has at most 1 sgpr operand
 417     * The optimizer will inline constants for us */
 418    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 419       src0 = as_vgpr(ctx, src0);
 420    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 421       src1 = as_vgpr(ctx, src1);
 422    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 423       src2 = as_vgpr(ctx, src2);
 424
 425    Builder bld(ctx->program, ctx->block);
 426    bld.vop3(op, Definition(dst), src0, src1, src2);
 427 }
 428
 429 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 430 {
 431    Builder bld(ctx->program, ctx->block);
 432    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 433 }
 434
 435 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 436 {
 437    Temp src0 = get_alu_src(ctx, instr->src[0]);
 438    Temp src1 = get_alu_src(ctx, instr->src[1]);
 439    aco_ptr<Instruction> vopc;
 440    if (src1.type() == RegType::sgpr) {
 441       if (src0.type() == RegType::vgpr) {
 442          /* to swap the operands, we might also have to change the opcode */
 443          switch (op) {
 444             case aco_opcode::v_cmp_lt_f32:
 445                op = aco_opcode::v_cmp_gt_f32;
 446                break;
 447             case aco_opcode::v_cmp_ge_f32:
 448                op = aco_opcode::v_cmp_le_f32;
 449                break;
 450             case aco_opcode::v_cmp_lt_i32:
 451                op = aco_opcode::v_cmp_gt_i32;
 452                break;
 453             case aco_opcode::v_cmp_ge_i32:
 454                op = aco_opcode::v_cmp_le_i32;
 455                break;
 456             case aco_opcode::v_cmp_lt_u32:
 457                op = aco_opcode::v_cmp_gt_u32;
 458                break;
 459             case aco_opcode::v_cmp_ge_u32:
 460                op = aco_opcode::v_cmp_le_u32;
 461                break;
 462             case aco_opcode::v_cmp_lt_f64:
 463                op = aco_opcode::v_cmp_gt_f64;
 464                break;
 465             case aco_opcode::v_cmp_ge_f64:
 466                op = aco_opcode::v_cmp_le_f64;
 467                break;
 468             case aco_opcode::v_cmp_lt_i64:
 469                op = aco_opcode::v_cmp_gt_i64;
 470                break;
 471             case aco_opcode::v_cmp_ge_i64:
 472                op = aco_opcode::v_cmp_le_i64;
 473                break;
 474             case aco_opcode::v_cmp_lt_u64:
 475                op = aco_opcode::v_cmp_gt_u64;
 476                break;
 477             case aco_opcode::v_cmp_ge_u64:
 478                op = aco_opcode::v_cmp_le_u64;
 479                break;
 480             default: /* eq and ne are commutative */
 481                break;
 482          }
 483          Temp t = src0;
 484          src0 = src1;
 485          src1 = t;
 486       } else {
 487          src1 = as_vgpr(ctx, src1);
 488       }
 489    }
 490    Builder bld(ctx->program, ctx->block);
 491    bld.vopc(op, Definition(dst), src0, src1).def(0).setHint(vcc);
 492 }
 493
 494 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 495 {
 496    if (dst.regClass() == s2) {
 497       emit_vopc_instruction(ctx, instr, op, dst);
 498       if (!ctx->divergent_vals[instr->dest.dest.ssa.index])
 499          emit_split_vector(ctx, dst, 2);
 500    } else if (dst.regClass() == s1) {
 501       Temp src0 = get_alu_src(ctx, instr->src[0]);
 502       Temp src1 = get_alu_src(ctx, instr->src[1]);
 503       assert(src0.type() == RegType::sgpr && src1.type() == RegType::sgpr);
 504
 505       Builder bld(ctx->program, ctx->block);
 506       bld.sopc(op, bld.scc(Definition(dst)), src0, src1);
 507
 508    } else {
 509       assert(false);
 510    }
 511 }
 512
 513 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 514 {
 515    Builder bld(ctx->program, ctx->block);
 516    Temp src0 = get_alu_src(ctx, instr->src[0]);
 517    Temp src1 = get_alu_src(ctx, instr->src[1]);
 518    if (dst.regClass() == s2) {
 519       bld.sop2(op64, Definition(dst), bld.def(s1, scc),
 520                as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
 521    } else {
 522       assert(dst.regClass() == s1);
 523       bld.sop2(op32, bld.def(s1), bld.scc(Definition(dst)),
 524                as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
 525    }
 526 }
 527
 528
 529 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 530 {
 531    Builder bld(ctx->program, ctx->block);
 532    Temp cond = get_alu_src(ctx, instr->src[0]);
 533    Temp then = get_alu_src(ctx, instr->src[1]);
 534    Temp els = get_alu_src(ctx, instr->src[2]);
 535
 536    if (dst.type() == RegType::vgpr) {
 537       cond = as_divergent_bool(ctx, cond, true);
 538
 539       aco_ptr<Instruction> bcsel;
 540       if (dst.size() == 1) {
 541          then = as_vgpr(ctx, then);
 542          els = as_vgpr(ctx, els);
 543
 544          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 545       } else if (dst.size() == 2) {
 546          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 547          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 548          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 549          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 550
 551          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 552          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 553
 554          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 555       } else {
 556          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 557          nir_print_instr(&instr->instr, stderr);
 558          fprintf(stderr, "\n");
 559       }
 560       return;
 561    }
 562
 563    if (instr->dest.dest.ssa.bit_size != 1) { /* uniform condition and values in sgpr */
 564       if (dst.regClass() == s1 || dst.regClass() == s2) {
 565          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 566          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 567          bld.sop2(op, Definition(dst), then, els, bld.scc(as_uniform_bool(ctx, cond)));
 568       } else {
 569          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 570          nir_print_instr(&instr->instr, stderr);
 571          fprintf(stderr, "\n");
 572       }
 573       return;
 574    }
 575
 576    /* boolean bcsel */
 577    assert(instr->dest.dest.ssa.bit_size == 1);
 578
 579    if (dst.regClass() == s1)
 580       cond = as_uniform_bool(ctx, cond);
 581
 582    if (cond.regClass() == s1) { /* uniform selection */
 583       aco_opcode op;
 584       if (dst.regClass() == s2) {
 585          op = aco_opcode::s_cselect_b64;
 586          then = as_divergent_bool(ctx, then, false);
 587          els = as_divergent_bool(ctx, els, false);
 588       } else {
 589          assert(dst.regClass() == s1);
 590          op = aco_opcode::s_cselect_b32;
 591          then = as_uniform_bool(ctx, then);
 592          els = as_uniform_bool(ctx, els);
 593       }
 594       bld.sop2(op, Definition(dst), then, els, bld.scc(cond));
 595       return;
 596    }
 597
 598    /* divergent boolean bcsel
 599     * this implements bcsel on bools: dst = s0 ? s1 : s2
 600     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 601    assert (dst.regClass() == s2);
 602    then = as_divergent_bool(ctx, then, false);
 603    els = as_divergent_bool(ctx, els, false);
 604
 605    if (cond.id() != then.id())
 606       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 607
 608    if (cond.id() == els.id())
 609       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 610    else
 611       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 612                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 613 }
 614
 615 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 616 {
 617    if (!instr->dest.dest.is_ssa) {
 618       fprintf(stderr, "nir alu dst not in ssa: ");
 619       nir_print_instr(&instr->instr, stderr);
 620       fprintf(stderr, "\n");
 621       abort();
 622    }
 623    Builder bld(ctx->program, ctx->block);
 624    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 625    switch(instr->op) {
 626    case nir_op_vec2:
 627    case nir_op_vec3:
 628    case nir_op_vec4: {
 629       std::array<Temp,4> elems;
 630       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 631       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 632          elems[i] = get_alu_src(ctx, instr->src[i]);
 633          vec->operands[i] = Operand{elems[i]};
 634       }
 635       vec->definitions[0] = Definition(dst);
 636       ctx->block->instructions.emplace_back(std::move(vec));
 637       ctx->allocated_vec.emplace(dst.id(), elems);
 638       break;
 639    }
 640    case nir_op_mov: {
 641       Temp src = get_alu_src(ctx, instr->src[0]);
 642       aco_ptr<Instruction> mov;
 643       if (dst.type() == RegType::sgpr) {
 644          if (src.type() == RegType::vgpr)
 645             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 646          else if (src.regClass() == s1)
 647             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 648          else if (src.regClass() == s2)
 649             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 650          else
 651             unreachable("wrong src register class for nir_op_imov");
 652       } else if (dst.regClass() == v1) {
 653          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 654       } else if (dst.regClass() == v2) {
 655          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 656       } else {
 657          nir_print_instr(&instr->instr, stderr);
 658          unreachable("Should have been lowered to scalar.");
 659       }
 660       break;
 661    }
 662    case nir_op_inot: {
 663       Temp src = get_alu_src(ctx, instr->src[0]);
 664       /* uniform booleans */
 665       if (instr->dest.dest.ssa.bit_size == 1 && dst.regClass() == s1) {
 666          if (src.regClass() == s1) {
 667             /* in this case, src is either 1 or 0 */
 668             bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.scc(Definition(dst)), Operand(1u), src);
 669          } else {
 670             /* src is either exec_mask or 0 */
 671             assert(src.regClass() == s2);
 672             bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(Definition(dst)), Operand(0u), src);
 673          }
 674       } else if (dst.regClass() == v1) {
 675          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 676       } else if (dst.type() == RegType::sgpr) {
 677          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 678          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 679       } else {
 680          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 681          nir_print_instr(&instr->instr, stderr);
 682          fprintf(stderr, "\n");
 683       }
 684       break;
 685    }
 686    case nir_op_ineg: {
 687       Temp src = get_alu_src(ctx, instr->src[0]);
 688       if (dst.regClass() == v1) {
 689          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 690       } else if (dst.regClass() == s1) {
 691          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 692       } else if (dst.size() == 2) {
 693          Temp src0 = bld.tmp(dst.type(), 1);
 694          Temp src1 = bld.tmp(dst.type(), 1);
 695          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 696
 697          if (dst.regClass() == s2) {
 698             Temp carry = bld.tmp(s1);
 699             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 700             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 701             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 702          } else {
 703             Temp lower = bld.tmp(v1);
 704             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 705             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 706             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 707          }
 708       } else {
 709          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 710          nir_print_instr(&instr->instr, stderr);
 711          fprintf(stderr, "\n");
 712       }
 713       break;
 714    }
 715    case nir_op_iabs: {
 716       if (dst.regClass() == s1) {
 717          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 718       } else if (dst.regClass() == v1) {
 719          Temp src = get_alu_src(ctx, instr->src[0]);
 720          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 721       } else {
 722          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 723          nir_print_instr(&instr->instr, stderr);
 724          fprintf(stderr, "\n");
 725       }
 726       break;
 727    }
 728    case nir_op_isign: {
 729       Temp src = get_alu_src(ctx, instr->src[0]);
 730       if (dst.regClass() == s1) {
 731          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 732          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 733          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 734       } else if (dst.regClass() == s2) {
 735          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 736          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 737          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 738       } else if (dst.regClass() == v1) {
 739          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 740          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 741          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 742       } else if (dst.regClass() == v2) {
 743          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 744          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 745          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 746          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 747          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 748          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 749       } else {
 750          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 751          nir_print_instr(&instr->instr, stderr);
 752          fprintf(stderr, "\n");
 753       }
 754       break;
 755    }
 756    case nir_op_imax: {
 757       if (dst.regClass() == v1) {
 758          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 759       } else if (dst.regClass() == s1) {
 760          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 761       } else {
 762          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 763          nir_print_instr(&instr->instr, stderr);
 764          fprintf(stderr, "\n");
 765       }
 766       break;
 767    }
 768    case nir_op_umax: {
 769       if (dst.regClass() == v1) {
 770          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 771       } else if (dst.regClass() == s1) {
 772          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 773       } else {
 774          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 775          nir_print_instr(&instr->instr, stderr);
 776          fprintf(stderr, "\n");
 777       }
 778       break;
 779    }
 780    case nir_op_imin: {
 781       if (dst.regClass() == v1) {
 782          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 783       } else if (dst.regClass() == s1) {
 784          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 785       } else {
 786          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 787          nir_print_instr(&instr->instr, stderr);
 788          fprintf(stderr, "\n");
 789       }
 790       break;
 791    }
 792    case nir_op_umin: {
 793       if (dst.regClass() == v1) {
 794          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 795       } else if (dst.regClass() == s1) {
 796          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 797       } else {
 798          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 799          nir_print_instr(&instr->instr, stderr);
 800          fprintf(stderr, "\n");
 801       }
 802       break;
 803    }
 804    case nir_op_ior: {
 805       if (instr->dest.dest.ssa.bit_size == 1) {
 806          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 807       } else if (dst.regClass() == v1) {
 808          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 809       } else if (dst.regClass() == s1) {
 810          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 811       } else if (dst.regClass() == s2) {
 812          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 813       } else {
 814          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 815          nir_print_instr(&instr->instr, stderr);
 816          fprintf(stderr, "\n");
 817       }
 818       break;
 819    }
 820    case nir_op_iand: {
 821       if (instr->dest.dest.ssa.bit_size == 1) {
 822          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 823       } else if (dst.regClass() == v1) {
 824          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 825       } else if (dst.regClass() == s1) {
 826          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 827       } else if (dst.regClass() == s2) {
 828          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 829       } else {
 830          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 831          nir_print_instr(&instr->instr, stderr);
 832          fprintf(stderr, "\n");
 833       }
 834       break;
 835    }
 836    case nir_op_ixor: {
 837       if (instr->dest.dest.ssa.bit_size == 1) {
 838          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 839       } else if (dst.regClass() == v1) {
 840          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 841       } else if (dst.regClass() == s1) {
 842          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 843       } else if (dst.regClass() == s2) {
 844          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 845       } else {
 846          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 847          nir_print_instr(&instr->instr, stderr);
 848          fprintf(stderr, "\n");
 849       }
 850       break;
 851    }
 852    case nir_op_ushr: {
 853       if (dst.regClass() == v1) {
 854          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 855       } else if (dst.regClass() == v2) {
 856          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 857                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 858       } else if (dst.regClass() == s2) {
 859          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 860       } else if (dst.regClass() == s1) {
 861          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 862       } else {
 863          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 864          nir_print_instr(&instr->instr, stderr);
 865          fprintf(stderr, "\n");
 866       }
 867       break;
 868    }
 869    case nir_op_ishl: {
 870       if (dst.regClass() == v1) {
 871          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 872       } else if (dst.regClass() == v2) {
 873          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 874                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 875       } else if (dst.regClass() == s1) {
 876          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 877       } else if (dst.regClass() == s2) {
 878          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 879       } else {
 880          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 881          nir_print_instr(&instr->instr, stderr);
 882          fprintf(stderr, "\n");
 883       }
 884       break;
 885    }
 886    case nir_op_ishr: {
 887       if (dst.regClass() == v1) {
 888          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 889       } else if (dst.regClass() == v2) {
 890          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 891                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 892       } else if (dst.regClass() == s1) {
 893          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 894       } else if (dst.regClass() == s2) {
 895          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 896       } else {
 897          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 898          nir_print_instr(&instr->instr, stderr);
 899          fprintf(stderr, "\n");
 900       }
 901       break;
 902    }
 903    case nir_op_find_lsb: {
 904       Temp src = get_alu_src(ctx, instr->src[0]);
 905       if (src.regClass() == s1) {
 906          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 907       } else if (src.regClass() == v1) {
 908          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 909       } else if (src.regClass() == s2) {
 910          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 911       } else {
 912          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 913          nir_print_instr(&instr->instr, stderr);
 914          fprintf(stderr, "\n");
 915       }
 916       break;
 917    }
 918    case nir_op_ufind_msb:
 919    case nir_op_ifind_msb: {
 920       Temp src = get_alu_src(ctx, instr->src[0]);
 921       if (src.regClass() == s1 || src.regClass() == s2) {
 922          aco_opcode op = src.regClass() == s2 ?
 923                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 924                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 925          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 926
 927          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 928                                         Operand(src.size() * 32u - 1u), msb_rev);
 929          Temp msb = sub.def(0).getTemp();
 930          Temp carry = sub.def(1).getTemp();
 931
 932          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 933       } else if (src.regClass() == v1) {
 934          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 935          Temp msb_rev = bld.tmp(v1);
 936          emit_vop1_instruction(ctx, instr, op, msb_rev);
 937          Temp msb = bld.tmp(v1);
 938          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 939          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 940       } else {
 941          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 942          nir_print_instr(&instr->instr, stderr);
 943          fprintf(stderr, "\n");
 944       }
 945       break;
 946    }
 947    case nir_op_bitfield_reverse: {
 948       if (dst.regClass() == s1) {
 949          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 950       } else if (dst.regClass() == v1) {
 951          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 952       } else {
 953          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 954          nir_print_instr(&instr->instr, stderr);
 955          fprintf(stderr, "\n");
 956       }
 957       break;
 958    }
 959    case nir_op_iadd: {
 960       if (dst.regClass() == s1) {
 961          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
 962          break;
 963       }
 964
 965       Temp src0 = get_alu_src(ctx, instr->src[0]);
 966       Temp src1 = get_alu_src(ctx, instr->src[1]);
 967       if (dst.regClass() == v1) {
 968          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
 969          break;
 970       }
 971
 972       assert(src0.size() == 2 && src1.size() == 2);
 973       Temp src00 = bld.tmp(src0.type(), 1);
 974       Temp src01 = bld.tmp(dst.type(), 1);
 975       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
 976       Temp src10 = bld.tmp(src1.type(), 1);
 977       Temp src11 = bld.tmp(dst.type(), 1);
 978       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
 979
 980       if (dst.regClass() == s2) {
 981          Temp carry = bld.tmp(s1);
 982          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
 983          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
 984          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 985       } else if (dst.regClass() == v2) {
 986          Temp dst0 = bld.tmp(v1);
 987          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
 988          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
 989          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 990       } else {
 991          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 992          nir_print_instr(&instr->instr, stderr);
 993          fprintf(stderr, "\n");
 994       }
 995       break;
 996    }
 997    case nir_op_uadd_sat: {
 998       Temp src0 = get_alu_src(ctx, instr->src[0]);
 999       Temp src1 = get_alu_src(ctx, instr->src[1]);
1000       if (dst.regClass() == s1) {
1001          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1002          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1003                   src0, src1);
1004          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1005       } else if (dst.regClass() == v1) {
1006          if (ctx->options->chip_class >= GFX9) {
1007             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1008             add->operands[0] = Operand(src0);
1009             add->operands[1] = Operand(src1);
1010             add->definitions[0] = Definition(dst);
1011             add->clamp = 1;
1012             ctx->block->instructions.emplace_back(std::move(add));
1013          } else {
1014             if (src1.regClass() != v1)
1015                std::swap(src0, src1);
1016             assert(src1.regClass() == v1);
1017             Temp tmp = bld.tmp(v1);
1018             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1019             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1020          }
1021       } else {
1022          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1023          nir_print_instr(&instr->instr, stderr);
1024          fprintf(stderr, "\n");
1025       }
1026       break;
1027    }
1028    case nir_op_uadd_carry: {
1029       Temp src0 = get_alu_src(ctx, instr->src[0]);
1030       Temp src1 = get_alu_src(ctx, instr->src[1]);
1031       if (dst.regClass() == s1) {
1032          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1033          break;
1034       }
1035       if (dst.regClass() == v1) {
1036          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1037          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1038          break;
1039       }
1040
1041       Temp src00 = bld.tmp(src0.type(), 1);
1042       Temp src01 = bld.tmp(dst.type(), 1);
1043       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1044       Temp src10 = bld.tmp(src1.type(), 1);
1045       Temp src11 = bld.tmp(dst.type(), 1);
1046       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1047       if (dst.regClass() == s2) {
1048          Temp carry = bld.tmp(s1);
1049          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1050          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1051          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1052       } else if (dst.regClass() == v2) {
1053          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1054          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1055          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1056          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1057       } else {
1058          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1059          nir_print_instr(&instr->instr, stderr);
1060          fprintf(stderr, "\n");
1061       }
1062       break;
1063    }
1064    case nir_op_isub: {
1065       if (dst.regClass() == s1) {
1066          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1067          break;
1068       }
1069
1070       Temp src0 = get_alu_src(ctx, instr->src[0]);
1071       Temp src1 = get_alu_src(ctx, instr->src[1]);
1072       if (dst.regClass() == v1) {
1073          bld.vsub32(Definition(dst), src0, src1);
1074          break;
1075       }
1076
1077       Temp src00 = bld.tmp(src0.type(), 1);
1078       Temp src01 = bld.tmp(dst.type(), 1);
1079       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1080       Temp src10 = bld.tmp(src1.type(), 1);
1081       Temp src11 = bld.tmp(dst.type(), 1);
1082       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1083       if (dst.regClass() == s2) {
1084          Temp carry = bld.tmp(s1);
1085          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1086          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1087          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1088       } else if (dst.regClass() == v2) {
1089          Temp lower = bld.tmp(v1);
1090          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1091          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1092          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1093       } else {
1094          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1095          nir_print_instr(&instr->instr, stderr);
1096          fprintf(stderr, "\n");
1097       }
1098       break;
1099    }
1100    case nir_op_usub_borrow: {
1101       Temp src0 = get_alu_src(ctx, instr->src[0]);
1102       Temp src1 = get_alu_src(ctx, instr->src[1]);
1103       if (dst.regClass() == s1) {
1104          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1105          break;
1106       } else if (dst.regClass() == v1) {
1107          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1108          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1109          break;
1110       }
1111
1112       Temp src00 = bld.tmp(src0.type(), 1);
1113       Temp src01 = bld.tmp(dst.type(), 1);
1114       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1115       Temp src10 = bld.tmp(src1.type(), 1);
1116       Temp src11 = bld.tmp(dst.type(), 1);
1117       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1118       if (dst.regClass() == s2) {
1119          Temp borrow = bld.tmp(s1);
1120          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1121          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1122          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1123       } else if (dst.regClass() == v2) {
1124          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1125          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1126          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1127          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1128       } else {
1129          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1130          nir_print_instr(&instr->instr, stderr);
1131          fprintf(stderr, "\n");
1132       }
1133       break;
1134    }
1135    case nir_op_imul: {
1136       if (dst.regClass() == v1) {
1137          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1138                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1139       } else if (dst.regClass() == s1) {
1140          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1141       } else {
1142          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1143          nir_print_instr(&instr->instr, stderr);
1144          fprintf(stderr, "\n");
1145       }
1146       break;
1147    }
1148    case nir_op_umul_high: {
1149       if (dst.regClass() == v1) {
1150          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1151       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1152          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1153       } else if (dst.regClass() == s1) {
1154          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1155                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1156          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1157       } else {
1158          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1159          nir_print_instr(&instr->instr, stderr);
1160          fprintf(stderr, "\n");
1161       }
1162       break;
1163    }
1164    case nir_op_imul_high: {
1165       if (dst.regClass() == v1) {
1166          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1167       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1168          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1169       } else if (dst.regClass() == s1) {
1170          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1171                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1172          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1173       } else {
1174          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1175          nir_print_instr(&instr->instr, stderr);
1176          fprintf(stderr, "\n");
1177       }
1178       break;
1179    }
1180    case nir_op_fmul: {
1181       if (dst.size() == 1) {
1182          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1183       } else if (dst.size() == 2) {
1184          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1185                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1186       } else {
1187          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1188          nir_print_instr(&instr->instr, stderr);
1189          fprintf(stderr, "\n");
1190       }
1191       break;
1192    }
1193    case nir_op_fadd: {
1194       if (dst.size() == 1) {
1195          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1196       } else if (dst.size() == 2) {
1197          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1198                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1199       } else {
1200          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1201          nir_print_instr(&instr->instr, stderr);
1202          fprintf(stderr, "\n");
1203       }
1204       break;
1205    }
1206    case nir_op_fsub: {
1207       Temp src0 = get_alu_src(ctx, instr->src[0]);
1208       Temp src1 = get_alu_src(ctx, instr->src[1]);
1209       if (dst.size() == 1) {
1210          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1211             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1212          else
1213             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1214       } else if (dst.size() == 2) {
1215          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1216                                      get_alu_src(ctx, instr->src[0]),
1217                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1218          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1219          sub->neg[1] = true;
1220       } else {
1221          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1222          nir_print_instr(&instr->instr, stderr);
1223          fprintf(stderr, "\n");
1224       }
1225       break;
1226    }
1227    case nir_op_fmod:
1228    case nir_op_frem: {
1229       if (dst.size() == 1) {
1230          Temp rcp = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_alu_src(ctx, instr->src[1]));
1231          Temp mul = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]), rcp);
1232
1233          aco_opcode op = instr->op == nir_op_fmod ? aco_opcode::v_floor_f32 : aco_opcode::v_trunc_f32;
1234          Temp floor = bld.vop1(op, bld.def(v1), mul);
1235
1236          mul = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), get_alu_src(ctx, instr->src[1]), floor);
1237          bld.vop2(aco_opcode::v_sub_f32, Definition(dst), get_alu_src(ctx, instr->src[0]), mul);
1238       } else if (dst.size() == 2) {
1239          Temp rcp = bld.vop1(aco_opcode::v_rcp_f64, bld.def(v2), get_alu_src(ctx, instr->src[1]));
1240          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), get_alu_src(ctx, instr->src[0]), rcp);
1241
1242          aco_opcode op = instr->op == nir_op_fmod ? aco_opcode::v_floor_f64 : aco_opcode::v_trunc_f64;
1243          Temp floor = bld.vop1(op, bld.def(v1), mul);
1244
1245          mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), get_alu_src(ctx, instr->src[1]), floor);
1246          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]), mul);
1247          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1248          sub->neg[1] = true;
1249       } else {
1250          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1251          nir_print_instr(&instr->instr, stderr);
1252          fprintf(stderr, "\n");
1253       }
1254       break;
1255    }
1256    case nir_op_fmax: {
1257       if (dst.size() == 1) {
1258          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1259       } else if (dst.size() == 2) {
1260          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1261                   get_alu_src(ctx, instr->src[0]),
1262                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1263       } else {
1264          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1265          nir_print_instr(&instr->instr, stderr);
1266          fprintf(stderr, "\n");
1267       }
1268       break;
1269    }
1270    case nir_op_fmin: {
1271       if (dst.size() == 1) {
1272          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1273       } else if (dst.size() == 2) {
1274          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1275                   get_alu_src(ctx, instr->src[0]),
1276                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1277       } else {
1278          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1279          nir_print_instr(&instr->instr, stderr);
1280          fprintf(stderr, "\n");
1281       }
1282       break;
1283    }
1284    case nir_op_fmax3: {
1285       if (dst.size() == 1) {
1286          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1287       } else {
1288          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1289          nir_print_instr(&instr->instr, stderr);
1290          fprintf(stderr, "\n");
1291       }
1292       break;
1293    }
1294    case nir_op_fmin3: {
1295       if (dst.size() == 1) {
1296          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1297       } else {
1298          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1299          nir_print_instr(&instr->instr, stderr);
1300          fprintf(stderr, "\n");
1301       }
1302       break;
1303    }
1304    case nir_op_fmed3: {
1305       if (dst.size() == 1) {
1306          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1307       } else {
1308          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1309          nir_print_instr(&instr->instr, stderr);
1310          fprintf(stderr, "\n");
1311       }
1312       break;
1313    }
1314    case nir_op_umax3: {
1315       if (dst.size() == 1) {
1316          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1317       } else {
1318          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1319          nir_print_instr(&instr->instr, stderr);
1320          fprintf(stderr, "\n");
1321       }
1322       break;
1323    }
1324    case nir_op_umin3: {
1325       if (dst.size() == 1) {
1326          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1327       } else {
1328          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1329          nir_print_instr(&instr->instr, stderr);
1330          fprintf(stderr, "\n");
1331       }
1332       break;
1333    }
1334    case nir_op_umed3: {
1335       if (dst.size() == 1) {
1336          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1337       } else {
1338          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1339          nir_print_instr(&instr->instr, stderr);
1340          fprintf(stderr, "\n");
1341       }
1342       break;
1343    }
1344    case nir_op_imax3: {
1345       if (dst.size() == 1) {
1346          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1347       } else {
1348          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1349          nir_print_instr(&instr->instr, stderr);
1350          fprintf(stderr, "\n");
1351       }
1352       break;
1353    }
1354    case nir_op_imin3: {
1355       if (dst.size() == 1) {
1356          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1357       } else {
1358          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1359          nir_print_instr(&instr->instr, stderr);
1360          fprintf(stderr, "\n");
1361       }
1362       break;
1363    }
1364    case nir_op_imed3: {
1365       if (dst.size() == 1) {
1366          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1367       } else {
1368          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1369          nir_print_instr(&instr->instr, stderr);
1370          fprintf(stderr, "\n");
1371       }
1372       break;
1373    }
1374    case nir_op_cube_face_coord: {
1375       Temp in = get_alu_src(ctx, instr->src[0], 3);
1376       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1377                       emit_extract_vector(ctx, in, 1, v1),
1378                       emit_extract_vector(ctx, in, 2, v1) };
1379       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1380       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1381       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1382       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1383       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1384       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1385       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1386       break;
1387    }
1388    case nir_op_cube_face_index: {
1389       Temp in = get_alu_src(ctx, instr->src[0], 3);
1390       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1391                       emit_extract_vector(ctx, in, 1, v1),
1392                       emit_extract_vector(ctx, in, 2, v1) };
1393       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1394       break;
1395    }
1396    case nir_op_bcsel: {
1397       emit_bcsel(ctx, instr, dst);
1398       break;
1399    }
1400    case nir_op_frsq: {
1401       if (dst.size() == 1) {
1402          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1403       } else if (dst.size() == 2) {
1404          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1405       } else {
1406          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1407          nir_print_instr(&instr->instr, stderr);
1408          fprintf(stderr, "\n");
1409       }
1410       break;
1411    }
1412    case nir_op_fneg: {
1413       Temp src = get_alu_src(ctx, instr->src[0]);
1414       if (dst.size() == 1) {
1415          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1416       } else if (dst.size() == 2) {
1417          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1418          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1419          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1420          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1421       } else {
1422          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1423          nir_print_instr(&instr->instr, stderr);
1424          fprintf(stderr, "\n");
1425       }
1426       break;
1427    }
1428    case nir_op_fabs: {
1429       Temp src = get_alu_src(ctx, instr->src[0]);
1430       if (dst.size() == 1) {
1431          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1432       } else if (dst.size() == 2) {
1433          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1434          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1435          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1436          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1437       } else {
1438          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1439          nir_print_instr(&instr->instr, stderr);
1440          fprintf(stderr, "\n");
1441       }
1442       break;
1443    }
1444    case nir_op_fsat: {
1445       Temp src = get_alu_src(ctx, instr->src[0]);
1446       if (dst.size() == 1) {
1447          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1448       } else if (dst.size() == 2) {
1449          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1450          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1451          vop3->clamp = true;
1452       } else {
1453          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1454          nir_print_instr(&instr->instr, stderr);
1455          fprintf(stderr, "\n");
1456       }
1457       break;
1458    }
1459    case nir_op_flog2: {
1460       if (dst.size() == 1) {
1461          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1462       } else {
1463          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1464          nir_print_instr(&instr->instr, stderr);
1465          fprintf(stderr, "\n");
1466       }
1467       break;
1468    }
1469    case nir_op_frcp: {
1470       if (dst.size() == 1) {
1471          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1472       } else if (dst.size() == 2) {
1473          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1474       } else {
1475          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1476          nir_print_instr(&instr->instr, stderr);
1477          fprintf(stderr, "\n");
1478       }
1479       break;
1480    }
1481    case nir_op_fexp2: {
1482       if (dst.size() == 1) {
1483          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1484       } else {
1485          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1486          nir_print_instr(&instr->instr, stderr);
1487          fprintf(stderr, "\n");
1488       }
1489       break;
1490    }
1491    case nir_op_fsqrt: {
1492       if (dst.size() == 1) {
1493          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1494       } else if (dst.size() == 2) {
1495          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1496       } else {
1497          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1498          nir_print_instr(&instr->instr, stderr);
1499          fprintf(stderr, "\n");
1500       }
1501       break;
1502    }
1503    case nir_op_ffract: {
1504       if (dst.size() == 1) {
1505          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1506       } else if (dst.size() == 2) {
1507          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1508       } else {
1509          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1510          nir_print_instr(&instr->instr, stderr);
1511          fprintf(stderr, "\n");
1512       }
1513       break;
1514    }
1515    case nir_op_ffloor: {
1516       if (dst.size() == 1) {
1517          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1518       } else if (dst.size() == 2) {
1519          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1520       } else {
1521          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1522          nir_print_instr(&instr->instr, stderr);
1523          fprintf(stderr, "\n");
1524       }
1525       break;
1526    }
1527    case nir_op_fceil: {
1528       if (dst.size() == 1) {
1529          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1530       } else if (dst.size() == 2) {
1531          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1532       } else {
1533          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1534          nir_print_instr(&instr->instr, stderr);
1535          fprintf(stderr, "\n");
1536       }
1537       break;
1538    }
1539    case nir_op_ftrunc: {
1540       if (dst.size() == 1) {
1541          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1542       } else if (dst.size() == 2) {
1543          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1544       } else {
1545          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1546          nir_print_instr(&instr->instr, stderr);
1547          fprintf(stderr, "\n");
1548       }
1549       break;
1550    }
1551    case nir_op_fround_even: {
1552       if (dst.size() == 1) {
1553          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1554       } else if (dst.size() == 2) {
1555          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1556       } else {
1557          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1558          nir_print_instr(&instr->instr, stderr);
1559          fprintf(stderr, "\n");
1560       }
1561       break;
1562    }
1563    case nir_op_fsin:
1564    case nir_op_fcos: {
1565       Temp src = get_alu_src(ctx, instr->src[0]);
1566       aco_ptr<Instruction> norm;
1567       if (dst.size() == 1) {
1568          Temp tmp;
1569          Operand half_pi(0x3e22f983u);
1570          if (src.type() == RegType::sgpr)
1571             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1572          else
1573             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1574
1575          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1576          if (ctx->options->chip_class < GFX9)
1577             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1578
1579          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1580          bld.vop1(opcode, Definition(dst), tmp);
1581       } else {
1582          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1583          nir_print_instr(&instr->instr, stderr);
1584          fprintf(stderr, "\n");
1585       }
1586       break;
1587    }
1588    case nir_op_ldexp: {
1589       if (dst.size() == 1) {
1590          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1591                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1592                   get_alu_src(ctx, instr->src[1]));
1593       } else if (dst.size() == 2) {
1594          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1595                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1596                   get_alu_src(ctx, instr->src[1]));
1597       } else {
1598          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1599          nir_print_instr(&instr->instr, stderr);
1600          fprintf(stderr, "\n");
1601       }
1602       break;
1603    }
1604    case nir_op_frexp_sig: {
1605       if (dst.size() == 1) {
1606          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1607                   get_alu_src(ctx, instr->src[0]));
1608       } else if (dst.size() == 2) {
1609          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1610                   get_alu_src(ctx, instr->src[0]));
1611       } else {
1612          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1613          nir_print_instr(&instr->instr, stderr);
1614          fprintf(stderr, "\n");
1615       }
1616       break;
1617    }
1618    case nir_op_frexp_exp: {
1619       if (instr->src[0].src.ssa->bit_size == 32) {
1620          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1621                   get_alu_src(ctx, instr->src[0]));
1622       } else if (instr->src[0].src.ssa->bit_size == 64) {
1623          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1624                   get_alu_src(ctx, instr->src[0]));
1625       } else {
1626          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1627          nir_print_instr(&instr->instr, stderr);
1628          fprintf(stderr, "\n");
1629       }
1630       break;
1631    }
1632    case nir_op_fsign: {
1633       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1634       if (dst.size() == 1) {
1635          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1636          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1637          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1638          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1639       } else if (dst.size() == 2) {
1640          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1641          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1642          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, src, cond);
1643
1644          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1645          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1646          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1647
1648          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1649       } else {
1650          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1651          nir_print_instr(&instr->instr, stderr);
1652          fprintf(stderr, "\n");
1653       }
1654       break;
1655    }
1656    case nir_op_f2f32: {
1657       if (instr->src[0].src.ssa->bit_size == 64) {
1658          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1659       } else {
1660          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1661          nir_print_instr(&instr->instr, stderr);
1662          fprintf(stderr, "\n");
1663       }
1664       break;
1665    }
1666    case nir_op_f2f64: {
1667       if (instr->src[0].src.ssa->bit_size == 32) {
1668          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1669       } else {
1670          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1671          nir_print_instr(&instr->instr, stderr);
1672          fprintf(stderr, "\n");
1673       }
1674       break;
1675    }
1676    case nir_op_i2f32: {
1677       assert(dst.size() == 1);
1678       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1679       break;
1680    }
1681    case nir_op_i2f64: {
1682       if (instr->src[0].src.ssa->bit_size == 32) {
1683          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1684       } else if (instr->src[0].src.ssa->bit_size == 64) {
1685          Temp src = get_alu_src(ctx, instr->src[0]);
1686          RegClass rc = RegClass(src.type(), 1);
1687          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1688          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1689          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1690          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1691          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1692          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1693
1694       } else {
1695          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1696          nir_print_instr(&instr->instr, stderr);
1697          fprintf(stderr, "\n");
1698       }
1699       break;
1700    }
1701    case nir_op_u2f32: {
1702       assert(dst.size() == 1);
1703       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1704       break;
1705    }
1706    case nir_op_u2f64: {
1707       if (instr->src[0].src.ssa->bit_size == 32) {
1708          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1709       } else if (instr->src[0].src.ssa->bit_size == 64) {
1710          Temp src = get_alu_src(ctx, instr->src[0]);
1711          RegClass rc = RegClass(src.type(), 1);
1712          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1713          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1714          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1715          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1716          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1717          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1718       } else {
1719          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1720          nir_print_instr(&instr->instr, stderr);
1721          fprintf(stderr, "\n");
1722       }
1723       break;
1724    }
1725    case nir_op_f2i32: {
1726       Temp src = get_alu_src(ctx, instr->src[0]);
1727       if (instr->src[0].src.ssa->bit_size == 32) {
1728          if (dst.type() == RegType::vgpr)
1729             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1730          else
1731             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1732                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1733
1734       } else if (instr->src[0].src.ssa->bit_size == 64) {
1735          if (dst.type() == RegType::vgpr)
1736             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1737          else
1738             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1739                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1740
1741       } else {
1742          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1743          nir_print_instr(&instr->instr, stderr);
1744          fprintf(stderr, "\n");
1745       }
1746       break;
1747    }
1748    case nir_op_f2u32: {
1749       Temp src = get_alu_src(ctx, instr->src[0]);
1750       if (instr->src[0].src.ssa->bit_size == 32) {
1751          if (dst.type() == RegType::vgpr)
1752             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1753          else
1754             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1755                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1756
1757       } else if (instr->src[0].src.ssa->bit_size == 64) {
1758          if (dst.type() == RegType::vgpr)
1759             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1760          else
1761             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1762                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1763
1764       } else {
1765          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1766          nir_print_instr(&instr->instr, stderr);
1767          fprintf(stderr, "\n");
1768       }
1769       break;
1770    }
1771    case nir_op_f2i64: {
1772       Temp src = get_alu_src(ctx, instr->src[0]);
1773       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1774          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1775          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1776          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1777          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1778          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1779          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1780          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1781          Temp new_exponent = bld.tmp(v1);
1782          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1783          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1784          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1785          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1786          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1787          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1788          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1789          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1790          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1791          Temp new_lower = bld.tmp(v1);
1792          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1793          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1794          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1795
1796       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1797          if (src.type() == RegType::vgpr)
1798             src = bld.as_uniform(src);
1799          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1800          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1801          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1802          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1803          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1804          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1805          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1806          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1807          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1808          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1809          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1810          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1811          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1812          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1813          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1814          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1815          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1816          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1817          Temp borrow = bld.tmp(s1);
1818          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1819          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1820          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1821
1822       } else if (instr->src[0].src.ssa->bit_size == 64) {
1823          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1824          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1825          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1826          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1827          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1828          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1829          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1830          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1831          if (dst.type() == RegType::sgpr) {
1832             lower = bld.as_uniform(lower);
1833             upper = bld.as_uniform(upper);
1834          }
1835          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1836
1837       } else {
1838          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1839          nir_print_instr(&instr->instr, stderr);
1840          fprintf(stderr, "\n");
1841       }
1842       break;
1843    }
1844    case nir_op_f2u64: {
1845       Temp src = get_alu_src(ctx, instr->src[0]);
1846       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1847          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1848          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1849          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1850          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1851          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1852          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1853          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1854          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1855          Temp new_exponent = bld.tmp(v1);
1856          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1857          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1858          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1859          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1860          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1861          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1862          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1863          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1864          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1865
1866       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1867          if (src.type() == RegType::vgpr)
1868             src = bld.as_uniform(src);
1869          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1870          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1871          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1872          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1873          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1874          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1875          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1876          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1877          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1878          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1879          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1880          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1881          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1882          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1883          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1884          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1885          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1886          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1887
1888       } else if (instr->src[0].src.ssa->bit_size == 64) {
1889          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1890          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1891          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1892          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1893          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1894          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1895          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1896          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1897          if (dst.type() == RegType::sgpr) {
1898             lower = bld.as_uniform(lower);
1899             upper = bld.as_uniform(upper);
1900          }
1901          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1902
1903       } else {
1904          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1905          nir_print_instr(&instr->instr, stderr);
1906          fprintf(stderr, "\n");
1907       }
1908       break;
1909    }
1910    case nir_op_b2f32: {
1911       Temp src = get_alu_src(ctx, instr->src[0]);
1912       if (dst.regClass() == s1) {
1913          src = as_uniform_bool(ctx, src);
1914          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1915       } else if (dst.regClass() == v1) {
1916          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u),
1917                       as_divergent_bool(ctx, src, true));
1918       } else {
1919          unreachable("Wrong destination register class for nir_op_b2f32.");
1920       }
1921       break;
1922    }
1923    case nir_op_b2f64: {
1924       Temp src = get_alu_src(ctx, instr->src[0]);
1925       if (dst.regClass() == s2) {
1926          src = as_uniform_bool(ctx, src);
1927          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1928       } else if (dst.regClass() == v2) {
1929          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1930          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one,
1931                       as_divergent_bool(ctx, src, true));
1932          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1933       } else {
1934          unreachable("Wrong destination register class for nir_op_b2f64.");
1935       }
1936       break;
1937    }
1938    case nir_op_i2i32: {
1939       Temp src = get_alu_src(ctx, instr->src[0]);
1940       if (instr->src[0].src.ssa->bit_size == 64) {
1941          /* we can actually just say dst = src, as it would map the lower register */
1942          emit_extract_vector(ctx, src, 0, dst);
1943       } else {
1944          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1945          nir_print_instr(&instr->instr, stderr);
1946          fprintf(stderr, "\n");
1947       }
1948       break;
1949    }
1950    case nir_op_u2u32: {
1951       Temp src = get_alu_src(ctx, instr->src[0]);
1952       if (instr->src[0].src.ssa->bit_size == 16) {
1953          if (dst.regClass() == s1) {
1954             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1955          } else {
1956             // TODO: do better with SDWA
1957             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1958          }
1959       } else if (instr->src[0].src.ssa->bit_size == 64) {
1960          /* we can actually just say dst = src, as it would map the lower register */
1961          emit_extract_vector(ctx, src, 0, dst);
1962       } else {
1963          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1964          nir_print_instr(&instr->instr, stderr);
1965          fprintf(stderr, "\n");
1966       }
1967       break;
1968    }
1969    case nir_op_i2i64: {
1970       Temp src = get_alu_src(ctx, instr->src[0]);
1971       if (instr->src[0].src.ssa->bit_size == 32) {
1972          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1973       } else {
1974          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1975          nir_print_instr(&instr->instr, stderr);
1976          fprintf(stderr, "\n");
1977       }
1978       break;
1979    }
1980    case nir_op_u2u64: {
1981       Temp src = get_alu_src(ctx, instr->src[0]);
1982       if (instr->src[0].src.ssa->bit_size == 32) {
1983          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1984       } else {
1985          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1986          nir_print_instr(&instr->instr, stderr);
1987          fprintf(stderr, "\n");
1988       }
1989       break;
1990    }
1991    case nir_op_b2i32: {
1992       Temp src = get_alu_src(ctx, instr->src[0]);
1993       if (dst.regClass() == s1) {
1994          if (src.regClass() == s1) {
1995             bld.copy(Definition(dst), src);
1996          } else {
1997             // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
1998             assert(src.regClass() == s2);
1999             bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(dst)), Operand(0u), src);
2000          }
2001       } else {
2002          assert(dst.regClass() == v1 && src.regClass() == s2);
2003          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2004       }
2005       break;
2006    }
2007    case nir_op_i2b1: {
2008       Temp src = get_alu_src(ctx, instr->src[0]);
2009       if (dst.regClass() == s2) {
2010          assert(src.regClass() == v1 || src.regClass() == v2);
2011          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2012                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2013       } else {
2014          assert(src.regClass() == s1 && dst.regClass() == s1);
2015          bld.sopc(aco_opcode::s_cmp_lg_u32, bld.scc(Definition(dst)), Operand(0u), src);
2016       }
2017       break;
2018    }
2019    case nir_op_pack_64_2x32_split: {
2020       Temp src0 = get_alu_src(ctx, instr->src[0]);
2021       Temp src1 = get_alu_src(ctx, instr->src[1]);
2022
2023       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2024       break;
2025    }
2026    case nir_op_unpack_64_2x32_split_x:
2027       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2028       break;
2029    case nir_op_unpack_64_2x32_split_y:
2030       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2031       break;
2032    case nir_op_pack_half_2x16: {
2033       Temp src = get_alu_src(ctx, instr->src[0], 2);
2034
2035       if (dst.regClass() == v1) {
2036          Temp src0 = bld.tmp(v1);
2037          Temp src1 = bld.tmp(v1);
2038          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2039          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2040
2041       } else {
2042          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2043          nir_print_instr(&instr->instr, stderr);
2044          fprintf(stderr, "\n");
2045       }
2046       break;
2047    }
2048    case nir_op_unpack_half_2x16_split_x: {
2049       if (dst.regClass() == v1) {
2050          Builder bld(ctx->program, ctx->block);
2051          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2052       } else {
2053          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2054          nir_print_instr(&instr->instr, stderr);
2055          fprintf(stderr, "\n");
2056       }
2057       break;
2058    }
2059    case nir_op_unpack_half_2x16_split_y: {
2060       if (dst.regClass() == v1) {
2061          Builder bld(ctx->program, ctx->block);
2062          /* TODO: use SDWA here */
2063          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2064                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2065       } else {
2066          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2067          nir_print_instr(&instr->instr, stderr);
2068          fprintf(stderr, "\n");
2069       }
2070       break;
2071    }
2072    case nir_op_fquantize2f16: {
2073       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2074
2075       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2076
2077       Temp cmp_res = bld.tmp(s2);
2078       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2079
2080       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2081
2082       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2083       break;
2084    }
2085    case nir_op_bfm: {
2086       Temp bits = get_alu_src(ctx, instr->src[0]);
2087       Temp offset = get_alu_src(ctx, instr->src[1]);
2088
2089       if (dst.regClass() == s1) {
2090          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2091       } else if (dst.regClass() == v1) {
2092          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2093       } else {
2094          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2095          nir_print_instr(&instr->instr, stderr);
2096          fprintf(stderr, "\n");
2097       }
2098       break;
2099    }
2100    case nir_op_bitfield_select: {
2101       /* (mask & insert) | (~mask & base) */
2102       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2103       Temp insert = get_alu_src(ctx, instr->src[1]);
2104       Temp base = get_alu_src(ctx, instr->src[2]);
2105
2106       /* dst = (insert & bitmask) | (base & ~bitmask) */
2107       if (dst.regClass() == s1) {
2108          aco_ptr<Instruction> sop2;
2109          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2110          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2111          Operand lhs;
2112          if (const_insert && const_bitmask) {
2113             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2114          } else {
2115             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2116             lhs = Operand(insert);
2117          }
2118
2119          Operand rhs;
2120          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2121          if (const_base && const_bitmask) {
2122             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2123          } else {
2124             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2125             rhs = Operand(base);
2126          }
2127
2128          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2129
2130       } else if (dst.regClass() == v1) {
2131          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2132             base = as_vgpr(ctx, base);
2133          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2134             insert = as_vgpr(ctx, insert);
2135
2136          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2137
2138       } else {
2139          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2140          nir_print_instr(&instr->instr, stderr);
2141          fprintf(stderr, "\n");
2142       }
2143       break;
2144    }
2145    case nir_op_ubfe:
2146    case nir_op_ibfe: {
2147       Temp base = get_alu_src(ctx, instr->src[0]);
2148       Temp offset = get_alu_src(ctx, instr->src[1]);
2149       Temp bits = get_alu_src(ctx, instr->src[2]);
2150
2151       if (dst.type() == RegType::sgpr) {
2152          Operand extract;
2153          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2154          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2155          if (const_offset && const_bits) {
2156             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2157             extract = Operand(const_extract);
2158          } else {
2159             Operand width;
2160             if (const_bits) {
2161                width = Operand(const_bits->u32 << 16);
2162             } else {
2163                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2164             }
2165             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2166          }
2167
2168          aco_opcode opcode;
2169          if (dst.regClass() == s1) {
2170             if (instr->op == nir_op_ubfe)
2171                opcode = aco_opcode::s_bfe_u32;
2172             else
2173                opcode = aco_opcode::s_bfe_i32;
2174          } else if (dst.regClass() == s2) {
2175             if (instr->op == nir_op_ubfe)
2176                opcode = aco_opcode::s_bfe_u64;
2177             else
2178                opcode = aco_opcode::s_bfe_i64;
2179          } else {
2180             unreachable("Unsupported BFE bit size");
2181          }
2182
2183          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2184
2185       } else {
2186          aco_opcode opcode;
2187          if (dst.regClass() == v1) {
2188             if (instr->op == nir_op_ubfe)
2189                opcode = aco_opcode::v_bfe_u32;
2190             else
2191                opcode = aco_opcode::v_bfe_i32;
2192          } else {
2193             unreachable("Unsupported BFE bit size");
2194          }
2195
2196          emit_vop3a_instruction(ctx, instr, opcode, dst);
2197       }
2198       break;
2199    }
2200    case nir_op_bit_count: {
2201       Temp src = get_alu_src(ctx, instr->src[0]);
2202       if (src.regClass() == s1) {
2203          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2204       } else if (src.regClass() == v1) {
2205          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2206       } else if (src.regClass() == v2) {
2207          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2208                   emit_extract_vector(ctx, src, 1, v1),
2209                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2210                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2211       } else if (src.regClass() == s2) {
2212          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2213       } else {
2214          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2215          nir_print_instr(&instr->instr, stderr);
2216          fprintf(stderr, "\n");
2217       }
2218       break;
2219    }
2220    case nir_op_flt: {
2221       if (instr->src[0].src.ssa->bit_size == 32)
2222          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f32, dst);
2223       else if (instr->src[0].src.ssa->bit_size == 64)
2224          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f64, dst);
2225       break;
2226    }
2227    case nir_op_fge: {
2228       if (instr->src[0].src.ssa->bit_size == 32)
2229          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f32, dst);
2230       else if (instr->src[0].src.ssa->bit_size == 64)
2231          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f64, dst);
2232       break;
2233    }
2234    case nir_op_feq: {
2235       if (instr->src[0].src.ssa->bit_size == 32)
2236          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f32, dst);
2237       else if (instr->src[0].src.ssa->bit_size == 64)
2238          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f64, dst);
2239       break;
2240    }
2241    case nir_op_fne: {
2242       if (instr->src[0].src.ssa->bit_size == 32)
2243          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f32, dst);
2244       else if (instr->src[0].src.ssa->bit_size == 64)
2245          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f64, dst);
2246       break;
2247    }
2248    case nir_op_ilt: {
2249       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2250          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i32, dst);
2251       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2252          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_i32, dst);
2253       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2254          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i64, dst);
2255       break;
2256    }
2257    case nir_op_ige: {
2258       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2259          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i32, dst);
2260       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2261          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_i32, dst);
2262       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2263          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i64, dst);
2264       break;
2265    }
2266    case nir_op_ieq: {
2267       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2268          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i32, dst);
2269       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2270          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_i32, dst);
2271       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2272          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i64, dst);
2273       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2274          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_u64, dst);
2275       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2276          Temp src0 = get_alu_src(ctx, instr->src[0]);
2277          Temp src1 = get_alu_src(ctx, instr->src[1]);
2278          bld.sopc(aco_opcode::s_cmp_eq_i32, bld.scc(Definition(dst)),
2279                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2280       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2281          Temp src0 = get_alu_src(ctx, instr->src[0]);
2282          Temp src1 = get_alu_src(ctx, instr->src[1]);
2283          bld.sop2(aco_opcode::s_xnor_b64, Definition(dst), bld.def(s1, scc),
2284                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2285       } else {
2286          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2287          nir_print_instr(&instr->instr, stderr);
2288          fprintf(stderr, "\n");
2289       }
2290       break;
2291    }
2292    case nir_op_ine: {
2293       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2294          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i32, dst);
2295       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2296          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i64, dst);
2297       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2298          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_i32, dst);
2299       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2300          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_u64, dst);
2301       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2302          Temp src0 = get_alu_src(ctx, instr->src[0]);
2303          Temp src1 = get_alu_src(ctx, instr->src[1]);
2304          bld.sopc(aco_opcode::s_cmp_lg_i32, bld.scc(Definition(dst)),
2305                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2306       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2307          Temp src0 = get_alu_src(ctx, instr->src[0]);
2308          Temp src1 = get_alu_src(ctx, instr->src[1]);
2309          bld.sop2(aco_opcode::s_xor_b64, Definition(dst), bld.def(s1, scc),
2310                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2311       } else {
2312          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2313          nir_print_instr(&instr->instr, stderr);
2314          fprintf(stderr, "\n");
2315       }
2316       break;
2317    }
2318    case nir_op_ult: {
2319       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2320          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u32, dst);
2321       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2322          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_u32, dst);
2323       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2324          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u64, dst);
2325       break;
2326    }
2327    case nir_op_uge: {
2328       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2329          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u32, dst);
2330       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2331          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_u32, dst);
2332       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2333          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u64, dst);
2334       break;
2335    }
2336    case nir_op_fddx:
2337    case nir_op_fddy:
2338    case nir_op_fddx_fine:
2339    case nir_op_fddy_fine:
2340    case nir_op_fddx_coarse:
2341    case nir_op_fddy_coarse: {
2342       Definition tl = bld.def(v1);
2343       uint16_t dpp_ctrl;
2344       if (instr->op == nir_op_fddx_fine) {
2345          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2346          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2347       } else if (instr->op == nir_op_fddy_fine) {
2348          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2349          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2350       } else {
2351          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2352          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2353             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2354          else
2355             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2356       }
2357
2358       Definition tmp = bld.def(v1);
2359       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2360       emit_wqm(ctx, tmp.getTemp(), dst, true);
2361       break;
2362    }
2363    default:
2364       fprintf(stderr, "Unknown NIR ALU instr: ");
2365       nir_print_instr(&instr->instr, stderr);
2366       fprintf(stderr, "\n");
2367    }
2368 }
2369
2370 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2371 {
2372    Temp dst = get_ssa_temp(ctx, &instr->def);
2373
2374    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2375    // which get truncated the lsb if double and msb if int
2376    // for now, we only use s_mov_b64 with 64bit inline constants
2377    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2378    assert(dst.type() == RegType::sgpr);
2379
2380    if (dst.size() == 1)
2381    {
2382       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(instr->value[0].u32));
2383    } else {
2384       assert(dst.size() != 1);
2385       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2386       if (instr->def.bit_size == 64)
2387          for (unsigned i = 0; i < dst.size(); i++)
2388             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2389       else {
2390          for (unsigned i = 0; i < dst.size(); i++)
2391             vec->operands[i] = Operand{instr->value[i].u32};
2392       }
2393       vec->definitions[0] = Definition(dst);
2394       ctx->block->instructions.emplace_back(std::move(vec));
2395    }
2396 }
2397
2398 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2399 {
2400    uint32_t new_mask = 0;
2401    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2402       if (mask & (1u << i))
2403          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2404    return new_mask;
2405 }
2406
2407 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2408 {
2409    /* This wouldn't work inside control flow or with indirect offsets but
2410     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2411
2412    unsigned write_mask = nir_intrinsic_write_mask(instr);
2413    unsigned component = nir_intrinsic_component(instr);
2414    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2415    unsigned idx = nir_intrinsic_base(instr) + component;
2416
2417    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2418    if (off_instr->type != nir_instr_type_load_const) {
2419       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2420       nir_print_instr(off_instr, stderr);
2421       fprintf(stderr, "\n");
2422    }
2423    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2424
2425    if (instr->src[0].ssa->bit_size == 64)
2426       write_mask = widen_mask(write_mask, 2);
2427
2428    for (unsigned i = 0; i < 8; ++i) {
2429       if (write_mask & (1 << i)) {
2430          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2431          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2432       }
2433       idx++;
2434    }
2435 }
2436
2437 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2438 {
2439    unsigned write_mask = nir_intrinsic_write_mask(instr);
2440    Operand values[4];
2441    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2442    for (unsigned i = 0; i < 4; ++i) {
2443       if (write_mask & (1 << i)) {
2444          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2445          values[i] = Operand(tmp);
2446       } else {
2447          values[i] = Operand(v1);
2448       }
2449    }
2450
2451    unsigned index = nir_intrinsic_base(instr) / 4;
2452    unsigned target, col_format;
2453    unsigned enabled_channels = 0xF;
2454    aco_opcode compr_op = (aco_opcode)0;
2455
2456    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2457    assert(offset && "Non-const offsets on exports not yet supported");
2458    index += offset->u32;
2459
2460    assert(index != FRAG_RESULT_COLOR);
2461
2462    /* Unlike vertex shader exports, it's fine to use multiple exports to
2463     * export separate channels of one target. So shaders which export both
2464     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2465     * TODO: combine the exports in those cases and create better code
2466     */
2467
2468    if (index == FRAG_RESULT_SAMPLE_MASK) {
2469
2470       if (ctx->program->info->ps.writes_z) {
2471          target = V_008DFC_SQ_EXP_MRTZ;
2472          enabled_channels = 0x4;
2473          col_format = (unsigned) -1;
2474
2475          values[2] = values[0];
2476          values[0] = Operand(v1);
2477       } else {
2478          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2479          exp->valid_mask = false;
2480          exp->done = false;
2481          exp->compressed = true;
2482          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2483          exp->enabled_mask = 0xc;
2484          for (int i = 0; i < 4; i++)
2485             exp->operands[i] = Operand(v1);
2486          exp->operands[1] = Operand(values[0]);
2487          ctx->block->instructions.emplace_back(std::move(exp));
2488          return;
2489       }
2490
2491    } else if (index == FRAG_RESULT_DEPTH) {
2492
2493       target = V_008DFC_SQ_EXP_MRTZ;
2494       enabled_channels = 0x1;
2495       col_format = (unsigned) -1;
2496
2497    } else if (index == FRAG_RESULT_STENCIL) {
2498
2499       if (ctx->program->info->ps.writes_z) {
2500          target = V_008DFC_SQ_EXP_MRTZ;
2501          enabled_channels = 0x2;
2502          col_format = (unsigned) -1;
2503
2504          values[1] = values[0];
2505          values[0] = Operand(v1);
2506       } else {
2507          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2508          shift->operands[0] = Operand((uint32_t) 16);
2509          shift->operands[1] = values[0];
2510          Temp tmp = {ctx->program->allocateId(), v1};
2511          shift->definitions[0] = Definition(tmp);
2512          ctx->block->instructions.emplace_back(std::move(shift));
2513
2514          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2515          exp->valid_mask = false;
2516          exp->done = false;
2517          exp->compressed = true;
2518          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2519          exp->enabled_mask = 0x3;
2520          exp->operands[0] = Operand(tmp);
2521          for (int i = 1; i < 4; i++)
2522             exp->operands[i] = Operand(v1);
2523          ctx->block->instructions.emplace_back(std::move(exp));
2524          return;
2525       }
2526
2527    } else {
2528       index -= FRAG_RESULT_DATA0;
2529       target = V_008DFC_SQ_EXP_MRT + index;
2530       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2531    }
2532    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2533    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2534    assert(!is_int8 && !is_int10);
2535
2536    switch (col_format)
2537    {
2538    case V_028714_SPI_SHADER_ZERO:
2539       enabled_channels = 0; /* writemask */
2540       target = V_008DFC_SQ_EXP_NULL;
2541       break;
2542
2543    case V_028714_SPI_SHADER_32_R:
2544       enabled_channels = 1;
2545       break;
2546
2547    case V_028714_SPI_SHADER_32_GR:
2548       enabled_channels = 0x3;
2549       break;
2550
2551    case V_028714_SPI_SHADER_32_AR:
2552       enabled_channels = 0x9;
2553       break;
2554
2555    case V_028714_SPI_SHADER_FP16_ABGR:
2556       enabled_channels = 0x5;
2557       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2558       break;
2559
2560    case V_028714_SPI_SHADER_UNORM16_ABGR:
2561       enabled_channels = 0x5;
2562       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2563       break;
2564
2565    case V_028714_SPI_SHADER_SNORM16_ABGR:
2566       enabled_channels = 0x5;
2567       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2568       break;
2569
2570    case V_028714_SPI_SHADER_UINT16_ABGR:
2571       enabled_channels = 0x5;
2572       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2573       break;
2574
2575    case V_028714_SPI_SHADER_SINT16_ABGR:
2576       enabled_channels = 0x5;
2577       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2578       break;
2579
2580    case V_028714_SPI_SHADER_32_ABGR:
2581       enabled_channels = 0xF;
2582       break;
2583
2584    default:
2585       break;
2586    }
2587
2588    if (target == V_008DFC_SQ_EXP_NULL)
2589       return;
2590
2591    if ((bool)compr_op)
2592    {
2593       for (int i = 0; i < 2; i++)
2594       {
2595          /* check if at least one of the values to be compressed is enabled */
2596          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2597          if (enabled) {
2598             enabled_channels |= enabled << (i*2);
2599             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2600             Temp tmp{ctx->program->allocateId(), v1};
2601             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2602             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2603             compr->definitions[0] = Definition(tmp);
2604             values[i] = Operand(tmp);
2605             ctx->block->instructions.emplace_back(std::move(compr));
2606          } else {
2607             values[i] = Operand(v1);
2608          }
2609       }
2610    }
2611
2612    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2613    exp->valid_mask = false;
2614    exp->done = false;
2615    exp->compressed = (bool) compr_op;
2616    exp->dest = target;
2617    exp->enabled_mask = enabled_channels;
2618    if ((bool) compr_op) {
2619       for (int i = 0; i < 2; i++)
2620          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2621       exp->operands[2] = Operand(v1);
2622       exp->operands[3] = Operand(v1);
2623    } else {
2624       for (int i = 0; i < 4; i++)
2625          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2626    }
2627
2628    ctx->block->instructions.emplace_back(std::move(exp));
2629 }
2630
2631 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2632 {
2633    if (ctx->stage == vertex_vs) {
2634       visit_store_vs_output(ctx, instr);
2635    } else if (ctx->stage == fragment_fs) {
2636       visit_store_fs_output(ctx, instr);
2637    } else {
2638       unreachable("Shader stage not implemented");
2639    }
2640 }
2641
2642 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2643 {
2644    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2645    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2646
2647    Builder bld(ctx->program, ctx->block);
2648    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2649    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2650 }
2651
2652 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2653 {
2654    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2655    for (unsigned i = 0; i < num_components; i++)
2656       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2657
2658    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2659       assert(num_components == 4);
2660       Builder bld(ctx->program, ctx->block);
2661       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2662    }
2663
2664    for (Operand& op : vec->operands)
2665       op = op.isUndefined() ? Operand(0u) : op;
2666
2667    vec->definitions[0] = Definition(dst);
2668    ctx->block->instructions.emplace_back(std::move(vec));
2669    emit_split_vector(ctx, dst, num_components);
2670    return;
2671 }
2672
2673 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2674 {
2675    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2676    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2677    unsigned idx = nir_intrinsic_base(instr);
2678    unsigned component = nir_intrinsic_component(instr);
2679    Temp prim_mask = ctx->prim_mask;
2680
2681    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2682    if (offset) {
2683       assert(offset->u32 == 0);
2684    } else {
2685       /* the lower 15bit of the prim_mask contain the offset into LDS
2686        * while the upper bits contain the number of prims */
2687       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2688       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2689       Builder bld(ctx->program, ctx->block);
2690       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2691       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2692       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2693       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2694       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2695    }
2696
2697    if (instr->dest.ssa.num_components == 1) {
2698       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2699    } else {
2700       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2701       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2702       {
2703          Temp tmp = {ctx->program->allocateId(), v1};
2704          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2705          vec->operands[i] = Operand(tmp);
2706       }
2707       vec->definitions[0] = Definition(dst);
2708       ctx->block->instructions.emplace_back(std::move(vec));
2709    }
2710 }
2711
2712 unsigned get_num_channels_from_data_format(unsigned data_format)
2713 {
2714    switch (data_format) {
2715    case V_008F0C_BUF_DATA_FORMAT_8:
2716    case V_008F0C_BUF_DATA_FORMAT_16:
2717    case V_008F0C_BUF_DATA_FORMAT_32:
2718       return 1;
2719    case V_008F0C_BUF_DATA_FORMAT_8_8:
2720    case V_008F0C_BUF_DATA_FORMAT_16_16:
2721    case V_008F0C_BUF_DATA_FORMAT_32_32:
2722       return 2;
2723    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2724    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2725    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2726       return 3;
2727    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2728    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2729    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2730    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2731    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2732       return 4;
2733    default:
2734       break;
2735    }
2736
2737    return 4;
2738 }
2739
2740 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2741  * so we may need to fix it up. */
2742 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
2743 {
2744    Builder bld(ctx->program, ctx->block);
2745
2746    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
2747       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
2748
2749    /* For the integer-like cases, do a natural sign extension.
2750     *
2751     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
2752     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
2753     * exponent.
2754     */
2755    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
2756    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
2757
2758    /* Convert back to the right type. */
2759    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
2760       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2761       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
2762       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
2763    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
2764       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2765    }
2766
2767    return alpha;
2768 }
2769
2770 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
2771 {
2772    Builder bld(ctx->program, ctx->block);
2773    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2774    if (ctx->stage & sw_vs) {
2775
2776       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2777       if (off_instr->type != nir_instr_type_load_const) {
2778          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2779          nir_print_instr(off_instr, stderr);
2780          fprintf(stderr, "\n");
2781       }
2782       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
2783
2784       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
2785
2786       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
2787       unsigned component = nir_intrinsic_component(instr);
2788       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
2789       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
2790       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
2791       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
2792
2793       unsigned dfmt = attrib_format & 0xf;
2794
2795       unsigned nfmt = (attrib_format >> 4) & 0x7;
2796       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
2797       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
2798       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
2799       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
2800       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
2801       if (post_shuffle)
2802          num_channels = MAX2(num_channels, 3);
2803
2804       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
2805
2806       Temp index;
2807       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
2808          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
2809          if (divisor) {
2810             ctx->needs_instance_id = true;
2811
2812             if (divisor != 1) {
2813                Temp divided = bld.tmp(v1);
2814                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
2815                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
2816             } else {
2817                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
2818             }
2819          } else {
2820             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
2821          }
2822       } else {
2823          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
2824       }
2825
2826       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
2827          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
2828          attrib_offset = attrib_offset % attrib_stride;
2829       }
2830
2831       Operand soffset(0u);
2832       if (attrib_offset >= 4096) {
2833          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
2834          attrib_offset = 0;
2835       }
2836
2837       aco_opcode opcode;
2838       switch (num_channels) {
2839       case 1:
2840          opcode = aco_opcode::tbuffer_load_format_x;
2841          break;
2842       case 2:
2843          opcode = aco_opcode::tbuffer_load_format_xy;
2844          break;
2845       case 3:
2846          opcode = aco_opcode::tbuffer_load_format_xyz;
2847          break;
2848       case 4:
2849          opcode = aco_opcode::tbuffer_load_format_xyzw;
2850          break;
2851       default:
2852          unreachable("Unimplemented load_input vector size");
2853       }
2854
2855       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
2856
2857       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
2858       mubuf->operands[0] = Operand(index);
2859       mubuf->operands[1] = Operand(list);
2860       mubuf->operands[2] = soffset;
2861       mubuf->definitions[0] = Definition(tmp);
2862       mubuf->idxen = true;
2863       mubuf->can_reorder = true;
2864       mubuf->dfmt = dfmt;
2865       mubuf->nfmt = nfmt;
2866       assert(attrib_offset < 4096);
2867       mubuf->offset = attrib_offset;
2868       ctx->block->instructions.emplace_back(std::move(mubuf));
2869
2870       emit_split_vector(ctx, tmp, tmp.size());
2871
2872       if (tmp.id() != dst.id()) {
2873          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
2874                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
2875
2876          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
2877          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
2878          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
2879
2880          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2881          for (unsigned i = 0; i < dst.size(); i++) {
2882             unsigned idx = i + component;
2883             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
2884                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
2885                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
2886             } else if (idx < num_channels) {
2887                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
2888             } else if (is_float && idx == 3) {
2889                vec->operands[i] = Operand(0x3f800000u);
2890             } else if (!is_float && idx == 3) {
2891                vec->operands[i] = Operand(1u);
2892             } else {
2893                vec->operands[i] = Operand(0u);
2894             }
2895          }
2896          vec->definitions[0] = Definition(dst);
2897          ctx->block->instructions.emplace_back(std::move(vec));
2898          emit_split_vector(ctx, dst, dst.size());
2899       }
2900
2901    } else if (ctx->stage == fragment_fs) {
2902       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2903       if (off_instr->type != nir_instr_type_load_const ||
2904           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
2905          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2906          nir_print_instr(off_instr, stderr);
2907          fprintf(stderr, "\n");
2908       }
2909
2910       Temp prim_mask = ctx->prim_mask;
2911       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
2912       if (offset) {
2913          assert(offset->u32 == 0);
2914       } else {
2915          /* the lower 15bit of the prim_mask contain the offset into LDS
2916           * while the upper bits contain the number of prims */
2917          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
2918          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2919          Builder bld(ctx->program, ctx->block);
2920          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2921          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2922          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2923          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2924          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2925       }
2926
2927       unsigned idx = nir_intrinsic_base(instr);
2928       unsigned component = nir_intrinsic_component(instr);
2929
2930       if (dst.size() == 1) {
2931          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
2932       } else {
2933          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2934          for (unsigned i = 0; i < dst.size(); i++)
2935             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
2936          vec->definitions[0] = Definition(dst);
2937          bld.insert(std::move(vec));
2938       }
2939
2940    } else {
2941       unreachable("Shader stage not implemented");
2942    }
2943 }
2944
2945 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
2946 {
2947    if (ctx->program->info->need_indirect_descriptor_sets) {
2948       Builder bld(ctx->program, ctx->block);
2949       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
2950       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
2951    }
2952
2953    return ctx->descriptor_sets[desc_set];
2954 }
2955
2956
2957 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
2958 {
2959    Builder bld(ctx->program, ctx->block);
2960    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
2961    unsigned desc_set = nir_intrinsic_desc_set(instr);
2962    unsigned binding = nir_intrinsic_binding(instr);
2963
2964    Temp desc_ptr;
2965    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
2966    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
2967    unsigned offset = layout->binding[binding].offset;
2968    unsigned stride;
2969    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
2970        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
2971       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
2972       desc_ptr = ctx->push_constants;
2973       offset = pipeline_layout->push_constant_size + 16 * idx;
2974       stride = 16;
2975    } else {
2976       desc_ptr = load_desc_ptr(ctx, desc_set);
2977       stride = layout->binding[binding].size;
2978    }
2979
2980    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
2981    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
2982    if (stride != 1) {
2983       if (nir_const_index) {
2984          const_index = const_index * stride;
2985       } else {
2986          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
2987       }
2988    }
2989    if (offset) {
2990       if (nir_const_index) {
2991          const_index = const_index + offset;
2992       } else {
2993          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
2994       }
2995    }
2996
2997    if (nir_const_index && const_index == 0) {
2998       index = desc_ptr;
2999    } else {
3000       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3001                        nir_const_index ? Operand(const_index) : Operand(index),
3002                        Operand(desc_ptr));
3003    }
3004
3005    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3006    bld.sop1(aco_opcode::s_mov_b32, Definition(dst), index);
3007 }
3008
3009 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst, Temp rsrc, Temp offset, bool glc=false)
3010 {
3011    Builder bld(ctx->program, ctx->block);
3012
3013    unsigned num_bytes = dst.size() * 4;
3014
3015    aco_opcode op;
3016    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3017       if (ctx->options->chip_class < GFX8)
3018          offset = as_vgpr(ctx, offset);
3019
3020       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3021       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3022       unsigned const_offset = 0;
3023
3024       Temp lower = Temp();
3025       if (num_bytes > 16) {
3026          assert(num_components == 3 || num_components == 4);
3027          op = aco_opcode::buffer_load_dwordx4;
3028          lower = bld.tmp(v4);
3029          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3030          mubuf->definitions[0] = Definition(lower);
3031          mubuf->operands[0] = vaddr;
3032          mubuf->operands[1] = Operand(rsrc);
3033          mubuf->operands[2] = soffset;
3034          mubuf->offen = (offset.type() == RegType::vgpr);
3035          mubuf->glc = glc;
3036          mubuf->barrier = barrier_buffer;
3037          bld.insert(std::move(mubuf));
3038          emit_split_vector(ctx, lower, 2);
3039          num_bytes -= 16;
3040          const_offset = 16;
3041       }
3042
3043       switch (num_bytes) {
3044          case 4:
3045             op = aco_opcode::buffer_load_dword;
3046             break;
3047          case 8:
3048             op = aco_opcode::buffer_load_dwordx2;
3049             break;
3050          case 12:
3051             op = aco_opcode::buffer_load_dwordx3;
3052             break;
3053          case 16:
3054             op = aco_opcode::buffer_load_dwordx4;
3055             break;
3056          default:
3057             unreachable("Load SSBO not implemented for this size.");
3058       }
3059       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3060       mubuf->operands[0] = vaddr;
3061       mubuf->operands[1] = Operand(rsrc);
3062       mubuf->operands[2] = soffset;
3063       mubuf->offen = (offset.type() == RegType::vgpr);
3064       mubuf->glc = glc;
3065       mubuf->barrier = barrier_buffer;
3066       mubuf->offset = const_offset;
3067       aco_ptr<Instruction> instr = std::move(mubuf);
3068
3069       if (dst.size() > 4) {
3070          assert(lower != Temp());
3071          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3072          instr->definitions[0] = Definition(upper);
3073          bld.insert(std::move(instr));
3074          if (dst.size() == 8)
3075             emit_split_vector(ctx, upper, 2);
3076          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3077          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3078          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3079          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3080          if (dst.size() == 8)
3081             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3082       }
3083
3084       if (dst.type() == RegType::sgpr) {
3085          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3086          instr->definitions[0] = Definition(vec);
3087          bld.insert(std::move(instr));
3088          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3089       } else {
3090          instr->definitions[0] = Definition(dst);
3091          bld.insert(std::move(instr));
3092       }
3093    } else {
3094       switch (num_bytes) {
3095          case 4:
3096             op = aco_opcode::s_buffer_load_dword;
3097             break;
3098          case 8:
3099             op = aco_opcode::s_buffer_load_dwordx2;
3100             break;
3101          case 12:
3102          case 16:
3103             op = aco_opcode::s_buffer_load_dwordx4;
3104             break;
3105          case 24:
3106          case 32:
3107             op = aco_opcode::s_buffer_load_dwordx8;
3108             break;
3109          default:
3110             unreachable("Load SSBO not implemented for this size.");
3111       }
3112       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3113       load->operands[0] = Operand(rsrc);
3114       load->operands[1] = Operand(bld.as_uniform(offset));
3115       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3116       load->definitions[0] = Definition(dst);
3117       load->glc = glc;
3118       load->barrier = barrier_buffer;
3119       assert(ctx->options->chip_class >= GFX8 || !glc);
3120
3121       /* trim vector */
3122       if (dst.size() == 3) {
3123          Temp vec = bld.tmp(s4);
3124          load->definitions[0] = Definition(vec);
3125          bld.insert(std::move(load));
3126          emit_split_vector(ctx, vec, 4);
3127
3128          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3129                     emit_extract_vector(ctx, vec, 0, s1),
3130                     emit_extract_vector(ctx, vec, 1, s1),
3131                     emit_extract_vector(ctx, vec, 2, s1));
3132       } else if (dst.size() == 6) {
3133          Temp vec = bld.tmp(s8);
3134          load->definitions[0] = Definition(vec);
3135          bld.insert(std::move(load));
3136          emit_split_vector(ctx, vec, 4);
3137
3138          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3139                     emit_extract_vector(ctx, vec, 0, s2),
3140                     emit_extract_vector(ctx, vec, 1, s2),
3141                     emit_extract_vector(ctx, vec, 2, s2));
3142       } else {
3143          bld.insert(std::move(load));
3144       }
3145
3146    }
3147    emit_split_vector(ctx, dst, num_components);
3148 }
3149
3150 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3151 {
3152    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3153    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3154
3155    Builder bld(ctx->program, ctx->block);
3156
3157    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3158    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3159    unsigned binding = nir_intrinsic_binding(idx_instr);
3160    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3161
3162    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3163       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3164                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3165                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3166                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3167                            S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3168                            S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3169       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3170                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3171                                      Operand(0xFFFFFFFFu),
3172                                      Operand(desc_type));
3173       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3174                         rsrc, upper_dwords);
3175    } else {
3176       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3177       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3178    }
3179
3180    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3181 }
3182
3183 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3184 {
3185    Builder bld(ctx->program, ctx->block);
3186    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3187
3188    unsigned offset = nir_intrinsic_base(instr);
3189    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3190    if (index_cv && instr->dest.ssa.bit_size == 32) {
3191
3192       unsigned count = instr->dest.ssa.num_components;
3193       unsigned start = (offset + index_cv->u32) / 4u;
3194       start -= ctx->base_inline_push_consts;
3195       if (start + count <= ctx->num_inline_push_consts) {
3196          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3197          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3198          for (unsigned i = 0; i < count; ++i) {
3199             elems[i] = ctx->inline_push_consts[start + i];
3200             vec->operands[i] = Operand{elems[i]};
3201          }
3202          vec->definitions[0] = Definition(dst);
3203          ctx->block->instructions.emplace_back(std::move(vec));
3204          ctx->allocated_vec.emplace(dst.id(), elems);
3205          return;
3206       }
3207    }
3208
3209    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3210    if (offset != 0) // TODO check if index != 0 as well
3211       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3212    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3213    Temp vec = dst;
3214    bool trim = false;
3215    aco_opcode op;
3216
3217    switch (dst.size()) {
3218    case 1:
3219       op = aco_opcode::s_load_dword;
3220       break;
3221    case 2:
3222       op = aco_opcode::s_load_dwordx2;
3223       break;
3224    case 3:
3225       vec = bld.tmp(s4);
3226       trim = true;
3227    case 4:
3228       op = aco_opcode::s_load_dwordx4;
3229       break;
3230    case 6:
3231       vec = bld.tmp(s8);
3232       trim = true;
3233    case 8:
3234       op = aco_opcode::s_load_dwordx8;
3235       break;
3236    default:
3237       unreachable("unimplemented or forbidden load_push_constant.");
3238    }
3239
3240    bld.smem(op, Definition(vec), ptr, index);
3241
3242    if (trim) {
3243       emit_split_vector(ctx, vec, 4);
3244       RegClass rc = dst.size() == 3 ? s1 : s2;
3245       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3246                  emit_extract_vector(ctx, vec, 0, rc),
3247                  emit_extract_vector(ctx, vec, 1, rc),
3248                  emit_extract_vector(ctx, vec, 2, rc));
3249
3250    }
3251    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3252 }
3253
3254 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3255 {
3256    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3257
3258    Builder bld(ctx->program, ctx->block);
3259
3260    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3261                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3262                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3263                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3264    if (ctx->options->chip_class >= GFX10) {
3265       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3266                    S_008F0C_OOB_SELECT(3) |
3267                    S_008F0C_RESOURCE_LEVEL(1);
3268    } else {
3269       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3270                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3271    }
3272
3273    unsigned base = nir_intrinsic_base(instr) + ctx->constant_data_offset;
3274    unsigned range = nir_intrinsic_range(instr);
3275
3276    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3277    if (base && offset.type() == RegType::sgpr)
3278       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3279    else if (base && offset.type() == RegType::vgpr)
3280       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3281
3282    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3283                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(0u)),
3284                           Operand(MIN2(range, ctx->shader->constant_data_size - nir_intrinsic_base(instr))),
3285                           Operand(desc_type));
3286
3287    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3288 }
3289
3290 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3291 {
3292    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3293       ctx->cf_info.exec_potentially_empty = true;
3294
3295    ctx->program->needs_exact = true;
3296
3297    Builder bld(ctx->program, ctx->block);
3298    Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
3299    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3300    bld.pseudo(aco_opcode::p_discard_if, src);
3301    ctx->block->kind |= block_kind_uses_discard_if;
3302    return;
3303 }
3304
3305 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3306 {
3307    Builder bld(ctx->program, ctx->block);
3308
3309    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3310       ctx->cf_info.exec_potentially_empty = true;
3311
3312    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3313                     ctx->cf_info.parent_loop.has_divergent_continue;
3314
3315    if (ctx->block->loop_nest_depth &&
3316        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3317       /* we handle discards the same way as jump instructions */
3318       append_logical_end(ctx->block);
3319
3320       /* in loops, discard behaves like break */
3321       Block *linear_target = ctx->cf_info.parent_loop.exit;
3322       ctx->block->kind |= block_kind_discard;
3323
3324       if (!divergent) {
3325          /* uniform discard - loop ends here */
3326          assert(nir_instr_is_last(&instr->instr));
3327          ctx->block->kind |= block_kind_uniform;
3328          ctx->cf_info.has_branch = true;
3329          bld.branch(aco_opcode::p_branch);
3330          add_linear_edge(ctx->block->index, linear_target);
3331          return;
3332       }
3333
3334       /* we add a break right behind the discard() instructions */
3335       ctx->block->kind |= block_kind_break;
3336       unsigned idx = ctx->block->index;
3337
3338       /* remove critical edges from linear CFG */
3339       bld.branch(aco_opcode::p_branch);
3340       Block* break_block = ctx->program->create_and_insert_block();
3341       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3342       break_block->kind |= block_kind_uniform;
3343       add_linear_edge(idx, break_block);
3344       add_linear_edge(break_block->index, linear_target);
3345       bld.reset(break_block);
3346       bld.branch(aco_opcode::p_branch);
3347
3348       Block* continue_block = ctx->program->create_and_insert_block();
3349       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3350       add_linear_edge(idx, continue_block);
3351       append_logical_start(continue_block);
3352       ctx->block = continue_block;
3353
3354       return;
3355    }
3356
3357    /* it can currently happen that NIR doesn't remove the unreachable code */
3358    if (!nir_instr_is_last(&instr->instr)) {
3359       ctx->program->needs_exact = true;
3360       /* save exec somewhere temporarily so that it doesn't get
3361        * overwritten before the discard from outer exec masks */
3362       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3363       bld.pseudo(aco_opcode::p_discard_if, cond);
3364       ctx->block->kind |= block_kind_uses_discard_if;
3365       return;
3366    }
3367
3368    /* This condition is incorrect for uniformly branched discards in a loop
3369     * predicated by a divergent condition, but the above code catches that case
3370     * and the discard would end up turning into a discard_if.
3371     * For example:
3372     * if (divergent) {
3373     *    while (...) {
3374     *       if (uniform) {
3375     *          discard;
3376     *       }
3377     *    }
3378     * }
3379     */
3380    if (!ctx->cf_info.parent_if.is_divergent) {
3381       /* program just ends here */
3382       ctx->block->kind |= block_kind_uniform;
3383       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3384               0 /* enabled mask */, 9 /* dest */,
3385               false /* compressed */, true/* done */, true /* valid mask */);
3386       bld.sopp(aco_opcode::s_endpgm);
3387       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3388    } else {
3389       ctx->block->kind |= block_kind_discard;
3390       /* branch and linear edge is added by visit_if() */
3391    }
3392 }
3393
3394 enum aco_descriptor_type {
3395    ACO_DESC_IMAGE,
3396    ACO_DESC_FMASK,
3397    ACO_DESC_SAMPLER,
3398    ACO_DESC_BUFFER,
3399    ACO_DESC_PLANE_0,
3400    ACO_DESC_PLANE_1,
3401    ACO_DESC_PLANE_2,
3402 };
3403
3404 enum aco_image_dim {
3405    aco_image_1d,
3406    aco_image_2d,
3407    aco_image_3d,
3408    aco_image_cube, // includes cube arrays
3409    aco_image_1darray,
3410    aco_image_2darray,
3411    aco_image_2dmsaa,
3412    aco_image_2darraymsaa,
3413 };
3414
3415 static enum aco_image_dim
3416 get_sampler_dim(isel_context *ctx, enum glsl_sampler_dim dim, bool is_array)
3417 {
3418    switch (dim) {
3419    case GLSL_SAMPLER_DIM_1D:
3420       if (ctx->options->chip_class >= GFX9)
3421          return is_array ? aco_image_2darray : aco_image_2d;
3422       return is_array ? aco_image_1darray : aco_image_1d;
3423    case GLSL_SAMPLER_DIM_2D:
3424    case GLSL_SAMPLER_DIM_RECT:
3425    case GLSL_SAMPLER_DIM_EXTERNAL:
3426       return is_array ? aco_image_2darray : aco_image_2d;
3427    case GLSL_SAMPLER_DIM_3D:
3428       return aco_image_3d;
3429    case GLSL_SAMPLER_DIM_CUBE:
3430       return aco_image_cube;
3431    case GLSL_SAMPLER_DIM_MS:
3432       return is_array ? aco_image_2darraymsaa : aco_image_2dmsaa;
3433    case GLSL_SAMPLER_DIM_SUBPASS:
3434       return aco_image_2darray;
3435    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3436       return aco_image_2darraymsaa;
3437    default:
3438       unreachable("bad sampler dim");
3439    }
3440 }
3441
3442 static bool
3443 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3444    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3445       return false;
3446    aco_image_dim dim = get_sampler_dim(ctx, sampler_dim, is_array);
3447    return dim == aco_image_cube ||
3448           dim == aco_image_1darray ||
3449           dim == aco_image_2darray ||
3450           dim == aco_image_2darraymsaa;
3451 }
3452
3453 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3454                       enum aco_descriptor_type desc_type,
3455                       const nir_tex_instr *tex_instr, bool image, bool write)
3456 {
3457 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3458    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3459    if (it != ctx->tex_desc.end())
3460       return it->second;
3461 */
3462    Temp index = Temp();
3463    bool index_set = false;
3464    unsigned constant_index = 0;
3465    unsigned descriptor_set;
3466    unsigned base_index;
3467    Builder bld(ctx->program, ctx->block);
3468
3469    if (!deref_instr) {
3470       assert(tex_instr && !image);
3471       descriptor_set = 0;
3472       base_index = tex_instr->sampler_index;
3473    } else {
3474       while(deref_instr->deref_type != nir_deref_type_var) {
3475          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3476          if (!array_size)
3477             array_size = 1;
3478
3479          assert(deref_instr->deref_type == nir_deref_type_array);
3480          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3481          if (const_value) {
3482             constant_index += array_size * const_value->u32;
3483          } else {
3484             Temp indirect = bld.as_uniform(get_ssa_temp(ctx, deref_instr->arr.index.ssa));
3485
3486             if (array_size != 1)
3487                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3488
3489             if (!index_set) {
3490                index = indirect;
3491                index_set = true;
3492             } else {
3493                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3494             }
3495          }
3496
3497          deref_instr = nir_src_as_deref(deref_instr->parent);
3498       }
3499       descriptor_set = deref_instr->var->data.descriptor_set;
3500       base_index = deref_instr->var->data.binding;
3501    }
3502
3503    Temp list = load_desc_ptr(ctx, descriptor_set);
3504    list = convert_pointer_to_64_bit(ctx, list);
3505
3506    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3507    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3508    unsigned offset = binding->offset;
3509    unsigned stride = binding->size;
3510    aco_opcode opcode;
3511    RegClass type;
3512
3513    assert(base_index < layout->binding_count);
3514
3515    switch (desc_type) {
3516    case ACO_DESC_IMAGE:
3517       type = s8;
3518       opcode = aco_opcode::s_load_dwordx8;
3519       break;
3520    case ACO_DESC_FMASK:
3521       type = s8;
3522       opcode = aco_opcode::s_load_dwordx8;
3523       offset += 32;
3524       break;
3525    case ACO_DESC_SAMPLER:
3526       type = s4;
3527       opcode = aco_opcode::s_load_dwordx4;
3528       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3529          offset += radv_combined_image_descriptor_sampler_offset(binding);
3530       break;
3531    case ACO_DESC_BUFFER:
3532       type = s4;
3533       opcode = aco_opcode::s_load_dwordx4;
3534       break;
3535    case ACO_DESC_PLANE_0:
3536    case ACO_DESC_PLANE_1:
3537       type = s8;
3538       opcode = aco_opcode::s_load_dwordx8;
3539       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3540       break;
3541    case ACO_DESC_PLANE_2:
3542       type = s4;
3543       opcode = aco_opcode::s_load_dwordx4;
3544       offset += 64;
3545       break;
3546    default:
3547       unreachable("invalid desc_type\n");
3548    }
3549
3550    offset += constant_index * stride;
3551
3552    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3553       (!index_set || binding->immutable_samplers_equal)) {
3554       if (binding->immutable_samplers_equal)
3555          constant_index = 0;
3556
3557       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3558       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3559                         Operand(samplers[constant_index * 4 + 0]),
3560                         Operand(samplers[constant_index * 4 + 1]),
3561                         Operand(samplers[constant_index * 4 + 2]),
3562                         Operand(samplers[constant_index * 4 + 3]));
3563    }
3564
3565    Operand off;
3566    if (!index_set) {
3567       off = Operand(offset);
3568    } else {
3569       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3570                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3571    }
3572
3573    Temp res = bld.smem(opcode, bld.def(type), list, off);
3574
3575    if (desc_type == ACO_DESC_PLANE_2) {
3576       Temp components[8];
3577       for (unsigned i = 0; i < 8; i++)
3578          components[i] = bld.tmp(s1);
3579       bld.pseudo(aco_opcode::p_split_vector,
3580                  Definition(components[0]),
3581                  Definition(components[1]),
3582                  Definition(components[2]),
3583                  Definition(components[3]),
3584                  res);
3585
3586       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3587       bld.pseudo(aco_opcode::p_split_vector,
3588                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3589                  Definition(components[4]),
3590                  Definition(components[5]),
3591                  Definition(components[6]),
3592                  Definition(components[7]),
3593                  desc2);
3594
3595       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3596                        components[0], components[1], components[2], components[3],
3597                        components[4], components[5], components[6], components[7]);
3598    }
3599
3600    return res;
3601 }
3602
3603 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3604 {
3605    switch (dim) {
3606    case GLSL_SAMPLER_DIM_BUF:
3607       return 1;
3608    case GLSL_SAMPLER_DIM_1D:
3609       return array ? 2 : 1;
3610    case GLSL_SAMPLER_DIM_2D:
3611       return array ? 3 : 2;
3612    case GLSL_SAMPLER_DIM_MS:
3613       return array ? 4 : 3;
3614    case GLSL_SAMPLER_DIM_3D:
3615    case GLSL_SAMPLER_DIM_CUBE:
3616       return 3;
3617    case GLSL_SAMPLER_DIM_RECT:
3618    case GLSL_SAMPLER_DIM_SUBPASS:
3619       return 2;
3620    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3621       return 3;
3622    default:
3623       break;
3624    }
3625    return 0;
3626 }
3627
3628
3629 /* Adjust the sample index according to FMASK.
3630  *
3631  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3632  * which is the identity mapping. Each nibble says which physical sample
3633  * should be fetched to get that sample.
3634  *
3635  * For example, 0x11111100 means there are only 2 samples stored and
3636  * the second sample covers 3/4 of the pixel. When reading samples 0
3637  * and 1, return physical sample 0 (determined by the first two 0s
3638  * in FMASK), otherwise return physical sample 1.
3639  *
3640  * The sample index should be adjusted as follows:
3641  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3642  */
3643 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3644 {
3645    Builder bld(ctx->program, ctx->block);
3646    Temp fmask = bld.tmp(v1);
3647
3648    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3649    load->operands[0] = Operand(coords);
3650    load->operands[1] = Operand(fmask_desc_ptr);
3651    load->definitions[0] = Definition(fmask);
3652    load->glc = false;
3653    load->dmask = 0x1;
3654    load->unrm = true;
3655    load->da = da;
3656    load->can_reorder = true; /* fmask images shouldn't be modified */
3657    ctx->block->instructions.emplace_back(std::move(load));
3658
3659    Operand sample_index4;
3660    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3661       sample_index4 = Operand(sample_index.constantValue() << 2);
3662    } else if (sample_index.regClass() == s1) {
3663       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3664    } else {
3665       assert(sample_index.regClass() == v1);
3666       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3667    }
3668
3669    Temp final_sample;
3670    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3671       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3672    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3673       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3674    else
3675       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3676
3677    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3678     * resource descriptor is 0 (invalid),
3679     */
3680    Temp compare = bld.tmp(s2);
3681    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3682                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3683
3684    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3685
3686    /* Replace the MSAA sample index. */
3687    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3688 }
3689
3690 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3691 {
3692
3693    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3694    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3695    bool is_array = glsl_sampler_type_is_array(type);
3696    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3697    assert(!add_frag_pos && "Input attachments should be lowered.");
3698    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3699    bool gfx9_1d = ctx->options->chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3700    int count = image_type_to_components_count(dim, is_array);
3701    std::vector<Operand> coords(count);
3702
3703    if (is_ms) {
3704       Operand sample_index;
3705       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3706       if (sample_cv)
3707          sample_index = Operand(sample_cv->u32);
3708       else
3709          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3710
3711       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3712          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3713          for (unsigned i = 0; i < vec->operands.size(); i++)
3714             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3715          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3716          vec->definitions[0] = Definition(fmask_load_address);
3717          ctx->block->instructions.emplace_back(std::move(vec));
3718
3719          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3720          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3721       }
3722       count--;
3723       coords[count] = sample_index;
3724    }
3725
3726    if (count == 1 && !gfx9_1d)
3727       return emit_extract_vector(ctx, src0, 0, v1);
3728
3729    if (gfx9_1d) {
3730       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3731       coords.resize(coords.size() + 1);
3732       coords[1] = Operand((uint32_t) 0);
3733       if (is_array)
3734          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3735    } else {
3736       for (int i = 0; i < count; i++)
3737          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3738    }
3739
3740    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3741    for (unsigned i = 0; i < coords.size(); i++)
3742       vec->operands[i] = coords[i];
3743    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3744    vec->definitions[0] = Definition(res);
3745    ctx->block->instructions.emplace_back(std::move(vec));
3746    return res;
3747 }
3748
3749
3750 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
3751 {
3752    Builder bld(ctx->program, ctx->block);
3753    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3754    const struct glsl_type *type = glsl_without_array(var->type);
3755    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3756    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3757
3758    if (dim == GLSL_SAMPLER_DIM_BUF) {
3759       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
3760       unsigned num_channels = util_last_bit(mask);
3761       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3762       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3763
3764       aco_opcode opcode;
3765       switch (num_channels) {
3766       case 1:
3767          opcode = aco_opcode::buffer_load_format_x;
3768          break;
3769       case 2:
3770          opcode = aco_opcode::buffer_load_format_xy;
3771          break;
3772       case 3:
3773          opcode = aco_opcode::buffer_load_format_xyz;
3774          break;
3775       case 4:
3776          opcode = aco_opcode::buffer_load_format_xyzw;
3777          break;
3778       default:
3779          unreachable(">4 channel buffer image load");
3780       }
3781       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
3782       load->operands[0] = Operand(vindex);
3783       load->operands[1] = Operand(rsrc);
3784       load->operands[2] = Operand((uint32_t) 0);
3785       Temp tmp;
3786       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3787          tmp = dst;
3788       else
3789          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
3790       load->definitions[0] = Definition(tmp);
3791       load->idxen = true;
3792       load->barrier = barrier_image;
3793       ctx->block->instructions.emplace_back(std::move(load));
3794
3795       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
3796       return;
3797    }
3798
3799    Temp coords = get_image_coords(ctx, instr, type);
3800    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3801    //aco_image_dim img_dim = get_image_dim(ctx, glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
3802
3803    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
3804    unsigned num_components = util_bitcount(dmask);
3805    Temp tmp;
3806    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3807       tmp = dst;
3808    else
3809       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
3810
3811    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3812    load->operands[0] = Operand(coords);
3813    load->operands[1] = Operand(resource);
3814    load->definitions[0] = Definition(tmp);
3815    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
3816    load->dmask = dmask;
3817    load->unrm = true;
3818    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3819    load->barrier = barrier_image;
3820    ctx->block->instructions.emplace_back(std::move(load));
3821
3822    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
3823    return;
3824 }
3825
3826 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
3827 {
3828    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3829    const struct glsl_type *type = glsl_without_array(var->type);
3830    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3831    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3832
3833    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
3834
3835    if (dim == GLSL_SAMPLER_DIM_BUF) {
3836       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3837       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3838       aco_opcode opcode;
3839       switch (data.size()) {
3840       case 1:
3841          opcode = aco_opcode::buffer_store_format_x;
3842          break;
3843       case 2:
3844          opcode = aco_opcode::buffer_store_format_xy;
3845          break;
3846       case 3:
3847          opcode = aco_opcode::buffer_store_format_xyz;
3848          break;
3849       case 4:
3850          opcode = aco_opcode::buffer_store_format_xyzw;
3851          break;
3852       default:
3853          unreachable(">4 channel buffer image store");
3854       }
3855       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
3856       store->operands[0] = Operand(vindex);
3857       store->operands[1] = Operand(rsrc);
3858       store->operands[2] = Operand((uint32_t) 0);
3859       store->operands[3] = Operand(data);
3860       store->idxen = true;
3861       store->glc = glc;
3862       store->disable_wqm = true;
3863       store->barrier = barrier_image;
3864       ctx->program->needs_exact = true;
3865       ctx->block->instructions.emplace_back(std::move(store));
3866       return;
3867    }
3868
3869    assert(data.type() == RegType::vgpr);
3870    Temp coords = get_image_coords(ctx, instr, type);
3871    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3872
3873    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
3874    store->operands[0] = Operand(coords);
3875    store->operands[1] = Operand(resource);
3876    store->operands[2] = Operand(s4);
3877    store->operands[3] = Operand(data);
3878    store->glc = glc;
3879    store->dmask = (1 << data.size()) - 1;
3880    store->unrm = true;
3881    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3882    store->disable_wqm = true;
3883    store->barrier = barrier_image;
3884    ctx->program->needs_exact = true;
3885    ctx->block->instructions.emplace_back(std::move(store));
3886    return;
3887 }
3888
3889 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
3890 {
3891    /* return the previous value if dest is ever used */
3892    bool return_previous = false;
3893    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
3894       return_previous = true;
3895       break;
3896    }
3897    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
3898       return_previous = true;
3899       break;
3900    }
3901
3902    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3903    const struct glsl_type *type = glsl_without_array(var->type);
3904    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3905    Builder bld(ctx->program, ctx->block);
3906
3907    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3908    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
3909
3910    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
3911       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
3912
3913    aco_opcode buf_op, image_op;
3914    switch (instr->intrinsic) {
3915       case nir_intrinsic_image_deref_atomic_add:
3916          buf_op = aco_opcode::buffer_atomic_add;
3917          image_op = aco_opcode::image_atomic_add;
3918          break;
3919       case nir_intrinsic_image_deref_atomic_umin:
3920          buf_op = aco_opcode::buffer_atomic_umin;
3921          image_op = aco_opcode::image_atomic_umin;
3922          break;
3923       case nir_intrinsic_image_deref_atomic_imin:
3924          buf_op = aco_opcode::buffer_atomic_smin;
3925          image_op = aco_opcode::image_atomic_smin;
3926          break;
3927       case nir_intrinsic_image_deref_atomic_umax:
3928          buf_op = aco_opcode::buffer_atomic_umax;
3929          image_op = aco_opcode::image_atomic_umax;
3930          break;
3931       case nir_intrinsic_image_deref_atomic_imax:
3932          buf_op = aco_opcode::buffer_atomic_smax;
3933          image_op = aco_opcode::image_atomic_smax;
3934          break;
3935       case nir_intrinsic_image_deref_atomic_and:
3936          buf_op = aco_opcode::buffer_atomic_and;
3937          image_op = aco_opcode::image_atomic_and;
3938          break;
3939       case nir_intrinsic_image_deref_atomic_or:
3940          buf_op = aco_opcode::buffer_atomic_or;
3941          image_op = aco_opcode::image_atomic_or;
3942          break;
3943       case nir_intrinsic_image_deref_atomic_xor:
3944          buf_op = aco_opcode::buffer_atomic_xor;
3945          image_op = aco_opcode::image_atomic_xor;
3946          break;
3947       case nir_intrinsic_image_deref_atomic_exchange:
3948          buf_op = aco_opcode::buffer_atomic_swap;
3949          image_op = aco_opcode::image_atomic_swap;
3950          break;
3951       case nir_intrinsic_image_deref_atomic_comp_swap:
3952          buf_op = aco_opcode::buffer_atomic_cmpswap;
3953          image_op = aco_opcode::image_atomic_cmpswap;
3954          break;
3955       default:
3956          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
3957    }
3958
3959    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3960
3961    if (dim == GLSL_SAMPLER_DIM_BUF) {
3962       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3963       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3964       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
3965       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
3966       mubuf->operands[0] = Operand(vindex);
3967       mubuf->operands[1] = Operand(resource);
3968       mubuf->operands[2] = Operand((uint32_t)0);
3969       mubuf->operands[3] = Operand(data);
3970       if (return_previous)
3971          mubuf->definitions[0] = Definition(dst);
3972       mubuf->offset = 0;
3973       mubuf->idxen = true;
3974       mubuf->glc = return_previous;
3975       mubuf->disable_wqm = true;
3976       mubuf->barrier = barrier_image;
3977       ctx->program->needs_exact = true;
3978       ctx->block->instructions.emplace_back(std::move(mubuf));
3979       return;
3980    }
3981
3982    Temp coords = get_image_coords(ctx, instr, type);
3983    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3984    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
3985    mimg->operands[0] = Operand(coords);
3986    mimg->operands[1] = Operand(resource);
3987    mimg->operands[2] = Operand(s4); /* no sampler */
3988    mimg->operands[3] = Operand(data);
3989    if (return_previous)
3990       mimg->definitions[0] = Definition(dst);
3991    mimg->glc = return_previous;
3992    mimg->dmask = (1 << data.size()) - 1;
3993    mimg->unrm = true;
3994    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3995    mimg->disable_wqm = true;
3996    mimg->barrier = barrier_image;
3997    ctx->program->needs_exact = true;
3998    ctx->block->instructions.emplace_back(std::move(mimg));
3999    return;
4000 }
4001
4002 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4003 {
4004    if (in_elements && ctx->options->chip_class == GFX8) {
4005       Builder bld(ctx->program, ctx->block);
4006
4007       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4008       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4009       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4010       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4011
4012       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4013       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4014
4015       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4016       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4017       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4018
4019       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4020       /* idea
4021        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4022        * in case 12 (or 3?), we have to divide by 3:
4023        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4024        * use v_mul_hi_u32 with magic number to divide
4025        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4026        * disable v_skip
4027        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4028        */
4029
4030    } else {
4031       emit_extract_vector(ctx, desc, 2, dst);
4032    }
4033 }
4034
4035 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4036 {
4037    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4038    const struct glsl_type *type = glsl_without_array(var->type);
4039    Builder bld(ctx->program, ctx->block);
4040
4041    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4042       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4043       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4044    }
4045
4046    /* LOD */
4047    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4048
4049    /* Resource */
4050    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4051
4052    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4053
4054    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4055    mimg->operands[0] = Operand(lod);
4056    mimg->operands[1] = Operand(resource);
4057    unsigned& dmask = mimg->dmask;
4058    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4059    mimg->da = glsl_sampler_type_is_array(type);
4060    mimg->can_reorder = true;
4061    Definition& def = mimg->definitions[0];
4062    ctx->block->instructions.emplace_back(std::move(mimg));
4063
4064    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4065        glsl_sampler_type_is_array(type)) {
4066
4067       assert(instr->dest.ssa.num_components == 3);
4068       Temp tmp = {ctx->program->allocateId(), v3};
4069       def = Definition(tmp);
4070       emit_split_vector(ctx, tmp, 3);
4071
4072       /* divide 3rd value by 6 by multiplying with magic number */
4073       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4074       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4075
4076       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4077                  emit_extract_vector(ctx, tmp, 0, v1),
4078                  emit_extract_vector(ctx, tmp, 1, v1),
4079                  by_6);
4080
4081    } else if (ctx->options->chip_class >= GFX9 &&
4082               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4083               glsl_sampler_type_is_array(type)) {
4084       assert(instr->dest.ssa.num_components == 2);
4085       def = Definition(dst);
4086       dmask = 0x5;
4087    } else {
4088       def = Definition(dst);
4089    }
4090
4091    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4092 }
4093
4094 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4095 {
4096    Builder bld(ctx->program, ctx->block);
4097    unsigned num_components = instr->num_components;
4098
4099    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4100    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4101    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4102
4103    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4104    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc);
4105 }
4106
4107 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4108 {
4109    Builder bld(ctx->program, ctx->block);
4110    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4111    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4112    unsigned writemask = nir_intrinsic_write_mask(instr);
4113
4114    Temp offset;
4115    if (ctx->options->chip_class < GFX8)
4116       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4117    else
4118       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4119
4120    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4121    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4122
4123    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4124                ctx->options->chip_class >= GFX8;
4125    if (smem)
4126       offset = bld.as_uniform(offset);
4127    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4128
4129    while (writemask) {
4130       int start, count;
4131       u_bit_scan_consecutive_range(&writemask, &start, &count);
4132       if (count == 3 && smem) {
4133          writemask |= 1u << (start + 2);
4134          count = 2;
4135       }
4136       int num_bytes = count * elem_size_bytes;
4137
4138       if (num_bytes > 16) {
4139          assert(elem_size_bytes == 8);
4140          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4141          count = 2;
4142          num_bytes = 16;
4143       }
4144
4145       // TODO: check alignment of sub-dword stores
4146       // TODO: split 3 bytes. there is no store instruction for that
4147
4148       Temp write_data;
4149       if (count != instr->num_components) {
4150          emit_split_vector(ctx, data, instr->num_components);
4151          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4152          for (int i = 0; i < count; i++) {
4153             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4154             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4155          }
4156          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4157          vec->definitions[0] = Definition(write_data);
4158          ctx->block->instructions.emplace_back(std::move(vec));
4159       } else if (!smem && data.type() != RegType::vgpr) {
4160          assert(num_bytes % 4 == 0);
4161          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4162       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4163          assert(num_bytes % 4 == 0);
4164          write_data = bld.as_uniform(data);
4165       } else {
4166          write_data = data;
4167       }
4168
4169       aco_opcode vmem_op, smem_op;
4170       switch (num_bytes) {
4171          case 4:
4172             vmem_op = aco_opcode::buffer_store_dword;
4173             smem_op = aco_opcode::s_buffer_store_dword;
4174             break;
4175          case 8:
4176             vmem_op = aco_opcode::buffer_store_dwordx2;
4177             smem_op = aco_opcode::s_buffer_store_dwordx2;
4178             break;
4179          case 12:
4180             vmem_op = aco_opcode::buffer_store_dwordx3;
4181             smem_op = aco_opcode::last_opcode;
4182             assert(!smem);
4183             break;
4184          case 16:
4185             vmem_op = aco_opcode::buffer_store_dwordx4;
4186             smem_op = aco_opcode::s_buffer_store_dwordx4;
4187             break;
4188          default:
4189             unreachable("Store SSBO not implemented for this size.");
4190       }
4191       if (ctx->stage == fragment_fs)
4192          smem_op = aco_opcode::p_fs_buffer_store_smem;
4193
4194       if (smem) {
4195          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4196          store->operands[0] = Operand(rsrc);
4197          if (start) {
4198             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4199                                 offset, Operand(start * elem_size_bytes));
4200             store->operands[1] = Operand(off);
4201          } else {
4202             store->operands[1] = Operand(offset);
4203          }
4204          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4205             store->operands[1].setFixed(m0);
4206          store->operands[2] = Operand(write_data);
4207          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4208          store->disable_wqm = true;
4209          store->barrier = barrier_buffer;
4210          ctx->block->instructions.emplace_back(std::move(store));
4211          ctx->program->wb_smem_l1_on_end = true;
4212          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4213             ctx->block->kind |= block_kind_needs_lowering;
4214             ctx->program->needs_exact = true;
4215          }
4216       } else {
4217          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4218          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4219          store->operands[1] = Operand(rsrc);
4220          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4221          store->operands[3] = Operand(write_data);
4222          store->offset = start * elem_size_bytes;
4223          store->offen = (offset.type() == RegType::vgpr);
4224          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4225          store->disable_wqm = true;
4226          store->barrier = barrier_buffer;
4227          ctx->program->needs_exact = true;
4228          ctx->block->instructions.emplace_back(std::move(store));
4229       }
4230    }
4231 }
4232
4233 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4234 {
4235    /* return the previous value if dest is ever used */
4236    bool return_previous = false;
4237    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4238       return_previous = true;
4239       break;
4240    }
4241    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4242       return_previous = true;
4243       break;
4244    }
4245
4246    Builder bld(ctx->program, ctx->block);
4247    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4248
4249    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4250       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4251                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4252
4253    Temp offset;
4254    if (ctx->options->chip_class < GFX8)
4255       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4256    else
4257       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4258
4259    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4260    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4261
4262    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4263
4264    aco_opcode op32, op64;
4265    switch (instr->intrinsic) {
4266       case nir_intrinsic_ssbo_atomic_add:
4267          op32 = aco_opcode::buffer_atomic_add;
4268          op64 = aco_opcode::buffer_atomic_add_x2;
4269          break;
4270       case nir_intrinsic_ssbo_atomic_imin:
4271          op32 = aco_opcode::buffer_atomic_smin;
4272          op64 = aco_opcode::buffer_atomic_smin_x2;
4273          break;
4274       case nir_intrinsic_ssbo_atomic_umin:
4275          op32 = aco_opcode::buffer_atomic_umin;
4276          op64 = aco_opcode::buffer_atomic_umin_x2;
4277          break;
4278       case nir_intrinsic_ssbo_atomic_imax:
4279          op32 = aco_opcode::buffer_atomic_smax;
4280          op64 = aco_opcode::buffer_atomic_smax_x2;
4281          break;
4282       case nir_intrinsic_ssbo_atomic_umax:
4283          op32 = aco_opcode::buffer_atomic_umax;
4284          op64 = aco_opcode::buffer_atomic_umax_x2;
4285          break;
4286       case nir_intrinsic_ssbo_atomic_and:
4287          op32 = aco_opcode::buffer_atomic_and;
4288          op64 = aco_opcode::buffer_atomic_and_x2;
4289          break;
4290       case nir_intrinsic_ssbo_atomic_or:
4291          op32 = aco_opcode::buffer_atomic_or;
4292          op64 = aco_opcode::buffer_atomic_or_x2;
4293          break;
4294       case nir_intrinsic_ssbo_atomic_xor:
4295          op32 = aco_opcode::buffer_atomic_xor;
4296          op64 = aco_opcode::buffer_atomic_xor_x2;
4297          break;
4298       case nir_intrinsic_ssbo_atomic_exchange:
4299          op32 = aco_opcode::buffer_atomic_swap;
4300          op64 = aco_opcode::buffer_atomic_swap_x2;
4301          break;
4302       case nir_intrinsic_ssbo_atomic_comp_swap:
4303          op32 = aco_opcode::buffer_atomic_cmpswap;
4304          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4305          break;
4306       default:
4307          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4308    }
4309    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4310    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4311    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4312    mubuf->operands[1] = Operand(rsrc);
4313    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4314    mubuf->operands[3] = Operand(data);
4315    if (return_previous)
4316       mubuf->definitions[0] = Definition(dst);
4317    mubuf->offset = 0;
4318    mubuf->offen = (offset.type() == RegType::vgpr);
4319    mubuf->glc = return_previous;
4320    mubuf->disable_wqm = true;
4321    mubuf->barrier = barrier_buffer;
4322    ctx->program->needs_exact = true;
4323    ctx->block->instructions.emplace_back(std::move(mubuf));
4324 }
4325
4326 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4327
4328    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4329    Builder bld(ctx->program, ctx->block);
4330    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4331    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4332 }
4333
4334 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4335 {
4336    Builder bld(ctx->program, ctx->block);
4337    unsigned num_components = instr->num_components;
4338    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4339
4340    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4341    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4342
4343    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4344    aco_opcode op;
4345    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4346       bool global = ctx->options->chip_class >= GFX9;
4347       aco_opcode op;
4348       switch (num_bytes) {
4349       case 4:
4350          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4351          break;
4352       case 8:
4353          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4354          break;
4355       case 12:
4356          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4357          break;
4358       case 16:
4359          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4360          break;
4361       default:
4362          unreachable("load_global not implemented for this size.");
4363       }
4364       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4365       flat->operands[0] = Operand(addr);
4366       flat->operands[1] = Operand(s1);
4367       flat->glc = glc;
4368
4369       if (dst.type() == RegType::sgpr) {
4370          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4371          flat->definitions[0] = Definition(vec);
4372          ctx->block->instructions.emplace_back(std::move(flat));
4373          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4374       } else {
4375          flat->definitions[0] = Definition(dst);
4376          ctx->block->instructions.emplace_back(std::move(flat));
4377       }
4378       emit_split_vector(ctx, dst, num_components);
4379    } else {
4380       switch (num_bytes) {
4381          case 4:
4382             op = aco_opcode::s_load_dword;
4383             break;
4384          case 8:
4385             op = aco_opcode::s_load_dwordx2;
4386             break;
4387          case 12:
4388          case 16:
4389             op = aco_opcode::s_load_dwordx4;
4390             break;
4391          default:
4392             unreachable("load_global not implemented for this size.");
4393       }
4394       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4395       load->operands[0] = Operand(addr);
4396       load->operands[1] = Operand(0u);
4397       load->definitions[0] = Definition(dst);
4398       load->glc = glc;
4399       load->barrier = barrier_buffer;
4400       assert(ctx->options->chip_class >= GFX8 || !glc);
4401
4402       if (dst.size() == 3) {
4403          /* trim vector */
4404          Temp vec = bld.tmp(s4);
4405          load->definitions[0] = Definition(vec);
4406          ctx->block->instructions.emplace_back(std::move(load));
4407          emit_split_vector(ctx, vec, 4);
4408
4409          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4410                     emit_extract_vector(ctx, vec, 0, s1),
4411                     emit_extract_vector(ctx, vec, 1, s1),
4412                     emit_extract_vector(ctx, vec, 2, s1));
4413       } else {
4414          ctx->block->instructions.emplace_back(std::move(load));
4415       }
4416    }
4417 }
4418
4419 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4420 {
4421    Builder bld(ctx->program, ctx->block);
4422    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4423
4424    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4425    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4426
4427    unsigned writemask = nir_intrinsic_write_mask(instr);
4428    while (writemask) {
4429       int start, count;
4430       u_bit_scan_consecutive_range(&writemask, &start, &count);
4431       unsigned num_bytes = count * elem_size_bytes;
4432
4433       Temp write_data = data;
4434       if (count != instr->num_components) {
4435          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4436          for (int i = 0; i < count; i++)
4437             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4438          write_data = bld.tmp(RegType::vgpr, count);
4439          vec->definitions[0] = Definition(write_data);
4440          ctx->block->instructions.emplace_back(std::move(vec));
4441       }
4442
4443       unsigned offset = start * elem_size_bytes;
4444       if (offset > 0 && ctx->options->chip_class < GFX9) {
4445          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4446          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4447          Temp carry = bld.tmp(s2);
4448          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4449
4450          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4451                   Operand(offset), addr0);
4452          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4453                   Operand(0u), addr1,
4454                   carry).def(1).setHint(vcc);
4455
4456          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4457
4458          offset = 0;
4459       }
4460
4461       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4462       bool global = ctx->options->chip_class >= GFX9;
4463       aco_opcode op;
4464       switch (num_bytes) {
4465       case 4:
4466          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4467          break;
4468       case 8:
4469          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4470          break;
4471       case 12:
4472          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4473          break;
4474       case 16:
4475          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4476          break;
4477       default:
4478          unreachable("store_global not implemented for this size.");
4479       }
4480       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4481       flat->operands[0] = Operand(addr);
4482       flat->operands[1] = Operand(s1);
4483       flat->operands[2] = Operand(data);
4484       flat->glc = glc;
4485       flat->offset = offset;
4486       ctx->block->instructions.emplace_back(std::move(flat));
4487    }
4488 }
4489
4490 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4491    Builder bld(ctx->program, ctx->block);
4492    switch(instr->intrinsic) {
4493       case nir_intrinsic_group_memory_barrier:
4494       case nir_intrinsic_memory_barrier:
4495          bld.barrier(aco_opcode::p_memory_barrier_all);
4496          break;
4497       case nir_intrinsic_memory_barrier_atomic_counter:
4498          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4499          break;
4500       case nir_intrinsic_memory_barrier_buffer:
4501          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4502          break;
4503       case nir_intrinsic_memory_barrier_image:
4504          bld.barrier(aco_opcode::p_memory_barrier_image);
4505          break;
4506       case nir_intrinsic_memory_barrier_shared:
4507          bld.barrier(aco_opcode::p_memory_barrier_shared);
4508          break;
4509       default:
4510          unreachable("Unimplemented memory barrier intrinsic");
4511          break;
4512    }
4513 }
4514
4515 Operand load_lds_size_m0(isel_context *ctx)
4516 {
4517    /* TODO: m0 does not need to be initialized on GFX9+ */
4518    Builder bld(ctx->program, ctx->block);
4519    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
4520 }
4521
4522
4523 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4524 {
4525    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4526    Operand m = load_lds_size_m0(ctx);
4527    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4528    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4529    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4530    Builder bld(ctx->program, ctx->block);
4531
4532    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4533    unsigned bytes_read = 0;
4534    unsigned result_size = 0;
4535    unsigned total_bytes = instr->num_components * elem_size_bytes;
4536    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : instr->dest.ssa.bit_size / 8;
4537    std::array<Temp, 4> result;
4538
4539    while (bytes_read < total_bytes) {
4540       unsigned todo = total_bytes - bytes_read;
4541       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
4542       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
4543
4544       aco_opcode op = aco_opcode::last_opcode;
4545       if (todo >= 16 && aligned16) {
4546          op = aco_opcode::ds_read_b128;
4547          todo = 16;
4548       } else if (todo >= 12 && aligned16) {
4549          op = aco_opcode::ds_read_b96;
4550          todo = 12;
4551       } else if (todo >= 8) {
4552          op = aligned8 ? aco_opcode::ds_read_b64 : aco_opcode::ds_read2_b32;
4553          todo = 8;
4554       } else if (todo >= 4) {
4555          op = aco_opcode::ds_read_b32;
4556          todo = 4;
4557       } else {
4558          assert(false);
4559       }
4560       assert(todo % elem_size_bytes == 0);
4561       unsigned num_elements = todo / elem_size_bytes;
4562       unsigned offset = nir_intrinsic_base(instr) + bytes_read;
4563       unsigned max_offset = op == aco_opcode::ds_read2_b32 ? 1019 : 65535;
4564
4565       Temp address_offset = address;
4566       if (offset > max_offset) {
4567          address_offset = bld.vadd32(bld.def(v1), Operand((uint32_t)nir_intrinsic_base(instr)), address_offset);
4568          offset = bytes_read;
4569       }
4570       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
4571
4572       Temp res;
4573       if (instr->num_components == 1 && dst.type() == RegType::vgpr)
4574          res = dst;
4575       else
4576          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
4577
4578       if (op == aco_opcode::ds_read2_b32)
4579          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
4580       else
4581          res = bld.ds(op, Definition(res), address_offset, m, offset);
4582
4583       if (instr->num_components == 1) {
4584          assert(todo == total_bytes);
4585          if (dst.type() == RegType::sgpr)
4586             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4587          return;
4588       }
4589
4590       if (dst.type() == RegType::sgpr)
4591          res = bld.as_uniform(res);
4592
4593       if (num_elements == 1) {
4594          result[result_size++] = res;
4595       } else {
4596          assert(res != dst && res.size() % num_elements == 0);
4597          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
4598          split->operands[0] = Operand(res);
4599          for (unsigned i = 0; i < num_elements; i++)
4600             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
4601          ctx->block->instructions.emplace_back(std::move(split));
4602       }
4603
4604       bytes_read += todo;
4605    }
4606
4607    assert(result_size == instr->num_components && result_size > 1);
4608    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
4609    for (unsigned i = 0; i < result_size; i++)
4610       vec->operands[i] = Operand(result[i]);
4611    vec->definitions[0] = Definition(dst);
4612    ctx->block->instructions.emplace_back(std::move(vec));
4613    ctx->allocated_vec.emplace(dst.id(), result);
4614 }
4615
4616 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned offset0, unsigned offset1, unsigned align)
4617 {
4618    Builder bld(ctx->program, ctx->block);
4619    unsigned bytes_written = 0;
4620    while (bytes_written < data.size() * 4) {
4621       unsigned todo = data.size() * 4 - bytes_written;
4622       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
4623       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
4624
4625       aco_opcode op = aco_opcode::last_opcode;
4626       unsigned size = 0;
4627       if (todo >= 16 && aligned16) {
4628          op = aco_opcode::ds_write_b128;
4629          size = 4;
4630       } else if (todo >= 12 && aligned16) {
4631          op = aco_opcode::ds_write_b96;
4632          size = 3;
4633       } else if (todo >= 8) {
4634          op = aligned8 ? aco_opcode::ds_write_b64 : aco_opcode::ds_write2_b32;
4635          size = 2;
4636       } else if (todo >= 4) {
4637          op = aco_opcode::ds_write_b32;
4638          size = 1;
4639       } else {
4640          assert(false);
4641       }
4642
4643       bool write2 = op == aco_opcode::ds_write2_b32;
4644       unsigned offset = offset0 + offset1 + bytes_written;
4645       unsigned max_offset = write2 ? 1020 : 65535;
4646       Temp address_offset = address;
4647       if (offset > max_offset) {
4648          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
4649          offset = offset1 + bytes_written;
4650       }
4651       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
4652
4653       if (write2) {
4654          Temp val0 = emit_extract_vector(ctx, data, bytes_written >> 2, v1);
4655          Temp val1 = emit_extract_vector(ctx, data, (bytes_written >> 2) + 1, v1);
4656          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
4657       } else {
4658          Temp val = emit_extract_vector(ctx, data, bytes_written >> 2, RegClass(RegType::vgpr, size));
4659          bld.ds(op, address_offset, val, m, offset);
4660       }
4661
4662       bytes_written += size * 4;
4663    }
4664 }
4665
4666 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4667 {
4668    unsigned offset = nir_intrinsic_base(instr);
4669    unsigned writemask = nir_intrinsic_write_mask(instr);
4670    Operand m = load_lds_size_m0(ctx);
4671    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4672    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4673    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4674    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4675
4676    /* we need at most two stores for 32bit variables */
4677    int start[2], count[2];
4678    u_bit_scan_consecutive_range(&writemask, &start[0], &count[0]);
4679    u_bit_scan_consecutive_range(&writemask, &start[1], &count[1]);
4680    assert(writemask == 0);
4681
4682    /* one combined store is sufficient */
4683    if (count[0] == count[1]) {
4684       Builder bld(ctx->program, ctx->block);
4685
4686       Temp address_offset = address;
4687       if ((offset >> 2) + start[1] > 255) {
4688          address_offset = bld.vadd32(bld.def(v1), Operand(offset), address_offset);
4689          offset = 0;
4690       }
4691
4692       assert(count[0] == 1);
4693       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
4694       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
4695       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
4696       offset = offset / elem_size_bytes;
4697       bld.ds(op, address_offset, val0, val1, m,
4698              offset + start[0], offset + start[1]);
4699       return;
4700    }
4701
4702    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4703    for (unsigned i = 0; i < 2; i++) {
4704       if (count[i] == 0)
4705          continue;
4706
4707       Temp write_data = emit_extract_vector(ctx, data, start[i], RegClass(RegType::vgpr, count[i] * elem_size_bytes / 4));
4708       ds_write_helper(ctx, m, address, write_data, offset, start[i] * elem_size_bytes, align);
4709    }
4710    return;
4711 }
4712
4713 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4714 {
4715    unsigned offset = nir_intrinsic_base(instr);
4716    Operand m = load_lds_size_m0(ctx);
4717    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4718    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4719
4720    unsigned num_operands = 3;
4721    aco_opcode op32, op64, op32_rtn, op64_rtn;
4722    switch(instr->intrinsic) {
4723       case nir_intrinsic_shared_atomic_add:
4724          op32 = aco_opcode::ds_add_u32;
4725          op64 = aco_opcode::ds_add_u64;
4726          op32_rtn = aco_opcode::ds_add_rtn_u32;
4727          op64_rtn = aco_opcode::ds_add_rtn_u64;
4728          break;
4729       case nir_intrinsic_shared_atomic_imin:
4730          op32 = aco_opcode::ds_min_i32;
4731          op64 = aco_opcode::ds_min_i64;
4732          op32_rtn = aco_opcode::ds_min_rtn_i32;
4733          op64_rtn = aco_opcode::ds_min_rtn_i64;
4734          break;
4735       case nir_intrinsic_shared_atomic_umin:
4736          op32 = aco_opcode::ds_min_u32;
4737          op64 = aco_opcode::ds_min_u64;
4738          op32_rtn = aco_opcode::ds_min_rtn_u32;
4739          op64_rtn = aco_opcode::ds_min_rtn_u64;
4740          break;
4741       case nir_intrinsic_shared_atomic_imax:
4742          op32 = aco_opcode::ds_max_i32;
4743          op64 = aco_opcode::ds_max_i64;
4744          op32_rtn = aco_opcode::ds_max_rtn_i32;
4745          op64_rtn = aco_opcode::ds_max_rtn_i64;
4746          break;
4747       case nir_intrinsic_shared_atomic_umax:
4748          op32 = aco_opcode::ds_max_u32;
4749          op64 = aco_opcode::ds_max_u64;
4750          op32_rtn = aco_opcode::ds_max_rtn_u32;
4751          op64_rtn = aco_opcode::ds_max_rtn_u64;
4752          break;
4753       case nir_intrinsic_shared_atomic_and:
4754          op32 = aco_opcode::ds_and_b32;
4755          op64 = aco_opcode::ds_and_b64;
4756          op32_rtn = aco_opcode::ds_and_rtn_b32;
4757          op64_rtn = aco_opcode::ds_and_rtn_b64;
4758          break;
4759       case nir_intrinsic_shared_atomic_or:
4760          op32 = aco_opcode::ds_or_b32;
4761          op64 = aco_opcode::ds_or_b64;
4762          op32_rtn = aco_opcode::ds_or_rtn_b32;
4763          op64_rtn = aco_opcode::ds_or_rtn_b64;
4764          break;
4765       case nir_intrinsic_shared_atomic_xor:
4766          op32 = aco_opcode::ds_xor_b32;
4767          op64 = aco_opcode::ds_xor_b64;
4768          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4769          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4770          break;
4771       case nir_intrinsic_shared_atomic_exchange:
4772          op32 = aco_opcode::ds_write_b32;
4773          op64 = aco_opcode::ds_write_b64;
4774          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4775          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4776          break;
4777       case nir_intrinsic_shared_atomic_comp_swap:
4778          op32 = aco_opcode::ds_cmpst_b32;
4779          op64 = aco_opcode::ds_cmpst_b64;
4780          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4781          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4782          num_operands = 4;
4783          break;
4784       default:
4785          unreachable("Unhandled shared atomic intrinsic");
4786    }
4787
4788    /* return the previous value if dest is ever used */
4789    bool return_previous = false;
4790    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4791       return_previous = true;
4792       break;
4793    }
4794    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4795       return_previous = true;
4796       break;
4797    }
4798
4799    aco_opcode op;
4800    if (data.size() == 1) {
4801       assert(instr->dest.ssa.bit_size == 32);
4802       op = return_previous ? op32_rtn : op32;
4803    } else {
4804       assert(instr->dest.ssa.bit_size == 64);
4805       op = return_previous ? op64_rtn : op64;
4806    }
4807
4808    if (offset > 65535) {
4809       Builder bld(ctx->program, ctx->block);
4810       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4811       offset = 0;
4812    }
4813
4814    aco_ptr<DS_instruction> ds;
4815    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4816    ds->operands[0] = Operand(address);
4817    ds->operands[1] = Operand(data);
4818    if (num_operands == 4)
4819       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4820    ds->operands[num_operands - 1] = m;
4821    ds->offset0 = offset;
4822    if (return_previous)
4823       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4824    ctx->block->instructions.emplace_back(std::move(ds));
4825 }
4826
4827 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4828    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4829    Builder bld(ctx->program, ctx->block);
4830    Temp scratch_addr = ctx->private_segment_buffer;
4831    if (ctx->stage != MESA_SHADER_COMPUTE)
4832       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), ctx->private_segment_buffer, Operand(0u));
4833    uint32_t rsrc_conf;
4834    /* older generations need element size = 16 bytes */
4835    if (ctx->program->chip_class >= GFX9)
4836       rsrc_conf = 0x00E00000u;
4837    else
4838       rsrc_conf = 0x00F80000u;
4839    /* buffer res = addr + num_records = -1, index_stride = 64, add_tid_enable = true */
4840    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4841    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4842    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4843
4844    aco_opcode op;
4845    switch (dst.size()) {
4846       case 1:
4847          op = aco_opcode::buffer_load_dword;
4848          break;
4849       case 2:
4850          op = aco_opcode::buffer_load_dwordx2;
4851          break;
4852       case 3:
4853          op = aco_opcode::buffer_load_dwordx3;
4854          break;
4855       case 4:
4856          op = aco_opcode::buffer_load_dwordx4;
4857          break;
4858       case 6:
4859       case 8: {
4860          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4861          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4862                                 bld.def(v4), offset, rsrc,
4863                                 ctx->scratch_offset, 0, true);
4864          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4865                                                   aco_opcode::buffer_load_dwordx4,
4866                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4867                                 offset, rsrc, ctx->scratch_offset, 16, true);
4868          emit_split_vector(ctx, lower, 2);
4869          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4870          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4871          if (dst.size() == 8) {
4872             emit_split_vector(ctx, upper, 2);
4873             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4874             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4875          } else {
4876             elems[2] = upper;
4877          }
4878
4879          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4880                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4881          for (unsigned i = 0; i < dst.size() / 2; i++)
4882             vec->operands[i] = Operand(elems[i]);
4883          vec->definitions[0] = Definition(dst);
4884          bld.insert(std::move(vec));
4885          ctx->allocated_vec.emplace(dst.id(), elems);
4886          return;
4887       }
4888       default:
4889          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
4890    }
4891
4892    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->scratch_offset, 0, true);
4893    emit_split_vector(ctx, dst, instr->num_components);
4894 }
4895
4896 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4897    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
4898    Builder bld(ctx->program, ctx->block);
4899    Temp scratch_addr = ctx->private_segment_buffer;
4900    if (ctx->stage != MESA_SHADER_COMPUTE)
4901       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), ctx->private_segment_buffer, Operand(0u));
4902    uint32_t rsrc_conf;
4903    /* older generations need element size = 16 bytes */
4904    if (ctx->program->chip_class >= GFX9)
4905       rsrc_conf = 0x00E00000u;
4906    else
4907       rsrc_conf = 0x00F80000u;
4908    /* buffer res = addr + num_records = -1, index_stride = 64, add_tid_enable = true */
4909    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4910    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4911    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4912
4913    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4914    unsigned writemask = nir_intrinsic_write_mask(instr);
4915
4916    while (writemask) {
4917       int start, count;
4918       u_bit_scan_consecutive_range(&writemask, &start, &count);
4919       int num_bytes = count * elem_size_bytes;
4920
4921       if (num_bytes > 16) {
4922          assert(elem_size_bytes == 8);
4923          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4924          count = 2;
4925          num_bytes = 16;
4926       }
4927
4928       // TODO: check alignment of sub-dword stores
4929       // TODO: split 3 bytes. there is no store instruction for that
4930
4931       Temp write_data;
4932       if (count != instr->num_components) {
4933          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4934          for (int i = 0; i < count; i++) {
4935             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
4936             vec->operands[i] = Operand(elem);
4937          }
4938          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
4939          vec->definitions[0] = Definition(write_data);
4940          ctx->block->instructions.emplace_back(std::move(vec));
4941       } else {
4942          write_data = data;
4943       }
4944
4945       aco_opcode op;
4946       switch (num_bytes) {
4947          case 4:
4948             op = aco_opcode::buffer_store_dword;
4949             break;
4950          case 8:
4951             op = aco_opcode::buffer_store_dwordx2;
4952             break;
4953          case 12:
4954             op = aco_opcode::buffer_store_dwordx3;
4955             break;
4956          case 16:
4957             op = aco_opcode::buffer_store_dwordx4;
4958             break;
4959          default:
4960             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
4961       }
4962
4963       bld.mubuf(op, offset, rsrc, ctx->scratch_offset, write_data, start * elem_size_bytes, true);
4964    }
4965 }
4966
4967 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
4968    uint8_t log2_ps_iter_samples;
4969    if (ctx->program->info->ps.force_persample) {
4970       log2_ps_iter_samples =
4971          util_logbase2(ctx->options->key.fs.num_samples);
4972    } else {
4973       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
4974    }
4975
4976    /* The bit pattern matches that used by fixed function fragment
4977     * processing. */
4978    static const unsigned ps_iter_masks[] = {
4979       0xffff, /* not used */
4980       0x5555,
4981       0x1111,
4982       0x0101,
4983       0x0001,
4984    };
4985    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
4986
4987    Builder bld(ctx->program, ctx->block);
4988
4989    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
4990    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
4991    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
4992    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4993    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
4994 }
4995
4996 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
4997 {
4998    Builder bld(ctx->program, ctx->block);
4999
5000    if (cluster_size == 1) {
5001       return src;
5002    } if (op == nir_op_iand && cluster_size == 4) {
5003       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5004       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5005       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
5006                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
5007    } else if (op == nir_op_ior && cluster_size == 4) {
5008       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5009       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
5010                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
5011    } else if (op == nir_op_iand && cluster_size == 64) {
5012       //subgroupAnd(val) -> (exec & ~val) == 0
5013       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5014       return bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), tmp, Operand(0u));
5015    } else if (op == nir_op_ior && cluster_size == 64) {
5016       //subgroupOr(val) -> (val & exec) != 0
5017       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
5018    } else if (op == nir_op_ixor && cluster_size == 64) {
5019       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5020       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5021       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
5022       return bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5023    } else {
5024       //subgroupClustered{And,Or,Xor}(val, n) ->
5025       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5026       //cluster_offset = ~(n - 1) & lane_id
5027       //cluster_mask = ((1 << n) - 1)
5028       //subgroupClusteredAnd():
5029       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5030       //subgroupClusteredOr():
5031       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5032       //subgroupClusteredXor():
5033       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5034       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5035                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5036       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5037
5038       Temp tmp;
5039       if (op == nir_op_iand)
5040          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5041       else
5042          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5043
5044       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5045       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5046       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5047       if (cluster_mask != 0xffffffff)
5048          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5049
5050       Definition cmp_def = Definition();
5051       if (op == nir_op_iand) {
5052          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5053       } else if (op == nir_op_ior) {
5054          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5055       } else if (op == nir_op_ixor) {
5056          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5057                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5058          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5059       }
5060       cmp_def.setHint(vcc);
5061       return cmp_def.getTemp();
5062    }
5063 }
5064
5065 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5066 {
5067    Builder bld(ctx->program, ctx->block);
5068
5069    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5070    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5071    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5072    Temp tmp;
5073    if (op == nir_op_iand)
5074       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5075    else
5076       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5077
5078    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5079    Temp lo = lohi.def(0).getTemp();
5080    Temp hi = lohi.def(1).getTemp();
5081    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5082                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5083
5084    Definition cmp_def = Definition();
5085    if (op == nir_op_iand)
5086       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5087    else if (op == nir_op_ior)
5088       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5089    else if (op == nir_op_ixor)
5090       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5091                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5092    cmp_def.setHint(vcc);
5093    return cmp_def.getTemp();
5094 }
5095
5096 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5097 {
5098    Builder bld(ctx->program, ctx->block);
5099
5100    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5101    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5102    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5103    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5104    if (op == nir_op_iand)
5105       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5106    else if (op == nir_op_ior)
5107       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5108    else if (op == nir_op_ixor)
5109       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5110
5111    assert(false);
5112    return Temp();
5113 }
5114
5115 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5116 {
5117    Builder bld(ctx->program, ctx->block);
5118    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5119    if (src.regClass().type() == RegType::vgpr) {
5120       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5121    } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5122       bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(dst), Operand(0u), Operand(src));
5123    } else if (src.regClass() == s1) {
5124       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5125    } else if (src.regClass() == s2) {
5126       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5127    } else {
5128       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5129       nir_print_instr(&instr->instr, stderr);
5130       fprintf(stderr, "\n");
5131    }
5132 }
5133
5134 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5135 {
5136    Builder bld(ctx->program, ctx->block);
5137    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5138    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5139
5140    /* Build DD X/Y */
5141    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5142    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5143    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5144    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5145    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5146    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5147
5148    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5149    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5150    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5151    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5152    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5153    Temp wqm1 = bld.tmp(v1);
5154    emit_wqm(ctx, tmp1, wqm1, true);
5155    Temp wqm2 = bld.tmp(v1);
5156    emit_wqm(ctx, tmp2, wqm2, true);
5157    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5158    return;
5159 }
5160
5161 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5162 {
5163    Builder bld(ctx->program, ctx->block);
5164    switch(instr->intrinsic) {
5165    case nir_intrinsic_load_barycentric_sample:
5166    case nir_intrinsic_load_barycentric_pixel:
5167    case nir_intrinsic_load_barycentric_centroid: {
5168       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5169       fs_input input = get_interp_input(instr->intrinsic, mode);
5170
5171       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5172       if (input == fs_input::max_inputs) {
5173          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5174                     Operand(0u), Operand(0u));
5175       } else {
5176          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5177                     ctx->fs_inputs[input],
5178                     ctx->fs_inputs[input + 1]);
5179       }
5180       emit_split_vector(ctx, dst, 2);
5181       break;
5182    }
5183    case nir_intrinsic_load_barycentric_at_sample: {
5184       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5185       switch (ctx->options->key.fs.num_samples) {
5186          case 2: sample_pos_offset += 1 << 3; break;
5187          case 4: sample_pos_offset += 3 << 3; break;
5188          case 8: sample_pos_offset += 7 << 3; break;
5189          default: break;
5190       }
5191       Temp sample_pos;
5192       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5193       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5194       if (addr.type() == RegType::sgpr) {
5195          Operand offset;
5196          if (const_addr) {
5197             sample_pos_offset += const_addr->u32 << 3;
5198             offset = Operand(sample_pos_offset);
5199          } else if (ctx->options->chip_class >= GFX9) {
5200             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5201          } else {
5202             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5203             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5204          }
5205          addr = ctx->private_segment_buffer;
5206          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), addr, Operand(offset));
5207
5208       } else if (ctx->options->chip_class >= GFX9) {
5209          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5210          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, ctx->private_segment_buffer, sample_pos_offset);
5211       } else {
5212          /* addr += ctx->private_segment_buffer + sample_pos_offset */
5213          Temp tmp0 = bld.tmp(s1);
5214          Temp tmp1 = bld.tmp(s1);
5215          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), ctx->private_segment_buffer);
5216          Definition scc_tmp = bld.def(s1, scc);
5217          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5218          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), scc_tmp.getTemp());
5219          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5220          Temp pck0 = bld.tmp(v1);
5221          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5222          tmp1 = as_vgpr(ctx, tmp1);
5223          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5224          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5225
5226          /* sample_pos = flat_load_dwordx2 addr */
5227          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5228       }
5229
5230       /* sample_pos -= 0.5 */
5231       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5232       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5233       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5234       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5235       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5236
5237       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5238       break;
5239    }
5240    case nir_intrinsic_load_barycentric_at_offset: {
5241       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5242       RegClass rc = RegClass(offset.type(), 1);
5243       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5244       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5245       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5246       break;
5247    }
5248    case nir_intrinsic_load_front_face: {
5249       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5250                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5251       break;
5252    }
5253    case nir_intrinsic_load_view_index:
5254    case nir_intrinsic_load_layer_id: {
5255       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5256          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5257          bld.copy(Definition(dst), Operand(ctx->view_index));
5258          break;
5259       }
5260
5261       unsigned idx = nir_intrinsic_base(instr);
5262       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5263                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5264       break;
5265    }
5266    case nir_intrinsic_load_frag_coord: {
5267       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5268       break;
5269    }
5270    case nir_intrinsic_load_sample_pos: {
5271       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5272       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5273       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5274                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5275                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5276       break;
5277    }
5278    case nir_intrinsic_load_interpolated_input:
5279       visit_load_interpolated_input(ctx, instr);
5280       break;
5281    case nir_intrinsic_store_output:
5282       visit_store_output(ctx, instr);
5283       break;
5284    case nir_intrinsic_load_input:
5285       visit_load_input(ctx, instr);
5286       break;
5287    case nir_intrinsic_load_ubo:
5288       visit_load_ubo(ctx, instr);
5289       break;
5290    case nir_intrinsic_load_push_constant:
5291       visit_load_push_constant(ctx, instr);
5292       break;
5293    case nir_intrinsic_load_constant:
5294       visit_load_constant(ctx, instr);
5295       break;
5296    case nir_intrinsic_vulkan_resource_index:
5297       visit_load_resource(ctx, instr);
5298       break;
5299    case nir_intrinsic_discard:
5300       visit_discard(ctx, instr);
5301       break;
5302    case nir_intrinsic_discard_if:
5303       visit_discard_if(ctx, instr);
5304       break;
5305    case nir_intrinsic_load_shared:
5306       visit_load_shared(ctx, instr);
5307       break;
5308    case nir_intrinsic_store_shared:
5309       visit_store_shared(ctx, instr);
5310       break;
5311    case nir_intrinsic_shared_atomic_add:
5312    case nir_intrinsic_shared_atomic_imin:
5313    case nir_intrinsic_shared_atomic_umin:
5314    case nir_intrinsic_shared_atomic_imax:
5315    case nir_intrinsic_shared_atomic_umax:
5316    case nir_intrinsic_shared_atomic_and:
5317    case nir_intrinsic_shared_atomic_or:
5318    case nir_intrinsic_shared_atomic_xor:
5319    case nir_intrinsic_shared_atomic_exchange:
5320    case nir_intrinsic_shared_atomic_comp_swap:
5321       visit_shared_atomic(ctx, instr);
5322       break;
5323    case nir_intrinsic_image_deref_load:
5324       visit_image_load(ctx, instr);
5325       break;
5326    case nir_intrinsic_image_deref_store:
5327       visit_image_store(ctx, instr);
5328       break;
5329    case nir_intrinsic_image_deref_atomic_add:
5330    case nir_intrinsic_image_deref_atomic_umin:
5331    case nir_intrinsic_image_deref_atomic_imin:
5332    case nir_intrinsic_image_deref_atomic_umax:
5333    case nir_intrinsic_image_deref_atomic_imax:
5334    case nir_intrinsic_image_deref_atomic_and:
5335    case nir_intrinsic_image_deref_atomic_or:
5336    case nir_intrinsic_image_deref_atomic_xor:
5337    case nir_intrinsic_image_deref_atomic_exchange:
5338    case nir_intrinsic_image_deref_atomic_comp_swap:
5339       visit_image_atomic(ctx, instr);
5340       break;
5341    case nir_intrinsic_image_deref_size:
5342       visit_image_size(ctx, instr);
5343       break;
5344    case nir_intrinsic_load_ssbo:
5345       visit_load_ssbo(ctx, instr);
5346       break;
5347    case nir_intrinsic_store_ssbo:
5348       visit_store_ssbo(ctx, instr);
5349       break;
5350    case nir_intrinsic_load_global:
5351       visit_load_global(ctx, instr);
5352       break;
5353    case nir_intrinsic_store_global:
5354       visit_store_global(ctx, instr);
5355       break;
5356    case nir_intrinsic_ssbo_atomic_add:
5357    case nir_intrinsic_ssbo_atomic_imin:
5358    case nir_intrinsic_ssbo_atomic_umin:
5359    case nir_intrinsic_ssbo_atomic_imax:
5360    case nir_intrinsic_ssbo_atomic_umax:
5361    case nir_intrinsic_ssbo_atomic_and:
5362    case nir_intrinsic_ssbo_atomic_or:
5363    case nir_intrinsic_ssbo_atomic_xor:
5364    case nir_intrinsic_ssbo_atomic_exchange:
5365    case nir_intrinsic_ssbo_atomic_comp_swap:
5366       visit_atomic_ssbo(ctx, instr);
5367       break;
5368    case nir_intrinsic_load_scratch:
5369       visit_load_scratch(ctx, instr);
5370       break;
5371    case nir_intrinsic_store_scratch:
5372       visit_store_scratch(ctx, instr);
5373       break;
5374    case nir_intrinsic_get_buffer_size:
5375       visit_get_buffer_size(ctx, instr);
5376       break;
5377    case nir_intrinsic_barrier: {
5378       unsigned* bsize = ctx->program->info->cs.block_size;
5379       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5380       if (workgroup_size > 64)
5381          bld.sopp(aco_opcode::s_barrier);
5382       break;
5383    }
5384    case nir_intrinsic_group_memory_barrier:
5385    case nir_intrinsic_memory_barrier:
5386    case nir_intrinsic_memory_barrier_atomic_counter:
5387    case nir_intrinsic_memory_barrier_buffer:
5388    case nir_intrinsic_memory_barrier_image:
5389    case nir_intrinsic_memory_barrier_shared:
5390       emit_memory_barrier(ctx, instr);
5391       break;
5392    case nir_intrinsic_load_num_work_groups:
5393    case nir_intrinsic_load_work_group_id:
5394    case nir_intrinsic_load_local_invocation_id: {
5395       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5396       Temp* ids;
5397       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5398          ids = ctx->num_workgroups;
5399       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5400          ids = ctx->workgroup_ids;
5401       else
5402          ids = ctx->local_invocation_ids;
5403       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5404                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5405                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5406                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5407       emit_split_vector(ctx, dst, 3);
5408       break;
5409    }
5410    case nir_intrinsic_load_local_invocation_index: {
5411       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5412                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5413       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5414       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5415       break;
5416    }
5417    case nir_intrinsic_load_subgroup_id: {
5418       if (ctx->stage == compute_cs) {
5419          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5420          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5421       } else {
5422          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5423       }
5424       break;
5425    }
5426    case nir_intrinsic_load_subgroup_invocation: {
5427       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5428                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5429       break;
5430    }
5431    case nir_intrinsic_load_num_subgroups: {
5432       if (ctx->stage == compute_cs)
5433          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5434       else
5435          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5436       break;
5437    }
5438    case nir_intrinsic_ballot: {
5439       Definition tmp = bld.def(s2);
5440       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5441       if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s2) {
5442          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5443       } else if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s1) {
5444          bld.sop2(aco_opcode::s_cselect_b64, tmp, Operand(exec, s2), Operand(0u), bld.scc(src));
5445       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5446          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5447       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5448          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5449       } else {
5450          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5451          nir_print_instr(&instr->instr, stderr);
5452          fprintf(stderr, "\n");
5453       }
5454       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5455       break;
5456    }
5457    case nir_intrinsic_shuffle: {
5458       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5459       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5460          emit_uniform_subgroup(ctx, instr, src);
5461       } else {
5462          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5463          assert(tid.regClass() == v1);
5464          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5465          if (src.regClass() == v1) {
5466             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5467             emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, src), dst);
5468          } else if (src.regClass() == v2) {
5469             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5470
5471             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5472             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5473             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, lo));
5474             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, hi));
5475             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5476             emit_split_vector(ctx, dst, 2);
5477          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5478             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5479             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5480             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5481             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5482          } else {
5483             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5484             nir_print_instr(&instr->instr, stderr);
5485             fprintf(stderr, "\n");
5486          }
5487       }
5488       break;
5489    }
5490    case nir_intrinsic_load_sample_id: {
5491       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5492                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5493       break;
5494    }
5495    case nir_intrinsic_load_sample_mask_in: {
5496       visit_load_sample_mask_in(ctx, instr);
5497       break;
5498    }
5499    case nir_intrinsic_read_first_invocation: {
5500       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5501       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5502       if (src.regClass() == v1) {
5503          emit_wqm(ctx,
5504                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5505                   dst);
5506       } else if (src.regClass() == v2) {
5507          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5508          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5509          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5510          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5511          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5512          emit_split_vector(ctx, dst, 2);
5513       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5514          emit_wqm(ctx,
5515                   bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5516                            bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2))),
5517                   dst);
5518       } else if (src.regClass() == s1) {
5519          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5520       } else if (src.regClass() == s2) {
5521          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5522       } else {
5523          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5524          nir_print_instr(&instr->instr, stderr);
5525          fprintf(stderr, "\n");
5526       }
5527       break;
5528    }
5529    case nir_intrinsic_read_invocation: {
5530       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5531       Temp lane = get_ssa_temp(ctx, instr->src[1].ssa);
5532       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5533       assert(lane.regClass() == s1);
5534       if (src.regClass() == v1) {
5535          emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), src, lane), dst);
5536       } else if (src.regClass() == v2) {
5537          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5538          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5539          lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), lo, lane));
5540          hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), hi, lane));
5541          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5542          emit_split_vector(ctx, dst, 2);
5543       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5544          emit_wqm(ctx, bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, lane), dst);
5545       } else if (src.regClass() == s1) {
5546          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5547       } else if (src.regClass() == s2) {
5548          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5549       } else {
5550          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5551          nir_print_instr(&instr->instr, stderr);
5552          fprintf(stderr, "\n");
5553       }
5554       break;
5555    }
5556    case nir_intrinsic_vote_all: {
5557       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5558       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5559       assert(src.regClass() == s2);
5560       assert(dst.regClass() == s1);
5561
5562       Definition tmp = bld.def(s1);
5563       bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(tmp),
5564                bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)),
5565                Operand(exec, s2));
5566       emit_wqm(ctx, tmp.getTemp(), dst);
5567       break;
5568    }
5569    case nir_intrinsic_vote_any: {
5570       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5571       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5572       assert(src.regClass() == s2);
5573       assert(dst.regClass() == s1);
5574
5575       Definition tmp = bld.def(s1);
5576       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(tmp), src, Operand(exec, s2));
5577       emit_wqm(ctx, tmp.getTemp(), dst);
5578       break;
5579    }
5580    case nir_intrinsic_reduce:
5581    case nir_intrinsic_inclusive_scan:
5582    case nir_intrinsic_exclusive_scan: {
5583       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5584       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5585       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5586       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5587          nir_intrinsic_cluster_size(instr) : 0;
5588       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5589
5590       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5591          emit_uniform_subgroup(ctx, instr, src);
5592       } else if (instr->dest.ssa.bit_size == 1) {
5593          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5594             op = nir_op_iand;
5595          else if (op == nir_op_iadd)
5596             op = nir_op_ixor;
5597          else if (op == nir_op_umax || op == nir_op_imax)
5598             op = nir_op_ior;
5599          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5600
5601          switch (instr->intrinsic) {
5602          case nir_intrinsic_reduce:
5603             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5604             break;
5605          case nir_intrinsic_exclusive_scan:
5606             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5607             break;
5608          case nir_intrinsic_inclusive_scan:
5609             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5610             break;
5611          default:
5612             assert(false);
5613          }
5614       } else if (cluster_size == 1) {
5615          bld.copy(Definition(dst), src);
5616       } else {
5617          src = as_vgpr(ctx, src);
5618
5619          ReduceOp reduce_op;
5620          switch (op) {
5621          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5622             CASE(iadd)
5623             CASE(imul)
5624             CASE(fadd)
5625             CASE(fmul)
5626             CASE(imin)
5627             CASE(umin)
5628             CASE(fmin)
5629             CASE(imax)
5630             CASE(umax)
5631             CASE(fmax)
5632             CASE(iand)
5633             CASE(ior)
5634             CASE(ixor)
5635             default:
5636                unreachable("unknown reduction op");
5637          #undef CASE
5638          }
5639
5640          aco_opcode aco_op;
5641          switch (instr->intrinsic) {
5642             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5643             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5644             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5645             default:
5646                unreachable("unknown reduce intrinsic");
5647          }
5648
5649          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5650          reduce->operands[0] = Operand(src);
5651          // filled in by aco_reduce_assign.cpp, used internally as part of the
5652          // reduce sequence
5653          assert(dst.size() == 1 || dst.size() == 2);
5654          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5655          reduce->operands[2] = Operand(v1.as_linear());
5656
5657          Temp tmp_dst = bld.tmp(dst.regClass());
5658          reduce->definitions[0] = Definition(tmp_dst);
5659          reduce->definitions[1] = bld.def(s2); // used internally
5660          reduce->definitions[2] = Definition();
5661          reduce->definitions[3] = Definition(scc, s1);
5662          reduce->definitions[4] = Definition();
5663          reduce->reduce_op = reduce_op;
5664          reduce->cluster_size = cluster_size;
5665          ctx->block->instructions.emplace_back(std::move(reduce));
5666
5667          emit_wqm(ctx, tmp_dst, dst);
5668       }
5669       break;
5670    }
5671    case nir_intrinsic_quad_broadcast: {
5672       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5673       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5674          emit_uniform_subgroup(ctx, instr, src);
5675       } else {
5676          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5677          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5678          if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5679             uint32_t half_mask = 0x11111111u << lane;
5680             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5681             Temp tmp = bld.tmp(s2);
5682             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5683                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5684                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5685             emit_wqm(ctx, tmp, dst);
5686          } else if (instr->dest.ssa.bit_size == 32) {
5687             emit_wqm(ctx,
5688                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5689                                   dpp_quad_perm(lane, lane, lane, lane)),
5690                      dst);
5691          } else if (instr->dest.ssa.bit_size == 64) {
5692             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5693             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5694             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5695             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5696             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5697             emit_split_vector(ctx, dst, 2);
5698          } else {
5699             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5700             nir_print_instr(&instr->instr, stderr);
5701             fprintf(stderr, "\n");
5702          }
5703       }
5704       break;
5705    }
5706    case nir_intrinsic_quad_swap_horizontal:
5707    case nir_intrinsic_quad_swap_vertical:
5708    case nir_intrinsic_quad_swap_diagonal:
5709    case nir_intrinsic_quad_swizzle_amd: {
5710       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5711       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5712          emit_uniform_subgroup(ctx, instr, src);
5713          break;
5714       }
5715       uint16_t dpp_ctrl = 0;
5716       switch (instr->intrinsic) {
5717       case nir_intrinsic_quad_swap_horizontal:
5718          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5719          break;
5720       case nir_intrinsic_quad_swap_vertical:
5721          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5722          break;
5723       case nir_intrinsic_quad_swap_diagonal:
5724          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5725          break;
5726       case nir_intrinsic_quad_swizzle_amd: {
5727          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5728          break;
5729       }
5730       default:
5731          break;
5732       }
5733
5734       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5735       if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5736          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5737          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5738          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5739          emit_wqm(ctx, tmp, dst);
5740       } else if (instr->dest.ssa.bit_size == 32) {
5741          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5742          emit_wqm(ctx, tmp, dst);
5743       } else if (instr->dest.ssa.bit_size == 64) {
5744          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5745          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5746          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5747          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5748          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5749          emit_split_vector(ctx, dst, 2);
5750       } else {
5751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5752          nir_print_instr(&instr->instr, stderr);
5753          fprintf(stderr, "\n");
5754       }
5755       break;
5756    }
5757    case nir_intrinsic_masked_swizzle_amd: {
5758       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5759       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5760          emit_uniform_subgroup(ctx, instr, src);
5761          break;
5762       }
5763       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5764       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5765       if (dst.regClass() == v1) {
5766          emit_wqm(ctx,
5767                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5768                   dst);
5769       } else if (dst.regClass() == v2) {
5770          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5771          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5772          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5773          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5774          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5775          emit_split_vector(ctx, dst, 2);
5776       } else {
5777          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5778          nir_print_instr(&instr->instr, stderr);
5779          fprintf(stderr, "\n");
5780       }
5781       break;
5782    }
5783    case nir_intrinsic_write_invocation_amd: {
5784       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5785       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5786       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5787       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5788       if (dst.regClass() == v1) {
5789          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5790          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5791       } else if (dst.regClass() == v2) {
5792          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5793          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5794          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5795          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5796          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5797          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5798          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5799          emit_split_vector(ctx, dst, 2);
5800       } else {
5801          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5802          nir_print_instr(&instr->instr, stderr);
5803          fprintf(stderr, "\n");
5804       }
5805       break;
5806    }
5807    case nir_intrinsic_mbcnt_amd: {
5808       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5809       RegClass rc = RegClass(src.type(), 1);
5810       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5811       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5812       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5813       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5814       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5815       emit_wqm(ctx, wqm_tmp, dst);
5816       break;
5817    }
5818    case nir_intrinsic_load_helper_invocation: {
5819       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5820       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5821       ctx->block->kind |= block_kind_needs_lowering;
5822       ctx->program->needs_exact = true;
5823       break;
5824    }
5825    case nir_intrinsic_is_helper_invocation: {
5826       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5827       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5828       ctx->block->kind |= block_kind_needs_lowering;
5829       ctx->program->needs_exact = true;
5830       break;
5831    }
5832    case nir_intrinsic_demote:
5833       bld.pseudo(aco_opcode::p_demote_to_helper);
5834       ctx->block->kind |= block_kind_needs_lowering;
5835       ctx->program->needs_exact = true;
5836       break;
5837    case nir_intrinsic_demote_if: {
5838       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc),
5839                            as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false),
5840                            Operand(exec, s2));
5841       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5842       ctx->block->kind |= block_kind_needs_lowering;
5843       ctx->program->needs_exact = true;
5844       break;
5845    }
5846    case nir_intrinsic_first_invocation: {
5847       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5848                get_ssa_temp(ctx, &instr->dest.ssa));
5849       break;
5850    }
5851    case nir_intrinsic_shader_clock:
5852       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5853       break;
5854    case nir_intrinsic_load_vertex_id_zero_base: {
5855       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5856       bld.copy(Definition(dst), ctx->vertex_id);
5857       break;
5858    }
5859    case nir_intrinsic_load_first_vertex: {
5860       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5861       bld.copy(Definition(dst), ctx->base_vertex);
5862       break;
5863    }
5864    case nir_intrinsic_load_base_instance: {
5865       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5866       bld.copy(Definition(dst), ctx->start_instance);
5867       break;
5868    }
5869    case nir_intrinsic_load_instance_id: {
5870       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5871       bld.copy(Definition(dst), ctx->instance_id);
5872       break;
5873    }
5874    case nir_intrinsic_load_draw_id: {
5875       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5876       bld.copy(Definition(dst), ctx->draw_id);
5877       break;
5878    }
5879    default:
5880       fprintf(stderr, "Unimplemented intrinsic instr: ");
5881       nir_print_instr(&instr->instr, stderr);
5882       fprintf(stderr, "\n");
5883       abort();
5884
5885       break;
5886    }
5887 }
5888
5889
5890 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5891                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5892                     enum glsl_base_type *stype)
5893 {
5894    nir_deref_instr *texture_deref_instr = NULL;
5895    nir_deref_instr *sampler_deref_instr = NULL;
5896    int plane = -1;
5897
5898    for (unsigned i = 0; i < instr->num_srcs; i++) {
5899       switch (instr->src[i].src_type) {
5900       case nir_tex_src_texture_deref:
5901          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
5902          break;
5903       case nir_tex_src_sampler_deref:
5904          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
5905          break;
5906       case nir_tex_src_plane:
5907          plane = nir_src_as_int(instr->src[i].src);
5908          break;
5909       default:
5910          break;
5911       }
5912    }
5913
5914    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
5915
5916    if (!sampler_deref_instr)
5917       sampler_deref_instr = texture_deref_instr;
5918
5919    if (plane >= 0) {
5920       assert(instr->op != nir_texop_txf_ms &&
5921              instr->op != nir_texop_samples_identical);
5922       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
5923       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
5924    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
5925       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
5926    } else {
5927       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
5928    }
5929    if (samp_ptr) {
5930       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
5931       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
5932          fprintf(stderr, "Unimplemented sampler descriptor: ");
5933          nir_print_instr(&instr->instr, stderr);
5934          fprintf(stderr, "\n");
5935          abort();
5936          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
5937       }
5938    }
5939    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
5940                      instr->op == nir_texop_samples_identical))
5941       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
5942 }
5943
5944 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
5945                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
5946 {
5947    Builder bld(ctx->program, ctx->block);
5948
5949    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
5950    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
5951    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
5952
5953    Operand neg_one(0xbf800000u);
5954    Operand one(0x3f800000u);
5955    Operand two(0x40000000u);
5956    Operand four(0x40800000u);
5957
5958    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
5959    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
5960    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
5961
5962    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
5963    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
5964    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
5965    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
5966
5967    // select sc
5968    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
5969    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
5970                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
5971                        one, is_ma_y);
5972    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5973
5974    // select tc
5975    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
5976    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
5977    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5978
5979    // select ma
5980    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
5981                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
5982                   deriv_z, is_ma_z);
5983    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
5984    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
5985 }
5986
5987 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
5988 {
5989    Builder bld(ctx->program, ctx->block);
5990    Temp coord_args[4], ma, tc, sc, id;
5991    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
5992       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
5993
5994    if (is_array) {
5995       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
5996
5997       // see comment in ac_prepare_cube_coords()
5998       if (ctx->options->chip_class <= GFX8)
5999          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6000    }
6001
6002    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6003
6004    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6005    vop3a->operands[0] = Operand(ma);
6006    vop3a->abs[0] = true;
6007    Temp invma = bld.tmp(v1);
6008    vop3a->definitions[0] = Definition(invma);
6009    ctx->block->instructions.emplace_back(std::move(vop3a));
6010
6011    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6012    if (!is_deriv)
6013       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6014
6015    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6016    if (!is_deriv)
6017       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6018
6019    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6020
6021    if (is_deriv) {
6022       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6023       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6024
6025       for (unsigned i = 0; i < 2; i++) {
6026          // see comment in ac_prepare_cube_coords()
6027          Temp deriv_ma;
6028          Temp deriv_sc, deriv_tc;
6029          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6030                            &deriv_ma, &deriv_sc, &deriv_tc);
6031
6032          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6033
6034          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6035                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6036                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6037          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6038                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6039                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6040          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6041       }
6042
6043       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6044       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6045    }
6046
6047    if (is_array)
6048       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6049    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6050
6051 }
6052
6053 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6054 {
6055    Temp coord_vec[3];
6056    for (unsigned i = 0; i < coords.size(); i++)
6057       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6058
6059    Builder bld(ctx->program, ctx->block);
6060    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6061
6062    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6063    for (unsigned i = 0; i < coords.size(); i++)
6064       vec->operands[i] = Operand(coord_vec[i]);
6065    Temp res = bld.tmp(RegType::vgpr, coords.size());
6066    vec->definitions[0] = Definition(res);
6067    ctx->block->instructions.emplace_back(std::move(vec));
6068    return res;
6069 }
6070
6071 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6072 {
6073    if (vec->parent_instr->type != nir_instr_type_alu)
6074       return;
6075    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6076    if (vec_instr->op != nir_op_vec(vec->num_components))
6077       return;
6078
6079    for (unsigned i = 0; i < vec->num_components; i++) {
6080       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6081               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6082    }
6083 }
6084
6085 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6086 {
6087    Builder bld(ctx->program, ctx->block);
6088    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6089         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6090    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6091         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6092    nir_const_value *sample_index_cv = NULL;
6093    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6094    enum glsl_base_type stype;
6095    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6096
6097    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6098                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6099    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6100                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6101
6102    for (unsigned i = 0; i < instr->num_srcs; i++) {
6103       switch (instr->src[i].src_type) {
6104       case nir_tex_src_coord:
6105          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6106          break;
6107       case nir_tex_src_bias:
6108          if (instr->op == nir_texop_txb) {
6109             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6110             has_bias = true;
6111          }
6112          break;
6113       case nir_tex_src_lod: {
6114          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6115
6116          if (val && val->f32 <= 0.0) {
6117             level_zero = true;
6118          } else {
6119             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6120             has_lod = true;
6121          }
6122          break;
6123       }
6124       case nir_tex_src_comparator:
6125          if (instr->is_shadow) {
6126             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6127             has_compare = true;
6128          }
6129          break;
6130       case nir_tex_src_offset:
6131          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6132          get_const_vec(instr->src[i].src.ssa, const_offset);
6133          has_offset = true;
6134          break;
6135       case nir_tex_src_ddx:
6136          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6137          has_ddx = true;
6138          break;
6139       case nir_tex_src_ddy:
6140          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6141          has_ddy = true;
6142          break;
6143       case nir_tex_src_ms_index:
6144          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6145          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6146          has_sample_index = true;
6147          break;
6148       case nir_tex_src_texture_offset:
6149       case nir_tex_src_sampler_offset:
6150       default:
6151          break;
6152       }
6153    }
6154 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6155    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6156       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6157
6158    if (instr->op == nir_texop_texture_samples) {
6159       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6160
6161       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6162       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6163       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6164       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6165
6166       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6167                samples, Operand(1u), bld.scc(is_msaa));
6168       return;
6169    }
6170
6171    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6172       aco_ptr<Instruction> tmp_instr;
6173       Temp acc, pack = Temp();
6174
6175       uint32_t pack_const = 0;
6176       for (unsigned i = 0; i < offset.size(); i++) {
6177          if (!const_offset[i])
6178             continue;
6179          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6180       }
6181
6182       if (offset.type() == RegType::sgpr) {
6183          for (unsigned i = 0; i < offset.size(); i++) {
6184             if (const_offset[i])
6185                continue;
6186
6187             acc = emit_extract_vector(ctx, offset, i, s1);
6188             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6189
6190             if (i) {
6191                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6192             }
6193
6194             if (pack == Temp()) {
6195                pack = acc;
6196             } else {
6197                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6198             }
6199          }
6200
6201          if (pack_const && pack != Temp())
6202             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6203       } else {
6204          for (unsigned i = 0; i < offset.size(); i++) {
6205             if (const_offset[i])
6206                continue;
6207
6208             acc = emit_extract_vector(ctx, offset, i, v1);
6209             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6210
6211             if (i) {
6212                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6213             }
6214
6215             if (pack == Temp()) {
6216                pack = acc;
6217             } else {
6218                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6219             }
6220          }
6221
6222          if (pack_const && pack != Temp())
6223             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6224       }
6225       if (pack_const && pack == Temp())
6226          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6227       else if (pack == Temp())
6228          has_offset = false;
6229       else
6230          offset = pack;
6231    }
6232
6233    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6234       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6235
6236    /* pack derivatives */
6237    if (has_ddx || has_ddy) {
6238       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class >= GFX9) {
6239          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6240                              ddx, Operand(0u), ddy, Operand(0u));
6241       } else {
6242          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6243       }
6244       has_derivs = true;
6245    }
6246
6247    if (instr->coord_components > 1 &&
6248        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6249        instr->is_array &&
6250        instr->op != nir_texop_txf)
6251       coords = apply_round_slice(ctx, coords, 1);
6252
6253    if (instr->coord_components > 2 &&
6254       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6255        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6256        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6257        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6258        instr->is_array &&
6259        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6260       coords = apply_round_slice(ctx, coords, 2);
6261
6262    if (ctx->options->chip_class >= GFX9 &&
6263        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6264        instr->op != nir_texop_lod && instr->coord_components) {
6265       assert(coords.size() > 0 && coords.size() < 3);
6266
6267       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6268       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6269       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6270       if (coords.size() > 1)
6271          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6272       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6273       vec->definitions[0] = Definition(coords);
6274       ctx->block->instructions.emplace_back(std::move(vec));
6275    }
6276
6277    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6278
6279    if (instr->op == nir_texop_samples_identical)
6280       resource = fmask_ptr;
6281
6282    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6283              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6284             instr->op != nir_texop_txs) {
6285       assert(has_sample_index);
6286       Operand op(sample_index);
6287       if (sample_index_cv)
6288          op = Operand(sample_index_cv->u32);
6289       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6290    }
6291
6292    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6293       Temp split_coords[coords.size()];
6294       emit_split_vector(ctx, coords, coords.size());
6295       for (unsigned i = 0; i < coords.size(); i++)
6296          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6297
6298       unsigned i = 0;
6299       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6300          Temp off = emit_extract_vector(ctx, offset, i, v1);
6301          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6302       }
6303
6304       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6305       for (unsigned i = 0; i < coords.size(); i++)
6306          vec->operands[i] = Operand(split_coords[i]);
6307       coords = bld.tmp(coords.regClass());
6308       vec->definitions[0] = Definition(coords);
6309       ctx->block->instructions.emplace_back(std::move(vec));
6310
6311       has_offset = false;
6312    }
6313
6314    /* Build tex instruction */
6315    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6316    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6317    Temp tmp_dst = dst;
6318
6319    /* gather4 selects the component by dmask and always returns vec4 */
6320    if (instr->op == nir_texop_tg4) {
6321       assert(instr->dest.ssa.num_components == 4);
6322       if (instr->is_shadow)
6323          dmask = 1;
6324       else
6325          dmask = 1 << instr->component;
6326       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6327          tmp_dst = bld.tmp(v4);
6328    } else if (instr->op == nir_texop_samples_identical) {
6329       tmp_dst = bld.tmp(v1);
6330    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6331       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6332    }
6333
6334    aco_ptr<MIMG_instruction> tex;
6335    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6336       if (!has_lod)
6337          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6338
6339       bool div_by_6 = instr->op == nir_texop_txs &&
6340                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6341                       instr->is_array &&
6342                       (dmask & (1 << 2));
6343       if (tmp_dst.id() == dst.id() && div_by_6)
6344          tmp_dst = bld.tmp(tmp_dst.regClass());
6345
6346       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6347       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6348       tex->operands[1] = Operand(resource);
6349       if (ctx->options->chip_class >= GFX9 &&
6350           instr->op == nir_texop_txs &&
6351           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6352           instr->is_array) {
6353          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6354       } else if (instr->op == nir_texop_query_levels) {
6355          tex->dmask = 1 << 3;
6356       } else {
6357          tex->dmask = dmask;
6358       }
6359       tex->da = da;
6360       tex->definitions[0] = Definition(tmp_dst);
6361       tex->can_reorder = true;
6362       ctx->block->instructions.emplace_back(std::move(tex));
6363
6364       if (div_by_6) {
6365          /* divide 3rd value by 6 by multiplying with magic number */
6366          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6367          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6368          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6369          assert(instr->dest.ssa.num_components == 3);
6370          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6371          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6372                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6373                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6374                               by_6);
6375
6376       }
6377
6378       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6379       return;
6380    }
6381
6382    Temp tg4_compare_cube_wa64 = Temp();
6383
6384    if (tg4_integer_workarounds) {
6385       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6386       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6387       tex->operands[1] = Operand(resource);
6388       tex->dmask = 0x3;
6389       tex->da = da;
6390       Temp size = bld.tmp(v2);
6391       tex->definitions[0] = Definition(size);
6392       tex->can_reorder = true;
6393       ctx->block->instructions.emplace_back(std::move(tex));
6394       emit_split_vector(ctx, size, size.size());
6395
6396       Temp half_texel[2];
6397       for (unsigned i = 0; i < 2; i++) {
6398          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6399          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6400          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6401          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6402       }
6403
6404       Temp orig_coords[2] = {
6405          emit_extract_vector(ctx, coords, 0, v1),
6406          emit_extract_vector(ctx, coords, 1, v1)};
6407       Temp new_coords[2] = {
6408          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6409          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6410       };
6411
6412       if (tg4_integer_cube_workaround) {
6413          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6414          Temp desc[resource.size()];
6415          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6416                                                                            Format::PSEUDO, 1, resource.size())};
6417          split->operands[0] = Operand(resource);
6418          for (unsigned i = 0; i < resource.size(); i++) {
6419             desc[i] = bld.tmp(s1);
6420             split->definitions[i] = Definition(desc[i]);
6421          }
6422          ctx->block->instructions.emplace_back(std::move(split));
6423
6424          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6425          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6426                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6427
6428          Temp nfmt;
6429          if (stype == GLSL_TYPE_UINT) {
6430             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6431                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6432                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6433                             bld.scc(compare_cube_wa));
6434          } else {
6435             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6436                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6437                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6438                             bld.scc(compare_cube_wa));
6439          }
6440          tg4_compare_cube_wa64 = as_divergent_bool(ctx, compare_cube_wa, true);
6441          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6442
6443          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6444                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6445          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6446
6447          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6448                                                                          Format::PSEUDO, resource.size(), 1)};
6449          for (unsigned i = 0; i < resource.size(); i++)
6450             vec->operands[i] = Operand(desc[i]);
6451          resource = bld.tmp(resource.regClass());
6452          vec->definitions[0] = Definition(resource);
6453          ctx->block->instructions.emplace_back(std::move(vec));
6454
6455          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6456                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6457          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6458                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6459       }
6460
6461       if (coords.size() == 3) {
6462          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6463                              new_coords[0], new_coords[1],
6464                              emit_extract_vector(ctx, coords, 2, v1));
6465       } else {
6466          assert(coords.size() == 2);
6467          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6468                              new_coords[0], new_coords[1]);
6469       }
6470    }
6471
6472    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6473        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6474        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6475       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6476
6477    std::vector<Operand> args;
6478    if (has_offset)
6479       args.emplace_back(Operand(offset));
6480    if (has_bias)
6481       args.emplace_back(Operand(bias));
6482    if (has_compare)
6483       args.emplace_back(Operand(compare));
6484    if (has_derivs)
6485       args.emplace_back(Operand(derivs));
6486    args.emplace_back(Operand(coords));
6487    if (has_sample_index)
6488       args.emplace_back(Operand(sample_index));
6489    if (has_lod)
6490       args.emplace_back(lod);
6491
6492    Operand arg;
6493    if (args.size() > 1) {
6494       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6495       unsigned size = 0;
6496       for (unsigned i = 0; i < args.size(); i++) {
6497          size += args[i].size();
6498          vec->operands[i] = args[i];
6499       }
6500       RegClass rc = RegClass(RegType::vgpr, size);
6501       Temp tmp = bld.tmp(rc);
6502       vec->definitions[0] = Definition(tmp);
6503       ctx->block->instructions.emplace_back(std::move(vec));
6504       arg = Operand(tmp);
6505    } else {
6506       assert(args[0].isTemp());
6507       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6508    }
6509
6510    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6511       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6512
6513       assert(coords.size() == 1);
6514       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6515       aco_opcode op;
6516       switch (last_bit) {
6517       case 1:
6518          op = aco_opcode::buffer_load_format_x; break;
6519       case 2:
6520          op = aco_opcode::buffer_load_format_xy; break;
6521       case 3:
6522          op = aco_opcode::buffer_load_format_xyz; break;
6523       case 4:
6524          op = aco_opcode::buffer_load_format_xyzw; break;
6525       default:
6526          unreachable("Tex instruction loads more than 4 components.");
6527       }
6528
6529       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6530       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6531          tmp_dst = dst;
6532       else
6533          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6534
6535       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6536       mubuf->operands[0] = Operand(coords);
6537       mubuf->operands[1] = Operand(resource);
6538       mubuf->operands[2] = Operand((uint32_t) 0);
6539       mubuf->definitions[0] = Definition(tmp_dst);
6540       mubuf->idxen = true;
6541       mubuf->can_reorder = true;
6542       ctx->block->instructions.emplace_back(std::move(mubuf));
6543
6544       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6545       return;
6546    }
6547
6548
6549    if (instr->op == nir_texop_txf ||
6550        instr->op == nir_texop_txf_ms ||
6551        instr->op == nir_texop_samples_identical) {
6552       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6553       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6554       tex->operands[0] = Operand(arg);
6555       tex->operands[1] = Operand(resource);
6556       tex->dmask = dmask;
6557       tex->unrm = true;
6558       tex->da = da;
6559       tex->definitions[0] = Definition(tmp_dst);
6560       tex->can_reorder = true;
6561       ctx->block->instructions.emplace_back(std::move(tex));
6562
6563       if (instr->op == nir_texop_samples_identical) {
6564          assert(dmask == 1 && dst.regClass() == v1);
6565          assert(dst.id() != tmp_dst.id());
6566
6567          Temp tmp = bld.tmp(s2);
6568          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6569          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6570
6571       } else {
6572          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6573       }
6574       return;
6575    }
6576
6577    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6578    aco_opcode opcode = aco_opcode::image_sample;
6579    if (has_offset) { /* image_sample_*_o */
6580       if (has_compare) {
6581          opcode = aco_opcode::image_sample_c_o;
6582          if (has_derivs)
6583             opcode = aco_opcode::image_sample_c_d_o;
6584          if (has_bias)
6585             opcode = aco_opcode::image_sample_c_b_o;
6586          if (level_zero)
6587             opcode = aco_opcode::image_sample_c_lz_o;
6588          if (has_lod)
6589             opcode = aco_opcode::image_sample_c_l_o;
6590       } else {
6591          opcode = aco_opcode::image_sample_o;
6592          if (has_derivs)
6593             opcode = aco_opcode::image_sample_d_o;
6594          if (has_bias)
6595             opcode = aco_opcode::image_sample_b_o;
6596          if (level_zero)
6597             opcode = aco_opcode::image_sample_lz_o;
6598          if (has_lod)
6599             opcode = aco_opcode::image_sample_l_o;
6600       }
6601    } else { /* no offset */
6602       if (has_compare) {
6603          opcode = aco_opcode::image_sample_c;
6604          if (has_derivs)
6605             opcode = aco_opcode::image_sample_c_d;
6606          if (has_bias)
6607             opcode = aco_opcode::image_sample_c_b;
6608          if (level_zero)
6609             opcode = aco_opcode::image_sample_c_lz;
6610          if (has_lod)
6611             opcode = aco_opcode::image_sample_c_l;
6612       } else {
6613          opcode = aco_opcode::image_sample;
6614          if (has_derivs)
6615             opcode = aco_opcode::image_sample_d;
6616          if (has_bias)
6617             opcode = aco_opcode::image_sample_b;
6618          if (level_zero)
6619             opcode = aco_opcode::image_sample_lz;
6620          if (has_lod)
6621             opcode = aco_opcode::image_sample_l;
6622       }
6623    }
6624
6625    if (instr->op == nir_texop_tg4) {
6626       if (has_offset) {
6627          opcode = aco_opcode::image_gather4_lz_o;
6628          if (has_compare)
6629             opcode = aco_opcode::image_gather4_c_lz_o;
6630       } else {
6631          opcode = aco_opcode::image_gather4_lz;
6632          if (has_compare)
6633             opcode = aco_opcode::image_gather4_c_lz;
6634       }
6635    } else if (instr->op == nir_texop_lod) {
6636       opcode = aco_opcode::image_get_lod;
6637    }
6638
6639    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6640    tex->operands[0] = arg;
6641    tex->operands[1] = Operand(resource);
6642    tex->operands[2] = Operand(sampler);
6643    tex->dmask = dmask;
6644    tex->da = da;
6645    tex->definitions[0] = Definition(tmp_dst);
6646    tex->can_reorder = true;
6647    ctx->block->instructions.emplace_back(std::move(tex));
6648
6649    if (tg4_integer_cube_workaround) {
6650       assert(tmp_dst.id() != dst.id());
6651       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6652
6653       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6654       Temp val[4];
6655       for (unsigned i = 0; i < dst.size(); i++) {
6656          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6657          Temp cvt_val;
6658          if (stype == GLSL_TYPE_UINT)
6659             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6660          else
6661             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6662          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6663       }
6664       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6665       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6666                            val[0], val[1], val[2], val[3]);
6667    }
6668    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6669    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6670
6671 }
6672
6673
6674 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6675 {
6676    Temp tmp = get_ssa_temp(ctx, ssa);
6677    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6678       return Operand(tmp.regClass());
6679    else
6680       return Operand(tmp);
6681 }
6682
6683 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6684 {
6685    aco_ptr<Pseudo_instruction> phi;
6686    unsigned num_src = exec_list_length(&instr->srcs);
6687    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6688
6689    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6690
6691    std::map<unsigned, nir_ssa_def*> phi_src;
6692    bool all_undef = true;
6693    nir_foreach_phi_src(src, instr) {
6694       phi_src[src->pred->index] = src->src.ssa;
6695       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6696          all_undef = false;
6697    }
6698    if (all_undef) {
6699       Builder bld(ctx->program, ctx->block);
6700       if (dst.regClass() == s1) {
6701          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6702       } else if (dst.regClass() == v1) {
6703          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6704       } else {
6705          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6706          for (unsigned i = 0; i < dst.size(); i++)
6707             vec->operands[i] = Operand(0u);
6708          vec->definitions[0] = Definition(dst);
6709          ctx->block->instructions.emplace_back(std::move(vec));
6710       }
6711       return;
6712    }
6713
6714    /* try to scalarize vector phis */
6715    if (dst.size() > 1) {
6716       // TODO: scalarize linear phis on divergent ifs
6717       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6718       std::array<Temp, 4> new_vec;
6719       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6720          Operand src = get_phi_operand(ctx, pair.second);
6721          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6722             can_scalarize = false;
6723             break;
6724          }
6725       }
6726       if (can_scalarize) {
6727          unsigned num_components = instr->dest.ssa.num_components;
6728          assert(dst.size() % num_components == 0);
6729          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6730
6731          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6732          for (unsigned k = 0; k < num_components; k++) {
6733             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6734             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6735             for (unsigned i = 0; i < num_src; i++) {
6736                Operand src = get_phi_operand(ctx, it->second);
6737                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6738                ++it;
6739             }
6740             Temp phi_dst = {ctx->program->allocateId(), rc};
6741             phi->definitions[0] = Definition(phi_dst);
6742             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6743             new_vec[k] = phi_dst;
6744             vec->operands[k] = Operand(phi_dst);
6745          }
6746          vec->definitions[0] = Definition(dst);
6747          ctx->block->instructions.emplace_back(std::move(vec));
6748          ctx->allocated_vec.emplace(dst.id(), new_vec);
6749          return;
6750       }
6751    }
6752
6753    unsigned extra_src = 0;
6754    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6755        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6756       extra_src++;
6757    }
6758
6759    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6760
6761    /* if we have a linear phi on a divergent if, we know that one src is undef */
6762    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6763       assert(extra_src == 0);
6764       Block* block;
6765       /* we place the phi either in the invert-block or in the current block */
6766       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6767          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6768          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6769          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6770          assert(block->kind & block_kind_invert);
6771          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6772       } else {
6773          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6774          block = ctx->block;
6775          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6776       }
6777       phi->operands[1] = Operand(dst.regClass());
6778       phi->definitions[0] = Definition(dst);
6779       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6780       return;
6781    }
6782
6783    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6784    for (unsigned i = 0; i < num_src; i++) {
6785       phi->operands[i] = get_phi_operand(ctx, it->second);
6786       ++it;
6787    }
6788    for (unsigned i = 0; i < extra_src; i++)
6789       phi->operands[num_src + i] = Operand(dst.regClass());
6790    phi->definitions[0] = Definition(dst);
6791    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6792 }
6793
6794
6795 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6796 {
6797    Temp dst = get_ssa_temp(ctx, &instr->def);
6798
6799    assert(dst.type() == RegType::sgpr);
6800
6801    if (dst.size() == 1) {
6802       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6803    } else {
6804       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6805       for (unsigned i = 0; i < dst.size(); i++)
6806          vec->operands[i] = Operand(0u);
6807       vec->definitions[0] = Definition(dst);
6808       ctx->block->instructions.emplace_back(std::move(vec));
6809    }
6810 }
6811
6812 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6813 {
6814    Builder bld(ctx->program, ctx->block);
6815    Block *logical_target;
6816    append_logical_end(ctx->block);
6817    unsigned idx = ctx->block->index;
6818
6819    switch (instr->type) {
6820    case nir_jump_break:
6821       logical_target = ctx->cf_info.parent_loop.exit;
6822       add_logical_edge(idx, logical_target);
6823       ctx->block->kind |= block_kind_break;
6824
6825       if (!ctx->cf_info.parent_if.is_divergent &&
6826           !ctx->cf_info.parent_loop.has_divergent_continue) {
6827          /* uniform break - directly jump out of the loop */
6828          ctx->block->kind |= block_kind_uniform;
6829          ctx->cf_info.has_branch = true;
6830          bld.branch(aco_opcode::p_branch);
6831          add_linear_edge(idx, logical_target);
6832          return;
6833       }
6834       ctx->cf_info.parent_loop.has_divergent_branch = true;
6835       break;
6836    case nir_jump_continue:
6837       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6838       add_logical_edge(idx, logical_target);
6839       ctx->block->kind |= block_kind_continue;
6840
6841       if (ctx->cf_info.parent_if.is_divergent) {
6842          /* for potential uniform breaks after this continue,
6843             we must ensure that they are handled correctly */
6844          ctx->cf_info.parent_loop.has_divergent_continue = true;
6845          ctx->cf_info.parent_loop.has_divergent_branch = true;
6846       } else {
6847          /* uniform continue - directly jump to the loop header */
6848          ctx->block->kind |= block_kind_uniform;
6849          ctx->cf_info.has_branch = true;
6850          bld.branch(aco_opcode::p_branch);
6851          add_linear_edge(idx, logical_target);
6852          return;
6853       }
6854       break;
6855    default:
6856       fprintf(stderr, "Unknown NIR jump instr: ");
6857       nir_print_instr(&instr->instr, stderr);
6858       fprintf(stderr, "\n");
6859       abort();
6860    }
6861
6862    /* remove critical edges from linear CFG */
6863    bld.branch(aco_opcode::p_branch);
6864    Block* break_block = ctx->program->create_and_insert_block();
6865    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6866    break_block->kind |= block_kind_uniform;
6867    add_linear_edge(idx, break_block);
6868    /* the loop_header pointer might be invalidated by this point */
6869    if (instr->type == nir_jump_continue)
6870       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6871    add_linear_edge(break_block->index, logical_target);
6872    bld.reset(break_block);
6873    bld.branch(aco_opcode::p_branch);
6874
6875    Block* continue_block = ctx->program->create_and_insert_block();
6876    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6877    add_linear_edge(idx, continue_block);
6878    append_logical_start(continue_block);
6879    ctx->block = continue_block;
6880    return;
6881 }
6882
6883 void visit_block(isel_context *ctx, nir_block *block)
6884 {
6885    nir_foreach_instr(instr, block) {
6886       switch (instr->type) {
6887       case nir_instr_type_alu:
6888          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6889          break;
6890       case nir_instr_type_load_const:
6891          visit_load_const(ctx, nir_instr_as_load_const(instr));
6892          break;
6893       case nir_instr_type_intrinsic:
6894          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
6895          break;
6896       case nir_instr_type_tex:
6897          visit_tex(ctx, nir_instr_as_tex(instr));
6898          break;
6899       case nir_instr_type_phi:
6900          visit_phi(ctx, nir_instr_as_phi(instr));
6901          break;
6902       case nir_instr_type_ssa_undef:
6903          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
6904          break;
6905       case nir_instr_type_deref:
6906          break;
6907       case nir_instr_type_jump:
6908          visit_jump(ctx, nir_instr_as_jump(instr));
6909          break;
6910       default:
6911          fprintf(stderr, "Unknown NIR instr type: ");
6912          nir_print_instr(instr, stderr);
6913          fprintf(stderr, "\n");
6914          //abort();
6915       }
6916    }
6917 }
6918
6919
6920
6921 static void visit_loop(isel_context *ctx, nir_loop *loop)
6922 {
6923    append_logical_end(ctx->block);
6924    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
6925    Builder bld(ctx->program, ctx->block);
6926    bld.branch(aco_opcode::p_branch);
6927    unsigned loop_preheader_idx = ctx->block->index;
6928
6929    Block loop_exit = Block();
6930    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
6931    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
6932
6933    Block* loop_header = ctx->program->create_and_insert_block();
6934    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
6935    loop_header->kind |= block_kind_loop_header;
6936    add_edge(loop_preheader_idx, loop_header);
6937    ctx->block = loop_header;
6938
6939    /* emit loop body */
6940    unsigned loop_header_idx = loop_header->index;
6941    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
6942    append_logical_start(ctx->block);
6943    visit_cf_list(ctx, &loop->body);
6944
6945    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
6946    if (!ctx->cf_info.has_branch) {
6947       append_logical_end(ctx->block);
6948       if (ctx->cf_info.exec_potentially_empty) {
6949          /* Discards can result in code running with an empty exec mask.
6950           * This would result in divergent breaks not ever being taken. As a
6951           * workaround, break the loop when the loop mask is empty instead of
6952           * always continuing. */
6953          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
6954
6955          /* create "loop_almost_exit" to avoid critical edges */
6956          unsigned block_idx = ctx->block->index;
6957          Block *loop_almost_exit = ctx->program->create_and_insert_block();
6958          loop_almost_exit->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6959          loop_almost_exit->kind = block_kind_uniform;
6960          bld.reset(loop_almost_exit);
6961          bld.branch(aco_opcode::p_branch);
6962
6963          add_linear_edge(block_idx, loop_almost_exit);
6964          add_linear_edge(loop_almost_exit->index, &loop_exit);
6965
6966          ctx->block = &ctx->program->blocks[block_idx];
6967       } else {
6968          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
6969       }
6970       if (!ctx->cf_info.parent_loop.has_divergent_branch)
6971          add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6972       else
6973          add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6974       bld.reset(ctx->block);
6975       bld.branch(aco_opcode::p_branch);
6976    }
6977
6978    /* fixup phis in loop header from unreachable blocks */
6979    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
6980       bool linear = ctx->cf_info.has_branch;
6981       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
6982       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
6983          if ((logical && instr->opcode == aco_opcode::p_phi) ||
6984              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
6985             /* the last operand should be the one that needs to be removed */
6986             instr->operands.pop_back();
6987          } else if (!is_phi(instr)) {
6988             break;
6989          }
6990       }
6991    }
6992
6993    ctx->cf_info.has_branch = false;
6994
6995    // TODO: if the loop has not a single exit, we must add one °°
6996    /* emit loop successor block */
6997    ctx->block = ctx->program->insert_block(std::move(loop_exit));
6998    append_logical_start(ctx->block);
6999
7000    #if 0
7001    // TODO: check if it is beneficial to not branch on continues
7002    /* trim linear phis in loop header */
7003    for (auto&& instr : loop_entry->instructions) {
7004       if (instr->opcode == aco_opcode::p_linear_phi) {
7005          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7006          new_phi->definitions[0] = instr->definitions[0];
7007          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7008             new_phi->operands[i] = instr->operands[i];
7009          /* check that the remaining operands are all the same */
7010          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7011             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7012          instr.swap(new_phi);
7013       } else if (instr->opcode == aco_opcode::p_phi) {
7014          continue;
7015       } else {
7016          break;
7017       }
7018    }
7019    #endif
7020 }
7021
7022 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7023 {
7024    ic->cond = cond;
7025
7026    append_logical_end(ctx->block);
7027    ctx->block->kind |= block_kind_branch;
7028
7029    /* branch to linear then block */
7030    assert(cond.regClass() == s2);
7031    aco_ptr<Pseudo_branch_instruction> branch;
7032    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7033    branch->operands[0] = Operand(cond);
7034    ctx->block->instructions.push_back(std::move(branch));
7035
7036    ic->BB_if_idx = ctx->block->index;
7037    ic->BB_invert = Block();
7038    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7039    /* Invert blocks are intentionally not marked as top level because they
7040     * are not part of the logical cfg. */
7041    ic->BB_invert.kind |= block_kind_invert;
7042    ic->BB_endif = Block();
7043    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7044    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7045
7046    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7047    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7048    ctx->cf_info.parent_if.is_divergent = true;
7049    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7050
7051    /** emit logical then block */
7052    Block* BB_then_logical = ctx->program->create_and_insert_block();
7053    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7054    add_edge(ic->BB_if_idx, BB_then_logical);
7055    ctx->block = BB_then_logical;
7056    append_logical_start(BB_then_logical);
7057 }
7058
7059 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7060 {
7061    Block *BB_then_logical = ctx->block;
7062    append_logical_end(BB_then_logical);
7063     /* branch from logical then block to invert block */
7064    aco_ptr<Pseudo_branch_instruction> branch;
7065    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7066    BB_then_logical->instructions.emplace_back(std::move(branch));
7067    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7068    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7069       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7070    BB_then_logical->kind |= block_kind_uniform;
7071    assert(!ctx->cf_info.has_branch);
7072    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7073    ctx->cf_info.parent_loop.has_divergent_branch = false;
7074
7075    /** emit linear then block */
7076    Block* BB_then_linear = ctx->program->create_and_insert_block();
7077    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7078    BB_then_linear->kind |= block_kind_uniform;
7079    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7080    /* branch from linear then block to invert block */
7081    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7082    BB_then_linear->instructions.emplace_back(std::move(branch));
7083    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7084
7085    /** emit invert merge block */
7086    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7087    ic->invert_idx = ctx->block->index;
7088
7089    /* branch to linear else block (skip else) */
7090    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7091    branch->operands[0] = Operand(ic->cond);
7092    ctx->block->instructions.push_back(std::move(branch));
7093
7094    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7095    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7096
7097    /** emit logical else block */
7098    Block* BB_else_logical = ctx->program->create_and_insert_block();
7099    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7100    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7101    add_linear_edge(ic->invert_idx, BB_else_logical);
7102    ctx->block = BB_else_logical;
7103    append_logical_start(BB_else_logical);
7104 }
7105
7106 static void end_divergent_if(isel_context *ctx, if_context *ic)
7107 {
7108    Block *BB_else_logical = ctx->block;
7109    append_logical_end(BB_else_logical);
7110
7111    /* branch from logical else block to endif block */
7112    aco_ptr<Pseudo_branch_instruction> branch;
7113    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7114    BB_else_logical->instructions.emplace_back(std::move(branch));
7115    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7116    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7117       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7118    BB_else_logical->kind |= block_kind_uniform;
7119
7120    assert(!ctx->cf_info.has_branch);
7121    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7122
7123
7124    /** emit linear else block */
7125    Block* BB_else_linear = ctx->program->create_and_insert_block();
7126    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7127    BB_else_linear->kind |= block_kind_uniform;
7128    add_linear_edge(ic->invert_idx, BB_else_linear);
7129
7130    /* branch from linear else block to endif block */
7131    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7132    BB_else_linear->instructions.emplace_back(std::move(branch));
7133    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7134
7135
7136    /** emit endif merge block */
7137    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7138    append_logical_start(ctx->block);
7139
7140
7141    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7142    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7143    /* uniform control flow never has an empty exec-mask */
7144    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7145       ctx->cf_info.exec_potentially_empty = false;
7146 }
7147
7148 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7149 {
7150    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7151    Builder bld(ctx->program, ctx->block);
7152    aco_ptr<Pseudo_branch_instruction> branch;
7153
7154    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7155       /**
7156        * Uniform conditionals are represented in the following way*) :
7157        *
7158        * The linear and logical CFG:
7159        *                        BB_IF
7160        *                        /    \
7161        *       BB_THEN (logical)      BB_ELSE (logical)
7162        *                        \    /
7163        *                        BB_ENDIF
7164        *
7165        * *) Exceptions may be due to break and continue statements within loops
7166        *    If a break/continue happens within uniform control flow, it branches
7167        *    to the loop exit/entry block. Otherwise, it branches to the next
7168        *    merge block.
7169        **/
7170       append_logical_end(ctx->block);
7171       ctx->block->kind |= block_kind_uniform;
7172
7173       /* emit branch */
7174       if (cond.regClass() == s2) {
7175          // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7176          cond = as_uniform_bool(ctx, cond);
7177       }
7178       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7179       branch->operands[0] = Operand(cond);
7180       branch->operands[0].setFixed(scc);
7181       ctx->block->instructions.emplace_back(std::move(branch));
7182
7183       unsigned BB_if_idx = ctx->block->index;
7184       Block BB_endif = Block();
7185       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7186       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7187
7188       /** emit then block */
7189       Block* BB_then = ctx->program->create_and_insert_block();
7190       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7191       add_edge(BB_if_idx, BB_then);
7192       append_logical_start(BB_then);
7193       ctx->block = BB_then;
7194       visit_cf_list(ctx, &if_stmt->then_list);
7195       BB_then = ctx->block;
7196       bool then_branch = ctx->cf_info.has_branch;
7197       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7198
7199       if (!then_branch) {
7200          append_logical_end(BB_then);
7201          /* branch from then block to endif block */
7202          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7203          BB_then->instructions.emplace_back(std::move(branch));
7204          add_linear_edge(BB_then->index, &BB_endif);
7205          if (!then_branch_divergent)
7206             add_logical_edge(BB_then->index, &BB_endif);
7207          BB_then->kind |= block_kind_uniform;
7208       }
7209
7210       ctx->cf_info.has_branch = false;
7211       ctx->cf_info.parent_loop.has_divergent_branch = false;
7212
7213       /** emit else block */
7214       Block* BB_else = ctx->program->create_and_insert_block();
7215       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7216       add_edge(BB_if_idx, BB_else);
7217       append_logical_start(BB_else);
7218       ctx->block = BB_else;
7219       visit_cf_list(ctx, &if_stmt->else_list);
7220       BB_else = ctx->block;
7221
7222       if (!ctx->cf_info.has_branch) {
7223          append_logical_end(BB_else);
7224          /* branch from then block to endif block */
7225          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7226          BB_else->instructions.emplace_back(std::move(branch));
7227          add_linear_edge(BB_else->index, &BB_endif);
7228          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7229             add_logical_edge(BB_else->index, &BB_endif);
7230          BB_else->kind |= block_kind_uniform;
7231       }
7232
7233       ctx->cf_info.has_branch &= then_branch;
7234       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7235
7236       /** emit endif merge block */
7237       if (!ctx->cf_info.has_branch) {
7238          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7239          append_logical_start(ctx->block);
7240       }
7241    } else { /* non-uniform condition */
7242       /**
7243        * To maintain a logical and linear CFG without critical edges,
7244        * non-uniform conditionals are represented in the following way*) :
7245        *
7246        * The linear CFG:
7247        *                        BB_IF
7248        *                        /    \
7249        *       BB_THEN (logical)      BB_THEN (linear)
7250        *                        \    /
7251        *                        BB_INVERT (linear)
7252        *                        /    \
7253        *       BB_ELSE (logical)      BB_ELSE (linear)
7254        *                        \    /
7255        *                        BB_ENDIF
7256        *
7257        * The logical CFG:
7258        *                        BB_IF
7259        *                        /    \
7260        *       BB_THEN (logical)      BB_ELSE (logical)
7261        *                        \    /
7262        *                        BB_ENDIF
7263        *
7264        * *) Exceptions may be due to break and continue statements within loops
7265        **/
7266
7267       if_context ic;
7268
7269       begin_divergent_if_then(ctx, &ic, cond);
7270       visit_cf_list(ctx, &if_stmt->then_list);
7271
7272       begin_divergent_if_else(ctx, &ic);
7273       visit_cf_list(ctx, &if_stmt->else_list);
7274
7275       end_divergent_if(ctx, &ic);
7276    }
7277 }
7278
7279 static void visit_cf_list(isel_context *ctx,
7280                           struct exec_list *list)
7281 {
7282    foreach_list_typed(nir_cf_node, node, node, list) {
7283       switch (node->type) {
7284       case nir_cf_node_block:
7285          visit_block(ctx, nir_cf_node_as_block(node));
7286          break;
7287       case nir_cf_node_if:
7288          visit_if(ctx, nir_cf_node_as_if(node));
7289          break;
7290       case nir_cf_node_loop:
7291          visit_loop(ctx, nir_cf_node_as_loop(node));
7292          break;
7293       default:
7294          unreachable("unimplemented cf list type");
7295       }
7296    }
7297 }
7298
7299 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7300 {
7301    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7302    uint64_t mask = ctx->vs_output.mask[slot];
7303    if (!is_pos && !mask)
7304       return;
7305    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7306       return;
7307    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7308    exp->enabled_mask = mask;
7309    for (unsigned i = 0; i < 4; ++i) {
7310       if (mask & (1 << i))
7311          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7312       else
7313          exp->operands[i] = Operand(v1);
7314    }
7315    exp->valid_mask = false;
7316    exp->done = false;
7317    exp->compressed = false;
7318    if (is_pos)
7319       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7320    else
7321       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7322    ctx->block->instructions.emplace_back(std::move(exp));
7323 }
7324
7325 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7326 {
7327    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7328    exp->enabled_mask = 0;
7329    for (unsigned i = 0; i < 4; ++i)
7330       exp->operands[i] = Operand(v1);
7331    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7332       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7333       exp->enabled_mask |= 0x1;
7334    }
7335    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7336       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7337       exp->enabled_mask |= 0x4;
7338    }
7339    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7340       if (ctx->options->chip_class < GFX9) {
7341          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7342          exp->enabled_mask |= 0x8;
7343       } else {
7344          Builder bld(ctx->program, ctx->block);
7345
7346          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7347                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7348          if (exp->operands[2].isTemp())
7349             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7350
7351          exp->operands[2] = Operand(out);
7352          exp->enabled_mask |= 0x4;
7353       }
7354    }
7355    exp->valid_mask = false;
7356    exp->done = false;
7357    exp->compressed = false;
7358    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7359    ctx->block->instructions.emplace_back(std::move(exp));
7360 }
7361
7362 static void create_vs_exports(isel_context *ctx)
7363 {
7364    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7365
7366    if (outinfo->export_prim_id) {
7367       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7368       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7369    }
7370
7371    if (ctx->options->key.has_multiview_view_index) {
7372       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7373       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7374    }
7375
7376    /* the order these position exports are created is important */
7377    int next_pos = 0;
7378    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7379    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7380       export_vs_psiz_layer_viewport(ctx, &next_pos);
7381    }
7382    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7383       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7384    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7385       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7386
7387    if (ctx->options->key.vs_common_out.export_clip_dists) {
7388       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7389          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7390       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7391          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7392    }
7393
7394    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7395       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7396           i != VARYING_SLOT_PRIMITIVE_ID)
7397          continue;
7398
7399       export_vs_varying(ctx, i, false, NULL);
7400    }
7401 }
7402
7403 static void emit_stream_output(isel_context *ctx,
7404                                Temp const *so_buffers,
7405                                Temp const *so_write_offset,
7406                                const struct radv_stream_output *output)
7407 {
7408    unsigned num_comps = util_bitcount(output->component_mask);
7409    unsigned loc = output->location;
7410    unsigned buf = output->buffer;
7411    unsigned offset = output->offset;
7412
7413    assert(num_comps && num_comps <= 4);
7414    if (!num_comps || num_comps > 4)
7415       return;
7416
7417    unsigned start = ffs(output->component_mask) - 1;
7418
7419    Temp out[4];
7420    bool all_undef = true;
7421    assert(ctx->stage == vertex_vs);
7422    for (unsigned i = 0; i < num_comps; i++) {
7423       out[i] = ctx->vs_output.outputs[loc][start + i];
7424       all_undef = all_undef && !out[i].id();
7425    }
7426    if (all_undef)
7427       return;
7428
7429    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7430    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7431    for (unsigned i = 0; i < num_comps; ++i)
7432       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7433    vec->definitions[0] = Definition(write_data);
7434    ctx->block->instructions.emplace_back(std::move(vec));
7435
7436    aco_opcode opcode;
7437    switch (num_comps) {
7438    case 1:
7439       opcode = aco_opcode::buffer_store_dword;
7440       break;
7441    case 2:
7442       opcode = aco_opcode::buffer_store_dwordx2;
7443       break;
7444    case 3:
7445       opcode = aco_opcode::buffer_store_dwordx3;
7446       break;
7447    case 4:
7448       opcode = aco_opcode::buffer_store_dwordx4;
7449       break;
7450    }
7451
7452    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7453    store->operands[0] = Operand(so_write_offset[buf]);
7454    store->operands[1] = Operand(so_buffers[buf]);
7455    store->operands[2] = Operand((uint32_t) 0);
7456    store->operands[3] = Operand(write_data);
7457    if (offset > 4095) {
7458       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7459       Builder bld(ctx->program, ctx->block);
7460       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7461    } else {
7462       store->offset = offset;
7463    }
7464    store->offen = true;
7465    store->glc = true;
7466    store->slc = true;
7467    store->can_reorder = true;
7468    ctx->block->instructions.emplace_back(std::move(store));
7469 }
7470
7471 static void emit_streamout(isel_context *ctx, unsigned stream)
7472 {
7473    Builder bld(ctx->program, ctx->block);
7474
7475    Temp so_buffers[4];
7476    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7477    for (unsigned i = 0; i < 4; i++) {
7478       unsigned stride = ctx->program->info->so.strides[i];
7479       if (!stride)
7480          continue;
7481
7482       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7483    }
7484
7485    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7486                                 ctx->streamout_config, Operand(0x70010u));
7487
7488    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7489                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7490
7491    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7492
7493    if_context ic;
7494    begin_divergent_if_then(ctx, &ic, can_emit);
7495
7496    bld.reset(ctx->block);
7497
7498    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7499
7500    Temp so_write_offset[4];
7501
7502    for (unsigned i = 0; i < 4; i++) {
7503       unsigned stride = ctx->program->info->so.strides[i];
7504       if (!stride)
7505          continue;
7506
7507       if (stride == 1) {
7508          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7509                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7510          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7511
7512          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7513       } else {
7514          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7515          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7516          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7517       }
7518    }
7519
7520    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7521       struct radv_stream_output *output =
7522          &ctx->program->info->so.outputs[i];
7523       if (stream != output->stream)
7524          continue;
7525
7526       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7527    }
7528
7529    begin_divergent_if_else(ctx, &ic);
7530    end_divergent_if(ctx, &ic);
7531 }
7532
7533 } /* end namespace */
7534
7535 void handle_bc_optimize(isel_context *ctx)
7536 {
7537    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7538    Builder bld(ctx->program, ctx->block);
7539    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7540    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7541    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7542    if (uses_center && uses_centroid) {
7543       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7544
7545       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7546          for (unsigned i = 0; i < 2; i++) {
7547             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7548                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7549                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7550                                       sel);
7551             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7552          }
7553       }
7554
7555       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7556          for (unsigned i = 0; i < 2; i++) {
7557             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7558                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7559                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7560                                       sel);
7561             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7562          }
7563       }
7564    }
7565 }
7566
7567 void select_program(Program *program,
7568                     unsigned shader_count,
7569                     struct nir_shader *const *shaders,
7570                     ac_shader_config* config,
7571                     struct radv_shader_info *info,
7572                     struct radv_nir_compiler_options *options)
7573 {
7574    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7575
7576    for (unsigned i = 0; i < shader_count; i++) {
7577       nir_shader *nir = shaders[i];
7578       init_context(&ctx, nir);
7579
7580       if (!i) {
7581          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7582          append_logical_start(ctx.block);
7583       }
7584
7585       if_context ic;
7586       if (shader_count >= 2) {
7587          Builder bld(ctx.program, ctx.block);
7588          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7589          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7590                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7591          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7592
7593          begin_divergent_if_then(&ctx, &ic, cond);
7594       }
7595
7596       if (i) {
7597          Builder bld(ctx.program, ctx.block);
7598          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7599          bld.sopp(aco_opcode::s_barrier);
7600       }
7601
7602       if (ctx.stage == fragment_fs)
7603          handle_bc_optimize(&ctx);
7604
7605       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7606       visit_cf_list(&ctx, &func->body);
7607
7608       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7609          emit_streamout(&ctx, 0);
7610
7611       if (ctx.stage == vertex_vs)
7612          create_vs_exports(&ctx);
7613
7614       if (shader_count >= 2) {
7615          begin_divergent_if_else(&ctx, &ic);
7616          end_divergent_if(&ctx, &ic);
7617       }
7618
7619       ralloc_free(ctx.divergent_vals);
7620    }
7621
7622    append_logical_end(ctx.block);
7623    ctx.block->kind |= block_kind_uniform;
7624    Builder bld(ctx.program, ctx.block);
7625    if (ctx.program->wb_smem_l1_on_end)
7626       bld.smem(aco_opcode::s_dcache_wb, false);
7627    bld.sopp(aco_opcode::s_endpgm);
7628
7629    /* cleanup CFG */
7630    for (Block& BB : program->blocks) {
7631       for (unsigned idx : BB.linear_preds)
7632          program->blocks[idx].linear_succs.emplace_back(BB.index);
7633       for (unsigned idx : BB.logical_preds)
7634          program->blocks[idx].logical_succs.emplace_back(BB.index);
7635    }
7636 }
7637 }