src/amd/compiler/aco_instruction_selection.cpp

   1 /*
   2  * Copyright © 2018 Valve Corporation
   3  * Copyright © 2018 Google
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining a
   6  * copy of this software and associated documentation files (the "Software"),
   7  * to deal in the Software without restriction, including without limitation
   8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   9  * and/or sell copies of the Software, and to permit persons to whom the
  10  * Software is furnished to do so, subject to the following conditions:
  11  *
  12  * The above copyright notice and this permission notice (including the next
  13  * paragraph) shall be included in all copies or substantial portions of the
  14  * Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  21  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  22  * IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <algorithm>
  27 #include <array>
  28 #include <map>
  29
  30 #include "aco_ir.h"
  31 #include "aco_builder.h"
  32 #include "aco_interface.h"
  33 #include "aco_instruction_selection_setup.cpp"
  34 #include "util/fast_idiv_by_const.h"
  35
  36 namespace aco {
  37 namespace {
  38
  39 class loop_info_RAII {
  40    isel_context* ctx;
  41    unsigned header_idx_old;
  42    Block* exit_old;
  43    bool divergent_cont_old;
  44    bool divergent_branch_old;
  45    bool divergent_if_old;
  46
  47 public:
  48    loop_info_RAII(isel_context* ctx, unsigned loop_header_idx, Block* loop_exit)
  49       : ctx(ctx),
  50         header_idx_old(ctx->cf_info.parent_loop.header_idx), exit_old(ctx->cf_info.parent_loop.exit),
  51         divergent_cont_old(ctx->cf_info.parent_loop.has_divergent_continue),
  52         divergent_branch_old(ctx->cf_info.parent_loop.has_divergent_branch),
  53         divergent_if_old(ctx->cf_info.parent_if.is_divergent)
  54    {
  55       ctx->cf_info.parent_loop.header_idx = loop_header_idx;
  56       ctx->cf_info.parent_loop.exit = loop_exit;
  57       ctx->cf_info.parent_loop.has_divergent_continue = false;
  58       ctx->cf_info.parent_loop.has_divergent_branch = false;
  59       ctx->cf_info.parent_if.is_divergent = false;
  60       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
  61    }
  62
  63    ~loop_info_RAII()
  64    {
  65       ctx->cf_info.parent_loop.header_idx = header_idx_old;
  66       ctx->cf_info.parent_loop.exit = exit_old;
  67       ctx->cf_info.parent_loop.has_divergent_continue = divergent_cont_old;
  68       ctx->cf_info.parent_loop.has_divergent_branch = divergent_branch_old;
  69       ctx->cf_info.parent_if.is_divergent = divergent_if_old;
  70       ctx->cf_info.loop_nest_depth = ctx->cf_info.loop_nest_depth - 1;
  71       if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
  72          ctx->cf_info.exec_potentially_empty = false;
  73    }
  74 };
  75
  76 struct if_context {
  77    Temp cond;
  78
  79    bool divergent_old;
  80    bool exec_potentially_empty_old;
  81
  82    unsigned BB_if_idx;
  83    unsigned invert_idx;
  84    bool then_branch_divergent;
  85    Block BB_invert;
  86    Block BB_endif;
  87 };
  88
  89 static void visit_cf_list(struct isel_context *ctx,
  90                           struct exec_list *list);
  91
  92 static void add_logical_edge(unsigned pred_idx, Block *succ)
  93 {
  94    succ->logical_preds.emplace_back(pred_idx);
  95 }
  96
  97
  98 static void add_linear_edge(unsigned pred_idx, Block *succ)
  99 {
 100    succ->linear_preds.emplace_back(pred_idx);
 101 }
 102
 103 static void add_edge(unsigned pred_idx, Block *succ)
 104 {
 105    add_logical_edge(pred_idx, succ);
 106    add_linear_edge(pred_idx, succ);
 107 }
 108
 109 static void append_logical_start(Block *b)
 110 {
 111    Builder(NULL, b).pseudo(aco_opcode::p_logical_start);
 112 }
 113
 114 static void append_logical_end(Block *b)
 115 {
 116    Builder(NULL, b).pseudo(aco_opcode::p_logical_end);
 117 }
 118
 119 Temp get_ssa_temp(struct isel_context *ctx, nir_ssa_def *def)
 120 {
 121    assert(ctx->allocated[def->index].id());
 122    return ctx->allocated[def->index];
 123 }
 124
 125 Temp emit_wqm(isel_context *ctx, Temp src, Temp dst=Temp(0, s1), bool program_needs_wqm = false)
 126 {
 127    Builder bld(ctx->program, ctx->block);
 128
 129    if (!dst.id())
 130       dst = bld.tmp(src.regClass());
 131
 132    if (ctx->stage != fragment_fs) {
 133       if (!dst.id())
 134          return src;
 135
 136       if (src.type() == RegType::vgpr || src.size() > 1)
 137          bld.copy(Definition(dst), src);
 138       else
 139          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 140       return dst;
 141    }
 142
 143    bld.pseudo(aco_opcode::p_wqm, Definition(dst), src);
 144    ctx->program->needs_wqm |= program_needs_wqm;
 145    return dst;
 146 }
 147
 148 Temp as_vgpr(isel_context *ctx, Temp val)
 149 {
 150    if (val.type() == RegType::sgpr) {
 151       Builder bld(ctx->program, ctx->block);
 152       return bld.copy(bld.def(RegType::vgpr, val.size()), val);
 153    }
 154    assert(val.type() == RegType::vgpr);
 155    return val;
 156 }
 157
 158 //assumes a != 0xffffffff
 159 void emit_v_div_u32(isel_context *ctx, Temp dst, Temp a, uint32_t b)
 160 {
 161    assert(b != 0);
 162    Builder bld(ctx->program, ctx->block);
 163
 164    if (util_is_power_of_two_or_zero(b)) {
 165       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)util_logbase2(b)), a);
 166       return;
 167    }
 168
 169    util_fast_udiv_info info = util_compute_fast_udiv_info(b, 32, 32);
 170
 171    assert(info.multiplier <= 0xffffffff);
 172
 173    bool pre_shift = info.pre_shift != 0;
 174    bool increment = info.increment != 0;
 175    bool multiply = true;
 176    bool post_shift = info.post_shift != 0;
 177
 178    if (!pre_shift && !increment && !multiply && !post_shift) {
 179       bld.vop1(aco_opcode::v_mov_b32, Definition(dst), a);
 180       return;
 181    }
 182
 183    Temp pre_shift_dst = a;
 184    if (pre_shift) {
 185       pre_shift_dst = (increment || multiply || post_shift) ? bld.tmp(v1) : dst;
 186       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(pre_shift_dst), Operand((uint32_t)info.pre_shift), a);
 187    }
 188
 189    Temp increment_dst = pre_shift_dst;
 190    if (increment) {
 191       increment_dst = (post_shift || multiply) ? bld.tmp(v1) : dst;
 192       bld.vadd32(Definition(increment_dst), Operand((uint32_t) info.increment), pre_shift_dst);
 193    }
 194
 195    Temp multiply_dst = increment_dst;
 196    if (multiply) {
 197       multiply_dst = post_shift ? bld.tmp(v1) : dst;
 198       bld.vop3(aco_opcode::v_mul_hi_u32, Definition(multiply_dst), increment_dst,
 199                bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand((uint32_t)info.multiplier)));
 200    }
 201
 202    if (post_shift) {
 203       bld.vop2(aco_opcode::v_lshrrev_b32, Definition(dst), Operand((uint32_t)info.post_shift), multiply_dst);
 204    }
 205 }
 206
 207 void emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, Temp dst)
 208 {
 209    Builder bld(ctx->program, ctx->block);
 210    bld.pseudo(aco_opcode::p_extract_vector, Definition(dst), src, Operand(idx));
 211 }
 212
 213
 214 Temp emit_extract_vector(isel_context* ctx, Temp src, uint32_t idx, RegClass dst_rc)
 215 {
 216    /* no need to extract the whole vector */
 217    if (src.regClass() == dst_rc) {
 218       assert(idx == 0);
 219       return src;
 220    }
 221    assert(src.size() > idx);
 222    Builder bld(ctx->program, ctx->block);
 223    auto it = ctx->allocated_vec.find(src.id());
 224    /* the size check needs to be early because elements other than 0 may be garbage */
 225    if (it != ctx->allocated_vec.end() && it->second[0].size() == dst_rc.size()) {
 226       if (it->second[idx].regClass() == dst_rc) {
 227          return it->second[idx];
 228       } else {
 229          assert(dst_rc.size() == it->second[idx].regClass().size());
 230          assert(dst_rc.type() == RegType::vgpr && it->second[idx].type() == RegType::sgpr);
 231          return bld.copy(bld.def(dst_rc), it->second[idx]);
 232       }
 233    }
 234
 235    if (src.size() == dst_rc.size()) {
 236       assert(idx == 0);
 237       return bld.copy(bld.def(dst_rc), src);
 238    } else {
 239       Temp dst = bld.tmp(dst_rc);
 240       emit_extract_vector(ctx, src, idx, dst);
 241       return dst;
 242    }
 243 }
 244
 245 void emit_split_vector(isel_context* ctx, Temp vec_src, unsigned num_components)
 246 {
 247    if (num_components == 1)
 248       return;
 249    if (ctx->allocated_vec.find(vec_src.id()) != ctx->allocated_vec.end())
 250       return;
 251    aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_components)};
 252    split->operands[0] = Operand(vec_src);
 253    std::array<Temp,4> elems;
 254    for (unsigned i = 0; i < num_components; i++) {
 255       elems[i] = {ctx->program->allocateId(), RegClass(vec_src.type(), vec_src.size() / num_components)};
 256       split->definitions[i] = Definition(elems[i]);
 257    }
 258    ctx->block->instructions.emplace_back(std::move(split));
 259    ctx->allocated_vec.emplace(vec_src.id(), elems);
 260 }
 261
 262 /* This vector expansion uses a mask to determine which elements in the new vector
 263  * come from the original vector. The other elements are undefined. */
 264 void expand_vector(isel_context* ctx, Temp vec_src, Temp dst, unsigned num_components, unsigned mask)
 265 {
 266    emit_split_vector(ctx, vec_src, util_bitcount(mask));
 267
 268    if (vec_src == dst)
 269       return;
 270
 271    Builder bld(ctx->program, ctx->block);
 272    if (num_components == 1) {
 273       if (dst.type() == RegType::sgpr)
 274          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec_src);
 275       else
 276          bld.copy(Definition(dst), vec_src);
 277       return;
 278    }
 279
 280    unsigned component_size = dst.size() / num_components;
 281    std::array<Temp,4> elems;
 282
 283    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
 284    vec->definitions[0] = Definition(dst);
 285    unsigned k = 0;
 286    for (unsigned i = 0; i < num_components; i++) {
 287       if (mask & (1 << i)) {
 288          Temp src = emit_extract_vector(ctx, vec_src, k++, RegClass(vec_src.type(), component_size));
 289          if (dst.type() == RegType::sgpr)
 290             src = bld.as_uniform(src);
 291          vec->operands[i] = Operand(src);
 292       } else {
 293          vec->operands[i] = Operand(0u);
 294       }
 295       elems[i] = vec->operands[i].getTemp();
 296    }
 297    ctx->block->instructions.emplace_back(std::move(vec));
 298    ctx->allocated_vec.emplace(dst.id(), elems);
 299 }
 300
 301 Temp as_divergent_bool(isel_context *ctx, Temp val, bool vcc_hint)
 302 {
 303    if (val.regClass() == s2) {
 304       return val;
 305    } else {
 306       assert(val.regClass() == s1);
 307       Builder bld(ctx->program, ctx->block);
 308       Definition& def = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2),
 309                                  Operand((uint32_t) -1), Operand(0u), bld.scc(val)).def(0);
 310       if (vcc_hint)
 311          def.setHint(vcc);
 312       return def.getTemp();
 313    }
 314 }
 315
 316 Temp as_uniform_bool(isel_context *ctx, Temp val)
 317 {
 318    if (val.regClass() == s1) {
 319       return val;
 320    } else {
 321       assert(val.regClass() == s2);
 322       Builder bld(ctx->program, ctx->block);
 323       return bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), Operand(0u), Operand(val));
 324    }
 325 }
 326
 327 Temp get_alu_src(struct isel_context *ctx, nir_alu_src src, unsigned size=1)
 328 {
 329    if (src.src.ssa->num_components == 1 && src.swizzle[0] == 0 && size == 1)
 330       return get_ssa_temp(ctx, src.src.ssa);
 331
 332    if (src.src.ssa->num_components == size) {
 333       bool identity_swizzle = true;
 334       for (unsigned i = 0; identity_swizzle && i < size; i++) {
 335          if (src.swizzle[i] != i)
 336             identity_swizzle = false;
 337       }
 338       if (identity_swizzle)
 339          return get_ssa_temp(ctx, src.src.ssa);
 340    }
 341
 342    Temp vec = get_ssa_temp(ctx, src.src.ssa);
 343    unsigned elem_size = vec.size() / src.src.ssa->num_components;
 344    assert(elem_size > 0); /* TODO: 8 and 16-bit vectors not supported */
 345    assert(vec.size() % elem_size == 0);
 346
 347    RegClass elem_rc = RegClass(vec.type(), elem_size);
 348    if (size == 1) {
 349       return emit_extract_vector(ctx, vec, src.swizzle[0], elem_rc);
 350    } else {
 351       assert(size <= 4);
 352       std::array<Temp,4> elems;
 353       aco_ptr<Pseudo_instruction> vec_instr{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, size, 1)};
 354       for (unsigned i = 0; i < size; ++i) {
 355          elems[i] = emit_extract_vector(ctx, vec, src.swizzle[i], elem_rc);
 356          vec_instr->operands[i] = Operand{elems[i]};
 357       }
 358       Temp dst{ctx->program->allocateId(), RegClass(vec.type(), elem_size * size)};
 359       vec_instr->definitions[0] = Definition(dst);
 360       ctx->block->instructions.emplace_back(std::move(vec_instr));
 361       ctx->allocated_vec.emplace(dst.id(), elems);
 362       return dst;
 363    }
 364 }
 365
 366 Temp convert_pointer_to_64_bit(isel_context *ctx, Temp ptr)
 367 {
 368    if (ptr.size() == 2)
 369       return ptr;
 370    Builder bld(ctx->program, ctx->block);
 371    return bld.pseudo(aco_opcode::p_create_vector, bld.def(s2),
 372                      ptr, Operand((unsigned)ctx->options->address32_hi));
 373 }
 374
 375 void emit_sop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool writes_scc)
 376 {
 377    aco_ptr<SOP2_instruction> sop2{create_instruction<SOP2_instruction>(op, Format::SOP2, 2, writes_scc ? 2 : 1)};
 378    sop2->operands[0] = Operand(get_alu_src(ctx, instr->src[0]));
 379    sop2->operands[1] = Operand(get_alu_src(ctx, instr->src[1]));
 380    sop2->definitions[0] = Definition(dst);
 381    if (writes_scc)
 382       sop2->definitions[1] = Definition(ctx->program->allocateId(), scc, s1);
 383    ctx->block->instructions.emplace_back(std::move(sop2));
 384 }
 385
 386 void emit_vop2_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst, bool commutative, bool swap_srcs=false)
 387 {
 388    Builder bld(ctx->program, ctx->block);
 389    Temp src0 = get_alu_src(ctx, instr->src[swap_srcs ? 1 : 0]);
 390    Temp src1 = get_alu_src(ctx, instr->src[swap_srcs ? 0 : 1]);
 391    if (src1.type() == RegType::sgpr) {
 392       if (commutative && src0.type() == RegType::vgpr) {
 393          Temp t = src0;
 394          src0 = src1;
 395          src1 = t;
 396       } else if (src0.type() == RegType::vgpr &&
 397                  op != aco_opcode::v_madmk_f32 &&
 398                  op != aco_opcode::v_madak_f32 &&
 399                  op != aco_opcode::v_madmk_f16 &&
 400                  op != aco_opcode::v_madak_f16) {
 401          /* If the instruction is not commutative, we emit a VOP3A instruction */
 402          bld.vop2_e64(op, Definition(dst), src0, src1);
 403          return;
 404       } else {
 405          src1 = bld.copy(bld.def(RegType::vgpr, src1.size()), src1); //TODO: as_vgpr
 406       }
 407    }
 408    bld.vop2(op, Definition(dst), src0, src1);
 409 }
 410
 411 void emit_vop3a_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 412 {
 413    Temp src0 = get_alu_src(ctx, instr->src[0]);
 414    Temp src1 = get_alu_src(ctx, instr->src[1]);
 415    Temp src2 = get_alu_src(ctx, instr->src[2]);
 416
 417    /* ensure that the instruction has at most 1 sgpr operand
 418     * The optimizer will inline constants for us */
 419    if (src0.type() == RegType::sgpr && src1.type() == RegType::sgpr)
 420       src0 = as_vgpr(ctx, src0);
 421    if (src1.type() == RegType::sgpr && src2.type() == RegType::sgpr)
 422       src1 = as_vgpr(ctx, src1);
 423    if (src2.type() == RegType::sgpr && src0.type() == RegType::sgpr)
 424       src2 = as_vgpr(ctx, src2);
 425
 426    Builder bld(ctx->program, ctx->block);
 427    bld.vop3(op, Definition(dst), src0, src1, src2);
 428 }
 429
 430 void emit_vop1_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 431 {
 432    Builder bld(ctx->program, ctx->block);
 433    bld.vop1(op, Definition(dst), get_alu_src(ctx, instr->src[0]));
 434 }
 435
 436 void emit_vopc_instruction(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 437 {
 438    Temp src0 = get_alu_src(ctx, instr->src[0]);
 439    Temp src1 = get_alu_src(ctx, instr->src[1]);
 440    aco_ptr<Instruction> vopc;
 441    if (src1.type() == RegType::sgpr) {
 442       if (src0.type() == RegType::vgpr) {
 443          /* to swap the operands, we might also have to change the opcode */
 444          switch (op) {
 445             case aco_opcode::v_cmp_lt_f32:
 446                op = aco_opcode::v_cmp_gt_f32;
 447                break;
 448             case aco_opcode::v_cmp_ge_f32:
 449                op = aco_opcode::v_cmp_le_f32;
 450                break;
 451             case aco_opcode::v_cmp_lt_i32:
 452                op = aco_opcode::v_cmp_gt_i32;
 453                break;
 454             case aco_opcode::v_cmp_ge_i32:
 455                op = aco_opcode::v_cmp_le_i32;
 456                break;
 457             case aco_opcode::v_cmp_lt_u32:
 458                op = aco_opcode::v_cmp_gt_u32;
 459                break;
 460             case aco_opcode::v_cmp_ge_u32:
 461                op = aco_opcode::v_cmp_le_u32;
 462                break;
 463             case aco_opcode::v_cmp_lt_f64:
 464                op = aco_opcode::v_cmp_gt_f64;
 465                break;
 466             case aco_opcode::v_cmp_ge_f64:
 467                op = aco_opcode::v_cmp_le_f64;
 468                break;
 469             case aco_opcode::v_cmp_lt_i64:
 470                op = aco_opcode::v_cmp_gt_i64;
 471                break;
 472             case aco_opcode::v_cmp_ge_i64:
 473                op = aco_opcode::v_cmp_le_i64;
 474                break;
 475             case aco_opcode::v_cmp_lt_u64:
 476                op = aco_opcode::v_cmp_gt_u64;
 477                break;
 478             case aco_opcode::v_cmp_ge_u64:
 479                op = aco_opcode::v_cmp_le_u64;
 480                break;
 481             default: /* eq and ne are commutative */
 482                break;
 483          }
 484          Temp t = src0;
 485          src0 = src1;
 486          src1 = t;
 487       } else {
 488          src1 = as_vgpr(ctx, src1);
 489       }
 490    }
 491    Builder bld(ctx->program, ctx->block);
 492    bld.vopc(op, Definition(dst), src0, src1).def(0).setHint(vcc);
 493 }
 494
 495 void emit_comparison(isel_context *ctx, nir_alu_instr *instr, aco_opcode op, Temp dst)
 496 {
 497    if (dst.regClass() == s2) {
 498       emit_vopc_instruction(ctx, instr, op, dst);
 499       if (!ctx->divergent_vals[instr->dest.dest.ssa.index])
 500          emit_split_vector(ctx, dst, 2);
 501    } else if (dst.regClass() == s1) {
 502       Temp src0 = get_alu_src(ctx, instr->src[0]);
 503       Temp src1 = get_alu_src(ctx, instr->src[1]);
 504       assert(src0.type() == RegType::sgpr && src1.type() == RegType::sgpr);
 505
 506       Builder bld(ctx->program, ctx->block);
 507       bld.sopc(op, bld.scc(Definition(dst)), src0, src1);
 508
 509    } else {
 510       assert(false);
 511    }
 512 }
 513
 514 void emit_boolean_logic(isel_context *ctx, nir_alu_instr *instr, aco_opcode op32, aco_opcode op64, Temp dst)
 515 {
 516    Builder bld(ctx->program, ctx->block);
 517    Temp src0 = get_alu_src(ctx, instr->src[0]);
 518    Temp src1 = get_alu_src(ctx, instr->src[1]);
 519    if (dst.regClass() == s2) {
 520       bld.sop2(op64, Definition(dst), bld.def(s1, scc),
 521                as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
 522    } else {
 523       assert(dst.regClass() == s1);
 524       bld.sop2(op32, bld.def(s1), bld.scc(Definition(dst)),
 525                as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
 526    }
 527 }
 528
 529
 530 void emit_bcsel(isel_context *ctx, nir_alu_instr *instr, Temp dst)
 531 {
 532    Builder bld(ctx->program, ctx->block);
 533    Temp cond = get_alu_src(ctx, instr->src[0]);
 534    Temp then = get_alu_src(ctx, instr->src[1]);
 535    Temp els = get_alu_src(ctx, instr->src[2]);
 536
 537    if (dst.type() == RegType::vgpr) {
 538       cond = as_divergent_bool(ctx, cond, true);
 539
 540       aco_ptr<Instruction> bcsel;
 541       if (dst.size() == 1) {
 542          then = as_vgpr(ctx, then);
 543          els = as_vgpr(ctx, els);
 544
 545          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), els, then, cond);
 546       } else if (dst.size() == 2) {
 547          Temp then_lo = bld.tmp(v1), then_hi = bld.tmp(v1);
 548          bld.pseudo(aco_opcode::p_split_vector, Definition(then_lo), Definition(then_hi), then);
 549          Temp else_lo = bld.tmp(v1), else_hi = bld.tmp(v1);
 550          bld.pseudo(aco_opcode::p_split_vector, Definition(else_lo), Definition(else_hi), els);
 551
 552          Temp dst0 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_lo, then_lo, cond);
 553          Temp dst1 = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), else_hi, then_hi, cond);
 554
 555          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 556       } else {
 557          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 558          nir_print_instr(&instr->instr, stderr);
 559          fprintf(stderr, "\n");
 560       }
 561       return;
 562    }
 563
 564    if (instr->dest.dest.ssa.bit_size != 1) { /* uniform condition and values in sgpr */
 565       if (dst.regClass() == s1 || dst.regClass() == s2) {
 566          assert((then.regClass() == s1 || then.regClass() == s2) && els.regClass() == then.regClass());
 567          aco_opcode op = dst.regClass() == s1 ? aco_opcode::s_cselect_b32 : aco_opcode::s_cselect_b64;
 568          bld.sop2(op, Definition(dst), then, els, bld.scc(as_uniform_bool(ctx, cond)));
 569       } else {
 570          fprintf(stderr, "Unimplemented uniform bcsel bit size: ");
 571          nir_print_instr(&instr->instr, stderr);
 572          fprintf(stderr, "\n");
 573       }
 574       return;
 575    }
 576
 577    /* boolean bcsel */
 578    assert(instr->dest.dest.ssa.bit_size == 1);
 579
 580    if (dst.regClass() == s1)
 581       cond = as_uniform_bool(ctx, cond);
 582
 583    if (cond.regClass() == s1) { /* uniform selection */
 584       aco_opcode op;
 585       if (dst.regClass() == s2) {
 586          op = aco_opcode::s_cselect_b64;
 587          then = as_divergent_bool(ctx, then, false);
 588          els = as_divergent_bool(ctx, els, false);
 589       } else {
 590          assert(dst.regClass() == s1);
 591          op = aco_opcode::s_cselect_b32;
 592          then = as_uniform_bool(ctx, then);
 593          els = as_uniform_bool(ctx, els);
 594       }
 595       bld.sop2(op, Definition(dst), then, els, bld.scc(cond));
 596       return;
 597    }
 598
 599    /* divergent boolean bcsel
 600     * this implements bcsel on bools: dst = s0 ? s1 : s2
 601     * are going to be: dst = (s0 & s1) | (~s0 & s2) */
 602    assert (dst.regClass() == s2);
 603    then = as_divergent_bool(ctx, then, false);
 604    els = as_divergent_bool(ctx, els, false);
 605
 606    if (cond.id() != then.id())
 607       then = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), cond, then);
 608
 609    if (cond.id() == els.id())
 610       bld.sop1(aco_opcode::s_mov_b64, Definition(dst), then);
 611    else
 612       bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), then,
 613                bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), els, cond));
 614 }
 615
 616 void visit_alu_instr(isel_context *ctx, nir_alu_instr *instr)
 617 {
 618    if (!instr->dest.dest.is_ssa) {
 619       fprintf(stderr, "nir alu dst not in ssa: ");
 620       nir_print_instr(&instr->instr, stderr);
 621       fprintf(stderr, "\n");
 622       abort();
 623    }
 624    Builder bld(ctx->program, ctx->block);
 625    Temp dst = get_ssa_temp(ctx, &instr->dest.dest.ssa);
 626    switch(instr->op) {
 627    case nir_op_vec2:
 628    case nir_op_vec3:
 629    case nir_op_vec4: {
 630       std::array<Temp,4> elems;
 631       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.dest.ssa.num_components, 1)};
 632       for (unsigned i = 0; i < instr->dest.dest.ssa.num_components; ++i) {
 633          elems[i] = get_alu_src(ctx, instr->src[i]);
 634          vec->operands[i] = Operand{elems[i]};
 635       }
 636       vec->definitions[0] = Definition(dst);
 637       ctx->block->instructions.emplace_back(std::move(vec));
 638       ctx->allocated_vec.emplace(dst.id(), elems);
 639       break;
 640    }
 641    case nir_op_mov: {
 642       Temp src = get_alu_src(ctx, instr->src[0]);
 643       aco_ptr<Instruction> mov;
 644       if (dst.type() == RegType::sgpr) {
 645          if (src.type() == RegType::vgpr)
 646             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), src);
 647          else if (src.regClass() == s1)
 648             bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
 649          else if (src.regClass() == s2)
 650             bld.sop1(aco_opcode::s_mov_b64, Definition(dst), src);
 651          else
 652             unreachable("wrong src register class for nir_op_imov");
 653       } else if (dst.regClass() == v1) {
 654          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), src);
 655       } else if (dst.regClass() == v2) {
 656          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
 657       } else {
 658          nir_print_instr(&instr->instr, stderr);
 659          unreachable("Should have been lowered to scalar.");
 660       }
 661       break;
 662    }
 663    case nir_op_inot: {
 664       Temp src = get_alu_src(ctx, instr->src[0]);
 665       /* uniform booleans */
 666       if (instr->dest.dest.ssa.bit_size == 1 && dst.regClass() == s1) {
 667          if (src.regClass() == s1) {
 668             /* in this case, src is either 1 or 0 */
 669             bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.scc(Definition(dst)), Operand(1u), src);
 670          } else {
 671             /* src is either exec_mask or 0 */
 672             assert(src.regClass() == s2);
 673             bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(Definition(dst)), Operand(0u), src);
 674          }
 675       } else if (dst.regClass() == v1) {
 676          emit_vop1_instruction(ctx, instr, aco_opcode::v_not_b32, dst);
 677       } else if (dst.type() == RegType::sgpr) {
 678          aco_opcode opcode = dst.size() == 1 ? aco_opcode::s_not_b32 : aco_opcode::s_not_b64;
 679          bld.sop1(opcode, Definition(dst), bld.def(s1, scc), src);
 680       } else {
 681          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 682          nir_print_instr(&instr->instr, stderr);
 683          fprintf(stderr, "\n");
 684       }
 685       break;
 686    }
 687    case nir_op_ineg: {
 688       Temp src = get_alu_src(ctx, instr->src[0]);
 689       if (dst.regClass() == v1) {
 690          bld.vsub32(Definition(dst), Operand(0u), Operand(src));
 691       } else if (dst.regClass() == s1) {
 692          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand((uint32_t) -1), src);
 693       } else if (dst.size() == 2) {
 694          Temp src0 = bld.tmp(dst.type(), 1);
 695          Temp src1 = bld.tmp(dst.type(), 1);
 696          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
 697
 698          if (dst.regClass() == s2) {
 699             Temp carry = bld.tmp(s1);
 700             Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), Operand(0u), src0);
 701             Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), src1, carry);
 702             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 703          } else {
 704             Temp lower = bld.tmp(v1);
 705             Temp borrow = bld.vsub32(Definition(lower), Operand(0u), src0, true).def(1).getTemp();
 706             Temp upper = bld.vsub32(bld.def(v1), Operand(0u), src1, false, borrow);
 707             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 708          }
 709       } else {
 710          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 711          nir_print_instr(&instr->instr, stderr);
 712          fprintf(stderr, "\n");
 713       }
 714       break;
 715    }
 716    case nir_op_iabs: {
 717       if (dst.regClass() == s1) {
 718          bld.sop1(aco_opcode::s_abs_i32, Definition(dst), bld.def(s1, scc), get_alu_src(ctx, instr->src[0]));
 719       } else if (dst.regClass() == v1) {
 720          Temp src = get_alu_src(ctx, instr->src[0]);
 721          bld.vop2(aco_opcode::v_max_i32, Definition(dst), src, bld.vsub32(bld.def(v1), Operand(0u), src));
 722       } else {
 723          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 724          nir_print_instr(&instr->instr, stderr);
 725          fprintf(stderr, "\n");
 726       }
 727       break;
 728    }
 729    case nir_op_isign: {
 730       Temp src = get_alu_src(ctx, instr->src[0]);
 731       if (dst.regClass() == s1) {
 732          Temp tmp = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
 733          Temp gtz = bld.sopc(aco_opcode::s_cmp_gt_i32, bld.def(s1, scc), src, Operand(0u));
 734          bld.sop2(aco_opcode::s_add_i32, Definition(dst), bld.def(s1, scc), gtz, tmp);
 735       } else if (dst.regClass() == s2) {
 736          Temp neg = bld.sop2(aco_opcode::s_ashr_i64, bld.def(s2), bld.def(s1, scc), src, Operand(63u));
 737          Temp neqz = bld.sopc(aco_opcode::s_cmp_lg_u64, bld.def(s1, scc), src, Operand(0u));
 738          bld.sop2(aco_opcode::s_or_b64, Definition(dst), bld.def(s1, scc), neg, neqz);
 739       } else if (dst.regClass() == v1) {
 740          Temp tmp = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
 741          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 742          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(1u), tmp, gtz);
 743       } else if (dst.regClass() == v2) {
 744          Temp upper = emit_extract_vector(ctx, src, 1, v1);
 745          Temp neg = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), upper);
 746          Temp gtz = bld.vopc(aco_opcode::v_cmp_ge_i64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
 747          Temp lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(1u), neg, gtz);
 748          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), neg, gtz);
 749          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
 750       } else {
 751          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 752          nir_print_instr(&instr->instr, stderr);
 753          fprintf(stderr, "\n");
 754       }
 755       break;
 756    }
 757    case nir_op_imax: {
 758       if (dst.regClass() == v1) {
 759          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_i32, dst, true);
 760       } else if (dst.regClass() == s1) {
 761          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_i32, dst, true);
 762       } else {
 763          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 764          nir_print_instr(&instr->instr, stderr);
 765          fprintf(stderr, "\n");
 766       }
 767       break;
 768    }
 769    case nir_op_umax: {
 770       if (dst.regClass() == v1) {
 771          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_u32, dst, true);
 772       } else if (dst.regClass() == s1) {
 773          emit_sop2_instruction(ctx, instr, aco_opcode::s_max_u32, dst, true);
 774       } else {
 775          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 776          nir_print_instr(&instr->instr, stderr);
 777          fprintf(stderr, "\n");
 778       }
 779       break;
 780    }
 781    case nir_op_imin: {
 782       if (dst.regClass() == v1) {
 783          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_i32, dst, true);
 784       } else if (dst.regClass() == s1) {
 785          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_i32, dst, true);
 786       } else {
 787          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 788          nir_print_instr(&instr->instr, stderr);
 789          fprintf(stderr, "\n");
 790       }
 791       break;
 792    }
 793    case nir_op_umin: {
 794       if (dst.regClass() == v1) {
 795          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_u32, dst, true);
 796       } else if (dst.regClass() == s1) {
 797          emit_sop2_instruction(ctx, instr, aco_opcode::s_min_u32, dst, true);
 798       } else {
 799          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 800          nir_print_instr(&instr->instr, stderr);
 801          fprintf(stderr, "\n");
 802       }
 803       break;
 804    }
 805    case nir_op_ior: {
 806       if (instr->dest.dest.ssa.bit_size == 1) {
 807          emit_boolean_logic(ctx, instr, aco_opcode::s_or_b32, aco_opcode::s_or_b64, dst);
 808       } else if (dst.regClass() == v1) {
 809          emit_vop2_instruction(ctx, instr, aco_opcode::v_or_b32, dst, true);
 810       } else if (dst.regClass() == s1) {
 811          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b32, dst, true);
 812       } else if (dst.regClass() == s2) {
 813          emit_sop2_instruction(ctx, instr, aco_opcode::s_or_b64, dst, true);
 814       } else {
 815          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 816          nir_print_instr(&instr->instr, stderr);
 817          fprintf(stderr, "\n");
 818       }
 819       break;
 820    }
 821    case nir_op_iand: {
 822       if (instr->dest.dest.ssa.bit_size == 1) {
 823          emit_boolean_logic(ctx, instr, aco_opcode::s_and_b32, aco_opcode::s_and_b64, dst);
 824       } else if (dst.regClass() == v1) {
 825          emit_vop2_instruction(ctx, instr, aco_opcode::v_and_b32, dst, true);
 826       } else if (dst.regClass() == s1) {
 827          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b32, dst, true);
 828       } else if (dst.regClass() == s2) {
 829          emit_sop2_instruction(ctx, instr, aco_opcode::s_and_b64, dst, true);
 830       } else {
 831          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 832          nir_print_instr(&instr->instr, stderr);
 833          fprintf(stderr, "\n");
 834       }
 835       break;
 836    }
 837    case nir_op_ixor: {
 838       if (instr->dest.dest.ssa.bit_size == 1) {
 839          emit_boolean_logic(ctx, instr, aco_opcode::s_xor_b32, aco_opcode::s_xor_b64, dst);
 840       } else if (dst.regClass() == v1) {
 841          emit_vop2_instruction(ctx, instr, aco_opcode::v_xor_b32, dst, true);
 842       } else if (dst.regClass() == s1) {
 843          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b32, dst, true);
 844       } else if (dst.regClass() == s2) {
 845          emit_sop2_instruction(ctx, instr, aco_opcode::s_xor_b64, dst, true);
 846       } else {
 847          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 848          nir_print_instr(&instr->instr, stderr);
 849          fprintf(stderr, "\n");
 850       }
 851       break;
 852    }
 853    case nir_op_ushr: {
 854       if (dst.regClass() == v1) {
 855          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshrrev_b32, dst, false, true);
 856       } else if (dst.regClass() == v2) {
 857          bld.vop3(aco_opcode::v_lshrrev_b64, Definition(dst),
 858                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 859       } else if (dst.regClass() == s2) {
 860          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b64, dst, true);
 861       } else if (dst.regClass() == s1) {
 862          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshr_b32, dst, true);
 863       } else {
 864          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 865          nir_print_instr(&instr->instr, stderr);
 866          fprintf(stderr, "\n");
 867       }
 868       break;
 869    }
 870    case nir_op_ishl: {
 871       if (dst.regClass() == v1) {
 872          emit_vop2_instruction(ctx, instr, aco_opcode::v_lshlrev_b32, dst, false, true);
 873       } else if (dst.regClass() == v2) {
 874          bld.vop3(aco_opcode::v_lshlrev_b64, Definition(dst),
 875                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 876       } else if (dst.regClass() == s1) {
 877          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b32, dst, true);
 878       } else if (dst.regClass() == s2) {
 879          emit_sop2_instruction(ctx, instr, aco_opcode::s_lshl_b64, dst, true);
 880       } else {
 881          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 882          nir_print_instr(&instr->instr, stderr);
 883          fprintf(stderr, "\n");
 884       }
 885       break;
 886    }
 887    case nir_op_ishr: {
 888       if (dst.regClass() == v1) {
 889          emit_vop2_instruction(ctx, instr, aco_opcode::v_ashrrev_i32, dst, false, true);
 890       } else if (dst.regClass() == v2) {
 891          bld.vop3(aco_opcode::v_ashrrev_i64, Definition(dst),
 892                   get_alu_src(ctx, instr->src[1]), get_alu_src(ctx, instr->src[0]));
 893       } else if (dst.regClass() == s1) {
 894          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i32, dst, true);
 895       } else if (dst.regClass() == s2) {
 896          emit_sop2_instruction(ctx, instr, aco_opcode::s_ashr_i64, dst, true);
 897       } else {
 898          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 899          nir_print_instr(&instr->instr, stderr);
 900          fprintf(stderr, "\n");
 901       }
 902       break;
 903    }
 904    case nir_op_find_lsb: {
 905       Temp src = get_alu_src(ctx, instr->src[0]);
 906       if (src.regClass() == s1) {
 907          bld.sop1(aco_opcode::s_ff1_i32_b32, Definition(dst), src);
 908       } else if (src.regClass() == v1) {
 909          emit_vop1_instruction(ctx, instr, aco_opcode::v_ffbl_b32, dst);
 910       } else if (src.regClass() == s2) {
 911          bld.sop1(aco_opcode::s_ff1_i32_b64, Definition(dst), src);
 912       } else {
 913          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 914          nir_print_instr(&instr->instr, stderr);
 915          fprintf(stderr, "\n");
 916       }
 917       break;
 918    }
 919    case nir_op_ufind_msb:
 920    case nir_op_ifind_msb: {
 921       Temp src = get_alu_src(ctx, instr->src[0]);
 922       if (src.regClass() == s1 || src.regClass() == s2) {
 923          aco_opcode op = src.regClass() == s2 ?
 924                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b64 : aco_opcode::s_flbit_i32_i64) :
 925                          (instr->op == nir_op_ufind_msb ? aco_opcode::s_flbit_i32_b32 : aco_opcode::s_flbit_i32);
 926          Temp msb_rev = bld.sop1(op, bld.def(s1), src);
 927
 928          Builder::Result sub = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc),
 929                                         Operand(src.size() * 32u - 1u), msb_rev);
 930          Temp msb = sub.def(0).getTemp();
 931          Temp carry = sub.def(1).getTemp();
 932
 933          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t)-1), msb, carry);
 934       } else if (src.regClass() == v1) {
 935          aco_opcode op = instr->op == nir_op_ufind_msb ? aco_opcode::v_ffbh_u32 : aco_opcode::v_ffbh_i32;
 936          Temp msb_rev = bld.tmp(v1);
 937          emit_vop1_instruction(ctx, instr, op, msb_rev);
 938          Temp msb = bld.tmp(v1);
 939          Temp carry = bld.vsub32(Definition(msb), Operand(31u), Operand(msb_rev), true).def(1).getTemp();
 940          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), msb, Operand((uint32_t)-1), carry);
 941       } else {
 942          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 943          nir_print_instr(&instr->instr, stderr);
 944          fprintf(stderr, "\n");
 945       }
 946       break;
 947    }
 948    case nir_op_bitfield_reverse: {
 949       if (dst.regClass() == s1) {
 950          bld.sop1(aco_opcode::s_brev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 951       } else if (dst.regClass() == v1) {
 952          bld.vop1(aco_opcode::v_bfrev_b32, Definition(dst), get_alu_src(ctx, instr->src[0]));
 953       } else {
 954          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 955          nir_print_instr(&instr->instr, stderr);
 956          fprintf(stderr, "\n");
 957       }
 958       break;
 959    }
 960    case nir_op_iadd: {
 961       if (dst.regClass() == s1) {
 962          emit_sop2_instruction(ctx, instr, aco_opcode::s_add_u32, dst, true);
 963          break;
 964       }
 965
 966       Temp src0 = get_alu_src(ctx, instr->src[0]);
 967       Temp src1 = get_alu_src(ctx, instr->src[1]);
 968       if (dst.regClass() == v1) {
 969          bld.vadd32(Definition(dst), Operand(src0), Operand(src1));
 970          break;
 971       }
 972
 973       assert(src0.size() == 2 && src1.size() == 2);
 974       Temp src00 = bld.tmp(src0.type(), 1);
 975       Temp src01 = bld.tmp(dst.type(), 1);
 976       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
 977       Temp src10 = bld.tmp(src1.type(), 1);
 978       Temp src11 = bld.tmp(dst.type(), 1);
 979       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
 980
 981       if (dst.regClass() == s2) {
 982          Temp carry = bld.tmp(s1);
 983          Temp dst0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
 984          Temp dst1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), src01, src11, bld.scc(carry));
 985          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 986       } else if (dst.regClass() == v2) {
 987          Temp dst0 = bld.tmp(v1);
 988          Temp carry = bld.vadd32(Definition(dst0), src00, src10, true).def(1).getTemp();
 989          Temp dst1 = bld.vadd32(bld.def(v1), src01, src11, false, carry);
 990          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
 991       } else {
 992          fprintf(stderr, "Unimplemented NIR instr bit size: ");
 993          nir_print_instr(&instr->instr, stderr);
 994          fprintf(stderr, "\n");
 995       }
 996       break;
 997    }
 998    case nir_op_uadd_sat: {
 999       Temp src0 = get_alu_src(ctx, instr->src[0]);
1000       Temp src1 = get_alu_src(ctx, instr->src[1]);
1001       if (dst.regClass() == s1) {
1002          Temp tmp = bld.tmp(s1), carry = bld.tmp(s1);
1003          bld.sop2(aco_opcode::s_add_u32, Definition(tmp), bld.scc(Definition(carry)),
1004                   src0, src1);
1005          bld.sop2(aco_opcode::s_cselect_b32, Definition(dst), Operand((uint32_t) -1), tmp, bld.scc(carry));
1006       } else if (dst.regClass() == v1) {
1007          if (ctx->options->chip_class >= GFX9) {
1008             aco_ptr<VOP3A_instruction> add{create_instruction<VOP3A_instruction>(aco_opcode::v_add_u32, asVOP3(Format::VOP2), 2, 1)};
1009             add->operands[0] = Operand(src0);
1010             add->operands[1] = Operand(src1);
1011             add->definitions[0] = Definition(dst);
1012             add->clamp = 1;
1013             ctx->block->instructions.emplace_back(std::move(add));
1014          } else {
1015             if (src1.regClass() != v1)
1016                std::swap(src0, src1);
1017             assert(src1.regClass() == v1);
1018             Temp tmp = bld.tmp(v1);
1019             Temp carry = bld.vadd32(Definition(tmp), src0, src1, true).def(1).getTemp();
1020             bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), tmp, Operand((uint32_t) -1), carry);
1021          }
1022       } else {
1023          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1024          nir_print_instr(&instr->instr, stderr);
1025          fprintf(stderr, "\n");
1026       }
1027       break;
1028    }
1029    case nir_op_uadd_carry: {
1030       Temp src0 = get_alu_src(ctx, instr->src[0]);
1031       Temp src1 = get_alu_src(ctx, instr->src[1]);
1032       if (dst.regClass() == s1) {
1033          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1034          break;
1035       }
1036       if (dst.regClass() == v1) {
1037          Temp carry = bld.vadd32(bld.def(v1), src0, src1, true).def(1).getTemp();
1038          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), carry);
1039          break;
1040       }
1041
1042       Temp src00 = bld.tmp(src0.type(), 1);
1043       Temp src01 = bld.tmp(dst.type(), 1);
1044       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1045       Temp src10 = bld.tmp(src1.type(), 1);
1046       Temp src11 = bld.tmp(dst.type(), 1);
1047       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1048       if (dst.regClass() == s2) {
1049          Temp carry = bld.tmp(s1);
1050          bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1051          carry = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(carry)).def(1).getTemp();
1052          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1053       } else if (dst.regClass() == v2) {
1054          Temp carry = bld.vadd32(bld.def(v1), src00, src10, true).def(1).getTemp();
1055          carry = bld.vadd32(bld.def(v1), src01, src11, true, carry).def(1).getTemp();
1056          carry = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), carry);
1057          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), carry, Operand(0u));
1058       } else {
1059          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1060          nir_print_instr(&instr->instr, stderr);
1061          fprintf(stderr, "\n");
1062       }
1063       break;
1064    }
1065    case nir_op_isub: {
1066       if (dst.regClass() == s1) {
1067          emit_sop2_instruction(ctx, instr, aco_opcode::s_sub_i32, dst, true);
1068          break;
1069       }
1070
1071       Temp src0 = get_alu_src(ctx, instr->src[0]);
1072       Temp src1 = get_alu_src(ctx, instr->src[1]);
1073       if (dst.regClass() == v1) {
1074          bld.vsub32(Definition(dst), src0, src1);
1075          break;
1076       }
1077
1078       Temp src00 = bld.tmp(src0.type(), 1);
1079       Temp src01 = bld.tmp(dst.type(), 1);
1080       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1081       Temp src10 = bld.tmp(src1.type(), 1);
1082       Temp src11 = bld.tmp(dst.type(), 1);
1083       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1084       if (dst.regClass() == s2) {
1085          Temp carry = bld.tmp(s1);
1086          Temp dst0 = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(carry)), src00, src10);
1087          Temp dst1 = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), src01, src11, carry);
1088          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), dst0, dst1);
1089       } else if (dst.regClass() == v2) {
1090          Temp lower = bld.tmp(v1);
1091          Temp borrow = bld.vsub32(Definition(lower), src00, src10, true).def(1).getTemp();
1092          Temp upper = bld.vsub32(bld.def(v1), src01, src11, false, borrow);
1093          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1094       } else {
1095          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1096          nir_print_instr(&instr->instr, stderr);
1097          fprintf(stderr, "\n");
1098       }
1099       break;
1100    }
1101    case nir_op_usub_borrow: {
1102       Temp src0 = get_alu_src(ctx, instr->src[0]);
1103       Temp src1 = get_alu_src(ctx, instr->src[1]);
1104       if (dst.regClass() == s1) {
1105          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(dst)), src0, src1);
1106          break;
1107       } else if (dst.regClass() == v1) {
1108          Temp borrow = bld.vsub32(bld.def(v1), src0, src1, true).def(1).getTemp();
1109          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), borrow);
1110          break;
1111       }
1112
1113       Temp src00 = bld.tmp(src0.type(), 1);
1114       Temp src01 = bld.tmp(dst.type(), 1);
1115       bld.pseudo(aco_opcode::p_split_vector, Definition(src00), Definition(src01), src0);
1116       Temp src10 = bld.tmp(src1.type(), 1);
1117       Temp src11 = bld.tmp(dst.type(), 1);
1118       bld.pseudo(aco_opcode::p_split_vector, Definition(src10), Definition(src11), src1);
1119       if (dst.regClass() == s2) {
1120          Temp borrow = bld.tmp(s1);
1121          bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), src00, src10);
1122          borrow = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.scc(bld.def(s1)), src01, src11, bld.scc(borrow)).def(1).getTemp();
1123          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1124       } else if (dst.regClass() == v2) {
1125          Temp borrow = bld.vsub32(bld.def(v1), src00, src10, true).def(1).getTemp();
1126          borrow = bld.vsub32(bld.def(v1), src01, src11, true, Operand(borrow)).def(1).getTemp();
1127          borrow = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand(1u), borrow);
1128          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), borrow, Operand(0u));
1129       } else {
1130          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1131          nir_print_instr(&instr->instr, stderr);
1132          fprintf(stderr, "\n");
1133       }
1134       break;
1135    }
1136    case nir_op_imul: {
1137       if (dst.regClass() == v1) {
1138          bld.vop3(aco_opcode::v_mul_lo_u32, Definition(dst),
1139                   get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1140       } else if (dst.regClass() == s1) {
1141          emit_sop2_instruction(ctx, instr, aco_opcode::s_mul_i32, dst, false);
1142       } else {
1143          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1144          nir_print_instr(&instr->instr, stderr);
1145          fprintf(stderr, "\n");
1146       }
1147       break;
1148    }
1149    case nir_op_umul_high: {
1150       if (dst.regClass() == v1) {
1151          bld.vop3(aco_opcode::v_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1152       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1153          bld.sop2(aco_opcode::s_mul_hi_u32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1154       } else if (dst.regClass() == s1) {
1155          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_u32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1156                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1157          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1158       } else {
1159          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1160          nir_print_instr(&instr->instr, stderr);
1161          fprintf(stderr, "\n");
1162       }
1163       break;
1164    }
1165    case nir_op_imul_high: {
1166       if (dst.regClass() == v1) {
1167          bld.vop3(aco_opcode::v_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1168       } else if (dst.regClass() == s1 && ctx->options->chip_class >= GFX9) {
1169          bld.sop2(aco_opcode::s_mul_hi_i32, Definition(dst), get_alu_src(ctx, instr->src[0]), get_alu_src(ctx, instr->src[1]));
1170       } else if (dst.regClass() == s1) {
1171          Temp tmp = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), get_alu_src(ctx, instr->src[0]),
1172                              as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1173          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), tmp);
1174       } else {
1175          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1176          nir_print_instr(&instr->instr, stderr);
1177          fprintf(stderr, "\n");
1178       }
1179       break;
1180    }
1181    case nir_op_fmul: {
1182       if (dst.size() == 1) {
1183          emit_vop2_instruction(ctx, instr, aco_opcode::v_mul_f32, dst, true);
1184       } else if (dst.size() == 2) {
1185          bld.vop3(aco_opcode::v_mul_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1186                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1187       } else {
1188          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1189          nir_print_instr(&instr->instr, stderr);
1190          fprintf(stderr, "\n");
1191       }
1192       break;
1193    }
1194    case nir_op_fadd: {
1195       if (dst.size() == 1) {
1196          emit_vop2_instruction(ctx, instr, aco_opcode::v_add_f32, dst, true);
1197       } else if (dst.size() == 2) {
1198          bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]),
1199                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1200       } else {
1201          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1202          nir_print_instr(&instr->instr, stderr);
1203          fprintf(stderr, "\n");
1204       }
1205       break;
1206    }
1207    case nir_op_fsub: {
1208       Temp src0 = get_alu_src(ctx, instr->src[0]);
1209       Temp src1 = get_alu_src(ctx, instr->src[1]);
1210       if (dst.size() == 1) {
1211          if (src1.type() == RegType::vgpr || src0.type() != RegType::vgpr)
1212             emit_vop2_instruction(ctx, instr, aco_opcode::v_sub_f32, dst, false);
1213          else
1214             emit_vop2_instruction(ctx, instr, aco_opcode::v_subrev_f32, dst, true);
1215       } else if (dst.size() == 2) {
1216          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst),
1217                                      get_alu_src(ctx, instr->src[0]),
1218                                      as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1219          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1220          sub->neg[1] = true;
1221       } else {
1222          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1223          nir_print_instr(&instr->instr, stderr);
1224          fprintf(stderr, "\n");
1225       }
1226       break;
1227    }
1228    case nir_op_fmod:
1229    case nir_op_frem: {
1230       if (dst.size() == 1) {
1231          Temp rcp = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), get_alu_src(ctx, instr->src[1]));
1232          Temp mul = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]), rcp);
1233
1234          aco_opcode op = instr->op == nir_op_fmod ? aco_opcode::v_floor_f32 : aco_opcode::v_trunc_f32;
1235          Temp floor = bld.vop1(op, bld.def(v1), mul);
1236
1237          mul = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), get_alu_src(ctx, instr->src[1]), floor);
1238          bld.vop2(aco_opcode::v_sub_f32, Definition(dst), get_alu_src(ctx, instr->src[0]), mul);
1239       } else if (dst.size() == 2) {
1240          Temp rcp = bld.vop1(aco_opcode::v_rcp_f64, bld.def(v2), get_alu_src(ctx, instr->src[1]));
1241          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), get_alu_src(ctx, instr->src[0]), rcp);
1242
1243          aco_opcode op = instr->op == nir_op_fmod ? aco_opcode::v_floor_f64 : aco_opcode::v_trunc_f64;
1244          Temp floor = bld.vop1(op, bld.def(v1), mul);
1245
1246          mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), get_alu_src(ctx, instr->src[1]), floor);
1247          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), get_alu_src(ctx, instr->src[0]), mul);
1248          VOP3A_instruction* sub = static_cast<VOP3A_instruction*>(add);
1249          sub->neg[1] = true;
1250       } else {
1251          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1252          nir_print_instr(&instr->instr, stderr);
1253          fprintf(stderr, "\n");
1254       }
1255       break;
1256    }
1257    case nir_op_fmax: {
1258       if (dst.size() == 1) {
1259          emit_vop2_instruction(ctx, instr, aco_opcode::v_max_f32, dst, true);
1260       } else if (dst.size() == 2) {
1261          bld.vop3(aco_opcode::v_max_f64, Definition(dst),
1262                   get_alu_src(ctx, instr->src[0]),
1263                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1264       } else {
1265          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1266          nir_print_instr(&instr->instr, stderr);
1267          fprintf(stderr, "\n");
1268       }
1269       break;
1270    }
1271    case nir_op_fmin: {
1272       if (dst.size() == 1) {
1273          emit_vop2_instruction(ctx, instr, aco_opcode::v_min_f32, dst, true);
1274       } else if (dst.size() == 2) {
1275          bld.vop3(aco_opcode::v_min_f64, Definition(dst),
1276                   get_alu_src(ctx, instr->src[0]),
1277                   as_vgpr(ctx, get_alu_src(ctx, instr->src[1])));
1278       } else {
1279          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1280          nir_print_instr(&instr->instr, stderr);
1281          fprintf(stderr, "\n");
1282       }
1283       break;
1284    }
1285    case nir_op_fmax3: {
1286       if (dst.size() == 1) {
1287          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_f32, dst);
1288       } else {
1289          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1290          nir_print_instr(&instr->instr, stderr);
1291          fprintf(stderr, "\n");
1292       }
1293       break;
1294    }
1295    case nir_op_fmin3: {
1296       if (dst.size() == 1) {
1297          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_f32, dst);
1298       } else {
1299          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1300          nir_print_instr(&instr->instr, stderr);
1301          fprintf(stderr, "\n");
1302       }
1303       break;
1304    }
1305    case nir_op_fmed3: {
1306       if (dst.size() == 1) {
1307          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_f32, dst);
1308       } else {
1309          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1310          nir_print_instr(&instr->instr, stderr);
1311          fprintf(stderr, "\n");
1312       }
1313       break;
1314    }
1315    case nir_op_umax3: {
1316       if (dst.size() == 1) {
1317          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_u32, dst);
1318       } else {
1319          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1320          nir_print_instr(&instr->instr, stderr);
1321          fprintf(stderr, "\n");
1322       }
1323       break;
1324    }
1325    case nir_op_umin3: {
1326       if (dst.size() == 1) {
1327          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_u32, dst);
1328       } else {
1329          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1330          nir_print_instr(&instr->instr, stderr);
1331          fprintf(stderr, "\n");
1332       }
1333       break;
1334    }
1335    case nir_op_umed3: {
1336       if (dst.size() == 1) {
1337          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_u32, dst);
1338       } else {
1339          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1340          nir_print_instr(&instr->instr, stderr);
1341          fprintf(stderr, "\n");
1342       }
1343       break;
1344    }
1345    case nir_op_imax3: {
1346       if (dst.size() == 1) {
1347          emit_vop3a_instruction(ctx, instr, aco_opcode::v_max3_i32, dst);
1348       } else {
1349          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1350          nir_print_instr(&instr->instr, stderr);
1351          fprintf(stderr, "\n");
1352       }
1353       break;
1354    }
1355    case nir_op_imin3: {
1356       if (dst.size() == 1) {
1357          emit_vop3a_instruction(ctx, instr, aco_opcode::v_min3_i32, dst);
1358       } else {
1359          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1360          nir_print_instr(&instr->instr, stderr);
1361          fprintf(stderr, "\n");
1362       }
1363       break;
1364    }
1365    case nir_op_imed3: {
1366       if (dst.size() == 1) {
1367          emit_vop3a_instruction(ctx, instr, aco_opcode::v_med3_i32, dst);
1368       } else {
1369          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1370          nir_print_instr(&instr->instr, stderr);
1371          fprintf(stderr, "\n");
1372       }
1373       break;
1374    }
1375    case nir_op_cube_face_coord: {
1376       Temp in = get_alu_src(ctx, instr->src[0], 3);
1377       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1378                       emit_extract_vector(ctx, in, 1, v1),
1379                       emit_extract_vector(ctx, in, 2, v1) };
1380       Temp ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), src[0], src[1], src[2]);
1381       ma = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ma);
1382       Temp sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), src[0], src[1], src[2]);
1383       Temp tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), src[0], src[1], src[2]);
1384       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, ma, Operand(0x3f000000u/*0.5*/));
1385       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, ma, Operand(0x3f000000u/*0.5*/));
1386       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), sc, tc);
1387       break;
1388    }
1389    case nir_op_cube_face_index: {
1390       Temp in = get_alu_src(ctx, instr->src[0], 3);
1391       Temp src[3] = { emit_extract_vector(ctx, in, 0, v1),
1392                       emit_extract_vector(ctx, in, 1, v1),
1393                       emit_extract_vector(ctx, in, 2, v1) };
1394       bld.vop3(aco_opcode::v_cubeid_f32, Definition(dst), src[0], src[1], src[2]);
1395       break;
1396    }
1397    case nir_op_bcsel: {
1398       emit_bcsel(ctx, instr, dst);
1399       break;
1400    }
1401    case nir_op_frsq: {
1402       if (dst.size() == 1) {
1403          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f32, dst);
1404       } else if (dst.size() == 2) {
1405          emit_vop1_instruction(ctx, instr, aco_opcode::v_rsq_f64, dst);
1406       } else {
1407          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1408          nir_print_instr(&instr->instr, stderr);
1409          fprintf(stderr, "\n");
1410       }
1411       break;
1412    }
1413    case nir_op_fneg: {
1414       Temp src = get_alu_src(ctx, instr->src[0]);
1415       if (dst.size() == 1) {
1416          bld.vop2(aco_opcode::v_xor_b32, Definition(dst), Operand(0x80000000u), as_vgpr(ctx, src));
1417       } else if (dst.size() == 2) {
1418          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1419          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1420          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), Operand(0x80000000u), upper);
1421          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1422       } else {
1423          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1424          nir_print_instr(&instr->instr, stderr);
1425          fprintf(stderr, "\n");
1426       }
1427       break;
1428    }
1429    case nir_op_fabs: {
1430       Temp src = get_alu_src(ctx, instr->src[0]);
1431       if (dst.size() == 1) {
1432          bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0x7FFFFFFFu), as_vgpr(ctx, src));
1433       } else if (dst.size() == 2) {
1434          Temp upper = bld.tmp(v1), lower = bld.tmp(v1);
1435          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1436          upper = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7FFFFFFFu), upper);
1437          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1438       } else {
1439          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1440          nir_print_instr(&instr->instr, stderr);
1441          fprintf(stderr, "\n");
1442       }
1443       break;
1444    }
1445    case nir_op_fsat: {
1446       Temp src = get_alu_src(ctx, instr->src[0]);
1447       if (dst.size() == 1) {
1448          bld.vop3(aco_opcode::v_med3_f32, Definition(dst), Operand(0u), Operand(0x3f800000u), src);
1449       } else if (dst.size() == 2) {
1450          Instruction* add = bld.vop3(aco_opcode::v_add_f64, Definition(dst), src, Operand(0u));
1451          VOP3A_instruction* vop3 = static_cast<VOP3A_instruction*>(add);
1452          vop3->clamp = true;
1453       } else {
1454          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1455          nir_print_instr(&instr->instr, stderr);
1456          fprintf(stderr, "\n");
1457       }
1458       break;
1459    }
1460    case nir_op_flog2: {
1461       if (dst.size() == 1) {
1462          emit_vop1_instruction(ctx, instr, aco_opcode::v_log_f32, dst);
1463       } else {
1464          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1465          nir_print_instr(&instr->instr, stderr);
1466          fprintf(stderr, "\n");
1467       }
1468       break;
1469    }
1470    case nir_op_frcp: {
1471       if (dst.size() == 1) {
1472          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f32, dst);
1473       } else if (dst.size() == 2) {
1474          emit_vop1_instruction(ctx, instr, aco_opcode::v_rcp_f64, dst);
1475       } else {
1476          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1477          nir_print_instr(&instr->instr, stderr);
1478          fprintf(stderr, "\n");
1479       }
1480       break;
1481    }
1482    case nir_op_fexp2: {
1483       if (dst.size() == 1) {
1484          emit_vop1_instruction(ctx, instr, aco_opcode::v_exp_f32, dst);
1485       } else {
1486          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1487          nir_print_instr(&instr->instr, stderr);
1488          fprintf(stderr, "\n");
1489       }
1490       break;
1491    }
1492    case nir_op_fsqrt: {
1493       if (dst.size() == 1) {
1494          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f32, dst);
1495       } else if (dst.size() == 2) {
1496          emit_vop1_instruction(ctx, instr, aco_opcode::v_sqrt_f64, dst);
1497       } else {
1498          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1499          nir_print_instr(&instr->instr, stderr);
1500          fprintf(stderr, "\n");
1501       }
1502       break;
1503    }
1504    case nir_op_ffract: {
1505       if (dst.size() == 1) {
1506          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f32, dst);
1507       } else if (dst.size() == 2) {
1508          emit_vop1_instruction(ctx, instr, aco_opcode::v_fract_f64, dst);
1509       } else {
1510          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1511          nir_print_instr(&instr->instr, stderr);
1512          fprintf(stderr, "\n");
1513       }
1514       break;
1515    }
1516    case nir_op_ffloor: {
1517       if (dst.size() == 1) {
1518          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f32, dst);
1519       } else if (dst.size() == 2) {
1520          emit_vop1_instruction(ctx, instr, aco_opcode::v_floor_f64, dst);
1521       } else {
1522          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1523          nir_print_instr(&instr->instr, stderr);
1524          fprintf(stderr, "\n");
1525       }
1526       break;
1527    }
1528    case nir_op_fceil: {
1529       if (dst.size() == 1) {
1530          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f32, dst);
1531       } else if (dst.size() == 2) {
1532          emit_vop1_instruction(ctx, instr, aco_opcode::v_ceil_f64, dst);
1533       } else {
1534          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1535          nir_print_instr(&instr->instr, stderr);
1536          fprintf(stderr, "\n");
1537       }
1538       break;
1539    }
1540    case nir_op_ftrunc: {
1541       if (dst.size() == 1) {
1542          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f32, dst);
1543       } else if (dst.size() == 2) {
1544          emit_vop1_instruction(ctx, instr, aco_opcode::v_trunc_f64, dst);
1545       } else {
1546          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1547          nir_print_instr(&instr->instr, stderr);
1548          fprintf(stderr, "\n");
1549       }
1550       break;
1551    }
1552    case nir_op_fround_even: {
1553       if (dst.size() == 1) {
1554          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f32, dst);
1555       } else if (dst.size() == 2) {
1556          emit_vop1_instruction(ctx, instr, aco_opcode::v_rndne_f64, dst);
1557       } else {
1558          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1559          nir_print_instr(&instr->instr, stderr);
1560          fprintf(stderr, "\n");
1561       }
1562       break;
1563    }
1564    case nir_op_fsin:
1565    case nir_op_fcos: {
1566       Temp src = get_alu_src(ctx, instr->src[0]);
1567       aco_ptr<Instruction> norm;
1568       if (dst.size() == 1) {
1569          Temp tmp;
1570          Operand half_pi(0x3e22f983u);
1571          if (src.type() == RegType::sgpr)
1572             tmp = bld.vop2_e64(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1573          else
1574             tmp = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), half_pi, src);
1575
1576          /* before GFX9, v_sin_f32 and v_cos_f32 had a valid input domain of [-256, +256] */
1577          if (ctx->options->chip_class < GFX9)
1578             tmp = bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), tmp);
1579
1580          aco_opcode opcode = instr->op == nir_op_fsin ? aco_opcode::v_sin_f32 : aco_opcode::v_cos_f32;
1581          bld.vop1(opcode, Definition(dst), tmp);
1582       } else {
1583          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1584          nir_print_instr(&instr->instr, stderr);
1585          fprintf(stderr, "\n");
1586       }
1587       break;
1588    }
1589    case nir_op_ldexp: {
1590       if (dst.size() == 1) {
1591          bld.vop3(aco_opcode::v_ldexp_f32, Definition(dst),
1592                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1593                   get_alu_src(ctx, instr->src[1]));
1594       } else if (dst.size() == 2) {
1595          bld.vop3(aco_opcode::v_ldexp_f64, Definition(dst),
1596                   as_vgpr(ctx, get_alu_src(ctx, instr->src[0])),
1597                   get_alu_src(ctx, instr->src[1]));
1598       } else {
1599          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1600          nir_print_instr(&instr->instr, stderr);
1601          fprintf(stderr, "\n");
1602       }
1603       break;
1604    }
1605    case nir_op_frexp_sig: {
1606       if (dst.size() == 1) {
1607          bld.vop1(aco_opcode::v_frexp_mant_f32, Definition(dst),
1608                   get_alu_src(ctx, instr->src[0]));
1609       } else if (dst.size() == 2) {
1610          bld.vop1(aco_opcode::v_frexp_mant_f64, Definition(dst),
1611                   get_alu_src(ctx, instr->src[0]));
1612       } else {
1613          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1614          nir_print_instr(&instr->instr, stderr);
1615          fprintf(stderr, "\n");
1616       }
1617       break;
1618    }
1619    case nir_op_frexp_exp: {
1620       if (instr->src[0].src.ssa->bit_size == 32) {
1621          bld.vop1(aco_opcode::v_frexp_exp_i32_f32, Definition(dst),
1622                   get_alu_src(ctx, instr->src[0]));
1623       } else if (instr->src[0].src.ssa->bit_size == 64) {
1624          bld.vop1(aco_opcode::v_frexp_exp_i32_f64, Definition(dst),
1625                   get_alu_src(ctx, instr->src[0]));
1626       } else {
1627          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1628          nir_print_instr(&instr->instr, stderr);
1629          fprintf(stderr, "\n");
1630       }
1631       break;
1632    }
1633    case nir_op_fsign: {
1634       Temp src = as_vgpr(ctx, get_alu_src(ctx, instr->src[0]));
1635       if (dst.size() == 1) {
1636          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1637          src = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0x3f800000u), src, cond);
1638          cond = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1639          bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0xbf800000u), src, cond);
1640       } else if (dst.size() == 2) {
1641          Temp cond = bld.vopc(aco_opcode::v_cmp_nlt_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1642          Temp tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0x3FF00000u));
1643          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, src, cond);
1644
1645          cond = bld.vopc(aco_opcode::v_cmp_le_f64, bld.hint_vcc(bld.def(s2)), Operand(0u), src);
1646          tmp = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0xBFF00000u));
1647          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), tmp, upper, cond);
1648
1649          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1650       } else {
1651          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1652          nir_print_instr(&instr->instr, stderr);
1653          fprintf(stderr, "\n");
1654       }
1655       break;
1656    }
1657    case nir_op_f2f32: {
1658       if (instr->src[0].src.ssa->bit_size == 64) {
1659          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_f64, dst);
1660       } else {
1661          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1662          nir_print_instr(&instr->instr, stderr);
1663          fprintf(stderr, "\n");
1664       }
1665       break;
1666    }
1667    case nir_op_f2f64: {
1668       if (instr->src[0].src.ssa->bit_size == 32) {
1669          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_f32, dst);
1670       } else {
1671          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1672          nir_print_instr(&instr->instr, stderr);
1673          fprintf(stderr, "\n");
1674       }
1675       break;
1676    }
1677    case nir_op_i2f32: {
1678       assert(dst.size() == 1);
1679       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_i32, dst);
1680       break;
1681    }
1682    case nir_op_i2f64: {
1683       if (instr->src[0].src.ssa->bit_size == 32) {
1684          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_i32, dst);
1685       } else if (instr->src[0].src.ssa->bit_size == 64) {
1686          Temp src = get_alu_src(ctx, instr->src[0]);
1687          RegClass rc = RegClass(src.type(), 1);
1688          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1689          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1690          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1691          upper = bld.vop1(aco_opcode::v_cvt_f64_i32, bld.def(v2), upper);
1692          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1693          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1694
1695       } else {
1696          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1697          nir_print_instr(&instr->instr, stderr);
1698          fprintf(stderr, "\n");
1699       }
1700       break;
1701    }
1702    case nir_op_u2f32: {
1703       assert(dst.size() == 1);
1704       emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f32_u32, dst);
1705       break;
1706    }
1707    case nir_op_u2f64: {
1708       if (instr->src[0].src.ssa->bit_size == 32) {
1709          emit_vop1_instruction(ctx, instr, aco_opcode::v_cvt_f64_u32, dst);
1710       } else if (instr->src[0].src.ssa->bit_size == 64) {
1711          Temp src = get_alu_src(ctx, instr->src[0]);
1712          RegClass rc = RegClass(src.type(), 1);
1713          Temp lower = bld.tmp(rc), upper = bld.tmp(rc);
1714          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), src);
1715          lower = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), lower);
1716          upper = bld.vop1(aco_opcode::v_cvt_f64_u32, bld.def(v2), upper);
1717          upper = bld.vop3(aco_opcode::v_ldexp_f64, bld.def(v2), upper, Operand(32u));
1718          bld.vop3(aco_opcode::v_add_f64, Definition(dst), lower, upper);
1719       } else {
1720          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1721          nir_print_instr(&instr->instr, stderr);
1722          fprintf(stderr, "\n");
1723       }
1724       break;
1725    }
1726    case nir_op_f2i32: {
1727       Temp src = get_alu_src(ctx, instr->src[0]);
1728       if (instr->src[0].src.ssa->bit_size == 32) {
1729          if (dst.type() == RegType::vgpr)
1730             bld.vop1(aco_opcode::v_cvt_i32_f32, Definition(dst), src);
1731          else
1732             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1733                        bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), src));
1734
1735       } else if (instr->src[0].src.ssa->bit_size == 64) {
1736          if (dst.type() == RegType::vgpr)
1737             bld.vop1(aco_opcode::v_cvt_i32_f64, Definition(dst), src);
1738          else
1739             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1740                        bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), src));
1741
1742       } else {
1743          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1744          nir_print_instr(&instr->instr, stderr);
1745          fprintf(stderr, "\n");
1746       }
1747       break;
1748    }
1749    case nir_op_f2u32: {
1750       Temp src = get_alu_src(ctx, instr->src[0]);
1751       if (instr->src[0].src.ssa->bit_size == 32) {
1752          if (dst.type() == RegType::vgpr)
1753             bld.vop1(aco_opcode::v_cvt_u32_f32, Definition(dst), src);
1754          else
1755             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1756                        bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), src));
1757
1758       } else if (instr->src[0].src.ssa->bit_size == 64) {
1759          if (dst.type() == RegType::vgpr)
1760             bld.vop1(aco_opcode::v_cvt_u32_f64, Definition(dst), src);
1761          else
1762             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst),
1763                        bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), src));
1764
1765       } else {
1766          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1767          nir_print_instr(&instr->instr, stderr);
1768          fprintf(stderr, "\n");
1769       }
1770       break;
1771    }
1772    case nir_op_f2i64: {
1773       Temp src = get_alu_src(ctx, instr->src[0]);
1774       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1775          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1776          exponent = bld.vop3(aco_opcode::v_med3_i32, bld.def(v1), Operand(0x0u), exponent, Operand(64u));
1777          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1778          Temp sign = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(31u), src);
1779          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1780          mantissa = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(7u), mantissa);
1781          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1782          Temp new_exponent = bld.tmp(v1);
1783          Temp borrow = bld.vsub32(Definition(new_exponent), Operand(63u), exponent, true).def(1).getTemp();
1784          mantissa = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), new_exponent, mantissa);
1785          Temp saturate = bld.vop1(aco_opcode::v_bfrev_b32, bld.def(v1), Operand(0xfffffffeu));
1786          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1787          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1788          lower = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), lower, Operand(0xffffffffu), borrow);
1789          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), upper, saturate, borrow);
1790          lower = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, lower);
1791          upper = bld.vop2(aco_opcode::v_xor_b32, bld.def(v1), sign, upper);
1792          Temp new_lower = bld.tmp(v1);
1793          borrow = bld.vsub32(Definition(new_lower), lower, sign, true).def(1).getTemp();
1794          Temp new_upper = bld.vsub32(bld.def(v1), upper, sign, false, borrow);
1795          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), new_lower, new_upper);
1796
1797       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1798          if (src.type() == RegType::vgpr)
1799             src = bld.as_uniform(src);
1800          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1801          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1802          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1803          exponent = bld.sop2(aco_opcode::s_min_u32, bld.def(s1), bld.def(s1, scc), Operand(64u), exponent);
1804          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1805          Temp sign = bld.sop2(aco_opcode::s_ashr_i32, bld.def(s1), bld.def(s1, scc), src, Operand(31u));
1806          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1807          mantissa = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), mantissa, Operand(7u));
1808          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1809          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(63u), exponent);
1810          mantissa = bld.sop2(aco_opcode::s_lshr_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent);
1811          Temp cond = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), exponent, Operand(0xffffffffu)); // exp >= 64
1812          Temp saturate = bld.sop1(aco_opcode::s_brev_b64, bld.def(s2), Operand(0xfffffffeu));
1813          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), saturate, mantissa, cond);
1814          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1815          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1816          lower = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, lower);
1817          upper = bld.sop2(aco_opcode::s_xor_b32, bld.def(s1), bld.def(s1, scc), sign, upper);
1818          Temp borrow = bld.tmp(s1);
1819          lower = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.scc(Definition(borrow)), lower, sign);
1820          upper = bld.sop2(aco_opcode::s_subb_u32, bld.def(s1), bld.def(s1, scc), upper, sign, borrow);
1821          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1822
1823       } else if (instr->src[0].src.ssa->bit_size == 64) {
1824          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1825          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1826          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1827          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1828          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1829          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1830          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1831          Temp upper = bld.vop1(aco_opcode::v_cvt_i32_f64, bld.def(v1), floor);
1832          if (dst.type() == RegType::sgpr) {
1833             lower = bld.as_uniform(lower);
1834             upper = bld.as_uniform(upper);
1835          }
1836          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1837
1838       } else {
1839          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1840          nir_print_instr(&instr->instr, stderr);
1841          fprintf(stderr, "\n");
1842       }
1843       break;
1844    }
1845    case nir_op_f2u64: {
1846       Temp src = get_alu_src(ctx, instr->src[0]);
1847       if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::vgpr) {
1848          Temp exponent = bld.vop1(aco_opcode::v_frexp_exp_i32_f32, bld.def(v1), src);
1849          Temp exponent_in_range = bld.vopc(aco_opcode::v_cmp_ge_i32, bld.hint_vcc(bld.def(s2)), Operand(64u), exponent);
1850          exponent = bld.vop2(aco_opcode::v_max_i32, bld.def(v1), Operand(0x0u), exponent);
1851          Temp mantissa = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffu), src);
1852          mantissa = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(0x800000u), mantissa);
1853          Temp exponent_small = bld.vsub32(bld.def(v1), Operand(24u), exponent);
1854          Temp small = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), exponent_small, mantissa);
1855          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), Operand(0u), mantissa);
1856          Temp new_exponent = bld.tmp(v1);
1857          Temp cond_small = bld.vsub32(Definition(new_exponent), exponent, Operand(24u), true).def(1).getTemp();
1858          mantissa = bld.vop3(aco_opcode::v_lshlrev_b64, bld.def(v2), new_exponent, mantissa);
1859          Temp lower = bld.tmp(v1), upper = bld.tmp(v1);
1860          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1861          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), lower, small, cond_small);
1862          upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), upper, Operand(0u), cond_small);
1863          lower = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), lower, exponent_in_range);
1864          upper = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xffffffffu), upper, exponent_in_range);
1865          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1866
1867       } else if (instr->src[0].src.ssa->bit_size == 32 && dst.type() == RegType::sgpr) {
1868          if (src.type() == RegType::vgpr)
1869             src = bld.as_uniform(src);
1870          Temp exponent = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), src, Operand(0x80017u));
1871          exponent = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(126u));
1872          exponent = bld.sop2(aco_opcode::s_max_u32, bld.def(s1), bld.def(s1, scc), Operand(0u), exponent);
1873          Temp mantissa = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0x7fffffu), src);
1874          mantissa = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(0x800000u), mantissa);
1875          Temp exponent_small = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), Operand(24u), exponent);
1876          Temp small = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), mantissa, exponent_small);
1877          mantissa = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), mantissa);
1878          Temp exponent_large = bld.sop2(aco_opcode::s_sub_u32, bld.def(s1), bld.def(s1, scc), exponent, Operand(24u));
1879          mantissa = bld.sop2(aco_opcode::s_lshl_b64, bld.def(s2), bld.def(s1, scc), mantissa, exponent_large);
1880          Temp cond = bld.sopc(aco_opcode::s_cmp_ge_i32, bld.def(s1, scc), Operand(64u), exponent);
1881          mantissa = bld.sop2(aco_opcode::s_cselect_b64, bld.def(s2), mantissa, Operand(0xffffffffu), cond);
1882          Temp lower = bld.tmp(s1), upper = bld.tmp(s1);
1883          bld.pseudo(aco_opcode::p_split_vector, Definition(lower), Definition(upper), mantissa);
1884          Temp cond_small = bld.sopc(aco_opcode::s_cmp_le_i32, bld.def(s1, scc), exponent, Operand(24u));
1885          lower = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), small, lower, cond_small);
1886          upper = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1), Operand(0u), upper, cond_small);
1887          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1888
1889       } else if (instr->src[0].src.ssa->bit_size == 64) {
1890          Temp vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0x3df00000u));
1891          Temp trunc = bld.vop1(aco_opcode::v_trunc_f64, bld.def(v2), src);
1892          Temp mul = bld.vop3(aco_opcode::v_mul_f64, bld.def(v2), trunc, vec);
1893          vec = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(0u), Operand(0xc1f00000u));
1894          Temp floor  = bld.vop1(aco_opcode::v_floor_f64, bld.def(v2), mul);
1895          Temp fma = bld.vop3(aco_opcode::v_fma_f64, bld.def(v2), floor, vec, trunc);
1896          Temp lower = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), fma);
1897          Temp upper = bld.vop1(aco_opcode::v_cvt_u32_f64, bld.def(v1), floor);
1898          if (dst.type() == RegType::sgpr) {
1899             lower = bld.as_uniform(lower);
1900             upper = bld.as_uniform(upper);
1901          }
1902          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lower, upper);
1903
1904       } else {
1905          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1906          nir_print_instr(&instr->instr, stderr);
1907          fprintf(stderr, "\n");
1908       }
1909       break;
1910    }
1911    case nir_op_b2f32: {
1912       Temp src = get_alu_src(ctx, instr->src[0]);
1913       if (dst.regClass() == s1) {
1914          src = as_uniform_bool(ctx, src);
1915          bld.sop2(aco_opcode::s_mul_i32, Definition(dst), Operand(0x3f800000u), src);
1916       } else if (dst.regClass() == v1) {
1917          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(0x3f800000u),
1918                       as_divergent_bool(ctx, src, true));
1919       } else {
1920          unreachable("Wrong destination register class for nir_op_b2f32.");
1921       }
1922       break;
1923    }
1924    case nir_op_b2f64: {
1925       Temp src = get_alu_src(ctx, instr->src[0]);
1926       if (dst.regClass() == s2) {
1927          src = as_uniform_bool(ctx, src);
1928          bld.sop2(aco_opcode::s_cselect_b64, Definition(dst), Operand(0x3f800000u), Operand(0u), bld.scc(src));
1929       } else if (dst.regClass() == v2) {
1930          Temp one = bld.vop1(aco_opcode::v_mov_b32, bld.def(v2), Operand(0x3FF00000u));
1931          Temp upper = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), one,
1932                       as_divergent_bool(ctx, src, true));
1933          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), Operand(0u), upper);
1934       } else {
1935          unreachable("Wrong destination register class for nir_op_b2f64.");
1936       }
1937       break;
1938    }
1939    case nir_op_i2i32: {
1940       Temp src = get_alu_src(ctx, instr->src[0]);
1941       if (instr->src[0].src.ssa->bit_size == 64) {
1942          /* we can actually just say dst = src, as it would map the lower register */
1943          emit_extract_vector(ctx, src, 0, dst);
1944       } else {
1945          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1946          nir_print_instr(&instr->instr, stderr);
1947          fprintf(stderr, "\n");
1948       }
1949       break;
1950    }
1951    case nir_op_u2u32: {
1952       Temp src = get_alu_src(ctx, instr->src[0]);
1953       if (instr->src[0].src.ssa->bit_size == 16) {
1954          if (dst.regClass() == s1) {
1955             bld.sop2(aco_opcode::s_and_b32, Definition(dst), bld.def(s1, scc), Operand(0xFFFFu), src);
1956          } else {
1957             // TODO: do better with SDWA
1958             bld.vop2(aco_opcode::v_and_b32, Definition(dst), Operand(0xFFFFu), src);
1959          }
1960       } else if (instr->src[0].src.ssa->bit_size == 64) {
1961          /* we can actually just say dst = src, as it would map the lower register */
1962          emit_extract_vector(ctx, src, 0, dst);
1963       } else {
1964          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1965          nir_print_instr(&instr->instr, stderr);
1966          fprintf(stderr, "\n");
1967       }
1968       break;
1969    }
1970    case nir_op_i2i64: {
1971       Temp src = get_alu_src(ctx, instr->src[0]);
1972       if (instr->src[0].src.ssa->bit_size == 32) {
1973          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1974       } else {
1975          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1976          nir_print_instr(&instr->instr, stderr);
1977          fprintf(stderr, "\n");
1978       }
1979       break;
1980    }
1981    case nir_op_u2u64: {
1982       Temp src = get_alu_src(ctx, instr->src[0]);
1983       if (instr->src[0].src.ssa->bit_size == 32) {
1984          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src, Operand(0u));
1985       } else {
1986          fprintf(stderr, "Unimplemented NIR instr bit size: ");
1987          nir_print_instr(&instr->instr, stderr);
1988          fprintf(stderr, "\n");
1989       }
1990       break;
1991    }
1992    case nir_op_b2i32: {
1993       Temp src = get_alu_src(ctx, instr->src[0]);
1994       if (dst.regClass() == s1) {
1995          if (src.regClass() == s1) {
1996             bld.copy(Definition(dst), src);
1997          } else {
1998             // TODO: in a post-RA optimization, we can check if src is in VCC, and directly use VCCNZ
1999             assert(src.regClass() == s2);
2000             bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(Definition(dst)), Operand(0u), src);
2001          }
2002       } else {
2003          assert(dst.regClass() == v1 && src.regClass() == s2);
2004          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand(1u), src);
2005       }
2006       break;
2007    }
2008    case nir_op_i2b1: {
2009       Temp src = get_alu_src(ctx, instr->src[0]);
2010       if (dst.regClass() == s2) {
2011          assert(src.regClass() == v1 || src.regClass() == v2);
2012          bld.vopc(src.size() == 2 ? aco_opcode::v_cmp_lg_u64 : aco_opcode::v_cmp_lg_u32,
2013                   Definition(dst), Operand(0u), src).def(0).setHint(vcc);
2014       } else {
2015          assert(src.regClass() == s1 && dst.regClass() == s1);
2016          bld.sopc(aco_opcode::s_cmp_lg_u32, bld.scc(Definition(dst)), Operand(0u), src);
2017       }
2018       break;
2019    }
2020    case nir_op_pack_64_2x32_split: {
2021       Temp src0 = get_alu_src(ctx, instr->src[0]);
2022       Temp src1 = get_alu_src(ctx, instr->src[1]);
2023
2024       bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src0, src1);
2025       break;
2026    }
2027    case nir_op_unpack_64_2x32_split_x:
2028       bld.pseudo(aco_opcode::p_split_vector, Definition(dst), bld.def(dst.regClass()), get_alu_src(ctx, instr->src[0]));
2029       break;
2030    case nir_op_unpack_64_2x32_split_y:
2031       bld.pseudo(aco_opcode::p_split_vector, bld.def(dst.regClass()), Definition(dst), get_alu_src(ctx, instr->src[0]));
2032       break;
2033    case nir_op_pack_half_2x16: {
2034       Temp src = get_alu_src(ctx, instr->src[0], 2);
2035
2036       if (dst.regClass() == v1) {
2037          Temp src0 = bld.tmp(v1);
2038          Temp src1 = bld.tmp(v1);
2039          bld.pseudo(aco_opcode::p_split_vector, Definition(src0), Definition(src1), src);
2040          bld.vop3(aco_opcode::v_cvt_pkrtz_f16_f32, Definition(dst), src0, src1);
2041
2042       } else {
2043          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2044          nir_print_instr(&instr->instr, stderr);
2045          fprintf(stderr, "\n");
2046       }
2047       break;
2048    }
2049    case nir_op_unpack_half_2x16_split_x: {
2050       if (dst.regClass() == v1) {
2051          Builder bld(ctx->program, ctx->block);
2052          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst), get_alu_src(ctx, instr->src[0]));
2053       } else {
2054          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2055          nir_print_instr(&instr->instr, stderr);
2056          fprintf(stderr, "\n");
2057       }
2058       break;
2059    }
2060    case nir_op_unpack_half_2x16_split_y: {
2061       if (dst.regClass() == v1) {
2062          Builder bld(ctx->program, ctx->block);
2063          /* TODO: use SDWA here */
2064          bld.vop1(aco_opcode::v_cvt_f32_f16, Definition(dst),
2065                   bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(16u), as_vgpr(ctx, get_alu_src(ctx, instr->src[0]))));
2066       } else {
2067          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2068          nir_print_instr(&instr->instr, stderr);
2069          fprintf(stderr, "\n");
2070       }
2071       break;
2072    }
2073    case nir_op_fquantize2f16: {
2074       Temp f16 = bld.vop1(aco_opcode::v_cvt_f16_f32, bld.def(v1), get_alu_src(ctx, instr->src[0]));
2075
2076       Temp mask = bld.copy(bld.def(s1), Operand(0x36Fu)); /* value is NOT negative/positive denormal value */
2077
2078       Temp cmp_res = bld.tmp(s2);
2079       bld.vopc_e64(aco_opcode::v_cmp_class_f16, Definition(cmp_res), f16, mask).def(0).setHint(vcc);
2080
2081       Temp f32 = bld.vop1(aco_opcode::v_cvt_f32_f16, bld.def(v1), f16);
2082
2083       bld.vop2(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), f32, cmp_res);
2084       break;
2085    }
2086    case nir_op_bfm: {
2087       Temp bits = get_alu_src(ctx, instr->src[0]);
2088       Temp offset = get_alu_src(ctx, instr->src[1]);
2089
2090       if (dst.regClass() == s1) {
2091          bld.sop2(aco_opcode::s_bfm_b32, Definition(dst), bits, offset);
2092       } else if (dst.regClass() == v1) {
2093          bld.vop3(aco_opcode::v_bfm_b32, Definition(dst), bits, offset);
2094       } else {
2095          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2096          nir_print_instr(&instr->instr, stderr);
2097          fprintf(stderr, "\n");
2098       }
2099       break;
2100    }
2101    case nir_op_bitfield_select: {
2102       /* (mask & insert) | (~mask & base) */
2103       Temp bitmask = get_alu_src(ctx, instr->src[0]);
2104       Temp insert = get_alu_src(ctx, instr->src[1]);
2105       Temp base = get_alu_src(ctx, instr->src[2]);
2106
2107       /* dst = (insert & bitmask) | (base & ~bitmask) */
2108       if (dst.regClass() == s1) {
2109          aco_ptr<Instruction> sop2;
2110          nir_const_value* const_bitmask = nir_src_as_const_value(instr->src[0].src);
2111          nir_const_value* const_insert = nir_src_as_const_value(instr->src[1].src);
2112          Operand lhs;
2113          if (const_insert && const_bitmask) {
2114             lhs = Operand(const_insert->u32 & const_bitmask->u32);
2115          } else {
2116             insert = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), insert, bitmask);
2117             lhs = Operand(insert);
2118          }
2119
2120          Operand rhs;
2121          nir_const_value* const_base = nir_src_as_const_value(instr->src[2].src);
2122          if (const_base && const_bitmask) {
2123             rhs = Operand(const_base->u32 & ~const_bitmask->u32);
2124          } else {
2125             base = bld.sop2(aco_opcode::s_andn2_b32, bld.def(s1), bld.def(s1, scc), base, bitmask);
2126             rhs = Operand(base);
2127          }
2128
2129          bld.sop2(aco_opcode::s_or_b32, Definition(dst), bld.def(s1, scc), rhs, lhs);
2130
2131       } else if (dst.regClass() == v1) {
2132          if (base.type() == RegType::sgpr && (bitmask.type() == RegType::sgpr || (insert.type() == RegType::sgpr)))
2133             base = as_vgpr(ctx, base);
2134          if (insert.type() == RegType::sgpr && bitmask.type() == RegType::sgpr)
2135             insert = as_vgpr(ctx, insert);
2136
2137          bld.vop3(aco_opcode::v_bfi_b32, Definition(dst), bitmask, insert, base);
2138
2139       } else {
2140          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2141          nir_print_instr(&instr->instr, stderr);
2142          fprintf(stderr, "\n");
2143       }
2144       break;
2145    }
2146    case nir_op_ubfe:
2147    case nir_op_ibfe: {
2148       Temp base = get_alu_src(ctx, instr->src[0]);
2149       Temp offset = get_alu_src(ctx, instr->src[1]);
2150       Temp bits = get_alu_src(ctx, instr->src[2]);
2151
2152       if (dst.type() == RegType::sgpr) {
2153          Operand extract;
2154          nir_const_value* const_offset = nir_src_as_const_value(instr->src[1].src);
2155          nir_const_value* const_bits = nir_src_as_const_value(instr->src[2].src);
2156          if (const_offset && const_bits) {
2157             uint32_t const_extract = (const_bits->u32 << 16) | const_offset->u32;
2158             extract = Operand(const_extract);
2159          } else {
2160             Operand width;
2161             if (const_bits) {
2162                width = Operand(const_bits->u32 << 16);
2163             } else {
2164                width = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), bits, Operand(16u));
2165             }
2166             extract = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), offset, width);
2167          }
2168
2169          aco_opcode opcode;
2170          if (dst.regClass() == s1) {
2171             if (instr->op == nir_op_ubfe)
2172                opcode = aco_opcode::s_bfe_u32;
2173             else
2174                opcode = aco_opcode::s_bfe_i32;
2175          } else if (dst.regClass() == s2) {
2176             if (instr->op == nir_op_ubfe)
2177                opcode = aco_opcode::s_bfe_u64;
2178             else
2179                opcode = aco_opcode::s_bfe_i64;
2180          } else {
2181             unreachable("Unsupported BFE bit size");
2182          }
2183
2184          bld.sop2(opcode, Definition(dst), bld.def(s1, scc), base, extract);
2185
2186       } else {
2187          aco_opcode opcode;
2188          if (dst.regClass() == v1) {
2189             if (instr->op == nir_op_ubfe)
2190                opcode = aco_opcode::v_bfe_u32;
2191             else
2192                opcode = aco_opcode::v_bfe_i32;
2193          } else {
2194             unreachable("Unsupported BFE bit size");
2195          }
2196
2197          emit_vop3a_instruction(ctx, instr, opcode, dst);
2198       }
2199       break;
2200    }
2201    case nir_op_bit_count: {
2202       Temp src = get_alu_src(ctx, instr->src[0]);
2203       if (src.regClass() == s1) {
2204          bld.sop1(aco_opcode::s_bcnt1_i32_b32, Definition(dst), bld.def(s1, scc), src);
2205       } else if (src.regClass() == v1) {
2206          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst), src, Operand(0u));
2207       } else if (src.regClass() == v2) {
2208          bld.vop3(aco_opcode::v_bcnt_u32_b32, Definition(dst),
2209                   emit_extract_vector(ctx, src, 1, v1),
2210                   bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1),
2211                            emit_extract_vector(ctx, src, 0, v1), Operand(0u)));
2212       } else if (src.regClass() == s2) {
2213          bld.sop1(aco_opcode::s_bcnt1_i32_b64, Definition(dst), bld.def(s1, scc), src);
2214       } else {
2215          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2216          nir_print_instr(&instr->instr, stderr);
2217          fprintf(stderr, "\n");
2218       }
2219       break;
2220    }
2221    case nir_op_flt: {
2222       if (instr->src[0].src.ssa->bit_size == 32)
2223          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f32, dst);
2224       else if (instr->src[0].src.ssa->bit_size == 64)
2225          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_f64, dst);
2226       break;
2227    }
2228    case nir_op_fge: {
2229       if (instr->src[0].src.ssa->bit_size == 32)
2230          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f32, dst);
2231       else if (instr->src[0].src.ssa->bit_size == 64)
2232          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_f64, dst);
2233       break;
2234    }
2235    case nir_op_feq: {
2236       if (instr->src[0].src.ssa->bit_size == 32)
2237          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f32, dst);
2238       else if (instr->src[0].src.ssa->bit_size == 64)
2239          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_f64, dst);
2240       break;
2241    }
2242    case nir_op_fne: {
2243       if (instr->src[0].src.ssa->bit_size == 32)
2244          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f32, dst);
2245       else if (instr->src[0].src.ssa->bit_size == 64)
2246          emit_comparison(ctx, instr, aco_opcode::v_cmp_neq_f64, dst);
2247       break;
2248    }
2249    case nir_op_ilt: {
2250       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2251          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i32, dst);
2252       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2253          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_i32, dst);
2254       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2255          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_i64, dst);
2256       break;
2257    }
2258    case nir_op_ige: {
2259       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2260          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i32, dst);
2261       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2262          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_i32, dst);
2263       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2264          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_i64, dst);
2265       break;
2266    }
2267    case nir_op_ieq: {
2268       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2269          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i32, dst);
2270       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2271          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_i32, dst);
2272       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2273          emit_comparison(ctx, instr, aco_opcode::v_cmp_eq_i64, dst);
2274       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2275          emit_comparison(ctx, instr, aco_opcode::s_cmp_eq_u64, dst);
2276       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2277          Temp src0 = get_alu_src(ctx, instr->src[0]);
2278          Temp src1 = get_alu_src(ctx, instr->src[1]);
2279          bld.sopc(aco_opcode::s_cmp_eq_i32, bld.scc(Definition(dst)),
2280                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2281       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2282          Temp src0 = get_alu_src(ctx, instr->src[0]);
2283          Temp src1 = get_alu_src(ctx, instr->src[1]);
2284          bld.sop2(aco_opcode::s_xnor_b64, Definition(dst), bld.def(s1, scc),
2285                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2286       } else {
2287          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2288          nir_print_instr(&instr->instr, stderr);
2289          fprintf(stderr, "\n");
2290       }
2291       break;
2292    }
2293    case nir_op_ine: {
2294       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32) {
2295          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i32, dst);
2296       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64) {
2297          emit_comparison(ctx, instr, aco_opcode::v_cmp_lg_i64, dst);
2298       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32) {
2299          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_i32, dst);
2300       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 64) {
2301          emit_comparison(ctx, instr, aco_opcode::s_cmp_lg_u64, dst);
2302       } else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 1) {
2303          Temp src0 = get_alu_src(ctx, instr->src[0]);
2304          Temp src1 = get_alu_src(ctx, instr->src[1]);
2305          bld.sopc(aco_opcode::s_cmp_lg_i32, bld.scc(Definition(dst)),
2306                   as_uniform_bool(ctx, src0), as_uniform_bool(ctx, src1));
2307       } else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 1) {
2308          Temp src0 = get_alu_src(ctx, instr->src[0]);
2309          Temp src1 = get_alu_src(ctx, instr->src[1]);
2310          bld.sop2(aco_opcode::s_xor_b64, Definition(dst), bld.def(s1, scc),
2311                   as_divergent_bool(ctx, src0, false), as_divergent_bool(ctx, src1, false));
2312       } else {
2313          fprintf(stderr, "Unimplemented NIR instr bit size: ");
2314          nir_print_instr(&instr->instr, stderr);
2315          fprintf(stderr, "\n");
2316       }
2317       break;
2318    }
2319    case nir_op_ult: {
2320       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2321          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u32, dst);
2322       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2323          emit_comparison(ctx, instr, aco_opcode::s_cmp_lt_u32, dst);
2324       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2325          emit_comparison(ctx, instr, aco_opcode::v_cmp_lt_u64, dst);
2326       break;
2327    }
2328    case nir_op_uge: {
2329       if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 32)
2330          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u32, dst);
2331       else if (dst.regClass() == s1 && instr->src[0].src.ssa->bit_size == 32)
2332          emit_comparison(ctx, instr, aco_opcode::s_cmp_ge_u32, dst);
2333       else if (dst.regClass() == s2 && instr->src[0].src.ssa->bit_size == 64)
2334          emit_comparison(ctx, instr, aco_opcode::v_cmp_ge_u64, dst);
2335       break;
2336    }
2337    case nir_op_fddx:
2338    case nir_op_fddy:
2339    case nir_op_fddx_fine:
2340    case nir_op_fddy_fine:
2341    case nir_op_fddx_coarse:
2342    case nir_op_fddy_coarse: {
2343       Definition tl = bld.def(v1);
2344       uint16_t dpp_ctrl;
2345       if (instr->op == nir_op_fddx_fine) {
2346          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 2, 2));
2347          dpp_ctrl = dpp_quad_perm(1, 1, 3, 3);
2348       } else if (instr->op == nir_op_fddy_fine) {
2349          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 1, 0, 1));
2350          dpp_ctrl = dpp_quad_perm(2, 3, 2, 3);
2351       } else {
2352          bld.vop1_dpp(aco_opcode::v_mov_b32, tl, get_alu_src(ctx, instr->src[0]), dpp_quad_perm(0, 0, 0, 0));
2353          if (instr->op == nir_op_fddx || instr->op == nir_op_fddx_coarse)
2354             dpp_ctrl = dpp_quad_perm(1, 1, 1, 1);
2355          else
2356             dpp_ctrl = dpp_quad_perm(2, 2, 2, 2);
2357       }
2358
2359       Definition tmp = bld.def(v1);
2360       bld.vop2_dpp(aco_opcode::v_sub_f32, tmp, get_alu_src(ctx, instr->src[0]), tl.getTemp(), dpp_ctrl);
2361       emit_wqm(ctx, tmp.getTemp(), dst, true);
2362       break;
2363    }
2364    default:
2365       fprintf(stderr, "Unknown NIR ALU instr: ");
2366       nir_print_instr(&instr->instr, stderr);
2367       fprintf(stderr, "\n");
2368    }
2369 }
2370
2371 void visit_load_const(isel_context *ctx, nir_load_const_instr *instr)
2372 {
2373    Temp dst = get_ssa_temp(ctx, &instr->def);
2374
2375    // TODO: we really want to have the resulting type as this would allow for 64bit literals
2376    // which get truncated the lsb if double and msb if int
2377    // for now, we only use s_mov_b64 with 64bit inline constants
2378    assert(instr->def.num_components == 1 && "Vector load_const should be lowered to scalar.");
2379    assert(dst.type() == RegType::sgpr);
2380
2381    if (dst.size() == 1)
2382    {
2383       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(instr->value[0].u32));
2384    } else {
2385       assert(dst.size() != 1);
2386       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2387       if (instr->def.bit_size == 64)
2388          for (unsigned i = 0; i < dst.size(); i++)
2389             vec->operands[i] = Operand{(uint32_t)(instr->value[0].u64 >> i * 32)};
2390       else {
2391          for (unsigned i = 0; i < dst.size(); i++)
2392             vec->operands[i] = Operand{instr->value[i].u32};
2393       }
2394       vec->definitions[0] = Definition(dst);
2395       ctx->block->instructions.emplace_back(std::move(vec));
2396    }
2397 }
2398
2399 uint32_t widen_mask(uint32_t mask, unsigned multiplier)
2400 {
2401    uint32_t new_mask = 0;
2402    for(unsigned i = 0; i < 32 && (1u << i) <= mask; ++i)
2403       if (mask & (1u << i))
2404          new_mask |= ((1u << multiplier) - 1u) << (i * multiplier);
2405    return new_mask;
2406 }
2407
2408 void visit_store_vs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2409 {
2410    /* This wouldn't work inside control flow or with indirect offsets but
2411     * that doesn't happen because of nir_lower_io_to_temporaries(). */
2412
2413    unsigned write_mask = nir_intrinsic_write_mask(instr);
2414    unsigned component = nir_intrinsic_component(instr);
2415    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2416    unsigned idx = nir_intrinsic_base(instr) + component;
2417
2418    nir_instr *off_instr = instr->src[1].ssa->parent_instr;
2419    if (off_instr->type != nir_instr_type_load_const) {
2420       fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2421       nir_print_instr(off_instr, stderr);
2422       fprintf(stderr, "\n");
2423    }
2424    idx += nir_instr_as_load_const(off_instr)->value[0].u32 * 4u;
2425
2426    if (instr->src[0].ssa->bit_size == 64)
2427       write_mask = widen_mask(write_mask, 2);
2428
2429    for (unsigned i = 0; i < 8; ++i) {
2430       if (write_mask & (1 << i)) {
2431          ctx->vs_output.mask[idx / 4u] |= 1 << (idx % 4u);
2432          ctx->vs_output.outputs[idx / 4u][idx % 4u] = emit_extract_vector(ctx, src, i, v1);
2433       }
2434       idx++;
2435    }
2436 }
2437
2438 void visit_store_fs_output(isel_context *ctx, nir_intrinsic_instr *instr)
2439 {
2440    unsigned write_mask = nir_intrinsic_write_mask(instr);
2441    Operand values[4];
2442    Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
2443    for (unsigned i = 0; i < 4; ++i) {
2444       if (write_mask & (1 << i)) {
2445          Temp tmp = emit_extract_vector(ctx, src, i, v1);
2446          values[i] = Operand(tmp);
2447       } else {
2448          values[i] = Operand(v1);
2449       }
2450    }
2451
2452    unsigned index = nir_intrinsic_base(instr) / 4;
2453    unsigned target, col_format;
2454    unsigned enabled_channels = 0xF;
2455    aco_opcode compr_op = (aco_opcode)0;
2456
2457    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2458    assert(offset && "Non-const offsets on exports not yet supported");
2459    index += offset->u32;
2460
2461    assert(index != FRAG_RESULT_COLOR);
2462
2463    /* Unlike vertex shader exports, it's fine to use multiple exports to
2464     * export separate channels of one target. So shaders which export both
2465     * FRAG_RESULT_SAMPLE_MASK and FRAG_RESULT_DEPTH should work fine.
2466     * TODO: combine the exports in those cases and create better code
2467     */
2468
2469    if (index == FRAG_RESULT_SAMPLE_MASK) {
2470
2471       if (ctx->program->info->ps.writes_z) {
2472          target = V_008DFC_SQ_EXP_MRTZ;
2473          enabled_channels = 0x4;
2474          col_format = (unsigned) -1;
2475
2476          values[2] = values[0];
2477          values[0] = Operand(v1);
2478       } else {
2479          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2480          exp->valid_mask = false;
2481          exp->done = false;
2482          exp->compressed = true;
2483          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2484          exp->enabled_mask = 0xc;
2485          for (int i = 0; i < 4; i++)
2486             exp->operands[i] = Operand(v1);
2487          exp->operands[1] = Operand(values[0]);
2488          ctx->block->instructions.emplace_back(std::move(exp));
2489          return;
2490       }
2491
2492    } else if (index == FRAG_RESULT_DEPTH) {
2493
2494       target = V_008DFC_SQ_EXP_MRTZ;
2495       enabled_channels = 0x1;
2496       col_format = (unsigned) -1;
2497
2498    } else if (index == FRAG_RESULT_STENCIL) {
2499
2500       if (ctx->program->info->ps.writes_z) {
2501          target = V_008DFC_SQ_EXP_MRTZ;
2502          enabled_channels = 0x2;
2503          col_format = (unsigned) -1;
2504
2505          values[1] = values[0];
2506          values[0] = Operand(v1);
2507       } else {
2508          aco_ptr<Instruction> shift{create_instruction<VOP2_instruction>(aco_opcode::v_lshlrev_b32, Format::VOP2, 2, 1)};
2509          shift->operands[0] = Operand((uint32_t) 16);
2510          shift->operands[1] = values[0];
2511          Temp tmp = {ctx->program->allocateId(), v1};
2512          shift->definitions[0] = Definition(tmp);
2513          ctx->block->instructions.emplace_back(std::move(shift));
2514
2515          aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2516          exp->valid_mask = false;
2517          exp->done = false;
2518          exp->compressed = true;
2519          exp->dest = V_008DFC_SQ_EXP_MRTZ;
2520          exp->enabled_mask = 0x3;
2521          exp->operands[0] = Operand(tmp);
2522          for (int i = 1; i < 4; i++)
2523             exp->operands[i] = Operand(v1);
2524          ctx->block->instructions.emplace_back(std::move(exp));
2525          return;
2526       }
2527
2528    } else {
2529       index -= FRAG_RESULT_DATA0;
2530       target = V_008DFC_SQ_EXP_MRT + index;
2531       col_format = (ctx->options->key.fs.col_format >> (4 * index)) & 0xf;
2532    }
2533    ASSERTED bool is_int8 = (ctx->options->key.fs.is_int8 >> index) & 1;
2534    ASSERTED bool is_int10 = (ctx->options->key.fs.is_int10 >> index) & 1;
2535    assert(!is_int8 && !is_int10);
2536
2537    switch (col_format)
2538    {
2539    case V_028714_SPI_SHADER_ZERO:
2540       enabled_channels = 0; /* writemask */
2541       target = V_008DFC_SQ_EXP_NULL;
2542       break;
2543
2544    case V_028714_SPI_SHADER_32_R:
2545       enabled_channels = 1;
2546       break;
2547
2548    case V_028714_SPI_SHADER_32_GR:
2549       enabled_channels = 0x3;
2550       break;
2551
2552    case V_028714_SPI_SHADER_32_AR:
2553       enabled_channels = 0x9;
2554       break;
2555
2556    case V_028714_SPI_SHADER_FP16_ABGR:
2557       enabled_channels = 0x5;
2558       compr_op = aco_opcode::v_cvt_pkrtz_f16_f32;
2559       break;
2560
2561    case V_028714_SPI_SHADER_UNORM16_ABGR:
2562       enabled_channels = 0x5;
2563       compr_op = aco_opcode::v_cvt_pknorm_u16_f32;
2564       break;
2565
2566    case V_028714_SPI_SHADER_SNORM16_ABGR:
2567       enabled_channels = 0x5;
2568       compr_op = aco_opcode::v_cvt_pknorm_i16_f32;
2569       break;
2570
2571    case V_028714_SPI_SHADER_UINT16_ABGR:
2572       enabled_channels = 0x5;
2573       compr_op = aco_opcode::v_cvt_pk_u16_u32;
2574       break;
2575
2576    case V_028714_SPI_SHADER_SINT16_ABGR:
2577       enabled_channels = 0x5;
2578       compr_op = aco_opcode::v_cvt_pk_i16_i32;
2579       break;
2580
2581    case V_028714_SPI_SHADER_32_ABGR:
2582       enabled_channels = 0xF;
2583       break;
2584
2585    default:
2586       break;
2587    }
2588
2589    if (target == V_008DFC_SQ_EXP_NULL)
2590       return;
2591
2592    if ((bool)compr_op)
2593    {
2594       for (int i = 0; i < 2; i++)
2595       {
2596          /* check if at least one of the values to be compressed is enabled */
2597          unsigned enabled = (write_mask >> (i*2) | write_mask >> (i*2+1)) & 0x1;
2598          if (enabled) {
2599             enabled_channels |= enabled << (i*2);
2600             aco_ptr<VOP3A_instruction> compr{create_instruction<VOP3A_instruction>(compr_op, Format::VOP3A, 2, 1)};
2601             Temp tmp{ctx->program->allocateId(), v1};
2602             compr->operands[0] = values[i*2].isUndefined() ? Operand(0u) : values[i*2];
2603             compr->operands[1] = values[i*2+1].isUndefined() ? Operand(0u): values[i*2+1];
2604             compr->definitions[0] = Definition(tmp);
2605             values[i] = Operand(tmp);
2606             ctx->block->instructions.emplace_back(std::move(compr));
2607          } else {
2608             values[i] = Operand(v1);
2609          }
2610       }
2611    }
2612
2613    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
2614    exp->valid_mask = false;
2615    exp->done = false;
2616    exp->compressed = (bool) compr_op;
2617    exp->dest = target;
2618    exp->enabled_mask = enabled_channels;
2619    if ((bool) compr_op) {
2620       for (int i = 0; i < 2; i++)
2621          exp->operands[i] = enabled_channels & (3 << (i * 2)) ? values[i] : Operand(v1);
2622       exp->operands[2] = Operand(v1);
2623       exp->operands[3] = Operand(v1);
2624    } else {
2625       for (int i = 0; i < 4; i++)
2626          exp->operands[i] = enabled_channels & (1 << i) ? values[i] : Operand(v1);
2627    }
2628
2629    ctx->block->instructions.emplace_back(std::move(exp));
2630 }
2631
2632 void visit_store_output(isel_context *ctx, nir_intrinsic_instr *instr)
2633 {
2634    if (ctx->stage == vertex_vs) {
2635       visit_store_vs_output(ctx, instr);
2636    } else if (ctx->stage == fragment_fs) {
2637       visit_store_fs_output(ctx, instr);
2638    } else {
2639       unreachable("Shader stage not implemented");
2640    }
2641 }
2642
2643 void emit_interp_instr(isel_context *ctx, unsigned idx, unsigned component, Temp src, Temp dst, Temp prim_mask)
2644 {
2645    Temp coord1 = emit_extract_vector(ctx, src, 0, v1);
2646    Temp coord2 = emit_extract_vector(ctx, src, 1, v1);
2647
2648    Builder bld(ctx->program, ctx->block);
2649    Temp tmp = bld.vintrp(aco_opcode::v_interp_p1_f32, bld.def(v1), coord1, bld.m0(prim_mask), idx, component);
2650    bld.vintrp(aco_opcode::v_interp_p2_f32, Definition(dst), coord2, bld.m0(prim_mask), tmp, idx, component);
2651 }
2652
2653 void emit_load_frag_coord(isel_context *ctx, Temp dst, unsigned num_components)
2654 {
2655    aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1));
2656    for (unsigned i = 0; i < num_components; i++)
2657       vec->operands[i] = Operand(ctx->fs_inputs[fs_input::frag_pos_0 + i]);
2658
2659    if (ctx->fs_vgpr_args[fs_input::frag_pos_3]) {
2660       assert(num_components == 4);
2661       Builder bld(ctx->program, ctx->block);
2662       vec->operands[3] = bld.vop1(aco_opcode::v_rcp_f32, bld.def(v1), ctx->fs_inputs[fs_input::frag_pos_3]);
2663    }
2664
2665    for (Operand& op : vec->operands)
2666       op = op.isUndefined() ? Operand(0u) : op;
2667
2668    vec->definitions[0] = Definition(dst);
2669    ctx->block->instructions.emplace_back(std::move(vec));
2670    emit_split_vector(ctx, dst, num_components);
2671    return;
2672 }
2673
2674 void visit_load_interpolated_input(isel_context *ctx, nir_intrinsic_instr *instr)
2675 {
2676    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2677    Temp coords = get_ssa_temp(ctx, instr->src[0].ssa);
2678    unsigned idx = nir_intrinsic_base(instr);
2679    unsigned component = nir_intrinsic_component(instr);
2680    Temp prim_mask = ctx->prim_mask;
2681
2682    nir_const_value* offset = nir_src_as_const_value(instr->src[1]);
2683    if (offset) {
2684       assert(offset->u32 == 0);
2685    } else {
2686       /* the lower 15bit of the prim_mask contain the offset into LDS
2687        * while the upper bits contain the number of prims */
2688       Temp offset_src = get_ssa_temp(ctx, instr->src[1].ssa);
2689       assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2690       Builder bld(ctx->program, ctx->block);
2691       Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2692       stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2693       stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2694       offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2695       prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2696    }
2697
2698    if (instr->dest.ssa.num_components == 1) {
2699       emit_interp_instr(ctx, idx, component, coords, dst, prim_mask);
2700    } else {
2701       aco_ptr<Pseudo_instruction> vec(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, instr->dest.ssa.num_components, 1));
2702       for (unsigned i = 0; i < instr->dest.ssa.num_components; i++)
2703       {
2704          Temp tmp = {ctx->program->allocateId(), v1};
2705          emit_interp_instr(ctx, idx, component+i, coords, tmp, prim_mask);
2706          vec->operands[i] = Operand(tmp);
2707       }
2708       vec->definitions[0] = Definition(dst);
2709       ctx->block->instructions.emplace_back(std::move(vec));
2710    }
2711 }
2712
2713 unsigned get_num_channels_from_data_format(unsigned data_format)
2714 {
2715    switch (data_format) {
2716    case V_008F0C_BUF_DATA_FORMAT_8:
2717    case V_008F0C_BUF_DATA_FORMAT_16:
2718    case V_008F0C_BUF_DATA_FORMAT_32:
2719       return 1;
2720    case V_008F0C_BUF_DATA_FORMAT_8_8:
2721    case V_008F0C_BUF_DATA_FORMAT_16_16:
2722    case V_008F0C_BUF_DATA_FORMAT_32_32:
2723       return 2;
2724    case V_008F0C_BUF_DATA_FORMAT_10_11_11:
2725    case V_008F0C_BUF_DATA_FORMAT_11_11_10:
2726    case V_008F0C_BUF_DATA_FORMAT_32_32_32:
2727       return 3;
2728    case V_008F0C_BUF_DATA_FORMAT_8_8_8_8:
2729    case V_008F0C_BUF_DATA_FORMAT_10_10_10_2:
2730    case V_008F0C_BUF_DATA_FORMAT_2_10_10_10:
2731    case V_008F0C_BUF_DATA_FORMAT_16_16_16_16:
2732    case V_008F0C_BUF_DATA_FORMAT_32_32_32_32:
2733       return 4;
2734    default:
2735       break;
2736    }
2737
2738    return 4;
2739 }
2740
2741 /* For 2_10_10_10 formats the alpha is handled as unsigned by pre-vega HW.
2742  * so we may need to fix it up. */
2743 Temp adjust_vertex_fetch_alpha(isel_context *ctx, unsigned adjustment, Temp alpha)
2744 {
2745    Builder bld(ctx->program, ctx->block);
2746
2747    if (adjustment == RADV_ALPHA_ADJUST_SSCALED)
2748       alpha = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), alpha);
2749
2750    /* For the integer-like cases, do a natural sign extension.
2751     *
2752     * For the SNORM case, the values are 0.0, 0.333, 0.666, 1.0
2753     * and happen to contain 0, 1, 2, 3 as the two LSBs of the
2754     * exponent.
2755     */
2756    alpha = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(adjustment == RADV_ALPHA_ADJUST_SNORM ? 7u : 30u), alpha);
2757    alpha = bld.vop2(aco_opcode::v_ashrrev_i32, bld.def(v1), Operand(30u), alpha);
2758
2759    /* Convert back to the right type. */
2760    if (adjustment == RADV_ALPHA_ADJUST_SNORM) {
2761       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2762       Temp clamp = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0xbf800000u), alpha);
2763       alpha = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0xbf800000u), alpha, clamp);
2764    } else if (adjustment == RADV_ALPHA_ADJUST_SSCALED) {
2765       alpha = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), alpha);
2766    }
2767
2768    return alpha;
2769 }
2770
2771 void visit_load_input(isel_context *ctx, nir_intrinsic_instr *instr)
2772 {
2773    Builder bld(ctx->program, ctx->block);
2774    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
2775    if (ctx->stage & sw_vs) {
2776
2777       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2778       if (off_instr->type != nir_instr_type_load_const) {
2779          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2780          nir_print_instr(off_instr, stderr);
2781          fprintf(stderr, "\n");
2782       }
2783       uint32_t offset = nir_instr_as_load_const(off_instr)->value[0].u32;
2784
2785       Temp vertex_buffers = convert_pointer_to_64_bit(ctx, ctx->vertex_buffers);
2786
2787       unsigned location = nir_intrinsic_base(instr) / 4 - VERT_ATTRIB_GENERIC0 + offset;
2788       unsigned component = nir_intrinsic_component(instr);
2789       unsigned attrib_binding = ctx->options->key.vs.vertex_attribute_bindings[location];
2790       uint32_t attrib_offset = ctx->options->key.vs.vertex_attribute_offsets[location];
2791       uint32_t attrib_stride = ctx->options->key.vs.vertex_attribute_strides[location];
2792       unsigned attrib_format = ctx->options->key.vs.vertex_attribute_formats[location];
2793
2794       unsigned dfmt = attrib_format & 0xf;
2795
2796       unsigned nfmt = (attrib_format >> 4) & 0x7;
2797       unsigned num_dfmt_channels = get_num_channels_from_data_format(dfmt);
2798       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa) << component;
2799       unsigned num_channels = MIN2(util_last_bit(mask), num_dfmt_channels);
2800       unsigned alpha_adjust = (ctx->options->key.vs.alpha_adjust >> (location * 2)) & 3;
2801       bool post_shuffle = ctx->options->key.vs.post_shuffle & (1 << location);
2802       if (post_shuffle)
2803          num_channels = MAX2(num_channels, 3);
2804
2805       Temp list = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), vertex_buffers, Operand(attrib_binding * 16u));
2806
2807       Temp index;
2808       if (ctx->options->key.vs.instance_rate_inputs & (1u << location)) {
2809          uint32_t divisor = ctx->options->key.vs.instance_rate_divisors[location];
2810          if (divisor) {
2811             ctx->needs_instance_id = true;
2812
2813             if (divisor != 1) {
2814                Temp divided = bld.tmp(v1);
2815                emit_v_div_u32(ctx, divided, as_vgpr(ctx, ctx->instance_id), divisor);
2816                index = bld.vadd32(bld.def(v1), ctx->start_instance, divided);
2817             } else {
2818                index = bld.vadd32(bld.def(v1), ctx->start_instance, ctx->instance_id);
2819             }
2820          } else {
2821             index = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), ctx->start_instance);
2822          }
2823       } else {
2824          index = bld.vadd32(bld.def(v1), ctx->base_vertex, ctx->vertex_id);
2825       }
2826
2827       if (attrib_stride != 0 && attrib_offset > attrib_stride) {
2828          index = bld.vadd32(bld.def(v1), Operand(attrib_offset / attrib_stride), index);
2829          attrib_offset = attrib_offset % attrib_stride;
2830       }
2831
2832       Operand soffset(0u);
2833       if (attrib_offset >= 4096) {
2834          soffset = bld.copy(bld.def(s1), Operand(attrib_offset));
2835          attrib_offset = 0;
2836       }
2837
2838       aco_opcode opcode;
2839       switch (num_channels) {
2840       case 1:
2841          opcode = aco_opcode::tbuffer_load_format_x;
2842          break;
2843       case 2:
2844          opcode = aco_opcode::tbuffer_load_format_xy;
2845          break;
2846       case 3:
2847          opcode = aco_opcode::tbuffer_load_format_xyz;
2848          break;
2849       case 4:
2850          opcode = aco_opcode::tbuffer_load_format_xyzw;
2851          break;
2852       default:
2853          unreachable("Unimplemented load_input vector size");
2854       }
2855
2856       Temp tmp = post_shuffle || num_channels != dst.size() || alpha_adjust != RADV_ALPHA_ADJUST_NONE || component ? bld.tmp(RegType::vgpr, num_channels) : dst;
2857
2858       aco_ptr<MTBUF_instruction> mubuf{create_instruction<MTBUF_instruction>(opcode, Format::MTBUF, 3, 1)};
2859       mubuf->operands[0] = Operand(index);
2860       mubuf->operands[1] = Operand(list);
2861       mubuf->operands[2] = soffset;
2862       mubuf->definitions[0] = Definition(tmp);
2863       mubuf->idxen = true;
2864       mubuf->can_reorder = true;
2865       mubuf->dfmt = dfmt;
2866       mubuf->nfmt = nfmt;
2867       assert(attrib_offset < 4096);
2868       mubuf->offset = attrib_offset;
2869       ctx->block->instructions.emplace_back(std::move(mubuf));
2870
2871       emit_split_vector(ctx, tmp, tmp.size());
2872
2873       if (tmp.id() != dst.id()) {
2874          bool is_float = nfmt != V_008F0C_BUF_NUM_FORMAT_UINT &&
2875                          nfmt != V_008F0C_BUF_NUM_FORMAT_SINT;
2876
2877          static const unsigned swizzle_normal[4] = {0, 1, 2, 3};
2878          static const unsigned swizzle_post_shuffle[4] = {2, 1, 0, 3};
2879          const unsigned *swizzle = post_shuffle ? swizzle_post_shuffle : swizzle_normal;
2880
2881          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2882          for (unsigned i = 0; i < dst.size(); i++) {
2883             unsigned idx = i + component;
2884             if (idx == 3 && alpha_adjust != RADV_ALPHA_ADJUST_NONE && num_channels >= 4) {
2885                Temp alpha = emit_extract_vector(ctx, tmp, swizzle[3], v1);
2886                vec->operands[3] = Operand(adjust_vertex_fetch_alpha(ctx, alpha_adjust, alpha));
2887             } else if (idx < num_channels) {
2888                vec->operands[i] = Operand(emit_extract_vector(ctx, tmp, swizzle[idx], v1));
2889             } else if (is_float && idx == 3) {
2890                vec->operands[i] = Operand(0x3f800000u);
2891             } else if (!is_float && idx == 3) {
2892                vec->operands[i] = Operand(1u);
2893             } else {
2894                vec->operands[i] = Operand(0u);
2895             }
2896          }
2897          vec->definitions[0] = Definition(dst);
2898          ctx->block->instructions.emplace_back(std::move(vec));
2899          emit_split_vector(ctx, dst, dst.size());
2900       }
2901
2902    } else if (ctx->stage == fragment_fs) {
2903       nir_instr *off_instr = instr->src[0].ssa->parent_instr;
2904       if (off_instr->type != nir_instr_type_load_const ||
2905           nir_instr_as_load_const(off_instr)->value[0].u32 != 0) {
2906          fprintf(stderr, "Unimplemented nir_intrinsic_load_input offset\n");
2907          nir_print_instr(off_instr, stderr);
2908          fprintf(stderr, "\n");
2909       }
2910
2911       Temp prim_mask = ctx->prim_mask;
2912       nir_const_value* offset = nir_src_as_const_value(instr->src[0]);
2913       if (offset) {
2914          assert(offset->u32 == 0);
2915       } else {
2916          /* the lower 15bit of the prim_mask contain the offset into LDS
2917           * while the upper bits contain the number of prims */
2918          Temp offset_src = get_ssa_temp(ctx, instr->src[0].ssa);
2919          assert(offset_src.regClass() == s1 && "TODO: divergent offsets...");
2920          Builder bld(ctx->program, ctx->block);
2921          Temp stride = bld.sop2(aco_opcode::s_lshr_b32, bld.def(s1), bld.def(s1, scc), prim_mask, Operand(16u));
2922          stride = bld.sop1(aco_opcode::s_bcnt1_i32_b32, bld.def(s1), bld.def(s1, scc), stride);
2923          stride = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, Operand(48u));
2924          offset_src = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), stride, offset_src);
2925          prim_mask = bld.sop2(aco_opcode::s_add_i32, bld.def(s1, m0), bld.def(s1, scc), offset_src, prim_mask);
2926       }
2927
2928       unsigned idx = nir_intrinsic_base(instr);
2929       unsigned component = nir_intrinsic_component(instr);
2930
2931       if (dst.size() == 1) {
2932          bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(dst), Operand(2u), bld.m0(prim_mask), idx, component);
2933       } else {
2934          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
2935          for (unsigned i = 0; i < dst.size(); i++)
2936             vec->operands[i] = bld.vintrp(aco_opcode::v_interp_mov_f32, bld.def(v1), Operand(2u), bld.m0(prim_mask), idx, component + i);
2937          vec->definitions[0] = Definition(dst);
2938          bld.insert(std::move(vec));
2939       }
2940
2941    } else {
2942       unreachable("Shader stage not implemented");
2943    }
2944 }
2945
2946 Temp load_desc_ptr(isel_context *ctx, unsigned desc_set)
2947 {
2948    if (ctx->program->info->need_indirect_descriptor_sets) {
2949       Builder bld(ctx->program, ctx->block);
2950       Temp ptr64 = convert_pointer_to_64_bit(ctx, ctx->descriptor_sets[0]);
2951       return bld.smem(aco_opcode::s_load_dword, bld.def(s1), ptr64, Operand(desc_set << 2));//, false, false, false);
2952    }
2953
2954    return ctx->descriptor_sets[desc_set];
2955 }
2956
2957
2958 void visit_load_resource(isel_context *ctx, nir_intrinsic_instr *instr)
2959 {
2960    Builder bld(ctx->program, ctx->block);
2961    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
2962    unsigned desc_set = nir_intrinsic_desc_set(instr);
2963    unsigned binding = nir_intrinsic_binding(instr);
2964
2965    Temp desc_ptr;
2966    radv_pipeline_layout *pipeline_layout = ctx->options->layout;
2967    radv_descriptor_set_layout *layout = pipeline_layout->set[desc_set].layout;
2968    unsigned offset = layout->binding[binding].offset;
2969    unsigned stride;
2970    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC ||
2971        layout->binding[binding].type == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC) {
2972       unsigned idx = pipeline_layout->set[desc_set].dynamic_offset_start + layout->binding[binding].dynamic_offset_offset;
2973       desc_ptr = ctx->push_constants;
2974       offset = pipeline_layout->push_constant_size + 16 * idx;
2975       stride = 16;
2976    } else {
2977       desc_ptr = load_desc_ptr(ctx, desc_set);
2978       stride = layout->binding[binding].size;
2979    }
2980
2981    nir_const_value* nir_const_index = nir_src_as_const_value(instr->src[0]);
2982    unsigned const_index = nir_const_index ? nir_const_index->u32 : 0;
2983    if (stride != 1) {
2984       if (nir_const_index) {
2985          const_index = const_index * stride;
2986       } else {
2987          index = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), Operand(index));
2988       }
2989    }
2990    if (offset) {
2991       if (nir_const_index) {
2992          const_index = const_index + offset;
2993       } else {
2994          index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), Operand(index));
2995       }
2996    }
2997
2998    if (nir_const_index && const_index == 0) {
2999       index = desc_ptr;
3000    } else {
3001       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
3002                        nir_const_index ? Operand(const_index) : Operand(index),
3003                        Operand(desc_ptr));
3004    }
3005
3006    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3007    bld.sop1(aco_opcode::s_mov_b32, Definition(dst), index);
3008 }
3009
3010 void load_buffer(isel_context *ctx, unsigned num_components, Temp dst, Temp rsrc, Temp offset, bool glc=false)
3011 {
3012    Builder bld(ctx->program, ctx->block);
3013
3014    unsigned num_bytes = dst.size() * 4;
3015
3016    aco_opcode op;
3017    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
3018       if (ctx->options->chip_class < GFX8)
3019          offset = as_vgpr(ctx, offset);
3020
3021       Operand vaddr = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
3022       Operand soffset = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
3023       unsigned const_offset = 0;
3024
3025       Temp lower = Temp();
3026       if (num_bytes > 16) {
3027          assert(num_components == 3 || num_components == 4);
3028          op = aco_opcode::buffer_load_dwordx4;
3029          lower = bld.tmp(v4);
3030          aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3031          mubuf->definitions[0] = Definition(lower);
3032          mubuf->operands[0] = vaddr;
3033          mubuf->operands[1] = Operand(rsrc);
3034          mubuf->operands[2] = soffset;
3035          mubuf->offen = (offset.type() == RegType::vgpr);
3036          mubuf->glc = glc;
3037          mubuf->barrier = barrier_buffer;
3038          bld.insert(std::move(mubuf));
3039          emit_split_vector(ctx, lower, 2);
3040          num_bytes -= 16;
3041          const_offset = 16;
3042       }
3043
3044       switch (num_bytes) {
3045          case 4:
3046             op = aco_opcode::buffer_load_dword;
3047             break;
3048          case 8:
3049             op = aco_opcode::buffer_load_dwordx2;
3050             break;
3051          case 12:
3052             op = aco_opcode::buffer_load_dwordx3;
3053             break;
3054          case 16:
3055             op = aco_opcode::buffer_load_dwordx4;
3056             break;
3057          default:
3058             unreachable("Load SSBO not implemented for this size.");
3059       }
3060       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
3061       mubuf->operands[0] = vaddr;
3062       mubuf->operands[1] = Operand(rsrc);
3063       mubuf->operands[2] = soffset;
3064       mubuf->offen = (offset.type() == RegType::vgpr);
3065       mubuf->glc = glc;
3066       mubuf->barrier = barrier_buffer;
3067       mubuf->offset = const_offset;
3068       aco_ptr<Instruction> instr = std::move(mubuf);
3069
3070       if (dst.size() > 4) {
3071          assert(lower != Temp());
3072          Temp upper = bld.tmp(RegType::vgpr, dst.size() - lower.size());
3073          instr->definitions[0] = Definition(upper);
3074          bld.insert(std::move(instr));
3075          if (dst.size() == 8)
3076             emit_split_vector(ctx, upper, 2);
3077          instr.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size() / 2, 1));
3078          instr->operands[0] = Operand(emit_extract_vector(ctx, lower, 0, v2));
3079          instr->operands[1] = Operand(emit_extract_vector(ctx, lower, 1, v2));
3080          instr->operands[2] = Operand(emit_extract_vector(ctx, upper, 0, v2));
3081          if (dst.size() == 8)
3082             instr->operands[3] = Operand(emit_extract_vector(ctx, upper, 1, v2));
3083       }
3084
3085       if (dst.type() == RegType::sgpr) {
3086          Temp vec = bld.tmp(RegType::vgpr, dst.size());
3087          instr->definitions[0] = Definition(vec);
3088          bld.insert(std::move(instr));
3089          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
3090       } else {
3091          instr->definitions[0] = Definition(dst);
3092          bld.insert(std::move(instr));
3093       }
3094    } else {
3095       switch (num_bytes) {
3096          case 4:
3097             op = aco_opcode::s_buffer_load_dword;
3098             break;
3099          case 8:
3100             op = aco_opcode::s_buffer_load_dwordx2;
3101             break;
3102          case 12:
3103          case 16:
3104             op = aco_opcode::s_buffer_load_dwordx4;
3105             break;
3106          case 24:
3107          case 32:
3108             op = aco_opcode::s_buffer_load_dwordx8;
3109             break;
3110          default:
3111             unreachable("Load SSBO not implemented for this size.");
3112       }
3113       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
3114       load->operands[0] = Operand(rsrc);
3115       load->operands[1] = Operand(bld.as_uniform(offset));
3116       assert(load->operands[1].getTemp().type() == RegType::sgpr);
3117       load->definitions[0] = Definition(dst);
3118       load->glc = glc;
3119       load->barrier = barrier_buffer;
3120       assert(ctx->options->chip_class >= GFX8 || !glc);
3121
3122       /* trim vector */
3123       if (dst.size() == 3) {
3124          Temp vec = bld.tmp(s4);
3125          load->definitions[0] = Definition(vec);
3126          bld.insert(std::move(load));
3127          emit_split_vector(ctx, vec, 4);
3128
3129          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3130                     emit_extract_vector(ctx, vec, 0, s1),
3131                     emit_extract_vector(ctx, vec, 1, s1),
3132                     emit_extract_vector(ctx, vec, 2, s1));
3133       } else if (dst.size() == 6) {
3134          Temp vec = bld.tmp(s8);
3135          load->definitions[0] = Definition(vec);
3136          bld.insert(std::move(load));
3137          emit_split_vector(ctx, vec, 4);
3138
3139          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3140                     emit_extract_vector(ctx, vec, 0, s2),
3141                     emit_extract_vector(ctx, vec, 1, s2),
3142                     emit_extract_vector(ctx, vec, 2, s2));
3143       } else {
3144          bld.insert(std::move(load));
3145       }
3146
3147    }
3148    emit_split_vector(ctx, dst, num_components);
3149 }
3150
3151 void visit_load_ubo(isel_context *ctx, nir_intrinsic_instr *instr)
3152 {
3153    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3154    Temp rsrc = get_ssa_temp(ctx, instr->src[0].ssa);
3155
3156    Builder bld(ctx->program, ctx->block);
3157
3158    nir_intrinsic_instr* idx_instr = nir_instr_as_intrinsic(instr->src[0].ssa->parent_instr);
3159    unsigned desc_set = nir_intrinsic_desc_set(idx_instr);
3160    unsigned binding = nir_intrinsic_binding(idx_instr);
3161    radv_descriptor_set_layout *layout = ctx->options->layout->set[desc_set].layout;
3162
3163    if (layout->binding[binding].type == VK_DESCRIPTOR_TYPE_INLINE_UNIFORM_BLOCK_EXT) {
3164       uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3165                            S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3166                            S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3167                            S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W) |
3168                            S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3169                            S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3170       Temp upper_dwords = bld.pseudo(aco_opcode::p_create_vector, bld.def(s3),
3171                                      Operand(S_008F04_BASE_ADDRESS_HI(ctx->options->address32_hi)),
3172                                      Operand(0xFFFFFFFFu),
3173                                      Operand(desc_type));
3174       rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3175                         rsrc, upper_dwords);
3176    } else {
3177       rsrc = convert_pointer_to_64_bit(ctx, rsrc);
3178       rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
3179    }
3180
3181    load_buffer(ctx, instr->num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa));
3182 }
3183
3184 void visit_load_push_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3185 {
3186    Builder bld(ctx->program, ctx->block);
3187    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3188
3189    unsigned offset = nir_intrinsic_base(instr);
3190    nir_const_value *index_cv = nir_src_as_const_value(instr->src[0]);
3191    if (index_cv && instr->dest.ssa.bit_size == 32) {
3192
3193       unsigned count = instr->dest.ssa.num_components;
3194       unsigned start = (offset + index_cv->u32) / 4u;
3195       start -= ctx->base_inline_push_consts;
3196       if (start + count <= ctx->num_inline_push_consts) {
3197          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
3198          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
3199          for (unsigned i = 0; i < count; ++i) {
3200             elems[i] = ctx->inline_push_consts[start + i];
3201             vec->operands[i] = Operand{elems[i]};
3202          }
3203          vec->definitions[0] = Definition(dst);
3204          ctx->block->instructions.emplace_back(std::move(vec));
3205          ctx->allocated_vec.emplace(dst.id(), elems);
3206          return;
3207       }
3208    }
3209
3210    Temp index = bld.as_uniform(get_ssa_temp(ctx, instr->src[0].ssa));
3211    if (offset != 0) // TODO check if index != 0 as well
3212       index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset), index);
3213    Temp ptr = convert_pointer_to_64_bit(ctx, ctx->push_constants);
3214    Temp vec = dst;
3215    bool trim = false;
3216    aco_opcode op;
3217
3218    switch (dst.size()) {
3219    case 1:
3220       op = aco_opcode::s_load_dword;
3221       break;
3222    case 2:
3223       op = aco_opcode::s_load_dwordx2;
3224       break;
3225    case 3:
3226       vec = bld.tmp(s4);
3227       trim = true;
3228    case 4:
3229       op = aco_opcode::s_load_dwordx4;
3230       break;
3231    case 6:
3232       vec = bld.tmp(s8);
3233       trim = true;
3234    case 8:
3235       op = aco_opcode::s_load_dwordx8;
3236       break;
3237    default:
3238       unreachable("unimplemented or forbidden load_push_constant.");
3239    }
3240
3241    bld.smem(op, Definition(vec), ptr, index);
3242
3243    if (trim) {
3244       emit_split_vector(ctx, vec, 4);
3245       RegClass rc = dst.size() == 3 ? s1 : s2;
3246       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
3247                  emit_extract_vector(ctx, vec, 0, rc),
3248                  emit_extract_vector(ctx, vec, 1, rc),
3249                  emit_extract_vector(ctx, vec, 2, rc));
3250
3251    }
3252    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
3253 }
3254
3255 void visit_load_constant(isel_context *ctx, nir_intrinsic_instr *instr)
3256 {
3257    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3258
3259    Builder bld(ctx->program, ctx->block);
3260
3261    uint32_t desc_type = S_008F0C_DST_SEL_X(V_008F0C_SQ_SEL_X) |
3262                         S_008F0C_DST_SEL_Y(V_008F0C_SQ_SEL_Y) |
3263                         S_008F0C_DST_SEL_Z(V_008F0C_SQ_SEL_Z) |
3264                         S_008F0C_DST_SEL_W(V_008F0C_SQ_SEL_W);
3265    if (ctx->options->chip_class >= GFX10) {
3266       desc_type |= S_008F0C_FORMAT(V_008F0C_IMG_FORMAT_32_FLOAT) |
3267                    S_008F0C_OOB_SELECT(3) |
3268                    S_008F0C_RESOURCE_LEVEL(1);
3269    } else {
3270       desc_type |= S_008F0C_NUM_FORMAT(V_008F0C_BUF_NUM_FORMAT_FLOAT) |
3271                    S_008F0C_DATA_FORMAT(V_008F0C_BUF_DATA_FORMAT_32);
3272    }
3273
3274    unsigned base = nir_intrinsic_base(instr) + ctx->constant_data_offset;
3275    unsigned range = nir_intrinsic_range(instr);
3276
3277    Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
3278    if (base && offset.type() == RegType::sgpr)
3279       offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), offset, Operand(base));
3280    else if (base && offset.type() == RegType::vgpr)
3281       offset = bld.vadd32(bld.def(v1), Operand(base), offset);
3282
3283    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3284                           bld.sop1(aco_opcode::p_constaddr, bld.def(s2), bld.def(s1, scc), Operand(0u)),
3285                           Operand(MIN2(range, ctx->shader->constant_data_size - nir_intrinsic_base(instr))),
3286                           Operand(desc_type));
3287
3288    load_buffer(ctx, instr->num_components, dst, rsrc, offset);
3289 }
3290
3291 void visit_discard_if(isel_context *ctx, nir_intrinsic_instr *instr)
3292 {
3293    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3294       ctx->cf_info.exec_potentially_empty = true;
3295
3296    ctx->program->needs_exact = true;
3297
3298    Builder bld(ctx->program, ctx->block);
3299    Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
3300    src = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
3301    bld.pseudo(aco_opcode::p_discard_if, src);
3302    ctx->block->kind |= block_kind_uses_discard_if;
3303    return;
3304 }
3305
3306 void visit_discard(isel_context* ctx, nir_intrinsic_instr *instr)
3307 {
3308    Builder bld(ctx->program, ctx->block);
3309
3310    if (ctx->cf_info.loop_nest_depth || ctx->cf_info.parent_if.is_divergent)
3311       ctx->cf_info.exec_potentially_empty = true;
3312
3313    bool divergent = ctx->cf_info.parent_if.is_divergent ||
3314                     ctx->cf_info.parent_loop.has_divergent_continue;
3315
3316    if (ctx->block->loop_nest_depth &&
3317        ((nir_instr_is_last(&instr->instr) && !divergent) || divergent)) {
3318       /* we handle discards the same way as jump instructions */
3319       append_logical_end(ctx->block);
3320
3321       /* in loops, discard behaves like break */
3322       Block *linear_target = ctx->cf_info.parent_loop.exit;
3323       ctx->block->kind |= block_kind_discard;
3324
3325       if (!divergent) {
3326          /* uniform discard - loop ends here */
3327          assert(nir_instr_is_last(&instr->instr));
3328          ctx->block->kind |= block_kind_uniform;
3329          ctx->cf_info.has_branch = true;
3330          bld.branch(aco_opcode::p_branch);
3331          add_linear_edge(ctx->block->index, linear_target);
3332          return;
3333       }
3334
3335       /* we add a break right behind the discard() instructions */
3336       ctx->block->kind |= block_kind_break;
3337       unsigned idx = ctx->block->index;
3338
3339       /* remove critical edges from linear CFG */
3340       bld.branch(aco_opcode::p_branch);
3341       Block* break_block = ctx->program->create_and_insert_block();
3342       break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3343       break_block->kind |= block_kind_uniform;
3344       add_linear_edge(idx, break_block);
3345       add_linear_edge(break_block->index, linear_target);
3346       bld.reset(break_block);
3347       bld.branch(aco_opcode::p_branch);
3348
3349       Block* continue_block = ctx->program->create_and_insert_block();
3350       continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
3351       add_linear_edge(idx, continue_block);
3352       append_logical_start(continue_block);
3353       ctx->block = continue_block;
3354
3355       return;
3356    }
3357
3358    /* it can currently happen that NIR doesn't remove the unreachable code */
3359    if (!nir_instr_is_last(&instr->instr)) {
3360       ctx->program->needs_exact = true;
3361       /* save exec somewhere temporarily so that it doesn't get
3362        * overwritten before the discard from outer exec masks */
3363       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), Operand(0xFFFFFFFF), Operand(exec, s2));
3364       bld.pseudo(aco_opcode::p_discard_if, cond);
3365       ctx->block->kind |= block_kind_uses_discard_if;
3366       return;
3367    }
3368
3369    /* This condition is incorrect for uniformly branched discards in a loop
3370     * predicated by a divergent condition, but the above code catches that case
3371     * and the discard would end up turning into a discard_if.
3372     * For example:
3373     * if (divergent) {
3374     *    while (...) {
3375     *       if (uniform) {
3376     *          discard;
3377     *       }
3378     *    }
3379     * }
3380     */
3381    if (!ctx->cf_info.parent_if.is_divergent) {
3382       /* program just ends here */
3383       ctx->block->kind |= block_kind_uniform;
3384       bld.exp(aco_opcode::exp, Operand(v1), Operand(v1), Operand(v1), Operand(v1),
3385               0 /* enabled mask */, 9 /* dest */,
3386               false /* compressed */, true/* done */, true /* valid mask */);
3387       bld.sopp(aco_opcode::s_endpgm);
3388       // TODO: it will potentially be followed by a branch which is dead code to sanitize NIR phis
3389    } else {
3390       ctx->block->kind |= block_kind_discard;
3391       /* branch and linear edge is added by visit_if() */
3392    }
3393 }
3394
3395 enum aco_descriptor_type {
3396    ACO_DESC_IMAGE,
3397    ACO_DESC_FMASK,
3398    ACO_DESC_SAMPLER,
3399    ACO_DESC_BUFFER,
3400    ACO_DESC_PLANE_0,
3401    ACO_DESC_PLANE_1,
3402    ACO_DESC_PLANE_2,
3403 };
3404
3405 enum aco_image_dim {
3406    aco_image_1d,
3407    aco_image_2d,
3408    aco_image_3d,
3409    aco_image_cube, // includes cube arrays
3410    aco_image_1darray,
3411    aco_image_2darray,
3412    aco_image_2dmsaa,
3413    aco_image_2darraymsaa,
3414 };
3415
3416 static enum aco_image_dim
3417 get_sampler_dim(isel_context *ctx, enum glsl_sampler_dim dim, bool is_array)
3418 {
3419    switch (dim) {
3420    case GLSL_SAMPLER_DIM_1D:
3421       if (ctx->options->chip_class >= GFX9)
3422          return is_array ? aco_image_2darray : aco_image_2d;
3423       return is_array ? aco_image_1darray : aco_image_1d;
3424    case GLSL_SAMPLER_DIM_2D:
3425    case GLSL_SAMPLER_DIM_RECT:
3426    case GLSL_SAMPLER_DIM_EXTERNAL:
3427       return is_array ? aco_image_2darray : aco_image_2d;
3428    case GLSL_SAMPLER_DIM_3D:
3429       return aco_image_3d;
3430    case GLSL_SAMPLER_DIM_CUBE:
3431       return aco_image_cube;
3432    case GLSL_SAMPLER_DIM_MS:
3433       return is_array ? aco_image_2darraymsaa : aco_image_2dmsaa;
3434    case GLSL_SAMPLER_DIM_SUBPASS:
3435       return aco_image_2darray;
3436    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3437       return aco_image_2darraymsaa;
3438    default:
3439       unreachable("bad sampler dim");
3440    }
3441 }
3442
3443 static bool
3444 should_declare_array(isel_context *ctx, enum glsl_sampler_dim sampler_dim, bool is_array) {
3445    if (sampler_dim == GLSL_SAMPLER_DIM_BUF)
3446       return false;
3447    aco_image_dim dim = get_sampler_dim(ctx, sampler_dim, is_array);
3448    return dim == aco_image_cube ||
3449           dim == aco_image_1darray ||
3450           dim == aco_image_2darray ||
3451           dim == aco_image_2darraymsaa;
3452 }
3453
3454 Temp get_sampler_desc(isel_context *ctx, nir_deref_instr *deref_instr,
3455                       enum aco_descriptor_type desc_type,
3456                       const nir_tex_instr *tex_instr, bool image, bool write)
3457 {
3458 /* FIXME: we should lower the deref with some new nir_intrinsic_load_desc
3459    std::unordered_map<uint64_t, Temp>::iterator it = ctx->tex_desc.find((uint64_t) desc_type << 32 | deref_instr->dest.ssa.index);
3460    if (it != ctx->tex_desc.end())
3461       return it->second;
3462 */
3463    Temp index = Temp();
3464    bool index_set = false;
3465    unsigned constant_index = 0;
3466    unsigned descriptor_set;
3467    unsigned base_index;
3468    Builder bld(ctx->program, ctx->block);
3469
3470    if (!deref_instr) {
3471       assert(tex_instr && !image);
3472       descriptor_set = 0;
3473       base_index = tex_instr->sampler_index;
3474    } else {
3475       while(deref_instr->deref_type != nir_deref_type_var) {
3476          unsigned array_size = glsl_get_aoa_size(deref_instr->type);
3477          if (!array_size)
3478             array_size = 1;
3479
3480          assert(deref_instr->deref_type == nir_deref_type_array);
3481          nir_const_value *const_value = nir_src_as_const_value(deref_instr->arr.index);
3482          if (const_value) {
3483             constant_index += array_size * const_value->u32;
3484          } else {
3485             Temp indirect = bld.as_uniform(get_ssa_temp(ctx, deref_instr->arr.index.ssa));
3486
3487             if (array_size != 1)
3488                indirect = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(array_size), indirect);
3489
3490             if (!index_set) {
3491                index = indirect;
3492                index_set = true;
3493             } else {
3494                index = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), index, indirect);
3495             }
3496          }
3497
3498          deref_instr = nir_src_as_deref(deref_instr->parent);
3499       }
3500       descriptor_set = deref_instr->var->data.descriptor_set;
3501       base_index = deref_instr->var->data.binding;
3502    }
3503
3504    Temp list = load_desc_ptr(ctx, descriptor_set);
3505    list = convert_pointer_to_64_bit(ctx, list);
3506
3507    struct radv_descriptor_set_layout *layout = ctx->options->layout->set[descriptor_set].layout;
3508    struct radv_descriptor_set_binding_layout *binding = layout->binding + base_index;
3509    unsigned offset = binding->offset;
3510    unsigned stride = binding->size;
3511    aco_opcode opcode;
3512    RegClass type;
3513
3514    assert(base_index < layout->binding_count);
3515
3516    switch (desc_type) {
3517    case ACO_DESC_IMAGE:
3518       type = s8;
3519       opcode = aco_opcode::s_load_dwordx8;
3520       break;
3521    case ACO_DESC_FMASK:
3522       type = s8;
3523       opcode = aco_opcode::s_load_dwordx8;
3524       offset += 32;
3525       break;
3526    case ACO_DESC_SAMPLER:
3527       type = s4;
3528       opcode = aco_opcode::s_load_dwordx4;
3529       if (binding->type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
3530          offset += radv_combined_image_descriptor_sampler_offset(binding);
3531       break;
3532    case ACO_DESC_BUFFER:
3533       type = s4;
3534       opcode = aco_opcode::s_load_dwordx4;
3535       break;
3536    case ACO_DESC_PLANE_0:
3537    case ACO_DESC_PLANE_1:
3538       type = s8;
3539       opcode = aco_opcode::s_load_dwordx8;
3540       offset += 32 * (desc_type - ACO_DESC_PLANE_0);
3541       break;
3542    case ACO_DESC_PLANE_2:
3543       type = s4;
3544       opcode = aco_opcode::s_load_dwordx4;
3545       offset += 64;
3546       break;
3547    default:
3548       unreachable("invalid desc_type\n");
3549    }
3550
3551    offset += constant_index * stride;
3552
3553    if (desc_type == ACO_DESC_SAMPLER && binding->immutable_samplers_offset &&
3554       (!index_set || binding->immutable_samplers_equal)) {
3555       if (binding->immutable_samplers_equal)
3556          constant_index = 0;
3557
3558       const uint32_t *samplers = radv_immutable_samplers(layout, binding);
3559       return bld.pseudo(aco_opcode::p_create_vector, bld.def(s4),
3560                         Operand(samplers[constant_index * 4 + 0]),
3561                         Operand(samplers[constant_index * 4 + 1]),
3562                         Operand(samplers[constant_index * 4 + 2]),
3563                         Operand(samplers[constant_index * 4 + 3]));
3564    }
3565
3566    Operand off;
3567    if (!index_set) {
3568       off = Operand(offset);
3569    } else {
3570       off = Operand((Temp)bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc), Operand(offset),
3571                                    bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(stride), index)));
3572    }
3573
3574    Temp res = bld.smem(opcode, bld.def(type), list, off);
3575
3576    if (desc_type == ACO_DESC_PLANE_2) {
3577       Temp components[8];
3578       for (unsigned i = 0; i < 8; i++)
3579          components[i] = bld.tmp(s1);
3580       bld.pseudo(aco_opcode::p_split_vector,
3581                  Definition(components[0]),
3582                  Definition(components[1]),
3583                  Definition(components[2]),
3584                  Definition(components[3]),
3585                  res);
3586
3587       Temp desc2 = get_sampler_desc(ctx, deref_instr, ACO_DESC_PLANE_1, tex_instr, image, write);
3588       bld.pseudo(aco_opcode::p_split_vector,
3589                  bld.def(s1), bld.def(s1), bld.def(s1), bld.def(s1),
3590                  Definition(components[4]),
3591                  Definition(components[5]),
3592                  Definition(components[6]),
3593                  Definition(components[7]),
3594                  desc2);
3595
3596       res = bld.pseudo(aco_opcode::p_create_vector, bld.def(s8),
3597                        components[0], components[1], components[2], components[3],
3598                        components[4], components[5], components[6], components[7]);
3599    }
3600
3601    return res;
3602 }
3603
3604 static int image_type_to_components_count(enum glsl_sampler_dim dim, bool array)
3605 {
3606    switch (dim) {
3607    case GLSL_SAMPLER_DIM_BUF:
3608       return 1;
3609    case GLSL_SAMPLER_DIM_1D:
3610       return array ? 2 : 1;
3611    case GLSL_SAMPLER_DIM_2D:
3612       return array ? 3 : 2;
3613    case GLSL_SAMPLER_DIM_MS:
3614       return array ? 4 : 3;
3615    case GLSL_SAMPLER_DIM_3D:
3616    case GLSL_SAMPLER_DIM_CUBE:
3617       return 3;
3618    case GLSL_SAMPLER_DIM_RECT:
3619    case GLSL_SAMPLER_DIM_SUBPASS:
3620       return 2;
3621    case GLSL_SAMPLER_DIM_SUBPASS_MS:
3622       return 3;
3623    default:
3624       break;
3625    }
3626    return 0;
3627 }
3628
3629
3630 /* Adjust the sample index according to FMASK.
3631  *
3632  * For uncompressed MSAA surfaces, FMASK should return 0x76543210,
3633  * which is the identity mapping. Each nibble says which physical sample
3634  * should be fetched to get that sample.
3635  *
3636  * For example, 0x11111100 means there are only 2 samples stored and
3637  * the second sample covers 3/4 of the pixel. When reading samples 0
3638  * and 1, return physical sample 0 (determined by the first two 0s
3639  * in FMASK), otherwise return physical sample 1.
3640  *
3641  * The sample index should be adjusted as follows:
3642  *   sample_index = (fmask >> (sample_index * 4)) & 0xF;
3643  */
3644 static Temp adjust_sample_index_using_fmask(isel_context *ctx, bool da, Temp coords, Operand sample_index, Temp fmask_desc_ptr)
3645 {
3646    Builder bld(ctx->program, ctx->block);
3647    Temp fmask = bld.tmp(v1);
3648
3649    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3650    load->operands[0] = Operand(coords);
3651    load->operands[1] = Operand(fmask_desc_ptr);
3652    load->definitions[0] = Definition(fmask);
3653    load->glc = false;
3654    load->dmask = 0x1;
3655    load->unrm = true;
3656    load->da = da;
3657    load->can_reorder = true; /* fmask images shouldn't be modified */
3658    ctx->block->instructions.emplace_back(std::move(load));
3659
3660    Operand sample_index4;
3661    if (sample_index.isConstant() && sample_index.constantValue() < 16) {
3662       sample_index4 = Operand(sample_index.constantValue() << 2);
3663    } else if (sample_index.regClass() == s1) {
3664       sample_index4 = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), sample_index, Operand(2u));
3665    } else {
3666       assert(sample_index.regClass() == v1);
3667       sample_index4 = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), sample_index);
3668    }
3669
3670    Temp final_sample;
3671    if (sample_index4.isConstant() && sample_index4.constantValue() == 0)
3672       final_sample = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(15u), fmask);
3673    else if (sample_index4.isConstant() && sample_index4.constantValue() == 28)
3674       final_sample = bld.vop2(aco_opcode::v_lshrrev_b32, bld.def(v1), Operand(28u), fmask);
3675    else
3676       final_sample = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), fmask, sample_index4, Operand(4u));
3677
3678    /* Don't rewrite the sample index if WORD1.DATA_FORMAT of the FMASK
3679     * resource descriptor is 0 (invalid),
3680     */
3681    Temp compare = bld.tmp(s2);
3682    bld.vopc_e64(aco_opcode::v_cmp_lg_u32, Definition(compare),
3683                 Operand(0u), emit_extract_vector(ctx, fmask_desc_ptr, 1, s1)).def(0).setHint(vcc);
3684
3685    Temp sample_index_v = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), sample_index);
3686
3687    /* Replace the MSAA sample index. */
3688    return bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), sample_index_v, final_sample, compare);
3689 }
3690
3691 static Temp get_image_coords(isel_context *ctx, const nir_intrinsic_instr *instr, const struct glsl_type *type)
3692 {
3693
3694    Temp src0 = get_ssa_temp(ctx, instr->src[1].ssa);
3695    enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3696    bool is_array = glsl_sampler_type_is_array(type);
3697    ASSERTED bool add_frag_pos = (dim == GLSL_SAMPLER_DIM_SUBPASS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3698    assert(!add_frag_pos && "Input attachments should be lowered.");
3699    bool is_ms = (dim == GLSL_SAMPLER_DIM_MS || dim == GLSL_SAMPLER_DIM_SUBPASS_MS);
3700    bool gfx9_1d = ctx->options->chip_class >= GFX9 && dim == GLSL_SAMPLER_DIM_1D;
3701    int count = image_type_to_components_count(dim, is_array);
3702    std::vector<Operand> coords(count);
3703
3704    if (is_ms) {
3705       Operand sample_index;
3706       nir_const_value *sample_cv = nir_src_as_const_value(instr->src[2]);
3707       if (sample_cv)
3708          sample_index = Operand(sample_cv->u32);
3709       else
3710          sample_index = Operand(emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[2].ssa), 0, v1));
3711
3712       if (instr->intrinsic == nir_intrinsic_image_deref_load) {
3713          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, is_array ? 3 : 2, 1)};
3714          for (unsigned i = 0; i < vec->operands.size(); i++)
3715             vec->operands[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3716          Temp fmask_load_address = {ctx->program->allocateId(), is_array ? v3 : v2};
3717          vec->definitions[0] = Definition(fmask_load_address);
3718          ctx->block->instructions.emplace_back(std::move(vec));
3719
3720          Temp fmask_desc_ptr = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_FMASK, nullptr, false, false);
3721          sample_index = Operand(adjust_sample_index_using_fmask(ctx, is_array, fmask_load_address, sample_index, fmask_desc_ptr));
3722       }
3723       count--;
3724       coords[count] = sample_index;
3725    }
3726
3727    if (count == 1 && !gfx9_1d)
3728       return emit_extract_vector(ctx, src0, 0, v1);
3729
3730    if (gfx9_1d) {
3731       coords[0] = Operand(emit_extract_vector(ctx, src0, 0, v1));
3732       coords.resize(coords.size() + 1);
3733       coords[1] = Operand((uint32_t) 0);
3734       if (is_array)
3735          coords[2] = Operand(emit_extract_vector(ctx, src0, 1, v1));
3736    } else {
3737       for (int i = 0; i < count; i++)
3738          coords[i] = Operand(emit_extract_vector(ctx, src0, i, v1));
3739    }
3740
3741    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
3742    for (unsigned i = 0; i < coords.size(); i++)
3743       vec->operands[i] = coords[i];
3744    Temp res = {ctx->program->allocateId(), RegClass(RegType::vgpr, coords.size())};
3745    vec->definitions[0] = Definition(res);
3746    ctx->block->instructions.emplace_back(std::move(vec));
3747    return res;
3748 }
3749
3750
3751 void visit_image_load(isel_context *ctx, nir_intrinsic_instr *instr)
3752 {
3753    Builder bld(ctx->program, ctx->block);
3754    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3755    const struct glsl_type *type = glsl_without_array(var->type);
3756    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3757    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3758
3759    if (dim == GLSL_SAMPLER_DIM_BUF) {
3760       unsigned mask = nir_ssa_def_components_read(&instr->dest.ssa);
3761       unsigned num_channels = util_last_bit(mask);
3762       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3763       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3764
3765       aco_opcode opcode;
3766       switch (num_channels) {
3767       case 1:
3768          opcode = aco_opcode::buffer_load_format_x;
3769          break;
3770       case 2:
3771          opcode = aco_opcode::buffer_load_format_xy;
3772          break;
3773       case 3:
3774          opcode = aco_opcode::buffer_load_format_xyz;
3775          break;
3776       case 4:
3777          opcode = aco_opcode::buffer_load_format_xyzw;
3778          break;
3779       default:
3780          unreachable(">4 channel buffer image load");
3781       }
3782       aco_ptr<MUBUF_instruction> load{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 3, 1)};
3783       load->operands[0] = Operand(vindex);
3784       load->operands[1] = Operand(rsrc);
3785       load->operands[2] = Operand((uint32_t) 0);
3786       Temp tmp;
3787       if (num_channels == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3788          tmp = dst;
3789       else
3790          tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_channels)};
3791       load->definitions[0] = Definition(tmp);
3792       load->idxen = true;
3793       load->barrier = barrier_image;
3794       ctx->block->instructions.emplace_back(std::move(load));
3795
3796       expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, (1 << num_channels) - 1);
3797       return;
3798    }
3799
3800    Temp coords = get_image_coords(ctx, instr, type);
3801    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3802    //aco_image_dim img_dim = get_image_dim(ctx, glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
3803
3804    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
3805    unsigned num_components = util_bitcount(dmask);
3806    Temp tmp;
3807    if (num_components == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
3808       tmp = dst;
3809    else
3810       tmp = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_components)};
3811
3812    aco_ptr<MIMG_instruction> load{create_instruction<MIMG_instruction>(aco_opcode::image_load, Format::MIMG, 2, 1)};
3813    load->operands[0] = Operand(coords);
3814    load->operands[1] = Operand(resource);
3815    load->definitions[0] = Definition(tmp);
3816    load->glc = var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT) ? 1 : 0;
3817    load->dmask = dmask;
3818    load->unrm = true;
3819    load->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3820    load->barrier = barrier_image;
3821    ctx->block->instructions.emplace_back(std::move(load));
3822
3823    expand_vector(ctx, tmp, dst, instr->dest.ssa.num_components, dmask);
3824    return;
3825 }
3826
3827 void visit_image_store(isel_context *ctx, nir_intrinsic_instr *instr)
3828 {
3829    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3830    const struct glsl_type *type = glsl_without_array(var->type);
3831    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3832    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3833
3834    bool glc = ctx->options->chip_class == GFX6 || var->data.image.access & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE) ? 1 : 0;
3835
3836    if (dim == GLSL_SAMPLER_DIM_BUF) {
3837       Temp rsrc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3838       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3839       aco_opcode opcode;
3840       switch (data.size()) {
3841       case 1:
3842          opcode = aco_opcode::buffer_store_format_x;
3843          break;
3844       case 2:
3845          opcode = aco_opcode::buffer_store_format_xy;
3846          break;
3847       case 3:
3848          opcode = aco_opcode::buffer_store_format_xyz;
3849          break;
3850       case 4:
3851          opcode = aco_opcode::buffer_store_format_xyzw;
3852          break;
3853       default:
3854          unreachable(">4 channel buffer image store");
3855       }
3856       aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
3857       store->operands[0] = Operand(vindex);
3858       store->operands[1] = Operand(rsrc);
3859       store->operands[2] = Operand((uint32_t) 0);
3860       store->operands[3] = Operand(data);
3861       store->idxen = true;
3862       store->glc = glc;
3863       store->disable_wqm = true;
3864       store->barrier = barrier_image;
3865       ctx->program->needs_exact = true;
3866       ctx->block->instructions.emplace_back(std::move(store));
3867       return;
3868    }
3869
3870    assert(data.type() == RegType::vgpr);
3871    Temp coords = get_image_coords(ctx, instr, type);
3872    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3873
3874    aco_ptr<MIMG_instruction> store{create_instruction<MIMG_instruction>(aco_opcode::image_store, Format::MIMG, 4, 0)};
3875    store->operands[0] = Operand(coords);
3876    store->operands[1] = Operand(resource);
3877    store->operands[2] = Operand(s4);
3878    store->operands[3] = Operand(data);
3879    store->glc = glc;
3880    store->dmask = (1 << data.size()) - 1;
3881    store->unrm = true;
3882    store->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3883    store->disable_wqm = true;
3884    store->barrier = barrier_image;
3885    ctx->program->needs_exact = true;
3886    ctx->block->instructions.emplace_back(std::move(store));
3887    return;
3888 }
3889
3890 void visit_image_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
3891 {
3892    /* return the previous value if dest is ever used */
3893    bool return_previous = false;
3894    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
3895       return_previous = true;
3896       break;
3897    }
3898    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
3899       return_previous = true;
3900       break;
3901    }
3902
3903    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
3904    const struct glsl_type *type = glsl_without_array(var->type);
3905    const enum glsl_sampler_dim dim = glsl_get_sampler_dim(type);
3906    Builder bld(ctx->program, ctx->block);
3907
3908    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[3].ssa));
3909    assert(data.size() == 1 && "64bit ssbo atomics not yet implemented.");
3910
3911    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap)
3912       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), get_ssa_temp(ctx, instr->src[4].ssa), data);
3913
3914    aco_opcode buf_op, image_op;
3915    switch (instr->intrinsic) {
3916       case nir_intrinsic_image_deref_atomic_add:
3917          buf_op = aco_opcode::buffer_atomic_add;
3918          image_op = aco_opcode::image_atomic_add;
3919          break;
3920       case nir_intrinsic_image_deref_atomic_umin:
3921          buf_op = aco_opcode::buffer_atomic_umin;
3922          image_op = aco_opcode::image_atomic_umin;
3923          break;
3924       case nir_intrinsic_image_deref_atomic_imin:
3925          buf_op = aco_opcode::buffer_atomic_smin;
3926          image_op = aco_opcode::image_atomic_smin;
3927          break;
3928       case nir_intrinsic_image_deref_atomic_umax:
3929          buf_op = aco_opcode::buffer_atomic_umax;
3930          image_op = aco_opcode::image_atomic_umax;
3931          break;
3932       case nir_intrinsic_image_deref_atomic_imax:
3933          buf_op = aco_opcode::buffer_atomic_smax;
3934          image_op = aco_opcode::image_atomic_smax;
3935          break;
3936       case nir_intrinsic_image_deref_atomic_and:
3937          buf_op = aco_opcode::buffer_atomic_and;
3938          image_op = aco_opcode::image_atomic_and;
3939          break;
3940       case nir_intrinsic_image_deref_atomic_or:
3941          buf_op = aco_opcode::buffer_atomic_or;
3942          image_op = aco_opcode::image_atomic_or;
3943          break;
3944       case nir_intrinsic_image_deref_atomic_xor:
3945          buf_op = aco_opcode::buffer_atomic_xor;
3946          image_op = aco_opcode::image_atomic_xor;
3947          break;
3948       case nir_intrinsic_image_deref_atomic_exchange:
3949          buf_op = aco_opcode::buffer_atomic_swap;
3950          image_op = aco_opcode::image_atomic_swap;
3951          break;
3952       case nir_intrinsic_image_deref_atomic_comp_swap:
3953          buf_op = aco_opcode::buffer_atomic_cmpswap;
3954          image_op = aco_opcode::image_atomic_cmpswap;
3955          break;
3956       default:
3957          unreachable("visit_image_atomic should only be called with nir_intrinsic_image_deref_atomic_* instructions.");
3958    }
3959
3960    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
3961
3962    if (dim == GLSL_SAMPLER_DIM_BUF) {
3963       Temp vindex = emit_extract_vector(ctx, get_ssa_temp(ctx, instr->src[1].ssa), 0, v1);
3964       Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, nullptr, true, true);
3965       //assert(ctx->options->chip_class < GFX9 && "GFX9 stride size workaround not yet implemented.");
3966       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(buf_op, Format::MUBUF, 4, return_previous ? 1 : 0)};
3967       mubuf->operands[0] = Operand(vindex);
3968       mubuf->operands[1] = Operand(resource);
3969       mubuf->operands[2] = Operand((uint32_t)0);
3970       mubuf->operands[3] = Operand(data);
3971       if (return_previous)
3972          mubuf->definitions[0] = Definition(dst);
3973       mubuf->offset = 0;
3974       mubuf->idxen = true;
3975       mubuf->glc = return_previous;
3976       mubuf->disable_wqm = true;
3977       mubuf->barrier = barrier_image;
3978       ctx->program->needs_exact = true;
3979       ctx->block->instructions.emplace_back(std::move(mubuf));
3980       return;
3981    }
3982
3983    Temp coords = get_image_coords(ctx, instr, type);
3984    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, nullptr, true, true);
3985    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(image_op, Format::MIMG, 4, return_previous ? 1 : 0)};
3986    mimg->operands[0] = Operand(coords);
3987    mimg->operands[1] = Operand(resource);
3988    mimg->operands[2] = Operand(s4); /* no sampler */
3989    mimg->operands[3] = Operand(data);
3990    if (return_previous)
3991       mimg->definitions[0] = Definition(dst);
3992    mimg->glc = return_previous;
3993    mimg->dmask = (1 << data.size()) - 1;
3994    mimg->unrm = true;
3995    mimg->da = should_declare_array(ctx, dim, glsl_sampler_type_is_array(type));
3996    mimg->disable_wqm = true;
3997    mimg->barrier = barrier_image;
3998    ctx->program->needs_exact = true;
3999    ctx->block->instructions.emplace_back(std::move(mimg));
4000    return;
4001 }
4002
4003 void get_buffer_size(isel_context *ctx, Temp desc, Temp dst, bool in_elements)
4004 {
4005    if (in_elements && ctx->options->chip_class == GFX8) {
4006       Builder bld(ctx->program, ctx->block);
4007
4008       Temp stride = emit_extract_vector(ctx, desc, 1, s1);
4009       stride = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), stride, Operand((5u << 16) | 16u));
4010       stride = bld.vop1(aco_opcode::v_cvt_f32_ubyte0, bld.def(v1), stride);
4011       stride = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), stride);
4012
4013       Temp size = emit_extract_vector(ctx, desc, 2, s1);
4014       size = bld.vop1(aco_opcode::v_cvt_f32_u32, bld.def(v1), size);
4015
4016       Temp res = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), size, stride);
4017       res = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), res);
4018       bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4019
4020       // TODO: we can probably calculate this faster on the scalar unit to do: size / stride{1,2,4,8,12,16}
4021       /* idea
4022        * for 1,2,4,8,16, the result is just (stride >> S_FF1_I32_B32)
4023        * in case 12 (or 3?), we have to divide by 3:
4024        * set v_skip in case it's 12 (if we also have to take care of 3, shift first)
4025        * use v_mul_hi_u32 with magic number to divide
4026        * we need some pseudo merge opcode to overwrite the original SALU result with readfirstlane
4027        * disable v_skip
4028        * total: 6 SALU + 2 VALU instructions vs 1 SALU + 6 VALU instructions
4029        */
4030
4031    } else {
4032       emit_extract_vector(ctx, desc, 2, dst);
4033    }
4034 }
4035
4036 void visit_image_size(isel_context *ctx, nir_intrinsic_instr *instr)
4037 {
4038    const nir_variable *var = nir_deref_instr_get_variable(nir_instr_as_deref(instr->src[0].ssa->parent_instr));
4039    const struct glsl_type *type = glsl_without_array(var->type);
4040    Builder bld(ctx->program, ctx->block);
4041
4042    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_BUF) {
4043       Temp desc = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_BUFFER, NULL, true, false);
4044       return get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), true);
4045    }
4046
4047    /* LOD */
4048    Temp lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
4049
4050    /* Resource */
4051    Temp resource = get_sampler_desc(ctx, nir_instr_as_deref(instr->src[0].ssa->parent_instr), ACO_DESC_IMAGE, NULL, true, false);
4052
4053    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4054
4055    aco_ptr<MIMG_instruction> mimg{create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1)};
4056    mimg->operands[0] = Operand(lod);
4057    mimg->operands[1] = Operand(resource);
4058    unsigned& dmask = mimg->dmask;
4059    mimg->dmask = (1 << instr->dest.ssa.num_components) - 1;
4060    mimg->da = glsl_sampler_type_is_array(type);
4061    mimg->can_reorder = true;
4062    Definition& def = mimg->definitions[0];
4063    ctx->block->instructions.emplace_back(std::move(mimg));
4064
4065    if (glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_CUBE &&
4066        glsl_sampler_type_is_array(type)) {
4067
4068       assert(instr->dest.ssa.num_components == 3);
4069       Temp tmp = {ctx->program->allocateId(), v3};
4070       def = Definition(tmp);
4071       emit_split_vector(ctx, tmp, 3);
4072
4073       /* divide 3rd value by 6 by multiplying with magic number */
4074       Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
4075       Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp, 2, v1), c);
4076
4077       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4078                  emit_extract_vector(ctx, tmp, 0, v1),
4079                  emit_extract_vector(ctx, tmp, 1, v1),
4080                  by_6);
4081
4082    } else if (ctx->options->chip_class >= GFX9 &&
4083               glsl_get_sampler_dim(type) == GLSL_SAMPLER_DIM_1D &&
4084               glsl_sampler_type_is_array(type)) {
4085       assert(instr->dest.ssa.num_components == 2);
4086       def = Definition(dst);
4087       dmask = 0x5;
4088    } else {
4089       def = Definition(dst);
4090    }
4091
4092    emit_split_vector(ctx, dst, instr->dest.ssa.num_components);
4093 }
4094
4095 void visit_load_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4096 {
4097    Builder bld(ctx->program, ctx->block);
4098    unsigned num_components = instr->num_components;
4099
4100    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4101    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4102    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4103
4104    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4105    load_buffer(ctx, num_components, dst, rsrc, get_ssa_temp(ctx, instr->src[1].ssa), glc);
4106 }
4107
4108 void visit_store_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4109 {
4110    Builder bld(ctx->program, ctx->block);
4111    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4112    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4113    unsigned writemask = nir_intrinsic_write_mask(instr);
4114
4115    Temp offset;
4116    if (ctx->options->chip_class < GFX8)
4117       offset = as_vgpr(ctx,get_ssa_temp(ctx, instr->src[2].ssa));
4118    else
4119       offset = get_ssa_temp(ctx, instr->src[2].ssa);
4120
4121    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4122    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4123
4124    bool smem = !ctx->divergent_vals[instr->src[2].ssa->index] &&
4125                ctx->options->chip_class >= GFX8;
4126    if (smem)
4127       offset = bld.as_uniform(offset);
4128    bool smem_nonfs = smem && ctx->stage != fragment_fs;
4129
4130    while (writemask) {
4131       int start, count;
4132       u_bit_scan_consecutive_range(&writemask, &start, &count);
4133       if (count == 3 && smem) {
4134          writemask |= 1u << (start + 2);
4135          count = 2;
4136       }
4137       int num_bytes = count * elem_size_bytes;
4138
4139       if (num_bytes > 16) {
4140          assert(elem_size_bytes == 8);
4141          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4142          count = 2;
4143          num_bytes = 16;
4144       }
4145
4146       // TODO: check alignment of sub-dword stores
4147       // TODO: split 3 bytes. there is no store instruction for that
4148
4149       Temp write_data;
4150       if (count != instr->num_components) {
4151          emit_split_vector(ctx, data, instr->num_components);
4152          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4153          for (int i = 0; i < count; i++) {
4154             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(data.type(), elem_size_bytes / 4));
4155             vec->operands[i] = Operand(smem_nonfs ? bld.as_uniform(elem) : elem);
4156          }
4157          write_data = bld.tmp(smem_nonfs ? RegType::sgpr : data.type(), count * elem_size_bytes / 4);
4158          vec->definitions[0] = Definition(write_data);
4159          ctx->block->instructions.emplace_back(std::move(vec));
4160       } else if (!smem && data.type() != RegType::vgpr) {
4161          assert(num_bytes % 4 == 0);
4162          write_data = bld.copy(bld.def(RegType::vgpr, num_bytes / 4), data);
4163       } else if (smem_nonfs && data.type() == RegType::vgpr) {
4164          assert(num_bytes % 4 == 0);
4165          write_data = bld.as_uniform(data);
4166       } else {
4167          write_data = data;
4168       }
4169
4170       aco_opcode vmem_op, smem_op;
4171       switch (num_bytes) {
4172          case 4:
4173             vmem_op = aco_opcode::buffer_store_dword;
4174             smem_op = aco_opcode::s_buffer_store_dword;
4175             break;
4176          case 8:
4177             vmem_op = aco_opcode::buffer_store_dwordx2;
4178             smem_op = aco_opcode::s_buffer_store_dwordx2;
4179             break;
4180          case 12:
4181             vmem_op = aco_opcode::buffer_store_dwordx3;
4182             smem_op = aco_opcode::last_opcode;
4183             assert(!smem);
4184             break;
4185          case 16:
4186             vmem_op = aco_opcode::buffer_store_dwordx4;
4187             smem_op = aco_opcode::s_buffer_store_dwordx4;
4188             break;
4189          default:
4190             unreachable("Store SSBO not implemented for this size.");
4191       }
4192       if (ctx->stage == fragment_fs)
4193          smem_op = aco_opcode::p_fs_buffer_store_smem;
4194
4195       if (smem) {
4196          aco_ptr<SMEM_instruction> store{create_instruction<SMEM_instruction>(smem_op, Format::SMEM, 3, 0)};
4197          store->operands[0] = Operand(rsrc);
4198          if (start) {
4199             Temp off = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
4200                                 offset, Operand(start * elem_size_bytes));
4201             store->operands[1] = Operand(off);
4202          } else {
4203             store->operands[1] = Operand(offset);
4204          }
4205          if (smem_op != aco_opcode::p_fs_buffer_store_smem)
4206             store->operands[1].setFixed(m0);
4207          store->operands[2] = Operand(write_data);
4208          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4209          store->disable_wqm = true;
4210          store->barrier = barrier_buffer;
4211          ctx->block->instructions.emplace_back(std::move(store));
4212          ctx->program->wb_smem_l1_on_end = true;
4213          if (smem_op == aco_opcode::p_fs_buffer_store_smem) {
4214             ctx->block->kind |= block_kind_needs_lowering;
4215             ctx->program->needs_exact = true;
4216          }
4217       } else {
4218          aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(vmem_op, Format::MUBUF, 4, 0)};
4219          store->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4220          store->operands[1] = Operand(rsrc);
4221          store->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4222          store->operands[3] = Operand(write_data);
4223          store->offset = start * elem_size_bytes;
4224          store->offen = (offset.type() == RegType::vgpr);
4225          store->glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4226          store->disable_wqm = true;
4227          store->barrier = barrier_buffer;
4228          ctx->program->needs_exact = true;
4229          ctx->block->instructions.emplace_back(std::move(store));
4230       }
4231    }
4232 }
4233
4234 void visit_atomic_ssbo(isel_context *ctx, nir_intrinsic_instr *instr)
4235 {
4236    /* return the previous value if dest is ever used */
4237    bool return_previous = false;
4238    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4239       return_previous = true;
4240       break;
4241    }
4242    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4243       return_previous = true;
4244       break;
4245    }
4246
4247    Builder bld(ctx->program, ctx->block);
4248    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[2].ssa));
4249
4250    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
4251       data = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, data.size() * 2),
4252                         get_ssa_temp(ctx, instr->src[3].ssa), data);
4253
4254    Temp offset;
4255    if (ctx->options->chip_class < GFX8)
4256       offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4257    else
4258       offset = get_ssa_temp(ctx, instr->src[1].ssa);
4259
4260    Temp rsrc = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4261    rsrc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), rsrc, Operand(0u));
4262
4263    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4264
4265    aco_opcode op32, op64;
4266    switch (instr->intrinsic) {
4267       case nir_intrinsic_ssbo_atomic_add:
4268          op32 = aco_opcode::buffer_atomic_add;
4269          op64 = aco_opcode::buffer_atomic_add_x2;
4270          break;
4271       case nir_intrinsic_ssbo_atomic_imin:
4272          op32 = aco_opcode::buffer_atomic_smin;
4273          op64 = aco_opcode::buffer_atomic_smin_x2;
4274          break;
4275       case nir_intrinsic_ssbo_atomic_umin:
4276          op32 = aco_opcode::buffer_atomic_umin;
4277          op64 = aco_opcode::buffer_atomic_umin_x2;
4278          break;
4279       case nir_intrinsic_ssbo_atomic_imax:
4280          op32 = aco_opcode::buffer_atomic_smax;
4281          op64 = aco_opcode::buffer_atomic_smax_x2;
4282          break;
4283       case nir_intrinsic_ssbo_atomic_umax:
4284          op32 = aco_opcode::buffer_atomic_umax;
4285          op64 = aco_opcode::buffer_atomic_umax_x2;
4286          break;
4287       case nir_intrinsic_ssbo_atomic_and:
4288          op32 = aco_opcode::buffer_atomic_and;
4289          op64 = aco_opcode::buffer_atomic_and_x2;
4290          break;
4291       case nir_intrinsic_ssbo_atomic_or:
4292          op32 = aco_opcode::buffer_atomic_or;
4293          op64 = aco_opcode::buffer_atomic_or_x2;
4294          break;
4295       case nir_intrinsic_ssbo_atomic_xor:
4296          op32 = aco_opcode::buffer_atomic_xor;
4297          op64 = aco_opcode::buffer_atomic_xor_x2;
4298          break;
4299       case nir_intrinsic_ssbo_atomic_exchange:
4300          op32 = aco_opcode::buffer_atomic_swap;
4301          op64 = aco_opcode::buffer_atomic_swap_x2;
4302          break;
4303       case nir_intrinsic_ssbo_atomic_comp_swap:
4304          op32 = aco_opcode::buffer_atomic_cmpswap;
4305          op64 = aco_opcode::buffer_atomic_cmpswap_x2;
4306          break;
4307       default:
4308          unreachable("visit_atomic_ssbo should only be called with nir_intrinsic_ssbo_atomic_* instructions.");
4309    }
4310    aco_opcode op = instr->dest.ssa.bit_size == 32 ? op32 : op64;
4311    aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 4, return_previous ? 1 : 0)};
4312    mubuf->operands[0] = offset.type() == RegType::vgpr ? Operand(offset) : Operand(v1);
4313    mubuf->operands[1] = Operand(rsrc);
4314    mubuf->operands[2] = offset.type() == RegType::sgpr ? Operand(offset) : Operand((uint32_t) 0);
4315    mubuf->operands[3] = Operand(data);
4316    if (return_previous)
4317       mubuf->definitions[0] = Definition(dst);
4318    mubuf->offset = 0;
4319    mubuf->offen = (offset.type() == RegType::vgpr);
4320    mubuf->glc = return_previous;
4321    mubuf->disable_wqm = true;
4322    mubuf->barrier = barrier_buffer;
4323    ctx->program->needs_exact = true;
4324    ctx->block->instructions.emplace_back(std::move(mubuf));
4325 }
4326
4327 void visit_get_buffer_size(isel_context *ctx, nir_intrinsic_instr *instr) {
4328
4329    Temp index = convert_pointer_to_64_bit(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4330    Builder bld(ctx->program, ctx->block);
4331    Temp desc = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), index, Operand(0u));
4332    get_buffer_size(ctx, desc, get_ssa_temp(ctx, &instr->dest.ssa), false);
4333 }
4334
4335 void visit_load_global(isel_context *ctx, nir_intrinsic_instr *instr)
4336 {
4337    Builder bld(ctx->program, ctx->block);
4338    unsigned num_components = instr->num_components;
4339    unsigned num_bytes = num_components * instr->dest.ssa.bit_size / 8;
4340
4341    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4342    Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
4343
4344    bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT);
4345    aco_opcode op;
4346    if (dst.type() == RegType::vgpr || (glc && ctx->options->chip_class < GFX8)) {
4347       bool global = ctx->options->chip_class >= GFX9;
4348       aco_opcode op;
4349       switch (num_bytes) {
4350       case 4:
4351          op = global ? aco_opcode::global_load_dword : aco_opcode::flat_load_dword;
4352          break;
4353       case 8:
4354          op = global ? aco_opcode::global_load_dwordx2 : aco_opcode::flat_load_dwordx2;
4355          break;
4356       case 12:
4357          op = global ? aco_opcode::global_load_dwordx3 : aco_opcode::flat_load_dwordx3;
4358          break;
4359       case 16:
4360          op = global ? aco_opcode::global_load_dwordx4 : aco_opcode::flat_load_dwordx4;
4361          break;
4362       default:
4363          unreachable("load_global not implemented for this size.");
4364       }
4365       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 2, 1)};
4366       flat->operands[0] = Operand(addr);
4367       flat->operands[1] = Operand(s1);
4368       flat->glc = glc;
4369
4370       if (dst.type() == RegType::sgpr) {
4371          Temp vec = bld.tmp(RegType::vgpr, dst.size());
4372          flat->definitions[0] = Definition(vec);
4373          ctx->block->instructions.emplace_back(std::move(flat));
4374          bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), vec);
4375       } else {
4376          flat->definitions[0] = Definition(dst);
4377          ctx->block->instructions.emplace_back(std::move(flat));
4378       }
4379       emit_split_vector(ctx, dst, num_components);
4380    } else {
4381       switch (num_bytes) {
4382          case 4:
4383             op = aco_opcode::s_load_dword;
4384             break;
4385          case 8:
4386             op = aco_opcode::s_load_dwordx2;
4387             break;
4388          case 12:
4389          case 16:
4390             op = aco_opcode::s_load_dwordx4;
4391             break;
4392          default:
4393             unreachable("load_global not implemented for this size.");
4394       }
4395       aco_ptr<SMEM_instruction> load{create_instruction<SMEM_instruction>(op, Format::SMEM, 2, 1)};
4396       load->operands[0] = Operand(addr);
4397       load->operands[1] = Operand(0u);
4398       load->definitions[0] = Definition(dst);
4399       load->glc = glc;
4400       load->barrier = barrier_buffer;
4401       assert(ctx->options->chip_class >= GFX8 || !glc);
4402
4403       if (dst.size() == 3) {
4404          /* trim vector */
4405          Temp vec = bld.tmp(s4);
4406          load->definitions[0] = Definition(vec);
4407          ctx->block->instructions.emplace_back(std::move(load));
4408          emit_split_vector(ctx, vec, 4);
4409
4410          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
4411                     emit_extract_vector(ctx, vec, 0, s1),
4412                     emit_extract_vector(ctx, vec, 1, s1),
4413                     emit_extract_vector(ctx, vec, 2, s1));
4414       } else {
4415          ctx->block->instructions.emplace_back(std::move(load));
4416       }
4417    }
4418 }
4419
4420 void visit_store_global(isel_context *ctx, nir_intrinsic_instr *instr)
4421 {
4422    Builder bld(ctx->program, ctx->block);
4423    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4424
4425    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4426    Temp addr = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4427
4428    unsigned writemask = nir_intrinsic_write_mask(instr);
4429    while (writemask) {
4430       int start, count;
4431       u_bit_scan_consecutive_range(&writemask, &start, &count);
4432       unsigned num_bytes = count * elem_size_bytes;
4433
4434       Temp write_data = data;
4435       if (count != instr->num_components) {
4436          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4437          for (int i = 0; i < count; i++)
4438             vec->operands[i] = Operand(emit_extract_vector(ctx, data, start + i, v1));
4439          write_data = bld.tmp(RegType::vgpr, count);
4440          vec->definitions[0] = Definition(write_data);
4441          ctx->block->instructions.emplace_back(std::move(vec));
4442       }
4443
4444       unsigned offset = start * elem_size_bytes;
4445       if (offset > 0 && ctx->options->chip_class < GFX9) {
4446          Temp addr0 = bld.tmp(v1), addr1 = bld.tmp(v1);
4447          Temp new_addr0 = bld.tmp(v1), new_addr1 = bld.tmp(v1);
4448          Temp carry = bld.tmp(s2);
4449          bld.pseudo(aco_opcode::p_split_vector, Definition(addr0), Definition(addr1), addr);
4450
4451          bld.vop2(aco_opcode::v_add_co_u32, Definition(new_addr0), bld.hint_vcc(Definition(carry)),
4452                   Operand(offset), addr0);
4453          bld.vop2(aco_opcode::v_addc_co_u32, Definition(new_addr1), bld.def(s2),
4454                   Operand(0u), addr1,
4455                   carry).def(1).setHint(vcc);
4456
4457          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), new_addr0, new_addr1);
4458
4459          offset = 0;
4460       }
4461
4462       bool glc = nir_intrinsic_access(instr) & (ACCESS_VOLATILE | ACCESS_COHERENT | ACCESS_NON_READABLE);
4463       bool global = ctx->options->chip_class >= GFX9;
4464       aco_opcode op;
4465       switch (num_bytes) {
4466       case 4:
4467          op = global ? aco_opcode::global_store_dword : aco_opcode::flat_store_dword;
4468          break;
4469       case 8:
4470          op = global ? aco_opcode::global_store_dwordx2 : aco_opcode::flat_store_dwordx2;
4471          break;
4472       case 12:
4473          op = global ? aco_opcode::global_store_dwordx3 : aco_opcode::flat_store_dwordx3;
4474          break;
4475       case 16:
4476          op = global ? aco_opcode::global_store_dwordx4 : aco_opcode::flat_store_dwordx4;
4477          break;
4478       default:
4479          unreachable("store_global not implemented for this size.");
4480       }
4481       aco_ptr<FLAT_instruction> flat{create_instruction<FLAT_instruction>(op, global ? Format::GLOBAL : Format::FLAT, 3, 0)};
4482       flat->operands[0] = Operand(addr);
4483       flat->operands[1] = Operand(s1);
4484       flat->operands[2] = Operand(data);
4485       flat->glc = glc;
4486       flat->offset = offset;
4487       ctx->block->instructions.emplace_back(std::move(flat));
4488    }
4489 }
4490
4491 void emit_memory_barrier(isel_context *ctx, nir_intrinsic_instr *instr) {
4492    Builder bld(ctx->program, ctx->block);
4493    switch(instr->intrinsic) {
4494       case nir_intrinsic_group_memory_barrier:
4495       case nir_intrinsic_memory_barrier:
4496          bld.barrier(aco_opcode::p_memory_barrier_all);
4497          break;
4498       case nir_intrinsic_memory_barrier_atomic_counter:
4499          bld.barrier(aco_opcode::p_memory_barrier_atomic);
4500          break;
4501       case nir_intrinsic_memory_barrier_buffer:
4502          bld.barrier(aco_opcode::p_memory_barrier_buffer);
4503          break;
4504       case nir_intrinsic_memory_barrier_image:
4505          bld.barrier(aco_opcode::p_memory_barrier_image);
4506          break;
4507       case nir_intrinsic_memory_barrier_shared:
4508          bld.barrier(aco_opcode::p_memory_barrier_shared);
4509          break;
4510       default:
4511          unreachable("Unimplemented memory barrier intrinsic");
4512          break;
4513    }
4514 }
4515
4516 Operand load_lds_size_m0(isel_context *ctx)
4517 {
4518    /* TODO: m0 does not need to be initialized on GFX9+ */
4519    Builder bld(ctx->program, ctx->block);
4520    return bld.m0((Temp)bld.sopk(aco_opcode::s_movk_i32, bld.def(s1, m0), 0xffff));
4521 }
4522
4523
4524 void visit_load_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4525 {
4526    // TODO: implement sparse reads using ds_read2_b32 and nir_ssa_def_components_read()
4527    Operand m = load_lds_size_m0(ctx);
4528    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4529    assert(instr->dest.ssa.bit_size >= 32 && "Bitsize not supported in load_shared.");
4530    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4531    Builder bld(ctx->program, ctx->block);
4532
4533    unsigned elem_size_bytes = instr->dest.ssa.bit_size / 8;
4534    unsigned bytes_read = 0;
4535    unsigned result_size = 0;
4536    unsigned total_bytes = instr->num_components * elem_size_bytes;
4537    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : instr->dest.ssa.bit_size / 8;
4538    std::array<Temp, 4> result;
4539
4540    while (bytes_read < total_bytes) {
4541       unsigned todo = total_bytes - bytes_read;
4542       bool aligned8 = bytes_read % 8 == 0 && align % 8 == 0;
4543       bool aligned16 = bytes_read % 16 == 0 && align % 16 == 0;
4544
4545       aco_opcode op = aco_opcode::last_opcode;
4546       if (todo >= 16 && aligned16) {
4547          op = aco_opcode::ds_read_b128;
4548          todo = 16;
4549       } else if (todo >= 12 && aligned16) {
4550          op = aco_opcode::ds_read_b96;
4551          todo = 12;
4552       } else if (todo >= 8) {
4553          op = aligned8 ? aco_opcode::ds_read_b64 : aco_opcode::ds_read2_b32;
4554          todo = 8;
4555       } else if (todo >= 4) {
4556          op = aco_opcode::ds_read_b32;
4557          todo = 4;
4558       } else {
4559          assert(false);
4560       }
4561       assert(todo % elem_size_bytes == 0);
4562       unsigned num_elements = todo / elem_size_bytes;
4563       unsigned offset = nir_intrinsic_base(instr) + bytes_read;
4564       unsigned max_offset = op == aco_opcode::ds_read2_b32 ? 1019 : 65535;
4565
4566       Temp address_offset = address;
4567       if (offset > max_offset) {
4568          address_offset = bld.vadd32(bld.def(v1), Operand((uint32_t)nir_intrinsic_base(instr)), address_offset);
4569          offset = bytes_read;
4570       }
4571       assert(offset <= max_offset); /* bytes_read shouldn't be large enough for this to happen */
4572
4573       Temp res;
4574       if (instr->num_components == 1 && dst.type() == RegType::vgpr)
4575          res = dst;
4576       else
4577          res = bld.tmp(RegClass(RegType::vgpr, todo / 4));
4578
4579       if (op == aco_opcode::ds_read2_b32)
4580          res = bld.ds(op, Definition(res), address_offset, m, offset >> 2, (offset >> 2) + 1);
4581       else
4582          res = bld.ds(op, Definition(res), address_offset, m, offset);
4583
4584       if (instr->num_components == 1) {
4585          assert(todo == total_bytes);
4586          if (dst.type() == RegType::sgpr)
4587             bld.pseudo(aco_opcode::p_as_uniform, Definition(dst), res);
4588          return;
4589       }
4590
4591       if (dst.type() == RegType::sgpr)
4592          res = bld.as_uniform(res);
4593
4594       if (num_elements == 1) {
4595          result[result_size++] = res;
4596       } else {
4597          assert(res != dst && res.size() % num_elements == 0);
4598          aco_ptr<Pseudo_instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector, Format::PSEUDO, 1, num_elements)};
4599          split->operands[0] = Operand(res);
4600          for (unsigned i = 0; i < num_elements; i++)
4601             split->definitions[i] = Definition(result[result_size++] = bld.tmp(res.type(), elem_size_bytes / 4));
4602          ctx->block->instructions.emplace_back(std::move(split));
4603       }
4604
4605       bytes_read += todo;
4606    }
4607
4608    assert(result_size == instr->num_components && result_size > 1);
4609    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, result_size, 1)};
4610    for (unsigned i = 0; i < result_size; i++)
4611       vec->operands[i] = Operand(result[i]);
4612    vec->definitions[0] = Definition(dst);
4613    ctx->block->instructions.emplace_back(std::move(vec));
4614    ctx->allocated_vec.emplace(dst.id(), result);
4615 }
4616
4617 void ds_write_helper(isel_context *ctx, Operand m, Temp address, Temp data, unsigned offset0, unsigned offset1, unsigned align)
4618 {
4619    Builder bld(ctx->program, ctx->block);
4620    unsigned bytes_written = 0;
4621    while (bytes_written < data.size() * 4) {
4622       unsigned todo = data.size() * 4 - bytes_written;
4623       bool aligned8 = bytes_written % 8 == 0 && align % 8 == 0;
4624       bool aligned16 = bytes_written % 16 == 0 && align % 16 == 0;
4625
4626       aco_opcode op = aco_opcode::last_opcode;
4627       unsigned size = 0;
4628       if (todo >= 16 && aligned16) {
4629          op = aco_opcode::ds_write_b128;
4630          size = 4;
4631       } else if (todo >= 12 && aligned16) {
4632          op = aco_opcode::ds_write_b96;
4633          size = 3;
4634       } else if (todo >= 8) {
4635          op = aligned8 ? aco_opcode::ds_write_b64 : aco_opcode::ds_write2_b32;
4636          size = 2;
4637       } else if (todo >= 4) {
4638          op = aco_opcode::ds_write_b32;
4639          size = 1;
4640       } else {
4641          assert(false);
4642       }
4643
4644       bool write2 = op == aco_opcode::ds_write2_b32;
4645       unsigned offset = offset0 + offset1 + bytes_written;
4646       unsigned max_offset = write2 ? 1020 : 65535;
4647       Temp address_offset = address;
4648       if (offset > max_offset) {
4649          address_offset = bld.vadd32(bld.def(v1), Operand(offset0), address_offset);
4650          offset = offset1 + bytes_written;
4651       }
4652       assert(offset <= max_offset); /* offset1 shouldn't be large enough for this to happen */
4653
4654       if (write2) {
4655          Temp val0 = emit_extract_vector(ctx, data, bytes_written >> 2, v1);
4656          Temp val1 = emit_extract_vector(ctx, data, (bytes_written >> 2) + 1, v1);
4657          bld.ds(op, address_offset, val0, val1, m, offset >> 2, (offset >> 2) + 1);
4658       } else {
4659          Temp val = emit_extract_vector(ctx, data, bytes_written >> 2, RegClass(RegType::vgpr, size));
4660          bld.ds(op, address_offset, val, m, offset);
4661       }
4662
4663       bytes_written += size * 4;
4664    }
4665 }
4666
4667 void visit_store_shared(isel_context *ctx, nir_intrinsic_instr *instr)
4668 {
4669    unsigned offset = nir_intrinsic_base(instr);
4670    unsigned writemask = nir_intrinsic_write_mask(instr);
4671    Operand m = load_lds_size_m0(ctx);
4672    Temp data = get_ssa_temp(ctx, instr->src[0].ssa);
4673    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4674    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4675    assert(elem_size_bytes >= 4 && "Only 32bit & 64bit store_shared currently supported.");
4676
4677    /* we need at most two stores for 32bit variables */
4678    int start[2], count[2];
4679    u_bit_scan_consecutive_range(&writemask, &start[0], &count[0]);
4680    u_bit_scan_consecutive_range(&writemask, &start[1], &count[1]);
4681    assert(writemask == 0);
4682
4683    /* one combined store is sufficient */
4684    if (count[0] == count[1]) {
4685       Builder bld(ctx->program, ctx->block);
4686
4687       Temp address_offset = address;
4688       if ((offset >> 2) + start[1] > 255) {
4689          address_offset = bld.vadd32(bld.def(v1), Operand(offset), address_offset);
4690          offset = 0;
4691       }
4692
4693       assert(count[0] == 1);
4694       Temp val0 = emit_extract_vector(ctx, data, start[0], v1);
4695       Temp val1 = emit_extract_vector(ctx, data, start[1], v1);
4696       aco_opcode op = elem_size_bytes == 4 ? aco_opcode::ds_write2_b32 : aco_opcode::ds_write2_b64;
4697       offset = offset / elem_size_bytes;
4698       bld.ds(op, address_offset, val0, val1, m,
4699              offset + start[0], offset + start[1]);
4700       return;
4701    }
4702
4703    unsigned align = nir_intrinsic_align_mul(instr) ? nir_intrinsic_align(instr) : elem_size_bytes;
4704    for (unsigned i = 0; i < 2; i++) {
4705       if (count[i] == 0)
4706          continue;
4707
4708       Temp write_data = emit_extract_vector(ctx, data, start[i], RegClass(RegType::vgpr, count[i] * elem_size_bytes / 4));
4709       ds_write_helper(ctx, m, address, write_data, offset, start[i] * elem_size_bytes, align);
4710    }
4711    return;
4712 }
4713
4714 void visit_shared_atomic(isel_context *ctx, nir_intrinsic_instr *instr)
4715 {
4716    unsigned offset = nir_intrinsic_base(instr);
4717    Operand m = load_lds_size_m0(ctx);
4718    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4719    Temp address = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4720
4721    unsigned num_operands = 3;
4722    aco_opcode op32, op64, op32_rtn, op64_rtn;
4723    switch(instr->intrinsic) {
4724       case nir_intrinsic_shared_atomic_add:
4725          op32 = aco_opcode::ds_add_u32;
4726          op64 = aco_opcode::ds_add_u64;
4727          op32_rtn = aco_opcode::ds_add_rtn_u32;
4728          op64_rtn = aco_opcode::ds_add_rtn_u64;
4729          break;
4730       case nir_intrinsic_shared_atomic_imin:
4731          op32 = aco_opcode::ds_min_i32;
4732          op64 = aco_opcode::ds_min_i64;
4733          op32_rtn = aco_opcode::ds_min_rtn_i32;
4734          op64_rtn = aco_opcode::ds_min_rtn_i64;
4735          break;
4736       case nir_intrinsic_shared_atomic_umin:
4737          op32 = aco_opcode::ds_min_u32;
4738          op64 = aco_opcode::ds_min_u64;
4739          op32_rtn = aco_opcode::ds_min_rtn_u32;
4740          op64_rtn = aco_opcode::ds_min_rtn_u64;
4741          break;
4742       case nir_intrinsic_shared_atomic_imax:
4743          op32 = aco_opcode::ds_max_i32;
4744          op64 = aco_opcode::ds_max_i64;
4745          op32_rtn = aco_opcode::ds_max_rtn_i32;
4746          op64_rtn = aco_opcode::ds_max_rtn_i64;
4747          break;
4748       case nir_intrinsic_shared_atomic_umax:
4749          op32 = aco_opcode::ds_max_u32;
4750          op64 = aco_opcode::ds_max_u64;
4751          op32_rtn = aco_opcode::ds_max_rtn_u32;
4752          op64_rtn = aco_opcode::ds_max_rtn_u64;
4753          break;
4754       case nir_intrinsic_shared_atomic_and:
4755          op32 = aco_opcode::ds_and_b32;
4756          op64 = aco_opcode::ds_and_b64;
4757          op32_rtn = aco_opcode::ds_and_rtn_b32;
4758          op64_rtn = aco_opcode::ds_and_rtn_b64;
4759          break;
4760       case nir_intrinsic_shared_atomic_or:
4761          op32 = aco_opcode::ds_or_b32;
4762          op64 = aco_opcode::ds_or_b64;
4763          op32_rtn = aco_opcode::ds_or_rtn_b32;
4764          op64_rtn = aco_opcode::ds_or_rtn_b64;
4765          break;
4766       case nir_intrinsic_shared_atomic_xor:
4767          op32 = aco_opcode::ds_xor_b32;
4768          op64 = aco_opcode::ds_xor_b64;
4769          op32_rtn = aco_opcode::ds_xor_rtn_b32;
4770          op64_rtn = aco_opcode::ds_xor_rtn_b64;
4771          break;
4772       case nir_intrinsic_shared_atomic_exchange:
4773          op32 = aco_opcode::ds_write_b32;
4774          op64 = aco_opcode::ds_write_b64;
4775          op32_rtn = aco_opcode::ds_wrxchg_rtn_b32;
4776          op64_rtn = aco_opcode::ds_wrxchg2_rtn_b64;
4777          break;
4778       case nir_intrinsic_shared_atomic_comp_swap:
4779          op32 = aco_opcode::ds_cmpst_b32;
4780          op64 = aco_opcode::ds_cmpst_b64;
4781          op32_rtn = aco_opcode::ds_cmpst_rtn_b32;
4782          op64_rtn = aco_opcode::ds_cmpst_rtn_b64;
4783          num_operands = 4;
4784          break;
4785       default:
4786          unreachable("Unhandled shared atomic intrinsic");
4787    }
4788
4789    /* return the previous value if dest is ever used */
4790    bool return_previous = false;
4791    nir_foreach_use_safe(use_src, &instr->dest.ssa) {
4792       return_previous = true;
4793       break;
4794    }
4795    nir_foreach_if_use_safe(use_src, &instr->dest.ssa) {
4796       return_previous = true;
4797       break;
4798    }
4799
4800    aco_opcode op;
4801    if (data.size() == 1) {
4802       assert(instr->dest.ssa.bit_size == 32);
4803       op = return_previous ? op32_rtn : op32;
4804    } else {
4805       assert(instr->dest.ssa.bit_size == 64);
4806       op = return_previous ? op64_rtn : op64;
4807    }
4808
4809    if (offset > 65535) {
4810       Builder bld(ctx->program, ctx->block);
4811       address = bld.vadd32(bld.def(v1), Operand(offset), address);
4812       offset = 0;
4813    }
4814
4815    aco_ptr<DS_instruction> ds;
4816    ds.reset(create_instruction<DS_instruction>(op, Format::DS, num_operands, return_previous ? 1 : 0));
4817    ds->operands[0] = Operand(address);
4818    ds->operands[1] = Operand(data);
4819    if (num_operands == 4)
4820       ds->operands[2] = Operand(get_ssa_temp(ctx, instr->src[2].ssa));
4821    ds->operands[num_operands - 1] = m;
4822    ds->offset0 = offset;
4823    if (return_previous)
4824       ds->definitions[0] = Definition(get_ssa_temp(ctx, &instr->dest.ssa));
4825    ctx->block->instructions.emplace_back(std::move(ds));
4826 }
4827
4828 void visit_load_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4829    assert(instr->dest.ssa.bit_size == 32 || instr->dest.ssa.bit_size == 64);
4830    Builder bld(ctx->program, ctx->block);
4831    Temp scratch_addr = ctx->private_segment_buffer;
4832    if (ctx->stage != MESA_SHADER_COMPUTE)
4833       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), ctx->private_segment_buffer, Operand(0u));
4834    uint32_t rsrc_conf;
4835    /* older generations need element size = 16 bytes */
4836    if (ctx->program->chip_class >= GFX9)
4837       rsrc_conf = 0x00E00000u;
4838    else
4839       rsrc_conf = 0x00F80000u;
4840    /* buffer res = addr + num_records = -1, index_stride = 64, add_tid_enable = true */
4841    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4842    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4843    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4844
4845    aco_opcode op;
4846    switch (dst.size()) {
4847       case 1:
4848          op = aco_opcode::buffer_load_dword;
4849          break;
4850       case 2:
4851          op = aco_opcode::buffer_load_dwordx2;
4852          break;
4853       case 3:
4854          op = aco_opcode::buffer_load_dwordx3;
4855          break;
4856       case 4:
4857          op = aco_opcode::buffer_load_dwordx4;
4858          break;
4859       case 6:
4860       case 8: {
4861          std::array<Temp,NIR_MAX_VEC_COMPONENTS> elems;
4862          Temp lower = bld.mubuf(aco_opcode::buffer_load_dwordx4,
4863                                 bld.def(v4), offset, rsrc,
4864                                 ctx->scratch_offset, 0, true);
4865          Temp upper = bld.mubuf(dst.size() == 6 ? aco_opcode::buffer_load_dwordx2 :
4866                                                   aco_opcode::buffer_load_dwordx4,
4867                                 dst.size() == 6 ? bld.def(v2) : bld.def(v4),
4868                                 offset, rsrc, ctx->scratch_offset, 16, true);
4869          emit_split_vector(ctx, lower, 2);
4870          elems[0] = emit_extract_vector(ctx, lower, 0, v2);
4871          elems[1] = emit_extract_vector(ctx, lower, 1, v2);
4872          if (dst.size() == 8) {
4873             emit_split_vector(ctx, upper, 2);
4874             elems[2] = emit_extract_vector(ctx, upper, 0, v2);
4875             elems[3] = emit_extract_vector(ctx, upper, 1, v2);
4876          } else {
4877             elems[2] = upper;
4878          }
4879
4880          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
4881                                                                          Format::PSEUDO, dst.size() / 2, 1)};
4882          for (unsigned i = 0; i < dst.size() / 2; i++)
4883             vec->operands[i] = Operand(elems[i]);
4884          vec->definitions[0] = Definition(dst);
4885          bld.insert(std::move(vec));
4886          ctx->allocated_vec.emplace(dst.id(), elems);
4887          return;
4888       }
4889       default:
4890          unreachable("Wrong dst size for nir_intrinsic_load_scratch");
4891    }
4892
4893    bld.mubuf(op, Definition(dst), offset, rsrc, ctx->scratch_offset, 0, true);
4894    emit_split_vector(ctx, dst, instr->num_components);
4895 }
4896
4897 void visit_store_scratch(isel_context *ctx, nir_intrinsic_instr *instr) {
4898    assert(instr->src[0].ssa->bit_size == 32 || instr->src[0].ssa->bit_size == 64);
4899    Builder bld(ctx->program, ctx->block);
4900    Temp scratch_addr = ctx->private_segment_buffer;
4901    if (ctx->stage != MESA_SHADER_COMPUTE)
4902       scratch_addr = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), ctx->private_segment_buffer, Operand(0u));
4903    uint32_t rsrc_conf;
4904    /* older generations need element size = 16 bytes */
4905    if (ctx->program->chip_class >= GFX9)
4906       rsrc_conf = 0x00E00000u;
4907    else
4908       rsrc_conf = 0x00F80000u;
4909    /* buffer res = addr + num_records = -1, index_stride = 64, add_tid_enable = true */
4910    Temp rsrc = bld.pseudo(aco_opcode::p_create_vector, bld.def(s4), scratch_addr, Operand(-1u), Operand(rsrc_conf));
4911    Temp data = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
4912    Temp offset = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[1].ssa));
4913
4914    unsigned elem_size_bytes = instr->src[0].ssa->bit_size / 8;
4915    unsigned writemask = nir_intrinsic_write_mask(instr);
4916
4917    while (writemask) {
4918       int start, count;
4919       u_bit_scan_consecutive_range(&writemask, &start, &count);
4920       int num_bytes = count * elem_size_bytes;
4921
4922       if (num_bytes > 16) {
4923          assert(elem_size_bytes == 8);
4924          writemask |= (((count - 2) << 1) - 1) << (start + 2);
4925          count = 2;
4926          num_bytes = 16;
4927       }
4928
4929       // TODO: check alignment of sub-dword stores
4930       // TODO: split 3 bytes. there is no store instruction for that
4931
4932       Temp write_data;
4933       if (count != instr->num_components) {
4934          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, count, 1)};
4935          for (int i = 0; i < count; i++) {
4936             Temp elem = emit_extract_vector(ctx, data, start + i, RegClass(RegType::vgpr, elem_size_bytes / 4));
4937             vec->operands[i] = Operand(elem);
4938          }
4939          write_data = bld.tmp(RegClass(RegType::vgpr, count * elem_size_bytes / 4));
4940          vec->definitions[0] = Definition(write_data);
4941          ctx->block->instructions.emplace_back(std::move(vec));
4942       } else {
4943          write_data = data;
4944       }
4945
4946       aco_opcode op;
4947       switch (num_bytes) {
4948          case 4:
4949             op = aco_opcode::buffer_store_dword;
4950             break;
4951          case 8:
4952             op = aco_opcode::buffer_store_dwordx2;
4953             break;
4954          case 12:
4955             op = aco_opcode::buffer_store_dwordx3;
4956             break;
4957          case 16:
4958             op = aco_opcode::buffer_store_dwordx4;
4959             break;
4960          default:
4961             unreachable("Invalid data size for nir_intrinsic_store_scratch.");
4962       }
4963
4964       bld.mubuf(op, offset, rsrc, ctx->scratch_offset, write_data, start * elem_size_bytes, true);
4965    }
4966 }
4967
4968 void visit_load_sample_mask_in(isel_context *ctx, nir_intrinsic_instr *instr) {
4969    uint8_t log2_ps_iter_samples;
4970    if (ctx->program->info->ps.force_persample) {
4971       log2_ps_iter_samples =
4972          util_logbase2(ctx->options->key.fs.num_samples);
4973    } else {
4974       log2_ps_iter_samples = ctx->options->key.fs.log2_ps_iter_samples;
4975    }
4976
4977    /* The bit pattern matches that used by fixed function fragment
4978     * processing. */
4979    static const unsigned ps_iter_masks[] = {
4980       0xffff, /* not used */
4981       0x5555,
4982       0x1111,
4983       0x0101,
4984       0x0001,
4985    };
4986    assert(log2_ps_iter_samples < ARRAY_SIZE(ps_iter_masks));
4987
4988    Builder bld(ctx->program, ctx->block);
4989
4990    Temp sample_id = bld.vop3(aco_opcode::v_bfe_u32, bld.def(v1), ctx->fs_inputs[fs_input::ancillary], Operand(8u), Operand(4u));
4991    Temp ps_iter_mask = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(ps_iter_masks[log2_ps_iter_samples]));
4992    Temp mask = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), sample_id, ps_iter_mask);
4993    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
4994    bld.vop2(aco_opcode::v_and_b32, Definition(dst), mask, ctx->fs_inputs[fs_input::sample_coverage]);
4995 }
4996
4997 Temp emit_boolean_reduce(isel_context *ctx, nir_op op, unsigned cluster_size, Temp src)
4998 {
4999    Builder bld(ctx->program, ctx->block);
5000
5001    if (cluster_size == 1) {
5002       return src;
5003    } if (op == nir_op_iand && cluster_size == 4) {
5004       //subgroupClusteredAnd(val, 4) -> ~wqm(exec & ~val)
5005       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5006       return bld.sop1(aco_opcode::s_not_b64, bld.def(s2), bld.def(s1, scc),
5007                       bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc), tmp));
5008    } else if (op == nir_op_ior && cluster_size == 4) {
5009       //subgroupClusteredOr(val, 4) -> wqm(val & exec)
5010       return bld.sop1(aco_opcode::s_wqm_b64, bld.def(s2), bld.def(s1, scc),
5011                       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)));
5012    } else if (op == nir_op_iand && cluster_size == 64) {
5013       //subgroupAnd(val) -> (exec & ~val) == 0
5014       Temp tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src).def(1).getTemp();
5015       return bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), tmp, Operand(0u));
5016    } else if (op == nir_op_ior && cluster_size == 64) {
5017       //subgroupOr(val) -> (val & exec) != 0
5018       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)).def(1).getTemp();
5019    } else if (op == nir_op_ixor && cluster_size == 64) {
5020       //subgroupXor(val) -> s_bcnt1_i32_b64(val & exec) & 1
5021       Temp tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5022       tmp = bld.sop1(aco_opcode::s_bcnt1_i32_b64, bld.def(s2), bld.def(s1, scc), tmp);
5023       return bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), tmp, Operand(1u)).def(1).getTemp();
5024    } else {
5025       //subgroupClustered{And,Or,Xor}(val, n) ->
5026       //lane_id = v_mbcnt_hi_u32_b32(-1, v_mbcnt_lo_u32_b32(-1, 0))
5027       //cluster_offset = ~(n - 1) & lane_id
5028       //cluster_mask = ((1 << n) - 1)
5029       //subgroupClusteredAnd():
5030       //   return ((val | ~exec) >> cluster_offset) & cluster_mask == cluster_mask
5031       //subgroupClusteredOr():
5032       //   return ((val & exec) >> cluster_offset) & cluster_mask != 0
5033       //subgroupClusteredXor():
5034       //   return v_bnt_u32_b32(((val & exec) >> cluster_offset) & cluster_mask, 0) & 1 != 0
5035       Temp lane_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5036                               bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5037       Temp cluster_offset = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(~uint32_t(cluster_size - 1)), lane_id);
5038
5039       Temp tmp;
5040       if (op == nir_op_iand)
5041          tmp = bld.sop2(aco_opcode::s_orn2_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5042       else
5043          tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5044
5045       uint32_t cluster_mask = cluster_size == 32 ? -1 : (1u << cluster_size) - 1u;
5046       tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), cluster_offset, tmp);
5047       tmp = emit_extract_vector(ctx, tmp, 0, v1);
5048       if (cluster_mask != 0xffffffff)
5049          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(cluster_mask), tmp);
5050
5051       Definition cmp_def = Definition();
5052       if (op == nir_op_iand) {
5053          cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(cluster_mask), tmp).def(0);
5054       } else if (op == nir_op_ior) {
5055          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5056       } else if (op == nir_op_ixor) {
5057          tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u),
5058                         bld.vop3(aco_opcode::v_bcnt_u32_b32, bld.def(v1), tmp, Operand(0u)));
5059          cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp).def(0);
5060       }
5061       cmp_def.setHint(vcc);
5062       return cmp_def.getTemp();
5063    }
5064 }
5065
5066 Temp emit_boolean_exclusive_scan(isel_context *ctx, nir_op op, Temp src)
5067 {
5068    Builder bld(ctx->program, ctx->block);
5069
5070    //subgroupExclusiveAnd(val) -> mbcnt(exec & ~val) == 0
5071    //subgroupExclusiveOr(val) -> mbcnt(val & exec) != 0
5072    //subgroupExclusiveXor(val) -> mbcnt(val & exec) & 1 != 0
5073    Temp tmp;
5074    if (op == nir_op_iand)
5075       tmp = bld.sop2(aco_opcode::s_andn2_b64, bld.def(s2), bld.def(s1, scc), Operand(exec, s2), src);
5076    else
5077       tmp = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2));
5078
5079    Builder::Result lohi = bld.pseudo(aco_opcode::p_split_vector, bld.def(s1), bld.def(s1), tmp);
5080    Temp lo = lohi.def(0).getTemp();
5081    Temp hi = lohi.def(1).getTemp();
5082    Temp mbcnt = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), hi,
5083                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), lo, Operand(0u)));
5084
5085    Definition cmp_def = Definition();
5086    if (op == nir_op_iand)
5087       cmp_def = bld.vopc(aco_opcode::v_cmp_eq_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5088    else if (op == nir_op_ior)
5089       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), mbcnt).def(0);
5090    else if (op == nir_op_ixor)
5091       cmp_def = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u),
5092                          bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), mbcnt)).def(0);
5093    cmp_def.setHint(vcc);
5094    return cmp_def.getTemp();
5095 }
5096
5097 Temp emit_boolean_inclusive_scan(isel_context *ctx, nir_op op, Temp src)
5098 {
5099    Builder bld(ctx->program, ctx->block);
5100
5101    //subgroupInclusiveAnd(val) -> subgroupExclusiveAnd(val) && val
5102    //subgroupInclusiveOr(val) -> subgroupExclusiveOr(val) || val
5103    //subgroupInclusiveXor(val) -> subgroupExclusiveXor(val) ^^ val
5104    Temp tmp = emit_boolean_exclusive_scan(ctx, op, src);
5105    if (op == nir_op_iand)
5106       return bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5107    else if (op == nir_op_ior)
5108       return bld.sop2(aco_opcode::s_or_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5109    else if (op == nir_op_ixor)
5110       return bld.sop2(aco_opcode::s_xor_b64, bld.def(s2), bld.def(s1, scc), tmp, src);
5111
5112    assert(false);
5113    return Temp();
5114 }
5115
5116 void emit_uniform_subgroup(isel_context *ctx, nir_intrinsic_instr *instr, Temp src)
5117 {
5118    Builder bld(ctx->program, ctx->block);
5119    Definition dst(get_ssa_temp(ctx, &instr->dest.ssa));
5120    if (src.regClass().type() == RegType::vgpr) {
5121       bld.pseudo(aco_opcode::p_as_uniform, dst, src);
5122    } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5123       bld.sopc(aco_opcode::s_cmp_lg_u64, bld.scc(dst), Operand(0u), Operand(src));
5124    } else if (src.regClass() == s1) {
5125       bld.sop1(aco_opcode::s_mov_b32, dst, src);
5126    } else if (src.regClass() == s2) {
5127       bld.sop1(aco_opcode::s_mov_b64, dst, src);
5128    } else {
5129       fprintf(stderr, "Unimplemented NIR instr bit size: ");
5130       nir_print_instr(&instr->instr, stderr);
5131       fprintf(stderr, "\n");
5132    }
5133 }
5134
5135 void emit_interp_center(isel_context *ctx, Temp dst, Temp pos1, Temp pos2)
5136 {
5137    Builder bld(ctx->program, ctx->block);
5138    Temp p1 = ctx->fs_inputs[fs_input::persp_center_p1];
5139    Temp p2 = ctx->fs_inputs[fs_input::persp_center_p2];
5140
5141    /* Build DD X/Y */
5142    Temp tl_1 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p1, dpp_quad_perm(0, 0, 0, 0));
5143    Temp ddx_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(1, 1, 1, 1));
5144    Temp ddy_1 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p1, tl_1, dpp_quad_perm(2, 2, 2, 2));
5145    Temp tl_2 = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), p2, dpp_quad_perm(0, 0, 0, 0));
5146    Temp ddx_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(1, 1, 1, 1));
5147    Temp ddy_2 = bld.vop2_dpp(aco_opcode::v_sub_f32, bld.def(v1), p2, tl_2, dpp_quad_perm(2, 2, 2, 2));
5148
5149    /* res_k = p_k + ddx_k * pos1 + ddy_k * pos2 */
5150    Temp tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_1, pos1, p1);
5151    Temp tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddx_2, pos1, p2);
5152    tmp1 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_1, pos2, tmp1);
5153    tmp2 = bld.vop3(aco_opcode::v_mad_f32, bld.def(v1), ddy_2, pos2, tmp2);
5154    Temp wqm1 = bld.tmp(v1);
5155    emit_wqm(ctx, tmp1, wqm1, true);
5156    Temp wqm2 = bld.tmp(v1);
5157    emit_wqm(ctx, tmp2, wqm2, true);
5158    bld.pseudo(aco_opcode::p_create_vector, Definition(dst), wqm1, wqm2);
5159    return;
5160 }
5161
5162 void visit_intrinsic(isel_context *ctx, nir_intrinsic_instr *instr)
5163 {
5164    Builder bld(ctx->program, ctx->block);
5165    switch(instr->intrinsic) {
5166    case nir_intrinsic_load_barycentric_sample:
5167    case nir_intrinsic_load_barycentric_pixel:
5168    case nir_intrinsic_load_barycentric_centroid: {
5169       glsl_interp_mode mode = (glsl_interp_mode)nir_intrinsic_interp_mode(instr);
5170       fs_input input = get_interp_input(instr->intrinsic, mode);
5171
5172       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5173       if (input == fs_input::max_inputs) {
5174          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5175                     Operand(0u), Operand(0u));
5176       } else {
5177          bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5178                     ctx->fs_inputs[input],
5179                     ctx->fs_inputs[input + 1]);
5180       }
5181       emit_split_vector(ctx, dst, 2);
5182       break;
5183    }
5184    case nir_intrinsic_load_barycentric_at_sample: {
5185       uint32_t sample_pos_offset = RING_PS_SAMPLE_POSITIONS * 16;
5186       switch (ctx->options->key.fs.num_samples) {
5187          case 2: sample_pos_offset += 1 << 3; break;
5188          case 4: sample_pos_offset += 3 << 3; break;
5189          case 8: sample_pos_offset += 7 << 3; break;
5190          default: break;
5191       }
5192       Temp sample_pos;
5193       Temp addr = get_ssa_temp(ctx, instr->src[0].ssa);
5194       nir_const_value* const_addr = nir_src_as_const_value(instr->src[0]);
5195       if (addr.type() == RegType::sgpr) {
5196          Operand offset;
5197          if (const_addr) {
5198             sample_pos_offset += const_addr->u32 << 3;
5199             offset = Operand(sample_pos_offset);
5200          } else if (ctx->options->chip_class >= GFX9) {
5201             offset = bld.sop2(aco_opcode::s_lshl3_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5202          } else {
5203             offset = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), addr, Operand(3u));
5204             offset = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), bld.def(s1, scc), addr, Operand(sample_pos_offset));
5205          }
5206          addr = ctx->private_segment_buffer;
5207          sample_pos = bld.smem(aco_opcode::s_load_dwordx2, bld.def(s2), addr, Operand(offset));
5208
5209       } else if (ctx->options->chip_class >= GFX9) {
5210          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5211          sample_pos = bld.global(aco_opcode::global_load_dwordx2, bld.def(v2), addr, ctx->private_segment_buffer, sample_pos_offset);
5212       } else {
5213          /* addr += ctx->private_segment_buffer + sample_pos_offset */
5214          Temp tmp0 = bld.tmp(s1);
5215          Temp tmp1 = bld.tmp(s1);
5216          bld.pseudo(aco_opcode::p_split_vector, Definition(tmp0), Definition(tmp1), ctx->private_segment_buffer);
5217          Definition scc_tmp = bld.def(s1, scc);
5218          tmp0 = bld.sop2(aco_opcode::s_add_u32, bld.def(s1), scc_tmp, tmp0, Operand(sample_pos_offset));
5219          tmp1 = bld.sop2(aco_opcode::s_addc_u32, bld.def(s1), bld.def(s1, scc), tmp1, Operand(0u), scc_tmp.getTemp());
5220          addr = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(3u), addr);
5221          Temp pck0 = bld.tmp(v1);
5222          Temp carry = bld.vadd32(Definition(pck0), tmp0, addr, true).def(1).getTemp();
5223          tmp1 = as_vgpr(ctx, tmp1);
5224          Temp pck1 = bld.vop2_e64(aco_opcode::v_addc_co_u32, bld.def(v1), bld.hint_vcc(bld.def(s2)), tmp1, Operand(0u), carry);
5225          addr = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), pck0, pck1);
5226
5227          /* sample_pos = flat_load_dwordx2 addr */
5228          sample_pos = bld.flat(aco_opcode::flat_load_dwordx2, bld.def(v2), addr, Operand(s1));
5229       }
5230
5231       /* sample_pos -= 0.5 */
5232       Temp pos1 = bld.tmp(RegClass(sample_pos.type(), 1));
5233       Temp pos2 = bld.tmp(RegClass(sample_pos.type(), 1));
5234       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), sample_pos);
5235       pos1 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos1, Operand(0x3f000000u));
5236       pos2 = bld.vop2_e64(aco_opcode::v_sub_f32, bld.def(v1), pos2, Operand(0x3f000000u));
5237
5238       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5239       break;
5240    }
5241    case nir_intrinsic_load_barycentric_at_offset: {
5242       Temp offset = get_ssa_temp(ctx, instr->src[0].ssa);
5243       RegClass rc = RegClass(offset.type(), 1);
5244       Temp pos1 = bld.tmp(rc), pos2 = bld.tmp(rc);
5245       bld.pseudo(aco_opcode::p_split_vector, Definition(pos1), Definition(pos2), offset);
5246       emit_interp_center(ctx, get_ssa_temp(ctx, &instr->dest.ssa), pos1, pos2);
5247       break;
5248    }
5249    case nir_intrinsic_load_front_face: {
5250       bld.vopc(aco_opcode::v_cmp_lg_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5251                Operand(0u), ctx->fs_inputs[fs_input::front_face]).def(0).setHint(vcc);
5252       break;
5253    }
5254    case nir_intrinsic_load_view_index:
5255    case nir_intrinsic_load_layer_id: {
5256       if (instr->intrinsic == nir_intrinsic_load_view_index && (ctx->stage & sw_vs)) {
5257          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5258          bld.copy(Definition(dst), Operand(ctx->view_index));
5259          break;
5260       }
5261
5262       unsigned idx = nir_intrinsic_base(instr);
5263       bld.vintrp(aco_opcode::v_interp_mov_f32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5264                  Operand(2u), bld.m0(ctx->prim_mask), idx, 0);
5265       break;
5266    }
5267    case nir_intrinsic_load_frag_coord: {
5268       emit_load_frag_coord(ctx, get_ssa_temp(ctx, &instr->dest.ssa), 4);
5269       break;
5270    }
5271    case nir_intrinsic_load_sample_pos: {
5272       Temp posx = ctx->fs_inputs[fs_input::frag_pos_0];
5273       Temp posy = ctx->fs_inputs[fs_input::frag_pos_1];
5274       bld.pseudo(aco_opcode::p_create_vector, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5275                  posx.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posx) : Operand(0u),
5276                  posy.id() ? bld.vop1(aco_opcode::v_fract_f32, bld.def(v1), posy) : Operand(0u));
5277       break;
5278    }
5279    case nir_intrinsic_load_interpolated_input:
5280       visit_load_interpolated_input(ctx, instr);
5281       break;
5282    case nir_intrinsic_store_output:
5283       visit_store_output(ctx, instr);
5284       break;
5285    case nir_intrinsic_load_input:
5286       visit_load_input(ctx, instr);
5287       break;
5288    case nir_intrinsic_load_ubo:
5289       visit_load_ubo(ctx, instr);
5290       break;
5291    case nir_intrinsic_load_push_constant:
5292       visit_load_push_constant(ctx, instr);
5293       break;
5294    case nir_intrinsic_load_constant:
5295       visit_load_constant(ctx, instr);
5296       break;
5297    case nir_intrinsic_vulkan_resource_index:
5298       visit_load_resource(ctx, instr);
5299       break;
5300    case nir_intrinsic_discard:
5301       visit_discard(ctx, instr);
5302       break;
5303    case nir_intrinsic_discard_if:
5304       visit_discard_if(ctx, instr);
5305       break;
5306    case nir_intrinsic_load_shared:
5307       visit_load_shared(ctx, instr);
5308       break;
5309    case nir_intrinsic_store_shared:
5310       visit_store_shared(ctx, instr);
5311       break;
5312    case nir_intrinsic_shared_atomic_add:
5313    case nir_intrinsic_shared_atomic_imin:
5314    case nir_intrinsic_shared_atomic_umin:
5315    case nir_intrinsic_shared_atomic_imax:
5316    case nir_intrinsic_shared_atomic_umax:
5317    case nir_intrinsic_shared_atomic_and:
5318    case nir_intrinsic_shared_atomic_or:
5319    case nir_intrinsic_shared_atomic_xor:
5320    case nir_intrinsic_shared_atomic_exchange:
5321    case nir_intrinsic_shared_atomic_comp_swap:
5322       visit_shared_atomic(ctx, instr);
5323       break;
5324    case nir_intrinsic_image_deref_load:
5325       visit_image_load(ctx, instr);
5326       break;
5327    case nir_intrinsic_image_deref_store:
5328       visit_image_store(ctx, instr);
5329       break;
5330    case nir_intrinsic_image_deref_atomic_add:
5331    case nir_intrinsic_image_deref_atomic_umin:
5332    case nir_intrinsic_image_deref_atomic_imin:
5333    case nir_intrinsic_image_deref_atomic_umax:
5334    case nir_intrinsic_image_deref_atomic_imax:
5335    case nir_intrinsic_image_deref_atomic_and:
5336    case nir_intrinsic_image_deref_atomic_or:
5337    case nir_intrinsic_image_deref_atomic_xor:
5338    case nir_intrinsic_image_deref_atomic_exchange:
5339    case nir_intrinsic_image_deref_atomic_comp_swap:
5340       visit_image_atomic(ctx, instr);
5341       break;
5342    case nir_intrinsic_image_deref_size:
5343       visit_image_size(ctx, instr);
5344       break;
5345    case nir_intrinsic_load_ssbo:
5346       visit_load_ssbo(ctx, instr);
5347       break;
5348    case nir_intrinsic_store_ssbo:
5349       visit_store_ssbo(ctx, instr);
5350       break;
5351    case nir_intrinsic_load_global:
5352       visit_load_global(ctx, instr);
5353       break;
5354    case nir_intrinsic_store_global:
5355       visit_store_global(ctx, instr);
5356       break;
5357    case nir_intrinsic_ssbo_atomic_add:
5358    case nir_intrinsic_ssbo_atomic_imin:
5359    case nir_intrinsic_ssbo_atomic_umin:
5360    case nir_intrinsic_ssbo_atomic_imax:
5361    case nir_intrinsic_ssbo_atomic_umax:
5362    case nir_intrinsic_ssbo_atomic_and:
5363    case nir_intrinsic_ssbo_atomic_or:
5364    case nir_intrinsic_ssbo_atomic_xor:
5365    case nir_intrinsic_ssbo_atomic_exchange:
5366    case nir_intrinsic_ssbo_atomic_comp_swap:
5367       visit_atomic_ssbo(ctx, instr);
5368       break;
5369    case nir_intrinsic_load_scratch:
5370       visit_load_scratch(ctx, instr);
5371       break;
5372    case nir_intrinsic_store_scratch:
5373       visit_store_scratch(ctx, instr);
5374       break;
5375    case nir_intrinsic_get_buffer_size:
5376       visit_get_buffer_size(ctx, instr);
5377       break;
5378    case nir_intrinsic_barrier: {
5379       unsigned* bsize = ctx->program->info->cs.block_size;
5380       unsigned workgroup_size = bsize[0] * bsize[1] * bsize[2];
5381       if (workgroup_size > 64)
5382          bld.sopp(aco_opcode::s_barrier);
5383       break;
5384    }
5385    case nir_intrinsic_group_memory_barrier:
5386    case nir_intrinsic_memory_barrier:
5387    case nir_intrinsic_memory_barrier_atomic_counter:
5388    case nir_intrinsic_memory_barrier_buffer:
5389    case nir_intrinsic_memory_barrier_image:
5390    case nir_intrinsic_memory_barrier_shared:
5391       emit_memory_barrier(ctx, instr);
5392       break;
5393    case nir_intrinsic_load_num_work_groups:
5394    case nir_intrinsic_load_work_group_id:
5395    case nir_intrinsic_load_local_invocation_id: {
5396       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5397       Temp* ids;
5398       if (instr->intrinsic == nir_intrinsic_load_num_work_groups)
5399          ids = ctx->num_workgroups;
5400       else if (instr->intrinsic == nir_intrinsic_load_work_group_id)
5401          ids = ctx->workgroup_ids;
5402       else
5403          ids = ctx->local_invocation_ids;
5404       bld.pseudo(aco_opcode::p_create_vector, Definition(dst),
5405                  ids[0].id() ? Operand(ids[0]) : Operand(1u),
5406                  ids[1].id() ? Operand(ids[1]) : Operand(1u),
5407                  ids[2].id() ? Operand(ids[2]) : Operand(1u));
5408       emit_split_vector(ctx, dst, 3);
5409       break;
5410    }
5411    case nir_intrinsic_load_local_invocation_index: {
5412       Temp id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
5413                          bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5414       Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5415       bld.vop2(aco_opcode::v_or_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), tg_num, id);
5416       break;
5417    }
5418    case nir_intrinsic_load_subgroup_id: {
5419       if (ctx->stage == compute_cs) {
5420          Temp tg_num = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), Operand(0xfc0u), ctx->tg_size);
5421          bld.sop2(aco_opcode::s_lshr_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), tg_num, Operand(0x6u));
5422       } else {
5423          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x0u));
5424       }
5425       break;
5426    }
5427    case nir_intrinsic_load_subgroup_invocation: {
5428       bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand((uint32_t) -1),
5429                bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
5430       break;
5431    }
5432    case nir_intrinsic_load_num_subgroups: {
5433       if (ctx->stage == compute_cs)
5434          bld.sop2(aco_opcode::s_and_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), bld.def(s1, scc), Operand(0x3fu), ctx->tg_size);
5435       else
5436          bld.sop1(aco_opcode::s_mov_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)), Operand(0x1u));
5437       break;
5438    }
5439    case nir_intrinsic_ballot: {
5440       Definition tmp = bld.def(s2);
5441       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5442       if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s2) {
5443          bld.sop2(aco_opcode::s_and_b64, tmp, bld.def(s1, scc), Operand(exec, s2), src);
5444       } else if (instr->src[0].ssa->bit_size == 1 && src.regClass() == s1) {
5445          bld.sop2(aco_opcode::s_cselect_b64, tmp, Operand(exec, s2), Operand(0u), bld.scc(src));
5446       } else if (instr->src[0].ssa->bit_size == 32 && src.regClass() == v1) {
5447          bld.vopc(aco_opcode::v_cmp_lg_u32, tmp, Operand(0u), src);
5448       } else if (instr->src[0].ssa->bit_size == 64 && src.regClass() == v2) {
5449          bld.vopc(aco_opcode::v_cmp_lg_u64, tmp, Operand(0u), src);
5450       } else {
5451          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5452          nir_print_instr(&instr->instr, stderr);
5453          fprintf(stderr, "\n");
5454       }
5455       emit_wqm(ctx, tmp.getTemp(), get_ssa_temp(ctx, &instr->dest.ssa));
5456       break;
5457    }
5458    case nir_intrinsic_shuffle: {
5459       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5460       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5461          emit_uniform_subgroup(ctx, instr, src);
5462       } else {
5463          Temp tid = get_ssa_temp(ctx, instr->src[1].ssa);
5464          assert(tid.regClass() == v1);
5465          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5466          if (src.regClass() == v1) {
5467             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5468             emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, src), dst);
5469          } else if (src.regClass() == v2) {
5470             tid = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), tid);
5471
5472             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5473             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5474             lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, lo));
5475             hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_bpermute_b32, bld.def(v1), tid, hi));
5476             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5477             emit_split_vector(ctx, dst, 2);
5478          } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5479             Temp tmp = bld.vop3(aco_opcode::v_lshrrev_b64, bld.def(v2), tid, src);
5480             tmp = emit_extract_vector(ctx, tmp, 0, v1);
5481             tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(1u), tmp);
5482             emit_wqm(ctx, bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), tmp), dst);
5483          } else {
5484             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5485             nir_print_instr(&instr->instr, stderr);
5486             fprintf(stderr, "\n");
5487          }
5488       }
5489       break;
5490    }
5491    case nir_intrinsic_load_sample_id: {
5492       bld.vop3(aco_opcode::v_bfe_u32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
5493                ctx->fs_inputs[ancillary], Operand(8u), Operand(4u));
5494       break;
5495    }
5496    case nir_intrinsic_load_sample_mask_in: {
5497       visit_load_sample_mask_in(ctx, instr);
5498       break;
5499    }
5500    case nir_intrinsic_read_first_invocation: {
5501       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5502       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5503       if (src.regClass() == v1) {
5504          emit_wqm(ctx,
5505                   bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), src),
5506                   dst);
5507       } else if (src.regClass() == v2) {
5508          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5509          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5510          lo = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), lo));
5511          hi = emit_wqm(ctx, bld.vop1(aco_opcode::v_readfirstlane_b32, bld.def(s1), hi));
5512          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5513          emit_split_vector(ctx, dst, 2);
5514       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5515          emit_wqm(ctx,
5516                   bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src,
5517                            bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2))),
5518                   dst);
5519       } else if (src.regClass() == s1) {
5520          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5521       } else if (src.regClass() == s2) {
5522          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5523       } else {
5524          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5525          nir_print_instr(&instr->instr, stderr);
5526          fprintf(stderr, "\n");
5527       }
5528       break;
5529    }
5530    case nir_intrinsic_read_invocation: {
5531       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5532       Temp lane = get_ssa_temp(ctx, instr->src[1].ssa);
5533       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5534       assert(lane.regClass() == s1);
5535       if (src.regClass() == v1) {
5536          emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), src, lane), dst);
5537       } else if (src.regClass() == v2) {
5538          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5539          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5540          lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), lo, lane));
5541          hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_readlane_b32, bld.def(s1), hi, lane));
5542          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5543          emit_split_vector(ctx, dst, 2);
5544       } else if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5545          emit_wqm(ctx, bld.sopc(aco_opcode::s_bitcmp1_b64, bld.def(s1, scc), src, lane), dst);
5546       } else if (src.regClass() == s1) {
5547          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), src);
5548       } else if (src.regClass() == s2) {
5549          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), src);
5550       } else {
5551          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5552          nir_print_instr(&instr->instr, stderr);
5553          fprintf(stderr, "\n");
5554       }
5555       break;
5556    }
5557    case nir_intrinsic_vote_all: {
5558       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5559       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5560       assert(src.regClass() == s2);
5561       assert(dst.regClass() == s1);
5562
5563       Definition tmp = bld.def(s1);
5564       bld.sopc(aco_opcode::s_cmp_eq_u64, bld.scc(tmp),
5565                bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2)),
5566                Operand(exec, s2));
5567       emit_wqm(ctx, tmp.getTemp(), dst);
5568       break;
5569    }
5570    case nir_intrinsic_vote_any: {
5571       Temp src = as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false);
5572       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5573       assert(src.regClass() == s2);
5574       assert(dst.regClass() == s1);
5575
5576       Definition tmp = bld.def(s1);
5577       bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.scc(tmp), src, Operand(exec, s2));
5578       emit_wqm(ctx, tmp.getTemp(), dst);
5579       break;
5580    }
5581    case nir_intrinsic_reduce:
5582    case nir_intrinsic_inclusive_scan:
5583    case nir_intrinsic_exclusive_scan: {
5584       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5585       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5586       nir_op op = (nir_op) nir_intrinsic_reduction_op(instr);
5587       unsigned cluster_size = instr->intrinsic == nir_intrinsic_reduce ?
5588          nir_intrinsic_cluster_size(instr) : 0;
5589       cluster_size = util_next_power_of_two(MIN2(cluster_size ? cluster_size : 64, 64));
5590
5591       if (!ctx->divergent_vals[instr->src[0].ssa->index] && (op == nir_op_ior || op == nir_op_iand)) {
5592          emit_uniform_subgroup(ctx, instr, src);
5593       } else if (instr->dest.ssa.bit_size == 1) {
5594          if (op == nir_op_imul || op == nir_op_umin || op == nir_op_imin)
5595             op = nir_op_iand;
5596          else if (op == nir_op_iadd)
5597             op = nir_op_ixor;
5598          else if (op == nir_op_umax || op == nir_op_imax)
5599             op = nir_op_ior;
5600          assert(op == nir_op_iand || op == nir_op_ior || op == nir_op_ixor);
5601
5602          switch (instr->intrinsic) {
5603          case nir_intrinsic_reduce:
5604             emit_wqm(ctx, emit_boolean_reduce(ctx, op, cluster_size, src), dst);
5605             break;
5606          case nir_intrinsic_exclusive_scan:
5607             emit_wqm(ctx, emit_boolean_exclusive_scan(ctx, op, src), dst);
5608             break;
5609          case nir_intrinsic_inclusive_scan:
5610             emit_wqm(ctx, emit_boolean_inclusive_scan(ctx, op, src), dst);
5611             break;
5612          default:
5613             assert(false);
5614          }
5615       } else if (cluster_size == 1) {
5616          bld.copy(Definition(dst), src);
5617       } else {
5618          src = as_vgpr(ctx, src);
5619
5620          ReduceOp reduce_op;
5621          switch (op) {
5622          #define CASE(name) case nir_op_##name: reduce_op = (src.regClass() == v1) ? name##32 : name##64; break;
5623             CASE(iadd)
5624             CASE(imul)
5625             CASE(fadd)
5626             CASE(fmul)
5627             CASE(imin)
5628             CASE(umin)
5629             CASE(fmin)
5630             CASE(imax)
5631             CASE(umax)
5632             CASE(fmax)
5633             CASE(iand)
5634             CASE(ior)
5635             CASE(ixor)
5636             default:
5637                unreachable("unknown reduction op");
5638          #undef CASE
5639          }
5640
5641          aco_opcode aco_op;
5642          switch (instr->intrinsic) {
5643             case nir_intrinsic_reduce: aco_op = aco_opcode::p_reduce; break;
5644             case nir_intrinsic_inclusive_scan: aco_op = aco_opcode::p_inclusive_scan; break;
5645             case nir_intrinsic_exclusive_scan: aco_op = aco_opcode::p_exclusive_scan; break;
5646             default:
5647                unreachable("unknown reduce intrinsic");
5648          }
5649
5650          aco_ptr<Pseudo_reduction_instruction> reduce{create_instruction<Pseudo_reduction_instruction>(aco_op, Format::PSEUDO_REDUCTION, 3, 5)};
5651          reduce->operands[0] = Operand(src);
5652          // filled in by aco_reduce_assign.cpp, used internally as part of the
5653          // reduce sequence
5654          assert(dst.size() == 1 || dst.size() == 2);
5655          reduce->operands[1] = Operand(RegClass(RegType::vgpr, dst.size()).as_linear());
5656          reduce->operands[2] = Operand(v1.as_linear());
5657
5658          Temp tmp_dst = bld.tmp(dst.regClass());
5659          reduce->definitions[0] = Definition(tmp_dst);
5660          reduce->definitions[1] = bld.def(s2); // used internally
5661          reduce->definitions[2] = Definition();
5662          reduce->definitions[3] = Definition(scc, s1);
5663          reduce->definitions[4] = Definition();
5664          reduce->reduce_op = reduce_op;
5665          reduce->cluster_size = cluster_size;
5666          ctx->block->instructions.emplace_back(std::move(reduce));
5667
5668          emit_wqm(ctx, tmp_dst, dst);
5669       }
5670       break;
5671    }
5672    case nir_intrinsic_quad_broadcast: {
5673       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5674       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5675          emit_uniform_subgroup(ctx, instr, src);
5676       } else {
5677          Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5678          unsigned lane = nir_src_as_const_value(instr->src[1])->u32;
5679          if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5680             uint32_t half_mask = 0x11111111u << lane;
5681             Temp mask_tmp = bld.pseudo(aco_opcode::p_create_vector, bld.def(s2), Operand(half_mask), Operand(half_mask));
5682             Temp tmp = bld.tmp(s2);
5683             bld.sop1(aco_opcode::s_wqm_b64, Definition(tmp),
5684                      bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), mask_tmp,
5685                               bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc), src, Operand(exec, s2))));
5686             emit_wqm(ctx, tmp, dst);
5687          } else if (instr->dest.ssa.bit_size == 32) {
5688             emit_wqm(ctx,
5689                      bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src,
5690                                   dpp_quad_perm(lane, lane, lane, lane)),
5691                      dst);
5692          } else if (instr->dest.ssa.bit_size == 64) {
5693             Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5694             bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5695             lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_quad_perm(lane, lane, lane, lane)));
5696             hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_quad_perm(lane, lane, lane, lane)));
5697             bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5698             emit_split_vector(ctx, dst, 2);
5699          } else {
5700             fprintf(stderr, "Unimplemented NIR instr bit size: ");
5701             nir_print_instr(&instr->instr, stderr);
5702             fprintf(stderr, "\n");
5703          }
5704       }
5705       break;
5706    }
5707    case nir_intrinsic_quad_swap_horizontal:
5708    case nir_intrinsic_quad_swap_vertical:
5709    case nir_intrinsic_quad_swap_diagonal:
5710    case nir_intrinsic_quad_swizzle_amd: {
5711       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5712       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5713          emit_uniform_subgroup(ctx, instr, src);
5714          break;
5715       }
5716       uint16_t dpp_ctrl = 0;
5717       switch (instr->intrinsic) {
5718       case nir_intrinsic_quad_swap_horizontal:
5719          dpp_ctrl = dpp_quad_perm(1, 0, 3, 2);
5720          break;
5721       case nir_intrinsic_quad_swap_vertical:
5722          dpp_ctrl = dpp_quad_perm(2, 3, 0, 1);
5723          break;
5724       case nir_intrinsic_quad_swap_diagonal:
5725          dpp_ctrl = dpp_quad_perm(3, 2, 1, 0);
5726          break;
5727       case nir_intrinsic_quad_swizzle_amd: {
5728          dpp_ctrl = nir_intrinsic_swizzle_mask(instr);
5729          break;
5730       }
5731       default:
5732          break;
5733       }
5734
5735       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5736       if (instr->dest.ssa.bit_size == 1 && src.regClass() == s2) {
5737          src = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), Operand(0u), Operand((uint32_t)-1), src);
5738          src = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5739          Temp tmp = bld.vopc(aco_opcode::v_cmp_lg_u32, bld.def(s2), Operand(0u), src);
5740          emit_wqm(ctx, tmp, dst);
5741       } else if (instr->dest.ssa.bit_size == 32) {
5742          Temp tmp = bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), src, dpp_ctrl);
5743          emit_wqm(ctx, tmp, dst);
5744       } else if (instr->dest.ssa.bit_size == 64) {
5745          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5746          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5747          lo = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), lo, dpp_ctrl));
5748          hi = emit_wqm(ctx, bld.vop1_dpp(aco_opcode::v_mov_b32, bld.def(v1), hi, dpp_ctrl));
5749          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5750          emit_split_vector(ctx, dst, 2);
5751       } else {
5752          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5753          nir_print_instr(&instr->instr, stderr);
5754          fprintf(stderr, "\n");
5755       }
5756       break;
5757    }
5758    case nir_intrinsic_masked_swizzle_amd: {
5759       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5760       if (!ctx->divergent_vals[instr->dest.ssa.index]) {
5761          emit_uniform_subgroup(ctx, instr, src);
5762          break;
5763       }
5764       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5765       uint32_t mask = nir_intrinsic_swizzle_mask(instr);
5766       if (dst.regClass() == v1) {
5767          emit_wqm(ctx,
5768                   bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), src, mask, 0, false),
5769                   dst);
5770       } else if (dst.regClass() == v2) {
5771          Temp lo = bld.tmp(v1), hi = bld.tmp(v1);
5772          bld.pseudo(aco_opcode::p_split_vector, Definition(lo), Definition(hi), src);
5773          lo = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), lo, mask, 0, false));
5774          hi = emit_wqm(ctx, bld.ds(aco_opcode::ds_swizzle_b32, bld.def(v1), hi, mask, 0, false));
5775          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5776          emit_split_vector(ctx, dst, 2);
5777       } else {
5778          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5779          nir_print_instr(&instr->instr, stderr);
5780          fprintf(stderr, "\n");
5781       }
5782       break;
5783    }
5784    case nir_intrinsic_write_invocation_amd: {
5785       Temp src = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[0].ssa));
5786       Temp val = bld.as_uniform(get_ssa_temp(ctx, instr->src[1].ssa));
5787       Temp lane = bld.as_uniform(get_ssa_temp(ctx, instr->src[2].ssa));
5788       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5789       if (dst.regClass() == v1) {
5790          /* src2 is ignored for writelane. RA assigns the same reg for dst */
5791          emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val, lane, src), dst);
5792       } else if (dst.regClass() == v2) {
5793          Temp src_lo = bld.tmp(v1), src_hi = bld.tmp(v1);
5794          Temp val_lo = bld.tmp(s1), val_hi = bld.tmp(s1);
5795          bld.pseudo(aco_opcode::p_split_vector, Definition(src_lo), Definition(src_hi), src);
5796          bld.pseudo(aco_opcode::p_split_vector, Definition(val_lo), Definition(val_hi), val);
5797          Temp lo = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_lo, lane, src_hi));
5798          Temp hi = emit_wqm(ctx, bld.vop3(aco_opcode::v_writelane_b32, bld.def(v1), val_hi, lane, src_hi));
5799          bld.pseudo(aco_opcode::p_create_vector, Definition(dst), lo, hi);
5800          emit_split_vector(ctx, dst, 2);
5801       } else {
5802          fprintf(stderr, "Unimplemented NIR instr bit size: ");
5803          nir_print_instr(&instr->instr, stderr);
5804          fprintf(stderr, "\n");
5805       }
5806       break;
5807    }
5808    case nir_intrinsic_mbcnt_amd: {
5809       Temp src = get_ssa_temp(ctx, instr->src[0].ssa);
5810       RegClass rc = RegClass(src.type(), 1);
5811       Temp mask_lo = bld.tmp(rc), mask_hi = bld.tmp(rc);
5812       bld.pseudo(aco_opcode::p_split_vector, Definition(mask_lo), Definition(mask_hi), src);
5813       Temp tmp = bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), mask_lo, Operand(0u));
5814       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5815       Temp wqm_tmp = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), mask_hi, tmp);
5816       emit_wqm(ctx, wqm_tmp, dst);
5817       break;
5818    }
5819    case nir_intrinsic_load_helper_invocation: {
5820       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5821       bld.pseudo(aco_opcode::p_load_helper, Definition(dst));
5822       ctx->block->kind |= block_kind_needs_lowering;
5823       ctx->program->needs_exact = true;
5824       break;
5825    }
5826    case nir_intrinsic_is_helper_invocation: {
5827       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5828       bld.pseudo(aco_opcode::p_is_helper, Definition(dst));
5829       ctx->block->kind |= block_kind_needs_lowering;
5830       ctx->program->needs_exact = true;
5831       break;
5832    }
5833    case nir_intrinsic_demote:
5834       bld.pseudo(aco_opcode::p_demote_to_helper);
5835       ctx->block->kind |= block_kind_uses_demote;
5836       ctx->program->needs_exact = true;
5837       break;
5838    case nir_intrinsic_demote_if: {
5839       Temp cond = bld.sop2(aco_opcode::s_and_b64, bld.def(s2), bld.def(s1, scc),
5840                            as_divergent_bool(ctx, get_ssa_temp(ctx, instr->src[0].ssa), false),
5841                            Operand(exec, s2));
5842       bld.pseudo(aco_opcode::p_demote_to_helper, cond);
5843       ctx->block->kind |= block_kind_uses_demote;
5844       ctx->program->needs_exact = true;
5845       break;
5846    }
5847    case nir_intrinsic_first_invocation: {
5848       emit_wqm(ctx, bld.sop1(aco_opcode::s_ff1_i32_b64, bld.def(s1), Operand(exec, s2)),
5849                get_ssa_temp(ctx, &instr->dest.ssa));
5850       break;
5851    }
5852    case nir_intrinsic_shader_clock:
5853       bld.smem(aco_opcode::s_memtime, Definition(get_ssa_temp(ctx, &instr->dest.ssa)));
5854       break;
5855    case nir_intrinsic_load_vertex_id_zero_base: {
5856       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5857       bld.copy(Definition(dst), ctx->vertex_id);
5858       break;
5859    }
5860    case nir_intrinsic_load_first_vertex: {
5861       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5862       bld.copy(Definition(dst), ctx->base_vertex);
5863       break;
5864    }
5865    case nir_intrinsic_load_base_instance: {
5866       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5867       bld.copy(Definition(dst), ctx->start_instance);
5868       break;
5869    }
5870    case nir_intrinsic_load_instance_id: {
5871       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5872       bld.copy(Definition(dst), ctx->instance_id);
5873       break;
5874    }
5875    case nir_intrinsic_load_draw_id: {
5876       Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
5877       bld.copy(Definition(dst), ctx->draw_id);
5878       break;
5879    }
5880    default:
5881       fprintf(stderr, "Unimplemented intrinsic instr: ");
5882       nir_print_instr(&instr->instr, stderr);
5883       fprintf(stderr, "\n");
5884       abort();
5885
5886       break;
5887    }
5888 }
5889
5890
5891 void tex_fetch_ptrs(isel_context *ctx, nir_tex_instr *instr,
5892                     Temp *res_ptr, Temp *samp_ptr, Temp *fmask_ptr,
5893                     enum glsl_base_type *stype)
5894 {
5895    nir_deref_instr *texture_deref_instr = NULL;
5896    nir_deref_instr *sampler_deref_instr = NULL;
5897    int plane = -1;
5898
5899    for (unsigned i = 0; i < instr->num_srcs; i++) {
5900       switch (instr->src[i].src_type) {
5901       case nir_tex_src_texture_deref:
5902          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
5903          break;
5904       case nir_tex_src_sampler_deref:
5905          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
5906          break;
5907       case nir_tex_src_plane:
5908          plane = nir_src_as_int(instr->src[i].src);
5909          break;
5910       default:
5911          break;
5912       }
5913    }
5914
5915    *stype = glsl_get_sampler_result_type(texture_deref_instr->type);
5916
5917    if (!sampler_deref_instr)
5918       sampler_deref_instr = texture_deref_instr;
5919
5920    if (plane >= 0) {
5921       assert(instr->op != nir_texop_txf_ms &&
5922              instr->op != nir_texop_samples_identical);
5923       assert(instr->sampler_dim  != GLSL_SAMPLER_DIM_BUF);
5924       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, (aco_descriptor_type)(ACO_DESC_PLANE_0 + plane), instr, false, false);
5925    } else if (instr->sampler_dim  == GLSL_SAMPLER_DIM_BUF) {
5926       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_BUFFER, instr, false, false);
5927    } else {
5928       *res_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_IMAGE, instr, false, false);
5929    }
5930    if (samp_ptr) {
5931       *samp_ptr = get_sampler_desc(ctx, sampler_deref_instr, ACO_DESC_SAMPLER, instr, false, false);
5932       if (instr->sampler_dim < GLSL_SAMPLER_DIM_RECT && ctx->options->chip_class < GFX8) {
5933          fprintf(stderr, "Unimplemented sampler descriptor: ");
5934          nir_print_instr(&instr->instr, stderr);
5935          fprintf(stderr, "\n");
5936          abort();
5937          // TODO: build samp_ptr = and(samp_ptr, res_ptr)
5938       }
5939    }
5940    if (fmask_ptr && (instr->op == nir_texop_txf_ms ||
5941                      instr->op == nir_texop_samples_identical))
5942       *fmask_ptr = get_sampler_desc(ctx, texture_deref_instr, ACO_DESC_FMASK, instr, false, false);
5943 }
5944
5945 void build_cube_select(isel_context *ctx, Temp ma, Temp id, Temp deriv,
5946                        Temp *out_ma, Temp *out_sc, Temp *out_tc)
5947 {
5948    Builder bld(ctx->program, ctx->block);
5949
5950    Temp deriv_x = emit_extract_vector(ctx, deriv, 0, v1);
5951    Temp deriv_y = emit_extract_vector(ctx, deriv, 1, v1);
5952    Temp deriv_z = emit_extract_vector(ctx, deriv, 2, v1);
5953
5954    Operand neg_one(0xbf800000u);
5955    Operand one(0x3f800000u);
5956    Operand two(0x40000000u);
5957    Operand four(0x40800000u);
5958
5959    Temp is_ma_positive = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), Operand(0u), ma);
5960    Temp sgn_ma = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, one, is_ma_positive);
5961    Temp neg_sgn_ma = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1), Operand(0u), sgn_ma);
5962
5963    Temp is_ma_z = bld.vopc(aco_opcode::v_cmp_le_f32, bld.hint_vcc(bld.def(s2)), four, id);
5964    Temp is_ma_y = bld.vopc(aco_opcode::v_cmp_le_f32, bld.def(s2), two, id);
5965    is_ma_y = bld.sop2(aco_opcode::s_andn2_b64, bld.hint_vcc(bld.def(s2)), is_ma_y, is_ma_z);
5966    Temp is_not_ma_x = bld.sop2(aco_opcode::s_or_b64, bld.hint_vcc(bld.def(s2)), bld.def(s1, scc), is_ma_z, is_ma_y);
5967
5968    // select sc
5969    Temp tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_z, deriv_x, is_not_ma_x);
5970    Temp sgn = bld.vop2_e64(aco_opcode::v_cndmask_b32, bld.def(v1),
5971                        bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_sgn_ma, sgn_ma, is_ma_z),
5972                        one, is_ma_y);
5973    *out_sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5974
5975    // select tc
5976    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_y, deriv_z, is_ma_y);
5977    sgn = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), neg_one, sgn_ma, is_ma_y);
5978    *out_tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tmp, sgn);
5979
5980    // select ma
5981    tmp = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
5982                   bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), deriv_x, deriv_y, is_ma_y),
5983                   deriv_z, is_ma_z);
5984    tmp = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x7fffffffu), tmp);
5985    *out_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), two, tmp);
5986 }
5987
5988 void prepare_cube_coords(isel_context *ctx, Temp* coords, Temp* ddx, Temp* ddy, bool is_deriv, bool is_array)
5989 {
5990    Builder bld(ctx->program, ctx->block);
5991    Temp coord_args[4], ma, tc, sc, id;
5992    for (unsigned i = 0; i < (is_array ? 4 : 3); i++)
5993       coord_args[i] = emit_extract_vector(ctx, *coords, i, v1);
5994
5995    if (is_array) {
5996       coord_args[3] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_args[3]);
5997
5998       // see comment in ac_prepare_cube_coords()
5999       if (ctx->options->chip_class <= GFX8)
6000          coord_args[3] = bld.vop2(aco_opcode::v_max_f32, bld.def(v1), Operand(0u), coord_args[3]);
6001    }
6002
6003    ma = bld.vop3(aco_opcode::v_cubema_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6004
6005    aco_ptr<VOP3A_instruction> vop3a{create_instruction<VOP3A_instruction>(aco_opcode::v_rcp_f32, asVOP3(Format::VOP1), 1, 1)};
6006    vop3a->operands[0] = Operand(ma);
6007    vop3a->abs[0] = true;
6008    Temp invma = bld.tmp(v1);
6009    vop3a->definitions[0] = Definition(invma);
6010    ctx->block->instructions.emplace_back(std::move(vop3a));
6011
6012    sc = bld.vop3(aco_opcode::v_cubesc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6013    if (!is_deriv)
6014       sc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), sc, invma, Operand(0x3fc00000u/*1.5*/));
6015
6016    tc = bld.vop3(aco_opcode::v_cubetc_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6017    if (!is_deriv)
6018       tc = bld.vop2(aco_opcode::v_madak_f32, bld.def(v1), tc, invma, Operand(0x3fc00000u/*1.5*/));
6019
6020    id = bld.vop3(aco_opcode::v_cubeid_f32, bld.def(v1), coord_args[0], coord_args[1], coord_args[2]);
6021
6022    if (is_deriv) {
6023       sc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), sc, invma);
6024       tc = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), tc, invma);
6025
6026       for (unsigned i = 0; i < 2; i++) {
6027          // see comment in ac_prepare_cube_coords()
6028          Temp deriv_ma;
6029          Temp deriv_sc, deriv_tc;
6030          build_cube_select(ctx, ma, id, i ? *ddy : *ddx,
6031                            &deriv_ma, &deriv_sc, &deriv_tc);
6032
6033          deriv_ma = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, invma);
6034
6035          Temp x = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6036                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_sc, invma),
6037                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, sc));
6038          Temp y = bld.vop2(aco_opcode::v_sub_f32, bld.def(v1),
6039                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_tc, invma),
6040                                bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), deriv_ma, tc));
6041          *(i ? ddy : ddx) = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2), x, y);
6042       }
6043
6044       sc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), sc);
6045       tc = bld.vop2(aco_opcode::v_add_f32, bld.def(v1), Operand(0x3fc00000u/*1.5*/), tc);
6046    }
6047
6048    if (is_array)
6049       id = bld.vop2(aco_opcode::v_madmk_f32, bld.def(v1), coord_args[3], id, Operand(0x41000000u/*8.0*/));
6050    *coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3), sc, tc, id);
6051
6052 }
6053
6054 Temp apply_round_slice(isel_context *ctx, Temp coords, unsigned idx)
6055 {
6056    Temp coord_vec[3];
6057    for (unsigned i = 0; i < coords.size(); i++)
6058       coord_vec[i] = emit_extract_vector(ctx, coords, i, v1);
6059
6060    Builder bld(ctx->program, ctx->block);
6061    coord_vec[idx] = bld.vop1(aco_opcode::v_rndne_f32, bld.def(v1), coord_vec[idx]);
6062
6063    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6064    for (unsigned i = 0; i < coords.size(); i++)
6065       vec->operands[i] = Operand(coord_vec[i]);
6066    Temp res = bld.tmp(RegType::vgpr, coords.size());
6067    vec->definitions[0] = Definition(res);
6068    ctx->block->instructions.emplace_back(std::move(vec));
6069    return res;
6070 }
6071
6072 void get_const_vec(nir_ssa_def *vec, nir_const_value *cv[4])
6073 {
6074    if (vec->parent_instr->type != nir_instr_type_alu)
6075       return;
6076    nir_alu_instr *vec_instr = nir_instr_as_alu(vec->parent_instr);
6077    if (vec_instr->op != nir_op_vec(vec->num_components))
6078       return;
6079
6080    for (unsigned i = 0; i < vec->num_components; i++) {
6081       cv[i] = vec_instr->src[i].swizzle[0] == 0 ?
6082               nir_src_as_const_value(vec_instr->src[i].src) : NULL;
6083    }
6084 }
6085
6086 void visit_tex(isel_context *ctx, nir_tex_instr *instr)
6087 {
6088    Builder bld(ctx->program, ctx->block);
6089    bool has_bias = false, has_lod = false, level_zero = false, has_compare = false,
6090         has_offset = false, has_ddx = false, has_ddy = false, has_derivs = false, has_sample_index = false;
6091    Temp resource, sampler, fmask_ptr, bias = Temp(), coords, compare = Temp(), sample_index = Temp(),
6092         lod = Temp(), offset = Temp(), ddx = Temp(), ddy = Temp(), derivs = Temp();
6093    nir_const_value *sample_index_cv = NULL;
6094    nir_const_value *const_offset[4] = {NULL, NULL, NULL, NULL};
6095    enum glsl_base_type stype;
6096    tex_fetch_ptrs(ctx, instr, &resource, &sampler, &fmask_ptr, &stype);
6097
6098    bool tg4_integer_workarounds = ctx->options->chip_class <= GFX8 && instr->op == nir_texop_tg4 &&
6099                                   (stype == GLSL_TYPE_UINT || stype == GLSL_TYPE_INT);
6100    bool tg4_integer_cube_workaround = tg4_integer_workarounds &&
6101                                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE;
6102
6103    for (unsigned i = 0; i < instr->num_srcs; i++) {
6104       switch (instr->src[i].src_type) {
6105       case nir_tex_src_coord:
6106          coords = as_vgpr(ctx, get_ssa_temp(ctx, instr->src[i].src.ssa));
6107          break;
6108       case nir_tex_src_bias:
6109          if (instr->op == nir_texop_txb) {
6110             bias = get_ssa_temp(ctx, instr->src[i].src.ssa);
6111             has_bias = true;
6112          }
6113          break;
6114       case nir_tex_src_lod: {
6115          nir_const_value *val = nir_src_as_const_value(instr->src[i].src);
6116
6117          if (val && val->f32 <= 0.0) {
6118             level_zero = true;
6119          } else {
6120             lod = get_ssa_temp(ctx, instr->src[i].src.ssa);
6121             has_lod = true;
6122          }
6123          break;
6124       }
6125       case nir_tex_src_comparator:
6126          if (instr->is_shadow) {
6127             compare = get_ssa_temp(ctx, instr->src[i].src.ssa);
6128             has_compare = true;
6129          }
6130          break;
6131       case nir_tex_src_offset:
6132          offset = get_ssa_temp(ctx, instr->src[i].src.ssa);
6133          get_const_vec(instr->src[i].src.ssa, const_offset);
6134          has_offset = true;
6135          break;
6136       case nir_tex_src_ddx:
6137          ddx = get_ssa_temp(ctx, instr->src[i].src.ssa);
6138          has_ddx = true;
6139          break;
6140       case nir_tex_src_ddy:
6141          ddy = get_ssa_temp(ctx, instr->src[i].src.ssa);
6142          has_ddy = true;
6143          break;
6144       case nir_tex_src_ms_index:
6145          sample_index = get_ssa_temp(ctx, instr->src[i].src.ssa);
6146          sample_index_cv = nir_src_as_const_value(instr->src[i].src);
6147          has_sample_index = true;
6148          break;
6149       case nir_tex_src_texture_offset:
6150       case nir_tex_src_sampler_offset:
6151       default:
6152          break;
6153       }
6154    }
6155 // TODO: all other cases: structure taken from ac_nir_to_llvm.c
6156    if (instr->op == nir_texop_txs && instr->sampler_dim == GLSL_SAMPLER_DIM_BUF)
6157       return get_buffer_size(ctx, resource, get_ssa_temp(ctx, &instr->dest.ssa), true);
6158
6159    if (instr->op == nir_texop_texture_samples) {
6160       Temp dword3 = emit_extract_vector(ctx, resource, 3, s1);
6161
6162       Temp samples_log2 = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(16u | 4u<<16));
6163       Temp samples = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), Operand(1u), samples_log2);
6164       Temp type = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), dword3, Operand(28u | 4u<<16 /* offset=28, width=4 */));
6165       Temp is_msaa = bld.sopc(aco_opcode::s_cmp_ge_u32, bld.def(s1, scc), type, Operand(14u));
6166
6167       bld.sop2(aco_opcode::s_cselect_b32, Definition(get_ssa_temp(ctx, &instr->dest.ssa)),
6168                samples, Operand(1u), bld.scc(is_msaa));
6169       return;
6170    }
6171
6172    if (has_offset && instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms) {
6173       aco_ptr<Instruction> tmp_instr;
6174       Temp acc, pack = Temp();
6175
6176       uint32_t pack_const = 0;
6177       for (unsigned i = 0; i < offset.size(); i++) {
6178          if (!const_offset[i])
6179             continue;
6180          pack_const |= (const_offset[i]->u32 & 0x3Fu) << (8u * i);
6181       }
6182
6183       if (offset.type() == RegType::sgpr) {
6184          for (unsigned i = 0; i < offset.size(); i++) {
6185             if (const_offset[i])
6186                continue;
6187
6188             acc = emit_extract_vector(ctx, offset, i, s1);
6189             acc = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(0x3Fu));
6190
6191             if (i) {
6192                acc = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), acc, Operand(8u * i));
6193             }
6194
6195             if (pack == Temp()) {
6196                pack = acc;
6197             } else {
6198                pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), pack, acc);
6199             }
6200          }
6201
6202          if (pack_const && pack != Temp())
6203             pack = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), Operand(pack_const), pack);
6204       } else {
6205          for (unsigned i = 0; i < offset.size(); i++) {
6206             if (const_offset[i])
6207                continue;
6208
6209             acc = emit_extract_vector(ctx, offset, i, v1);
6210             acc = bld.vop2(aco_opcode::v_and_b32, bld.def(v1), Operand(0x3Fu), acc);
6211
6212             if (i) {
6213                acc = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(8u * i), acc);
6214             }
6215
6216             if (pack == Temp()) {
6217                pack = acc;
6218             } else {
6219                pack = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), pack, acc);
6220             }
6221          }
6222
6223          if (pack_const && pack != Temp())
6224             pack = bld.sop2(aco_opcode::v_or_b32, bld.def(v1), Operand(pack_const), pack);
6225       }
6226       if (pack_const && pack == Temp())
6227          offset = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(pack_const));
6228       else if (pack == Temp())
6229          has_offset = false;
6230       else
6231          offset = pack;
6232    }
6233
6234    if (instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE && instr->coord_components)
6235       prepare_cube_coords(ctx, &coords, &ddx, &ddy, instr->op == nir_texop_txd, instr->is_array && instr->op != nir_texop_lod);
6236
6237    /* pack derivatives */
6238    if (has_ddx || has_ddy) {
6239       if (instr->sampler_dim == GLSL_SAMPLER_DIM_1D && ctx->options->chip_class >= GFX9) {
6240          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(v4),
6241                              ddx, Operand(0u), ddy, Operand(0u));
6242       } else {
6243          derivs = bld.pseudo(aco_opcode::p_create_vector, bld.def(RegType::vgpr, ddx.size() + ddy.size()), ddx, ddy);
6244       }
6245       has_derivs = true;
6246    }
6247
6248    if (instr->coord_components > 1 &&
6249        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6250        instr->is_array &&
6251        instr->op != nir_texop_txf)
6252       coords = apply_round_slice(ctx, coords, 1);
6253
6254    if (instr->coord_components > 2 &&
6255       (instr->sampler_dim == GLSL_SAMPLER_DIM_2D ||
6256        instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6257        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS ||
6258        instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6259        instr->is_array &&
6260        instr->op != nir_texop_txf && instr->op != nir_texop_txf_ms)
6261       coords = apply_round_slice(ctx, coords, 2);
6262
6263    if (ctx->options->chip_class >= GFX9 &&
6264        instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6265        instr->op != nir_texop_lod && instr->coord_components) {
6266       assert(coords.size() > 0 && coords.size() < 3);
6267
6268       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size() + 1, 1)};
6269       vec->operands[0] = Operand(emit_extract_vector(ctx, coords, 0, v1));
6270       vec->operands[1] = instr->op == nir_texop_txf ? Operand((uint32_t) 0) : Operand((uint32_t) 0x3f000000);
6271       if (coords.size() > 1)
6272          vec->operands[2] = Operand(emit_extract_vector(ctx, coords, 1, v1));
6273       coords = bld.tmp(RegType::vgpr, coords.size() + 1);
6274       vec->definitions[0] = Definition(coords);
6275       ctx->block->instructions.emplace_back(std::move(vec));
6276    }
6277
6278    bool da = should_declare_array(ctx, instr->sampler_dim, instr->is_array);
6279
6280    if (instr->op == nir_texop_samples_identical)
6281       resource = fmask_ptr;
6282
6283    else if ((instr->sampler_dim == GLSL_SAMPLER_DIM_MS ||
6284              instr->sampler_dim == GLSL_SAMPLER_DIM_SUBPASS_MS) &&
6285             instr->op != nir_texop_txs) {
6286       assert(has_sample_index);
6287       Operand op(sample_index);
6288       if (sample_index_cv)
6289          op = Operand(sample_index_cv->u32);
6290       sample_index = adjust_sample_index_using_fmask(ctx, da, coords, op, fmask_ptr);
6291    }
6292
6293    if (has_offset && (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)) {
6294       Temp split_coords[coords.size()];
6295       emit_split_vector(ctx, coords, coords.size());
6296       for (unsigned i = 0; i < coords.size(); i++)
6297          split_coords[i] = emit_extract_vector(ctx, coords, i, v1);
6298
6299       unsigned i = 0;
6300       for (; i < std::min(offset.size(), instr->coord_components); i++) {
6301          Temp off = emit_extract_vector(ctx, offset, i, v1);
6302          split_coords[i] = bld.vadd32(bld.def(v1), split_coords[i], off);
6303       }
6304
6305       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, coords.size(), 1)};
6306       for (unsigned i = 0; i < coords.size(); i++)
6307          vec->operands[i] = Operand(split_coords[i]);
6308       coords = bld.tmp(coords.regClass());
6309       vec->definitions[0] = Definition(coords);
6310       ctx->block->instructions.emplace_back(std::move(vec));
6311
6312       has_offset = false;
6313    }
6314
6315    /* Build tex instruction */
6316    unsigned dmask = nir_ssa_def_components_read(&instr->dest.ssa);
6317    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6318    Temp tmp_dst = dst;
6319
6320    /* gather4 selects the component by dmask and always returns vec4 */
6321    if (instr->op == nir_texop_tg4) {
6322       assert(instr->dest.ssa.num_components == 4);
6323       if (instr->is_shadow)
6324          dmask = 1;
6325       else
6326          dmask = 1 << instr->component;
6327       if (tg4_integer_cube_workaround || dst.type() == RegType::sgpr)
6328          tmp_dst = bld.tmp(v4);
6329    } else if (instr->op == nir_texop_samples_identical) {
6330       tmp_dst = bld.tmp(v1);
6331    } else if (util_bitcount(dmask) != instr->dest.ssa.num_components || dst.type() == RegType::sgpr) {
6332       tmp_dst = bld.tmp(RegClass(RegType::vgpr, util_bitcount(dmask)));
6333    }
6334
6335    aco_ptr<MIMG_instruction> tex;
6336    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
6337       if (!has_lod)
6338          lod = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6339
6340       bool div_by_6 = instr->op == nir_texop_txs &&
6341                       instr->sampler_dim == GLSL_SAMPLER_DIM_CUBE &&
6342                       instr->is_array &&
6343                       (dmask & (1 << 2));
6344       if (tmp_dst.id() == dst.id() && div_by_6)
6345          tmp_dst = bld.tmp(tmp_dst.regClass());
6346
6347       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6348       tex->operands[0] = Operand(as_vgpr(ctx,lod));
6349       tex->operands[1] = Operand(resource);
6350       if (ctx->options->chip_class >= GFX9 &&
6351           instr->op == nir_texop_txs &&
6352           instr->sampler_dim == GLSL_SAMPLER_DIM_1D &&
6353           instr->is_array) {
6354          tex->dmask = (dmask & 0x1) | ((dmask & 0x2) << 1);
6355       } else if (instr->op == nir_texop_query_levels) {
6356          tex->dmask = 1 << 3;
6357       } else {
6358          tex->dmask = dmask;
6359       }
6360       tex->da = da;
6361       tex->definitions[0] = Definition(tmp_dst);
6362       tex->can_reorder = true;
6363       ctx->block->instructions.emplace_back(std::move(tex));
6364
6365       if (div_by_6) {
6366          /* divide 3rd value by 6 by multiplying with magic number */
6367          emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6368          Temp c = bld.copy(bld.def(s1), Operand((uint32_t) 0x2AAAAAAB));
6369          Temp by_6 = bld.vop3(aco_opcode::v_mul_hi_i32, bld.def(v1), emit_extract_vector(ctx, tmp_dst, 2, v1), c);
6370          assert(instr->dest.ssa.num_components == 3);
6371          Temp tmp = dst.type() == RegType::vgpr ? dst : bld.tmp(v3);
6372          tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6373                               emit_extract_vector(ctx, tmp_dst, 0, v1),
6374                               emit_extract_vector(ctx, tmp_dst, 1, v1),
6375                               by_6);
6376
6377       }
6378
6379       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6380       return;
6381    }
6382
6383    Temp tg4_compare_cube_wa64 = Temp();
6384
6385    if (tg4_integer_workarounds) {
6386       tex.reset(create_instruction<MIMG_instruction>(aco_opcode::image_get_resinfo, Format::MIMG, 2, 1));
6387       tex->operands[0] = bld.vop1(aco_opcode::v_mov_b32, bld.def(v1), Operand(0u));
6388       tex->operands[1] = Operand(resource);
6389       tex->dmask = 0x3;
6390       tex->da = da;
6391       Temp size = bld.tmp(v2);
6392       tex->definitions[0] = Definition(size);
6393       tex->can_reorder = true;
6394       ctx->block->instructions.emplace_back(std::move(tex));
6395       emit_split_vector(ctx, size, size.size());
6396
6397       Temp half_texel[2];
6398       for (unsigned i = 0; i < 2; i++) {
6399          half_texel[i] = emit_extract_vector(ctx, size, i, v1);
6400          half_texel[i] = bld.vop1(aco_opcode::v_cvt_f32_i32, bld.def(v1), half_texel[i]);
6401          half_texel[i] = bld.vop1(aco_opcode::v_rcp_iflag_f32, bld.def(v1), half_texel[i]);
6402          half_texel[i] = bld.vop2(aco_opcode::v_mul_f32, bld.def(v1), Operand(0xbf000000/*-0.5*/), half_texel[i]);
6403       }
6404
6405       Temp orig_coords[2] = {
6406          emit_extract_vector(ctx, coords, 0, v1),
6407          emit_extract_vector(ctx, coords, 1, v1)};
6408       Temp new_coords[2] = {
6409          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[0], half_texel[0]),
6410          bld.vop2(aco_opcode::v_add_f32, bld.def(v1), orig_coords[1], half_texel[1])
6411       };
6412
6413       if (tg4_integer_cube_workaround) {
6414          // see comment in ac_nir_to_llvm.c's lower_gather4_integer()
6415          Temp desc[resource.size()];
6416          aco_ptr<Instruction> split{create_instruction<Pseudo_instruction>(aco_opcode::p_split_vector,
6417                                                                            Format::PSEUDO, 1, resource.size())};
6418          split->operands[0] = Operand(resource);
6419          for (unsigned i = 0; i < resource.size(); i++) {
6420             desc[i] = bld.tmp(s1);
6421             split->definitions[i] = Definition(desc[i]);
6422          }
6423          ctx->block->instructions.emplace_back(std::move(split));
6424
6425          Temp dfmt = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), desc[1], Operand(20u | (6u << 16)));
6426          Temp compare_cube_wa = bld.sopc(aco_opcode::s_cmp_eq_u32, bld.def(s1, scc), dfmt,
6427                                          Operand((uint32_t)V_008F14_IMG_DATA_FORMAT_8_8_8_8));
6428
6429          Temp nfmt;
6430          if (stype == GLSL_TYPE_UINT) {
6431             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6432                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_USCALED),
6433                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_UINT),
6434                             bld.scc(compare_cube_wa));
6435          } else {
6436             nfmt = bld.sop2(aco_opcode::s_cselect_b32, bld.def(s1),
6437                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SSCALED),
6438                             Operand((uint32_t)V_008F14_IMG_NUM_FORMAT_SINT),
6439                             bld.scc(compare_cube_wa));
6440          }
6441          tg4_compare_cube_wa64 = as_divergent_bool(ctx, compare_cube_wa, true);
6442          nfmt = bld.sop2(aco_opcode::s_lshl_b32, bld.def(s1), bld.def(s1, scc), nfmt, Operand(26u));
6443
6444          desc[1] = bld.sop2(aco_opcode::s_and_b32, bld.def(s1), bld.def(s1, scc), desc[1],
6445                             Operand((uint32_t)C_008F14_NUM_FORMAT));
6446          desc[1] = bld.sop2(aco_opcode::s_or_b32, bld.def(s1), bld.def(s1, scc), desc[1], nfmt);
6447
6448          aco_ptr<Instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector,
6449                                                                          Format::PSEUDO, resource.size(), 1)};
6450          for (unsigned i = 0; i < resource.size(); i++)
6451             vec->operands[i] = Operand(desc[i]);
6452          resource = bld.tmp(resource.regClass());
6453          vec->definitions[0] = Definition(resource);
6454          ctx->block->instructions.emplace_back(std::move(vec));
6455
6456          new_coords[0] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6457                                   new_coords[0], orig_coords[0], tg4_compare_cube_wa64);
6458          new_coords[1] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
6459                                   new_coords[1], orig_coords[1], tg4_compare_cube_wa64);
6460       }
6461
6462       if (coords.size() == 3) {
6463          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v3),
6464                              new_coords[0], new_coords[1],
6465                              emit_extract_vector(ctx, coords, 2, v1));
6466       } else {
6467          assert(coords.size() == 2);
6468          coords = bld.pseudo(aco_opcode::p_create_vector, bld.def(v2),
6469                              new_coords[0], new_coords[1]);
6470       }
6471    }
6472
6473    if (!(has_ddx && has_ddy) && !has_lod && !level_zero &&
6474        instr->sampler_dim != GLSL_SAMPLER_DIM_MS &&
6475        instr->sampler_dim != GLSL_SAMPLER_DIM_SUBPASS_MS)
6476       coords = emit_wqm(ctx, coords, bld.tmp(coords.regClass()), true);
6477
6478    std::vector<Operand> args;
6479    if (has_offset)
6480       args.emplace_back(Operand(offset));
6481    if (has_bias)
6482       args.emplace_back(Operand(bias));
6483    if (has_compare)
6484       args.emplace_back(Operand(compare));
6485    if (has_derivs)
6486       args.emplace_back(Operand(derivs));
6487    args.emplace_back(Operand(coords));
6488    if (has_sample_index)
6489       args.emplace_back(Operand(sample_index));
6490    if (has_lod)
6491       args.emplace_back(lod);
6492
6493    Operand arg;
6494    if (args.size() > 1) {
6495       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, args.size(), 1)};
6496       unsigned size = 0;
6497       for (unsigned i = 0; i < args.size(); i++) {
6498          size += args[i].size();
6499          vec->operands[i] = args[i];
6500       }
6501       RegClass rc = RegClass(RegType::vgpr, size);
6502       Temp tmp = bld.tmp(rc);
6503       vec->definitions[0] = Definition(tmp);
6504       ctx->block->instructions.emplace_back(std::move(vec));
6505       arg = Operand(tmp);
6506    } else {
6507       assert(args[0].isTemp());
6508       arg = Operand(as_vgpr(ctx, args[0].getTemp()));
6509    }
6510
6511    if (instr->sampler_dim == GLSL_SAMPLER_DIM_BUF) {
6512       //FIXME: if (ctx->abi->gfx9_stride_size_workaround) return ac_build_buffer_load_format_gfx9_safe()
6513
6514       assert(coords.size() == 1);
6515       unsigned last_bit = util_last_bit(nir_ssa_def_components_read(&instr->dest.ssa));
6516       aco_opcode op;
6517       switch (last_bit) {
6518       case 1:
6519          op = aco_opcode::buffer_load_format_x; break;
6520       case 2:
6521          op = aco_opcode::buffer_load_format_xy; break;
6522       case 3:
6523          op = aco_opcode::buffer_load_format_xyz; break;
6524       case 4:
6525          op = aco_opcode::buffer_load_format_xyzw; break;
6526       default:
6527          unreachable("Tex instruction loads more than 4 components.");
6528       }
6529
6530       /* if the instruction return value matches exactly the nir dest ssa, we can use it directly */
6531       if (last_bit == instr->dest.ssa.num_components && dst.type() == RegType::vgpr)
6532          tmp_dst = dst;
6533       else
6534          tmp_dst = bld.tmp(RegType::vgpr, last_bit);
6535
6536       aco_ptr<MUBUF_instruction> mubuf{create_instruction<MUBUF_instruction>(op, Format::MUBUF, 3, 1)};
6537       mubuf->operands[0] = Operand(coords);
6538       mubuf->operands[1] = Operand(resource);
6539       mubuf->operands[2] = Operand((uint32_t) 0);
6540       mubuf->definitions[0] = Definition(tmp_dst);
6541       mubuf->idxen = true;
6542       mubuf->can_reorder = true;
6543       ctx->block->instructions.emplace_back(std::move(mubuf));
6544
6545       expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, (1 << last_bit) - 1);
6546       return;
6547    }
6548
6549
6550    if (instr->op == nir_texop_txf ||
6551        instr->op == nir_texop_txf_ms ||
6552        instr->op == nir_texop_samples_identical) {
6553       aco_opcode op = level_zero || instr->sampler_dim == GLSL_SAMPLER_DIM_MS ? aco_opcode::image_load : aco_opcode::image_load_mip;
6554       tex.reset(create_instruction<MIMG_instruction>(op, Format::MIMG, 2, 1));
6555       tex->operands[0] = Operand(arg);
6556       tex->operands[1] = Operand(resource);
6557       tex->dmask = dmask;
6558       tex->unrm = true;
6559       tex->da = da;
6560       tex->definitions[0] = Definition(tmp_dst);
6561       tex->can_reorder = true;
6562       ctx->block->instructions.emplace_back(std::move(tex));
6563
6564       if (instr->op == nir_texop_samples_identical) {
6565          assert(dmask == 1 && dst.regClass() == v1);
6566          assert(dst.id() != tmp_dst.id());
6567
6568          Temp tmp = bld.tmp(s2);
6569          bld.vopc(aco_opcode::v_cmp_eq_u32, Definition(tmp), Operand(0u), tmp_dst).def(0).setHint(vcc);
6570          bld.vop2_e64(aco_opcode::v_cndmask_b32, Definition(dst), Operand(0u), Operand((uint32_t)-1), tmp);
6571
6572       } else {
6573          expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, dmask);
6574       }
6575       return;
6576    }
6577
6578    // TODO: would be better to do this by adding offsets, but needs the opcodes ordered.
6579    aco_opcode opcode = aco_opcode::image_sample;
6580    if (has_offset) { /* image_sample_*_o */
6581       if (has_compare) {
6582          opcode = aco_opcode::image_sample_c_o;
6583          if (has_derivs)
6584             opcode = aco_opcode::image_sample_c_d_o;
6585          if (has_bias)
6586             opcode = aco_opcode::image_sample_c_b_o;
6587          if (level_zero)
6588             opcode = aco_opcode::image_sample_c_lz_o;
6589          if (has_lod)
6590             opcode = aco_opcode::image_sample_c_l_o;
6591       } else {
6592          opcode = aco_opcode::image_sample_o;
6593          if (has_derivs)
6594             opcode = aco_opcode::image_sample_d_o;
6595          if (has_bias)
6596             opcode = aco_opcode::image_sample_b_o;
6597          if (level_zero)
6598             opcode = aco_opcode::image_sample_lz_o;
6599          if (has_lod)
6600             opcode = aco_opcode::image_sample_l_o;
6601       }
6602    } else { /* no offset */
6603       if (has_compare) {
6604          opcode = aco_opcode::image_sample_c;
6605          if (has_derivs)
6606             opcode = aco_opcode::image_sample_c_d;
6607          if (has_bias)
6608             opcode = aco_opcode::image_sample_c_b;
6609          if (level_zero)
6610             opcode = aco_opcode::image_sample_c_lz;
6611          if (has_lod)
6612             opcode = aco_opcode::image_sample_c_l;
6613       } else {
6614          opcode = aco_opcode::image_sample;
6615          if (has_derivs)
6616             opcode = aco_opcode::image_sample_d;
6617          if (has_bias)
6618             opcode = aco_opcode::image_sample_b;
6619          if (level_zero)
6620             opcode = aco_opcode::image_sample_lz;
6621          if (has_lod)
6622             opcode = aco_opcode::image_sample_l;
6623       }
6624    }
6625
6626    if (instr->op == nir_texop_tg4) {
6627       if (has_offset) {
6628          opcode = aco_opcode::image_gather4_lz_o;
6629          if (has_compare)
6630             opcode = aco_opcode::image_gather4_c_lz_o;
6631       } else {
6632          opcode = aco_opcode::image_gather4_lz;
6633          if (has_compare)
6634             opcode = aco_opcode::image_gather4_c_lz;
6635       }
6636    } else if (instr->op == nir_texop_lod) {
6637       opcode = aco_opcode::image_get_lod;
6638    }
6639
6640    tex.reset(create_instruction<MIMG_instruction>(opcode, Format::MIMG, 3, 1));
6641    tex->operands[0] = arg;
6642    tex->operands[1] = Operand(resource);
6643    tex->operands[2] = Operand(sampler);
6644    tex->dmask = dmask;
6645    tex->da = da;
6646    tex->definitions[0] = Definition(tmp_dst);
6647    tex->can_reorder = true;
6648    ctx->block->instructions.emplace_back(std::move(tex));
6649
6650    if (tg4_integer_cube_workaround) {
6651       assert(tmp_dst.id() != dst.id());
6652       assert(tmp_dst.size() == dst.size() && dst.size() == 4);
6653
6654       emit_split_vector(ctx, tmp_dst, tmp_dst.size());
6655       Temp val[4];
6656       for (unsigned i = 0; i < dst.size(); i++) {
6657          val[i] = emit_extract_vector(ctx, tmp_dst, i, v1);
6658          Temp cvt_val;
6659          if (stype == GLSL_TYPE_UINT)
6660             cvt_val = bld.vop1(aco_opcode::v_cvt_u32_f32, bld.def(v1), val[i]);
6661          else
6662             cvt_val = bld.vop1(aco_opcode::v_cvt_i32_f32, bld.def(v1), val[i]);
6663          val[i] = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1), val[i], cvt_val, tg4_compare_cube_wa64);
6664       }
6665       Temp tmp = dst.regClass() == v4 ? dst : bld.tmp(v4);
6666       tmp_dst = bld.pseudo(aco_opcode::p_create_vector, Definition(tmp),
6667                            val[0], val[1], val[2], val[3]);
6668    }
6669    unsigned mask = instr->op == nir_texop_tg4 ? 0xF : dmask;
6670    expand_vector(ctx, tmp_dst, dst, instr->dest.ssa.num_components, mask);
6671
6672 }
6673
6674
6675 Operand get_phi_operand(isel_context *ctx, nir_ssa_def *ssa)
6676 {
6677    Temp tmp = get_ssa_temp(ctx, ssa);
6678    if (ssa->parent_instr->type == nir_instr_type_ssa_undef)
6679       return Operand(tmp.regClass());
6680    else
6681       return Operand(tmp);
6682 }
6683
6684 void visit_phi(isel_context *ctx, nir_phi_instr *instr)
6685 {
6686    aco_ptr<Pseudo_instruction> phi;
6687    unsigned num_src = exec_list_length(&instr->srcs);
6688    Temp dst = get_ssa_temp(ctx, &instr->dest.ssa);
6689
6690    aco_opcode opcode = !dst.is_linear() || ctx->divergent_vals[instr->dest.ssa.index] ? aco_opcode::p_phi : aco_opcode::p_linear_phi;
6691
6692    std::map<unsigned, nir_ssa_def*> phi_src;
6693    bool all_undef = true;
6694    nir_foreach_phi_src(src, instr) {
6695       phi_src[src->pred->index] = src->src.ssa;
6696       if (src->src.ssa->parent_instr->type != nir_instr_type_ssa_undef)
6697          all_undef = false;
6698    }
6699    if (all_undef) {
6700       Builder bld(ctx->program, ctx->block);
6701       if (dst.regClass() == s1) {
6702          bld.sop1(aco_opcode::s_mov_b32, Definition(dst), Operand(0u));
6703       } else if (dst.regClass() == v1) {
6704          bld.vop1(aco_opcode::v_mov_b32, Definition(dst), Operand(0u));
6705       } else {
6706          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6707          for (unsigned i = 0; i < dst.size(); i++)
6708             vec->operands[i] = Operand(0u);
6709          vec->definitions[0] = Definition(dst);
6710          ctx->block->instructions.emplace_back(std::move(vec));
6711       }
6712       return;
6713    }
6714
6715    /* try to scalarize vector phis */
6716    if (dst.size() > 1) {
6717       // TODO: scalarize linear phis on divergent ifs
6718       bool can_scalarize = (opcode == aco_opcode::p_phi || !(ctx->block->kind & block_kind_merge));
6719       std::array<Temp, 4> new_vec;
6720       for (std::pair<const unsigned, nir_ssa_def*>& pair : phi_src) {
6721          Operand src = get_phi_operand(ctx, pair.second);
6722          if (src.isTemp() && ctx->allocated_vec.find(src.tempId()) == ctx->allocated_vec.end()) {
6723             can_scalarize = false;
6724             break;
6725          }
6726       }
6727       if (can_scalarize) {
6728          unsigned num_components = instr->dest.ssa.num_components;
6729          assert(dst.size() % num_components == 0);
6730          RegClass rc = RegClass(dst.type(), dst.size() / num_components);
6731
6732          aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_components, 1)};
6733          for (unsigned k = 0; k < num_components; k++) {
6734             phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src, 1));
6735             std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6736             for (unsigned i = 0; i < num_src; i++) {
6737                Operand src = get_phi_operand(ctx, it->second);
6738                phi->operands[i] = src.isTemp() ? Operand(ctx->allocated_vec[src.tempId()][k]) : Operand(rc);
6739                ++it;
6740             }
6741             Temp phi_dst = {ctx->program->allocateId(), rc};
6742             phi->definitions[0] = Definition(phi_dst);
6743             ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6744             new_vec[k] = phi_dst;
6745             vec->operands[k] = Operand(phi_dst);
6746          }
6747          vec->definitions[0] = Definition(dst);
6748          ctx->block->instructions.emplace_back(std::move(vec));
6749          ctx->allocated_vec.emplace(dst.id(), new_vec);
6750          return;
6751       }
6752    }
6753
6754    unsigned extra_src = 0;
6755    if (opcode == aco_opcode::p_linear_phi && (ctx->block->kind & block_kind_loop_exit) &&
6756        ctx->program->blocks[ctx->block->index-2].kind & block_kind_continue_or_break) {
6757       extra_src++;
6758    }
6759
6760    phi.reset(create_instruction<Pseudo_instruction>(opcode, Format::PSEUDO, num_src + extra_src, 1));
6761
6762    /* if we have a linear phi on a divergent if, we know that one src is undef */
6763    if (opcode == aco_opcode::p_linear_phi && ctx->block->kind & block_kind_merge) {
6764       assert(extra_src == 0);
6765       Block* block;
6766       /* we place the phi either in the invert-block or in the current block */
6767       if (phi_src.begin()->second->parent_instr->type != nir_instr_type_ssa_undef) {
6768          assert((++phi_src.begin())->second->parent_instr->type == nir_instr_type_ssa_undef);
6769          Block& linear_else = ctx->program->blocks[ctx->block->linear_preds[1]];
6770          block = &ctx->program->blocks[linear_else.linear_preds[0]];
6771          assert(block->kind & block_kind_invert);
6772          phi->operands[0] = get_phi_operand(ctx, phi_src.begin()->second);
6773       } else {
6774          assert((++phi_src.begin())->second->parent_instr->type != nir_instr_type_ssa_undef);
6775          block = ctx->block;
6776          phi->operands[0] = get_phi_operand(ctx, (++phi_src.begin())->second);
6777       }
6778       phi->operands[1] = Operand(dst.regClass());
6779       phi->definitions[0] = Definition(dst);
6780       block->instructions.emplace(block->instructions.begin(), std::move(phi));
6781       return;
6782    }
6783
6784    std::map<unsigned, nir_ssa_def*>::iterator it = phi_src.begin();
6785    for (unsigned i = 0; i < num_src; i++) {
6786       phi->operands[i] = get_phi_operand(ctx, it->second);
6787       ++it;
6788    }
6789    for (unsigned i = 0; i < extra_src; i++)
6790       phi->operands[num_src + i] = Operand(dst.regClass());
6791    phi->definitions[0] = Definition(dst);
6792    ctx->block->instructions.emplace(ctx->block->instructions.begin(), std::move(phi));
6793 }
6794
6795
6796 void visit_undef(isel_context *ctx, nir_ssa_undef_instr *instr)
6797 {
6798    Temp dst = get_ssa_temp(ctx, &instr->def);
6799
6800    assert(dst.type() == RegType::sgpr);
6801
6802    if (dst.size() == 1) {
6803       Builder(ctx->program, ctx->block).copy(Definition(dst), Operand(0u));
6804    } else {
6805       aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, dst.size(), 1)};
6806       for (unsigned i = 0; i < dst.size(); i++)
6807          vec->operands[i] = Operand(0u);
6808       vec->definitions[0] = Definition(dst);
6809       ctx->block->instructions.emplace_back(std::move(vec));
6810    }
6811 }
6812
6813 void visit_jump(isel_context *ctx, nir_jump_instr *instr)
6814 {
6815    Builder bld(ctx->program, ctx->block);
6816    Block *logical_target;
6817    append_logical_end(ctx->block);
6818    unsigned idx = ctx->block->index;
6819
6820    switch (instr->type) {
6821    case nir_jump_break:
6822       logical_target = ctx->cf_info.parent_loop.exit;
6823       add_logical_edge(idx, logical_target);
6824       ctx->block->kind |= block_kind_break;
6825
6826       if (!ctx->cf_info.parent_if.is_divergent &&
6827           !ctx->cf_info.parent_loop.has_divergent_continue) {
6828          /* uniform break - directly jump out of the loop */
6829          ctx->block->kind |= block_kind_uniform;
6830          ctx->cf_info.has_branch = true;
6831          bld.branch(aco_opcode::p_branch);
6832          add_linear_edge(idx, logical_target);
6833          return;
6834       }
6835       ctx->cf_info.parent_loop.has_divergent_branch = true;
6836       break;
6837    case nir_jump_continue:
6838       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6839       add_logical_edge(idx, logical_target);
6840       ctx->block->kind |= block_kind_continue;
6841
6842       if (ctx->cf_info.parent_if.is_divergent) {
6843          /* for potential uniform breaks after this continue,
6844             we must ensure that they are handled correctly */
6845          ctx->cf_info.parent_loop.has_divergent_continue = true;
6846          ctx->cf_info.parent_loop.has_divergent_branch = true;
6847       } else {
6848          /* uniform continue - directly jump to the loop header */
6849          ctx->block->kind |= block_kind_uniform;
6850          ctx->cf_info.has_branch = true;
6851          bld.branch(aco_opcode::p_branch);
6852          add_linear_edge(idx, logical_target);
6853          return;
6854       }
6855       break;
6856    default:
6857       fprintf(stderr, "Unknown NIR jump instr: ");
6858       nir_print_instr(&instr->instr, stderr);
6859       fprintf(stderr, "\n");
6860       abort();
6861    }
6862
6863    /* remove critical edges from linear CFG */
6864    bld.branch(aco_opcode::p_branch);
6865    Block* break_block = ctx->program->create_and_insert_block();
6866    break_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6867    break_block->kind |= block_kind_uniform;
6868    add_linear_edge(idx, break_block);
6869    /* the loop_header pointer might be invalidated by this point */
6870    if (instr->type == nir_jump_continue)
6871       logical_target = &ctx->program->blocks[ctx->cf_info.parent_loop.header_idx];
6872    add_linear_edge(break_block->index, logical_target);
6873    bld.reset(break_block);
6874    bld.branch(aco_opcode::p_branch);
6875
6876    Block* continue_block = ctx->program->create_and_insert_block();
6877    continue_block->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6878    add_linear_edge(idx, continue_block);
6879    append_logical_start(continue_block);
6880    ctx->block = continue_block;
6881    return;
6882 }
6883
6884 void visit_block(isel_context *ctx, nir_block *block)
6885 {
6886    nir_foreach_instr(instr, block) {
6887       switch (instr->type) {
6888       case nir_instr_type_alu:
6889          visit_alu_instr(ctx, nir_instr_as_alu(instr));
6890          break;
6891       case nir_instr_type_load_const:
6892          visit_load_const(ctx, nir_instr_as_load_const(instr));
6893          break;
6894       case nir_instr_type_intrinsic:
6895          visit_intrinsic(ctx, nir_instr_as_intrinsic(instr));
6896          break;
6897       case nir_instr_type_tex:
6898          visit_tex(ctx, nir_instr_as_tex(instr));
6899          break;
6900       case nir_instr_type_phi:
6901          visit_phi(ctx, nir_instr_as_phi(instr));
6902          break;
6903       case nir_instr_type_ssa_undef:
6904          visit_undef(ctx, nir_instr_as_ssa_undef(instr));
6905          break;
6906       case nir_instr_type_deref:
6907          break;
6908       case nir_instr_type_jump:
6909          visit_jump(ctx, nir_instr_as_jump(instr));
6910          break;
6911       default:
6912          fprintf(stderr, "Unknown NIR instr type: ");
6913          nir_print_instr(instr, stderr);
6914          fprintf(stderr, "\n");
6915          //abort();
6916       }
6917    }
6918 }
6919
6920
6921
6922 static void visit_loop(isel_context *ctx, nir_loop *loop)
6923 {
6924    append_logical_end(ctx->block);
6925    ctx->block->kind |= block_kind_loop_preheader | block_kind_uniform;
6926    Builder bld(ctx->program, ctx->block);
6927    bld.branch(aco_opcode::p_branch);
6928    unsigned loop_preheader_idx = ctx->block->index;
6929
6930    Block loop_exit = Block();
6931    loop_exit.loop_nest_depth = ctx->cf_info.loop_nest_depth;
6932    loop_exit.kind |= (block_kind_loop_exit | (ctx->block->kind & block_kind_top_level));
6933
6934    Block* loop_header = ctx->program->create_and_insert_block();
6935    loop_header->loop_nest_depth = ctx->cf_info.loop_nest_depth + 1;
6936    loop_header->kind |= block_kind_loop_header;
6937    add_edge(loop_preheader_idx, loop_header);
6938    ctx->block = loop_header;
6939
6940    /* emit loop body */
6941    unsigned loop_header_idx = loop_header->index;
6942    loop_info_RAII loop_raii(ctx, loop_header_idx, &loop_exit);
6943    append_logical_start(ctx->block);
6944    visit_cf_list(ctx, &loop->body);
6945
6946    //TODO: what if a loop ends with a unconditional or uniformly branched continue and this branch is never taken?
6947    if (!ctx->cf_info.has_branch) {
6948       append_logical_end(ctx->block);
6949       if (ctx->cf_info.exec_potentially_empty) {
6950          /* Discards can result in code running with an empty exec mask.
6951           * This would result in divergent breaks not ever being taken. As a
6952           * workaround, break the loop when the loop mask is empty instead of
6953           * always continuing. */
6954          ctx->block->kind |= (block_kind_continue_or_break | block_kind_uniform);
6955
6956          /* create "loop_almost_exit" to avoid critical edges */
6957          unsigned block_idx = ctx->block->index;
6958          Block *loop_almost_exit = ctx->program->create_and_insert_block();
6959          loop_almost_exit->loop_nest_depth = ctx->cf_info.loop_nest_depth;
6960          loop_almost_exit->kind = block_kind_uniform;
6961          bld.reset(loop_almost_exit);
6962          bld.branch(aco_opcode::p_branch);
6963
6964          add_linear_edge(block_idx, loop_almost_exit);
6965          add_linear_edge(loop_almost_exit->index, &loop_exit);
6966
6967          ctx->block = &ctx->program->blocks[block_idx];
6968       } else {
6969          ctx->block->kind |= (block_kind_continue | block_kind_uniform);
6970       }
6971       if (!ctx->cf_info.parent_loop.has_divergent_branch)
6972          add_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6973       else
6974          add_linear_edge(ctx->block->index, &ctx->program->blocks[loop_header_idx]);
6975       bld.reset(ctx->block);
6976       bld.branch(aco_opcode::p_branch);
6977    }
6978
6979    /* fixup phis in loop header from unreachable blocks */
6980    if (ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch) {
6981       bool linear = ctx->cf_info.has_branch;
6982       bool logical = ctx->cf_info.has_branch || ctx->cf_info.parent_loop.has_divergent_branch;
6983       for (aco_ptr<Instruction>& instr : ctx->program->blocks[loop_header_idx].instructions) {
6984          if ((logical && instr->opcode == aco_opcode::p_phi) ||
6985              (linear && instr->opcode == aco_opcode::p_linear_phi)) {
6986             /* the last operand should be the one that needs to be removed */
6987             instr->operands.pop_back();
6988          } else if (!is_phi(instr)) {
6989             break;
6990          }
6991       }
6992    }
6993
6994    ctx->cf_info.has_branch = false;
6995
6996    // TODO: if the loop has not a single exit, we must add one °°
6997    /* emit loop successor block */
6998    ctx->block = ctx->program->insert_block(std::move(loop_exit));
6999    append_logical_start(ctx->block);
7000
7001    #if 0
7002    // TODO: check if it is beneficial to not branch on continues
7003    /* trim linear phis in loop header */
7004    for (auto&& instr : loop_entry->instructions) {
7005       if (instr->opcode == aco_opcode::p_linear_phi) {
7006          aco_ptr<Pseudo_instruction> new_phi{create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi, Format::PSEUDO, loop_entry->linear_predecessors.size(), 1)};
7007          new_phi->definitions[0] = instr->definitions[0];
7008          for (unsigned i = 0; i < new_phi->operands.size(); i++)
7009             new_phi->operands[i] = instr->operands[i];
7010          /* check that the remaining operands are all the same */
7011          for (unsigned i = new_phi->operands.size(); i < instr->operands.size(); i++)
7012             assert(instr->operands[i].tempId() == instr->operands.back().tempId());
7013          instr.swap(new_phi);
7014       } else if (instr->opcode == aco_opcode::p_phi) {
7015          continue;
7016       } else {
7017          break;
7018       }
7019    }
7020    #endif
7021 }
7022
7023 static void begin_divergent_if_then(isel_context *ctx, if_context *ic, Temp cond)
7024 {
7025    ic->cond = cond;
7026
7027    append_logical_end(ctx->block);
7028    ctx->block->kind |= block_kind_branch;
7029
7030    /* branch to linear then block */
7031    assert(cond.regClass() == s2);
7032    aco_ptr<Pseudo_branch_instruction> branch;
7033    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7034    branch->operands[0] = Operand(cond);
7035    ctx->block->instructions.push_back(std::move(branch));
7036
7037    ic->BB_if_idx = ctx->block->index;
7038    ic->BB_invert = Block();
7039    ic->BB_invert.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7040    /* Invert blocks are intentionally not marked as top level because they
7041     * are not part of the logical cfg. */
7042    ic->BB_invert.kind |= block_kind_invert;
7043    ic->BB_endif = Block();
7044    ic->BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7045    ic->BB_endif.kind |= (block_kind_merge | (ctx->block->kind & block_kind_top_level));
7046
7047    ic->exec_potentially_empty_old = ctx->cf_info.exec_potentially_empty;
7048    ic->divergent_old = ctx->cf_info.parent_if.is_divergent;
7049    ctx->cf_info.parent_if.is_divergent = true;
7050    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7051
7052    /** emit logical then block */
7053    Block* BB_then_logical = ctx->program->create_and_insert_block();
7054    BB_then_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7055    add_edge(ic->BB_if_idx, BB_then_logical);
7056    ctx->block = BB_then_logical;
7057    append_logical_start(BB_then_logical);
7058 }
7059
7060 static void begin_divergent_if_else(isel_context *ctx, if_context *ic)
7061 {
7062    Block *BB_then_logical = ctx->block;
7063    append_logical_end(BB_then_logical);
7064     /* branch from logical then block to invert block */
7065    aco_ptr<Pseudo_branch_instruction> branch;
7066    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7067    BB_then_logical->instructions.emplace_back(std::move(branch));
7068    add_linear_edge(BB_then_logical->index, &ic->BB_invert);
7069    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7070       add_logical_edge(BB_then_logical->index, &ic->BB_endif);
7071    BB_then_logical->kind |= block_kind_uniform;
7072    assert(!ctx->cf_info.has_branch);
7073    ic->then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7074    ctx->cf_info.parent_loop.has_divergent_branch = false;
7075
7076    /** emit linear then block */
7077    Block* BB_then_linear = ctx->program->create_and_insert_block();
7078    BB_then_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7079    BB_then_linear->kind |= block_kind_uniform;
7080    add_linear_edge(ic->BB_if_idx, BB_then_linear);
7081    /* branch from linear then block to invert block */
7082    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7083    BB_then_linear->instructions.emplace_back(std::move(branch));
7084    add_linear_edge(BB_then_linear->index, &ic->BB_invert);
7085
7086    /** emit invert merge block */
7087    ctx->block = ctx->program->insert_block(std::move(ic->BB_invert));
7088    ic->invert_idx = ctx->block->index;
7089
7090    /* branch to linear else block (skip else) */
7091    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_nz, Format::PSEUDO_BRANCH, 1, 0));
7092    branch->operands[0] = Operand(ic->cond);
7093    ctx->block->instructions.push_back(std::move(branch));
7094
7095    ic->exec_potentially_empty_old |= ctx->cf_info.exec_potentially_empty;
7096    ctx->cf_info.exec_potentially_empty = false; /* divergent branches use cbranch_execz */
7097
7098    /** emit logical else block */
7099    Block* BB_else_logical = ctx->program->create_and_insert_block();
7100    BB_else_logical->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7101    add_logical_edge(ic->BB_if_idx, BB_else_logical);
7102    add_linear_edge(ic->invert_idx, BB_else_logical);
7103    ctx->block = BB_else_logical;
7104    append_logical_start(BB_else_logical);
7105 }
7106
7107 static void end_divergent_if(isel_context *ctx, if_context *ic)
7108 {
7109    Block *BB_else_logical = ctx->block;
7110    append_logical_end(BB_else_logical);
7111
7112    /* branch from logical else block to endif block */
7113    aco_ptr<Pseudo_branch_instruction> branch;
7114    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7115    BB_else_logical->instructions.emplace_back(std::move(branch));
7116    add_linear_edge(BB_else_logical->index, &ic->BB_endif);
7117    if (!ctx->cf_info.parent_loop.has_divergent_branch)
7118       add_logical_edge(BB_else_logical->index, &ic->BB_endif);
7119    BB_else_logical->kind |= block_kind_uniform;
7120
7121    assert(!ctx->cf_info.has_branch);
7122    ctx->cf_info.parent_loop.has_divergent_branch &= ic->then_branch_divergent;
7123
7124
7125    /** emit linear else block */
7126    Block* BB_else_linear = ctx->program->create_and_insert_block();
7127    BB_else_linear->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7128    BB_else_linear->kind |= block_kind_uniform;
7129    add_linear_edge(ic->invert_idx, BB_else_linear);
7130
7131    /* branch from linear else block to endif block */
7132    branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7133    BB_else_linear->instructions.emplace_back(std::move(branch));
7134    add_linear_edge(BB_else_linear->index, &ic->BB_endif);
7135
7136
7137    /** emit endif merge block */
7138    ctx->block = ctx->program->insert_block(std::move(ic->BB_endif));
7139    append_logical_start(ctx->block);
7140
7141
7142    ctx->cf_info.parent_if.is_divergent = ic->divergent_old;
7143    ctx->cf_info.exec_potentially_empty |= ic->exec_potentially_empty_old;
7144    /* uniform control flow never has an empty exec-mask */
7145    if (!ctx->cf_info.loop_nest_depth && !ctx->cf_info.parent_if.is_divergent)
7146       ctx->cf_info.exec_potentially_empty = false;
7147 }
7148
7149 static void visit_if(isel_context *ctx, nir_if *if_stmt)
7150 {
7151    Temp cond = get_ssa_temp(ctx, if_stmt->condition.ssa);
7152    Builder bld(ctx->program, ctx->block);
7153    aco_ptr<Pseudo_branch_instruction> branch;
7154
7155    if (!ctx->divergent_vals[if_stmt->condition.ssa->index]) { /* uniform condition */
7156       /**
7157        * Uniform conditionals are represented in the following way*) :
7158        *
7159        * The linear and logical CFG:
7160        *                        BB_IF
7161        *                        /    \
7162        *       BB_THEN (logical)      BB_ELSE (logical)
7163        *                        \    /
7164        *                        BB_ENDIF
7165        *
7166        * *) Exceptions may be due to break and continue statements within loops
7167        *    If a break/continue happens within uniform control flow, it branches
7168        *    to the loop exit/entry block. Otherwise, it branches to the next
7169        *    merge block.
7170        **/
7171       append_logical_end(ctx->block);
7172       ctx->block->kind |= block_kind_uniform;
7173
7174       /* emit branch */
7175       if (cond.regClass() == s2) {
7176          // TODO: in a post-RA optimizer, we could check if the condition is in VCC and omit this instruction
7177          cond = as_uniform_bool(ctx, cond);
7178       }
7179       branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_cbranch_z, Format::PSEUDO_BRANCH, 1, 0));
7180       branch->operands[0] = Operand(cond);
7181       branch->operands[0].setFixed(scc);
7182       ctx->block->instructions.emplace_back(std::move(branch));
7183
7184       unsigned BB_if_idx = ctx->block->index;
7185       Block BB_endif = Block();
7186       BB_endif.loop_nest_depth = ctx->cf_info.loop_nest_depth;
7187       BB_endif.kind |= ctx->block->kind & block_kind_top_level;
7188
7189       /** emit then block */
7190       Block* BB_then = ctx->program->create_and_insert_block();
7191       BB_then->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7192       add_edge(BB_if_idx, BB_then);
7193       append_logical_start(BB_then);
7194       ctx->block = BB_then;
7195       visit_cf_list(ctx, &if_stmt->then_list);
7196       BB_then = ctx->block;
7197       bool then_branch = ctx->cf_info.has_branch;
7198       bool then_branch_divergent = ctx->cf_info.parent_loop.has_divergent_branch;
7199
7200       if (!then_branch) {
7201          append_logical_end(BB_then);
7202          /* branch from then block to endif block */
7203          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7204          BB_then->instructions.emplace_back(std::move(branch));
7205          add_linear_edge(BB_then->index, &BB_endif);
7206          if (!then_branch_divergent)
7207             add_logical_edge(BB_then->index, &BB_endif);
7208          BB_then->kind |= block_kind_uniform;
7209       }
7210
7211       ctx->cf_info.has_branch = false;
7212       ctx->cf_info.parent_loop.has_divergent_branch = false;
7213
7214       /** emit else block */
7215       Block* BB_else = ctx->program->create_and_insert_block();
7216       BB_else->loop_nest_depth = ctx->cf_info.loop_nest_depth;
7217       add_edge(BB_if_idx, BB_else);
7218       append_logical_start(BB_else);
7219       ctx->block = BB_else;
7220       visit_cf_list(ctx, &if_stmt->else_list);
7221       BB_else = ctx->block;
7222
7223       if (!ctx->cf_info.has_branch) {
7224          append_logical_end(BB_else);
7225          /* branch from then block to endif block */
7226          branch.reset(create_instruction<Pseudo_branch_instruction>(aco_opcode::p_branch, Format::PSEUDO_BRANCH, 0, 0));
7227          BB_else->instructions.emplace_back(std::move(branch));
7228          add_linear_edge(BB_else->index, &BB_endif);
7229          if (!ctx->cf_info.parent_loop.has_divergent_branch)
7230             add_logical_edge(BB_else->index, &BB_endif);
7231          BB_else->kind |= block_kind_uniform;
7232       }
7233
7234       ctx->cf_info.has_branch &= then_branch;
7235       ctx->cf_info.parent_loop.has_divergent_branch &= then_branch_divergent;
7236
7237       /** emit endif merge block */
7238       if (!ctx->cf_info.has_branch) {
7239          ctx->block = ctx->program->insert_block(std::move(BB_endif));
7240          append_logical_start(ctx->block);
7241       }
7242    } else { /* non-uniform condition */
7243       /**
7244        * To maintain a logical and linear CFG without critical edges,
7245        * non-uniform conditionals are represented in the following way*) :
7246        *
7247        * The linear CFG:
7248        *                        BB_IF
7249        *                        /    \
7250        *       BB_THEN (logical)      BB_THEN (linear)
7251        *                        \    /
7252        *                        BB_INVERT (linear)
7253        *                        /    \
7254        *       BB_ELSE (logical)      BB_ELSE (linear)
7255        *                        \    /
7256        *                        BB_ENDIF
7257        *
7258        * The logical CFG:
7259        *                        BB_IF
7260        *                        /    \
7261        *       BB_THEN (logical)      BB_ELSE (logical)
7262        *                        \    /
7263        *                        BB_ENDIF
7264        *
7265        * *) Exceptions may be due to break and continue statements within loops
7266        **/
7267
7268       if_context ic;
7269
7270       begin_divergent_if_then(ctx, &ic, cond);
7271       visit_cf_list(ctx, &if_stmt->then_list);
7272
7273       begin_divergent_if_else(ctx, &ic);
7274       visit_cf_list(ctx, &if_stmt->else_list);
7275
7276       end_divergent_if(ctx, &ic);
7277    }
7278 }
7279
7280 static void visit_cf_list(isel_context *ctx,
7281                           struct exec_list *list)
7282 {
7283    foreach_list_typed(nir_cf_node, node, node, list) {
7284       switch (node->type) {
7285       case nir_cf_node_block:
7286          visit_block(ctx, nir_cf_node_as_block(node));
7287          break;
7288       case nir_cf_node_if:
7289          visit_if(ctx, nir_cf_node_as_if(node));
7290          break;
7291       case nir_cf_node_loop:
7292          visit_loop(ctx, nir_cf_node_as_loop(node));
7293          break;
7294       default:
7295          unreachable("unimplemented cf list type");
7296       }
7297    }
7298 }
7299
7300 static void export_vs_varying(isel_context *ctx, int slot, bool is_pos, int *next_pos)
7301 {
7302    int offset = ctx->program->info->vs.outinfo.vs_output_param_offset[slot];
7303    uint64_t mask = ctx->vs_output.mask[slot];
7304    if (!is_pos && !mask)
7305       return;
7306    if (!is_pos && offset == AC_EXP_PARAM_UNDEFINED)
7307       return;
7308    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7309    exp->enabled_mask = mask;
7310    for (unsigned i = 0; i < 4; ++i) {
7311       if (mask & (1 << i))
7312          exp->operands[i] = Operand(ctx->vs_output.outputs[slot][i]);
7313       else
7314          exp->operands[i] = Operand(v1);
7315    }
7316    exp->valid_mask = false;
7317    exp->done = false;
7318    exp->compressed = false;
7319    if (is_pos)
7320       exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7321    else
7322       exp->dest = V_008DFC_SQ_EXP_PARAM + offset;
7323    ctx->block->instructions.emplace_back(std::move(exp));
7324 }
7325
7326 static void export_vs_psiz_layer_viewport(isel_context *ctx, int *next_pos)
7327 {
7328    aco_ptr<Export_instruction> exp{create_instruction<Export_instruction>(aco_opcode::exp, Format::EXP, 4, 0)};
7329    exp->enabled_mask = 0;
7330    for (unsigned i = 0; i < 4; ++i)
7331       exp->operands[i] = Operand(v1);
7332    if (ctx->vs_output.mask[VARYING_SLOT_PSIZ]) {
7333       exp->operands[0] = Operand(ctx->vs_output.outputs[VARYING_SLOT_PSIZ][0]);
7334       exp->enabled_mask |= 0x1;
7335    }
7336    if (ctx->vs_output.mask[VARYING_SLOT_LAYER]) {
7337       exp->operands[2] = Operand(ctx->vs_output.outputs[VARYING_SLOT_LAYER][0]);
7338       exp->enabled_mask |= 0x4;
7339    }
7340    if (ctx->vs_output.mask[VARYING_SLOT_VIEWPORT]) {
7341       if (ctx->options->chip_class < GFX9) {
7342          exp->operands[3] = Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]);
7343          exp->enabled_mask |= 0x8;
7344       } else {
7345          Builder bld(ctx->program, ctx->block);
7346
7347          Temp out = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(16u),
7348                              Operand(ctx->vs_output.outputs[VARYING_SLOT_VIEWPORT][0]));
7349          if (exp->operands[2].isTemp())
7350             out = bld.vop2(aco_opcode::v_or_b32, bld.def(v1), Operand(out), exp->operands[2]);
7351
7352          exp->operands[2] = Operand(out);
7353          exp->enabled_mask |= 0x4;
7354       }
7355    }
7356    exp->valid_mask = false;
7357    exp->done = false;
7358    exp->compressed = false;
7359    exp->dest = V_008DFC_SQ_EXP_POS + (*next_pos)++;
7360    ctx->block->instructions.emplace_back(std::move(exp));
7361 }
7362
7363 static void create_vs_exports(isel_context *ctx)
7364 {
7365    radv_vs_output_info *outinfo = &ctx->program->info->vs.outinfo;
7366
7367    if (outinfo->export_prim_id) {
7368       ctx->vs_output.mask[VARYING_SLOT_PRIMITIVE_ID] |= 0x1;
7369       ctx->vs_output.outputs[VARYING_SLOT_PRIMITIVE_ID][0] = ctx->vs_prim_id;
7370    }
7371
7372    if (ctx->options->key.has_multiview_view_index) {
7373       ctx->vs_output.mask[VARYING_SLOT_LAYER] |= 0x1;
7374       ctx->vs_output.outputs[VARYING_SLOT_LAYER][0] = as_vgpr(ctx, ctx->view_index);
7375    }
7376
7377    /* the order these position exports are created is important */
7378    int next_pos = 0;
7379    export_vs_varying(ctx, VARYING_SLOT_POS, true, &next_pos);
7380    if (outinfo->writes_pointsize || outinfo->writes_layer || outinfo->writes_viewport_index) {
7381       export_vs_psiz_layer_viewport(ctx, &next_pos);
7382    }
7383    if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7384       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, true, &next_pos);
7385    if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7386       export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, true, &next_pos);
7387
7388    if (ctx->options->key.vs_common_out.export_clip_dists) {
7389       if (ctx->num_clip_distances + ctx->num_cull_distances > 0)
7390          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST0, false, &next_pos);
7391       if (ctx->num_clip_distances + ctx->num_cull_distances > 4)
7392          export_vs_varying(ctx, VARYING_SLOT_CLIP_DIST1, false, &next_pos);
7393    }
7394
7395    for (unsigned i = 0; i <= VARYING_SLOT_VAR31; ++i) {
7396       if (i < VARYING_SLOT_VAR0 && i != VARYING_SLOT_LAYER &&
7397           i != VARYING_SLOT_PRIMITIVE_ID)
7398          continue;
7399
7400       export_vs_varying(ctx, i, false, NULL);
7401    }
7402 }
7403
7404 static void emit_stream_output(isel_context *ctx,
7405                                Temp const *so_buffers,
7406                                Temp const *so_write_offset,
7407                                const struct radv_stream_output *output)
7408 {
7409    unsigned num_comps = util_bitcount(output->component_mask);
7410    unsigned loc = output->location;
7411    unsigned buf = output->buffer;
7412    unsigned offset = output->offset;
7413
7414    assert(num_comps && num_comps <= 4);
7415    if (!num_comps || num_comps > 4)
7416       return;
7417
7418    unsigned start = ffs(output->component_mask) - 1;
7419
7420    Temp out[4];
7421    bool all_undef = true;
7422    assert(ctx->stage == vertex_vs);
7423    for (unsigned i = 0; i < num_comps; i++) {
7424       out[i] = ctx->vs_output.outputs[loc][start + i];
7425       all_undef = all_undef && !out[i].id();
7426    }
7427    if (all_undef)
7428       return;
7429
7430    Temp write_data = {ctx->program->allocateId(), RegClass(RegType::vgpr, num_comps)};
7431    aco_ptr<Pseudo_instruction> vec{create_instruction<Pseudo_instruction>(aco_opcode::p_create_vector, Format::PSEUDO, num_comps, 1)};
7432    for (unsigned i = 0; i < num_comps; ++i)
7433       vec->operands[i] = (ctx->vs_output.mask[loc] & 1 << i) ? Operand(out[i]) : Operand(0u);
7434    vec->definitions[0] = Definition(write_data);
7435    ctx->block->instructions.emplace_back(std::move(vec));
7436
7437    aco_opcode opcode;
7438    switch (num_comps) {
7439    case 1:
7440       opcode = aco_opcode::buffer_store_dword;
7441       break;
7442    case 2:
7443       opcode = aco_opcode::buffer_store_dwordx2;
7444       break;
7445    case 3:
7446       opcode = aco_opcode::buffer_store_dwordx3;
7447       break;
7448    case 4:
7449       opcode = aco_opcode::buffer_store_dwordx4;
7450       break;
7451    }
7452
7453    aco_ptr<MUBUF_instruction> store{create_instruction<MUBUF_instruction>(opcode, Format::MUBUF, 4, 0)};
7454    store->operands[0] = Operand(so_write_offset[buf]);
7455    store->operands[1] = Operand(so_buffers[buf]);
7456    store->operands[2] = Operand((uint32_t) 0);
7457    store->operands[3] = Operand(write_data);
7458    if (offset > 4095) {
7459       /* Don't think this can happen in RADV, but maybe GL? It's easy to do this anyway. */
7460       Builder bld(ctx->program, ctx->block);
7461       store->operands[0] = bld.vadd32(bld.def(v1), Operand(offset), Operand(so_write_offset[buf]));
7462    } else {
7463       store->offset = offset;
7464    }
7465    store->offen = true;
7466    store->glc = true;
7467    store->slc = true;
7468    store->can_reorder = true;
7469    ctx->block->instructions.emplace_back(std::move(store));
7470 }
7471
7472 static void emit_streamout(isel_context *ctx, unsigned stream)
7473 {
7474    Builder bld(ctx->program, ctx->block);
7475
7476    Temp so_buffers[4];
7477    Temp buf_ptr = convert_pointer_to_64_bit(ctx, ctx->streamout_buffers);
7478    for (unsigned i = 0; i < 4; i++) {
7479       unsigned stride = ctx->program->info->so.strides[i];
7480       if (!stride)
7481          continue;
7482
7483       so_buffers[i] = bld.smem(aco_opcode::s_load_dwordx4, bld.def(s4), buf_ptr, Operand(i * 16u));
7484    }
7485
7486    Temp so_vtx_count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc),
7487                                 ctx->streamout_config, Operand(0x70010u));
7488
7489    Temp tid = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7490                        bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7491
7492    Temp can_emit = bld.vopc(aco_opcode::v_cmp_gt_i32, bld.def(s2), so_vtx_count, tid);
7493
7494    if_context ic;
7495    begin_divergent_if_then(ctx, &ic, can_emit);
7496
7497    bld.reset(ctx->block);
7498
7499    Temp so_write_index = bld.vadd32(bld.def(v1), ctx->streamout_write_idx, tid);
7500
7501    Temp so_write_offset[4];
7502
7503    for (unsigned i = 0; i < 4; i++) {
7504       unsigned stride = ctx->program->info->so.strides[i];
7505       if (!stride)
7506          continue;
7507
7508       if (stride == 1) {
7509          Temp offset = bld.sop2(aco_opcode::s_add_i32, bld.def(s1), bld.def(s1, scc),
7510                                 ctx->streamout_write_idx, ctx->streamout_offset[i]);
7511          Temp new_offset = bld.vadd32(bld.def(v1), offset, tid);
7512
7513          so_write_offset[i] = bld.vop2(aco_opcode::v_lshlrev_b32, bld.def(v1), Operand(2u), new_offset);
7514       } else {
7515          Temp offset = bld.v_mul_imm(bld.def(v1), so_write_index, stride * 4u);
7516          Temp offset2 = bld.sop2(aco_opcode::s_mul_i32, bld.def(s1), Operand(4u), ctx->streamout_offset[i]);
7517          so_write_offset[i] = bld.vadd32(bld.def(v1), offset, offset2);
7518       }
7519    }
7520
7521    for (unsigned i = 0; i < ctx->program->info->so.num_outputs; i++) {
7522       struct radv_stream_output *output =
7523          &ctx->program->info->so.outputs[i];
7524       if (stream != output->stream)
7525          continue;
7526
7527       emit_stream_output(ctx, so_buffers, so_write_offset, output);
7528    }
7529
7530    begin_divergent_if_else(ctx, &ic);
7531    end_divergent_if(ctx, &ic);
7532 }
7533
7534 } /* end namespace */
7535
7536 void handle_bc_optimize(isel_context *ctx)
7537 {
7538    /* needed when SPI_PS_IN_CONTROL.BC_OPTIMIZE_DISABLE is set to 0 */
7539    Builder bld(ctx->program, ctx->block);
7540    uint32_t spi_ps_input_ena = ctx->program->config->spi_ps_input_ena;
7541    bool uses_center = G_0286CC_PERSP_CENTER_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTER_ENA(spi_ps_input_ena);
7542    bool uses_centroid = G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena) || G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena);
7543    if (uses_center && uses_centroid) {
7544       Temp sel = bld.vopc_e64(aco_opcode::v_cmp_lt_i32, bld.hint_vcc(bld.def(s2)), ctx->prim_mask, Operand(0u));
7545
7546       if (G_0286CC_PERSP_CENTROID_ENA(spi_ps_input_ena)) {
7547          for (unsigned i = 0; i < 2; i++) {
7548             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7549                                       ctx->fs_inputs[fs_input::persp_centroid_p1 + i],
7550                                       ctx->fs_inputs[fs_input::persp_center_p1 + i],
7551                                       sel);
7552             ctx->fs_inputs[fs_input::persp_centroid_p1 + i] = new_coord;
7553          }
7554       }
7555
7556       if (G_0286CC_LINEAR_CENTROID_ENA(spi_ps_input_ena)) {
7557          for (unsigned i = 0; i < 2; i++) {
7558             Temp new_coord = bld.vop2(aco_opcode::v_cndmask_b32, bld.def(v1),
7559                                       ctx->fs_inputs[fs_input::linear_centroid_p1 + i],
7560                                       ctx->fs_inputs[fs_input::linear_center_p1 + i],
7561                                       sel);
7562             ctx->fs_inputs[fs_input::linear_centroid_p1 + i] = new_coord;
7563          }
7564       }
7565    }
7566 }
7567
7568 void select_program(Program *program,
7569                     unsigned shader_count,
7570                     struct nir_shader *const *shaders,
7571                     ac_shader_config* config,
7572                     struct radv_shader_info *info,
7573                     struct radv_nir_compiler_options *options)
7574 {
7575    isel_context ctx = setup_isel_context(program, shader_count, shaders, config, info, options);
7576
7577    for (unsigned i = 0; i < shader_count; i++) {
7578       nir_shader *nir = shaders[i];
7579       init_context(&ctx, nir);
7580
7581       if (!i) {
7582          add_startpgm(&ctx); /* needs to be after init_context() for FS */
7583          append_logical_start(ctx.block);
7584       }
7585
7586       if_context ic;
7587       if (shader_count >= 2) {
7588          Builder bld(ctx.program, ctx.block);
7589          Temp count = bld.sop2(aco_opcode::s_bfe_u32, bld.def(s1), bld.def(s1, scc), ctx.merged_wave_info, Operand((8u << 16) | (i * 8u)));
7590          Temp thread_id = bld.vop3(aco_opcode::v_mbcnt_hi_u32_b32, bld.def(v1), Operand((uint32_t) -1),
7591                                    bld.vop3(aco_opcode::v_mbcnt_lo_u32_b32, bld.def(v1), Operand((uint32_t) -1), Operand(0u)));
7592          Temp cond = bld.vopc(aco_opcode::v_cmp_gt_u32, bld.hint_vcc(bld.def(s2)), count, thread_id);
7593
7594          begin_divergent_if_then(&ctx, &ic, cond);
7595       }
7596
7597       if (i) {
7598          Builder bld(ctx.program, ctx.block);
7599          bld.barrier(aco_opcode::p_memory_barrier_shared); //TODO: different barriers are needed for different stages
7600          bld.sopp(aco_opcode::s_barrier);
7601       }
7602
7603       if (ctx.stage == fragment_fs)
7604          handle_bc_optimize(&ctx);
7605
7606       nir_function_impl *func = nir_shader_get_entrypoint(nir);
7607       visit_cf_list(&ctx, &func->body);
7608
7609       if (ctx.program->info->so.num_outputs/*&& !ctx->is_gs_copy_shader */)
7610          emit_streamout(&ctx, 0);
7611
7612       if (ctx.stage == vertex_vs)
7613          create_vs_exports(&ctx);
7614
7615       if (shader_count >= 2) {
7616          begin_divergent_if_else(&ctx, &ic);
7617          end_divergent_if(&ctx, &ic);
7618       }
7619
7620       ralloc_free(ctx.divergent_vals);
7621    }
7622
7623    append_logical_end(ctx.block);
7624    ctx.block->kind |= block_kind_uniform;
7625    Builder bld(ctx.program, ctx.block);
7626    if (ctx.program->wb_smem_l1_on_end)
7627       bld.smem(aco_opcode::s_dcache_wb, false);
7628    bld.sopp(aco_opcode::s_endpgm);
7629
7630    /* cleanup CFG */
7631    for (Block& BB : program->blocks) {
7632       for (unsigned idx : BB.linear_preds)
7633          program->blocks[idx].linear_succs.emplace_back(BB.index);
7634       for (unsigned idx : BB.logical_preds)
7635          program->blocks[idx].logical_succs.emplace_back(BB.index);
7636    }
7637 }
7638 }