src/gallium/auxiliary/gallivm/lp_bld_nir.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2019 Red Hat.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  **************************************************************************/
  25
  26 #include "lp_bld_nir.h"
  27 #include "lp_bld_arit.h"
  28 #include "lp_bld_bitarit.h"
  29 #include "lp_bld_const.h"
  30 #include "lp_bld_gather.h"
  31 #include "lp_bld_logic.h"
  32 #include "lp_bld_quad.h"
  33 #include "lp_bld_flow.h"
  34 #include "lp_bld_struct.h"
  35 #include "lp_bld_debug.h"
  36 #include "lp_bld_printf.h"
  37 #include "nir_deref.h"
  38
  39 static void visit_cf_list(struct lp_build_nir_context *bld_base,
  40                           struct exec_list *list);
  41
  42 static LLVMValueRef cast_type(struct lp_build_nir_context *bld_base, LLVMValueRef val,
  43                               nir_alu_type alu_type, unsigned bit_size)
  44 {
  45    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
  46    switch (alu_type) {
  47    case nir_type_float:
  48       switch (bit_size) {
  49       case 32:
  50          return LLVMBuildBitCast(builder, val, bld_base->base.vec_type, "");
  51       case 64:
  52          return LLVMBuildBitCast(builder, val, bld_base->dbl_bld.vec_type, "");
  53       default:
  54          assert(0);
  55          break;
  56       }
  57       break;
  58    case nir_type_int:
  59       switch (bit_size) {
  60       case 32:
  61          return LLVMBuildBitCast(builder, val, bld_base->int_bld.vec_type, "");
  62       case 64:
  63          return LLVMBuildBitCast(builder, val, bld_base->int64_bld.vec_type, "");
  64       default:
  65          assert(0);
  66          break;
  67       }
  68       break;
  69    case nir_type_uint:
  70       switch (bit_size) {
  71       case 32:
  72          return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
  73       case 64:
  74          return LLVMBuildBitCast(builder, val, bld_base->uint64_bld.vec_type, "");
  75       default:
  76          assert(0);
  77          break;
  78       }
  79       break;
  80    case nir_type_uint32:
  81       return LLVMBuildBitCast(builder, val, bld_base->uint_bld.vec_type, "");
  82    default:
  83       return val;
  84    }
  85    return NULL;
  86 }
  87
  88 static struct lp_build_context *get_int_bld(struct lp_build_nir_context *bld_base,
  89                                             bool is_unsigned,
  90                                             unsigned op_bit_size)
  91 {
  92    if (is_unsigned)
  93       if (op_bit_size == 64)
  94          return &bld_base->uint64_bld;
  95       else
  96          return &bld_base->uint_bld;
  97    else if (op_bit_size == 64)
  98       return &bld_base->int64_bld;
  99    else
 100       return &bld_base->int_bld;
 101 }
 102
 103 static struct lp_build_context *get_flt_bld(struct lp_build_nir_context *bld_base,
 104                                             unsigned op_bit_size)
 105 {
 106    if (op_bit_size == 64)
 107       return &bld_base->dbl_bld;
 108    else
 109       return &bld_base->base;
 110 }
 111
 112 static unsigned glsl_sampler_to_pipe(int sampler_dim, bool is_array)
 113 {
 114    unsigned pipe_target = PIPE_BUFFER;
 115    switch (sampler_dim) {
 116    case GLSL_SAMPLER_DIM_1D:
 117       pipe_target = is_array ? PIPE_TEXTURE_1D_ARRAY : PIPE_TEXTURE_1D;
 118       break;
 119    case GLSL_SAMPLER_DIM_2D:
 120       pipe_target = is_array ? PIPE_TEXTURE_2D_ARRAY : PIPE_TEXTURE_2D;
 121       break;
 122    case GLSL_SAMPLER_DIM_3D:
 123       pipe_target = PIPE_TEXTURE_3D;
 124       break;
 125    case GLSL_SAMPLER_DIM_CUBE:
 126       pipe_target = is_array ? PIPE_TEXTURE_CUBE_ARRAY : PIPE_TEXTURE_CUBE;
 127       break;
 128    case GLSL_SAMPLER_DIM_RECT:
 129       pipe_target = PIPE_TEXTURE_RECT;
 130       break;
 131    case GLSL_SAMPLER_DIM_BUF:
 132       pipe_target = PIPE_BUFFER;
 133       break;
 134    default:
 135       break;
 136    }
 137    return pipe_target;
 138 }
 139
 140 static LLVMValueRef get_ssa_src(struct lp_build_nir_context *bld_base, nir_ssa_def *ssa)
 141 {
 142    return bld_base->ssa_defs[ssa->index];
 143 }
 144
 145 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src);
 146
 147 static LLVMValueRef get_reg_src(struct lp_build_nir_context *bld_base, nir_reg_src src)
 148 {
 149    struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, src.reg);
 150    LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
 151    struct lp_build_context *reg_bld = get_int_bld(bld_base, true, src.reg->bit_size);
 152    LLVMValueRef indir_src = NULL;
 153    if (src.indirect)
 154       indir_src = get_src(bld_base, *src.indirect);
 155    return bld_base->load_reg(bld_base, reg_bld, &src, indir_src, reg_storage);
 156 }
 157
 158 static LLVMValueRef get_src(struct lp_build_nir_context *bld_base, nir_src src)
 159 {
 160    if (src.is_ssa)
 161       return get_ssa_src(bld_base, src.ssa);
 162    else
 163       return get_reg_src(bld_base, src.reg);
 164 }
 165
 166 static void assign_ssa(struct lp_build_nir_context *bld_base, int idx, LLVMValueRef ptr)
 167 {
 168    bld_base->ssa_defs[idx] = ptr;
 169 }
 170
 171 static void assign_ssa_dest(struct lp_build_nir_context *bld_base, const nir_ssa_def *ssa,
 172                             LLVMValueRef vals[4])
 173 {
 174    assign_ssa(bld_base, ssa->index, ssa->num_components == 1 ? vals[0] : lp_nir_array_build_gather_values(bld_base->base.gallivm->builder, vals, ssa->num_components));
 175 }
 176
 177 static void assign_reg(struct lp_build_nir_context *bld_base, const nir_reg_dest *reg,
 178                        unsigned write_mask,
 179                        LLVMValueRef vals[4])
 180 {
 181    struct hash_entry *entry = _mesa_hash_table_search(bld_base->regs, reg->reg);
 182    LLVMValueRef reg_storage = (LLVMValueRef)entry->data;
 183    struct lp_build_context *reg_bld = get_int_bld(bld_base, true, reg->reg->bit_size);
 184    LLVMValueRef indir_src = NULL;
 185    if (reg->indirect)
 186       indir_src = get_src(bld_base, *reg->indirect);
 187    bld_base->store_reg(bld_base, reg_bld, reg, write_mask ? write_mask : 0xf, indir_src, reg_storage, vals);
 188 }
 189
 190 static void assign_dest(struct lp_build_nir_context *bld_base, const nir_dest *dest, LLVMValueRef vals[4])
 191 {
 192    if (dest->is_ssa)
 193       assign_ssa_dest(bld_base, &dest->ssa, vals);
 194    else
 195       assign_reg(bld_base, &dest->reg, 0, vals);
 196 }
 197
 198 static void assign_alu_dest(struct lp_build_nir_context *bld_base, const nir_alu_dest *dest, LLVMValueRef vals[4])
 199 {
 200    if (dest->dest.is_ssa)
 201       assign_ssa_dest(bld_base, &dest->dest.ssa, vals);
 202    else
 203       assign_reg(bld_base, &dest->dest.reg, dest->write_mask, vals);
 204 }
 205
 206 static LLVMValueRef int_to_bool32(struct lp_build_nir_context *bld_base,
 207                                 uint32_t src_bit_size,
 208                                 bool is_unsigned,
 209                                 LLVMValueRef val)
 210 {
 211    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 212    struct lp_build_context *int_bld = get_int_bld(bld_base, is_unsigned, src_bit_size);
 213    LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, int_bld->type, PIPE_FUNC_NOTEQUAL, val, int_bld->zero);
 214    if (src_bit_size == 64)
 215       result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
 216    return result;
 217 }
 218
 219 static LLVMValueRef flt_to_bool32(struct lp_build_nir_context *bld_base,
 220                                   uint32_t src_bit_size,
 221                                   LLVMValueRef val)
 222 {
 223    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 224    struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size);
 225    LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, flt_bld->type, PIPE_FUNC_NOTEQUAL, val, flt_bld->zero);
 226    if (src_bit_size == 64)
 227       result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
 228    return result;
 229 }
 230
 231 static LLVMValueRef fcmp32(struct lp_build_nir_context *bld_base,
 232                            enum pipe_compare_func compare,
 233                            uint32_t src_bit_size,
 234                            LLVMValueRef src[4])
 235 {
 236    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 237    LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, get_flt_bld(bld_base, src_bit_size)->type, compare, src[0], src[1]);
 238    if (src_bit_size == 64)
 239       result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
 240    return result;
 241 }
 242
 243 static LLVMValueRef icmp32(struct lp_build_nir_context *bld_base,
 244                            enum pipe_compare_func compare,
 245                            bool is_unsigned,
 246                            uint32_t src_bit_size,
 247                            LLVMValueRef src[4])
 248 {
 249    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 250    LLVMValueRef result = lp_build_compare(bld_base->base.gallivm, get_int_bld(bld_base, is_unsigned, src_bit_size)->type, compare, src[0], src[1]);
 251    if (src_bit_size == 64)
 252       result = LLVMBuildTrunc(builder, result, bld_base->int_bld.vec_type, "");
 253    return result;
 254 }
 255
 256 static LLVMValueRef get_alu_src(struct lp_build_nir_context *bld_base,
 257                                 nir_alu_src src,
 258                                 unsigned num_components)
 259 {
 260    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 261    struct gallivm_state *gallivm = bld_base->base.gallivm;
 262    LLVMValueRef value = get_src(bld_base, src.src);
 263    bool need_swizzle = false;
 264
 265    assert(value);
 266    unsigned src_components = nir_src_num_components(src.src);
 267    for (unsigned i = 0; i < num_components; ++i) {
 268       assert(src.swizzle[i] < src_components);
 269       if (src.swizzle[i] != i)
 270          need_swizzle = true;
 271    }
 272
 273    if (need_swizzle || num_components != src_components) {
 274       if (src_components > 1 && num_components == 1) {
 275          value = LLVMBuildExtractValue(gallivm->builder, value,
 276                                        src.swizzle[0], "");
 277       } else if (src_components == 1 && num_components > 1) {
 278          LLVMValueRef values[] = {value, value, value, value};
 279          value = lp_nir_array_build_gather_values(builder, values, num_components);
 280       } else {
 281          LLVMValueRef arr = LLVMGetUndef(LLVMArrayType(LLVMTypeOf(LLVMBuildExtractValue(builder, value, 0, "")), num_components));
 282          for (unsigned i = 0; i < num_components; i++)
 283             arr = LLVMBuildInsertValue(builder, arr, LLVMBuildExtractValue(builder, value, src.swizzle[i], ""), i, "");
 284          value = arr;
 285       }
 286    }
 287    assert(!src.negate);
 288    assert(!src.abs);
 289    return value;
 290 }
 291
 292 static LLVMValueRef emit_b2f(struct lp_build_nir_context *bld_base,
 293                              LLVMValueRef src0,
 294                              unsigned bitsize)
 295 {
 296    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 297    LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
 298                                       LLVMBuildBitCast(builder, lp_build_const_vec(bld_base->base.gallivm, bld_base->base.type,
 299                                                                                    1.0), bld_base->int_bld.vec_type, ""),
 300                                       "");
 301    result = LLVMBuildBitCast(builder, result, bld_base->base.vec_type, "");
 302    switch (bitsize) {
 303    case 32:
 304       break;
 305    case 64:
 306       result = LLVMBuildFPExt(builder, result, bld_base->dbl_bld.vec_type, "");
 307       break;
 308    default:
 309       unreachable("unsupported bit size.");
 310    }
 311    return result;
 312 }
 313
 314 static LLVMValueRef emit_b2i(struct lp_build_nir_context *bld_base,
 315                              LLVMValueRef src0,
 316                              unsigned bitsize)
 317 {
 318    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 319    LLVMValueRef result = LLVMBuildAnd(builder, cast_type(bld_base, src0, nir_type_int, 32),
 320                                       lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, 1), "");
 321    switch (bitsize) {
 322    case 32:
 323       return result;
 324    case 64:
 325       return LLVMBuildZExt(builder, result, bld_base->int64_bld.vec_type, "");
 326    default:
 327       unreachable("unsupported bit size.");
 328    }
 329 }
 330
 331 static LLVMValueRef emit_b32csel(struct lp_build_nir_context *bld_base,
 332                                unsigned src_bit_size[4],
 333                                LLVMValueRef src[4])
 334 {
 335    LLVMValueRef sel = cast_type(bld_base, src[0], nir_type_int, 32);
 336    LLVMValueRef v = lp_build_compare(bld_base->base.gallivm, bld_base->int_bld.type, PIPE_FUNC_NOTEQUAL, sel, bld_base->int_bld.zero);
 337    struct lp_build_context *bld = get_int_bld(bld_base, false, src_bit_size[1]);
 338    return lp_build_select(bld, v, src[1], src[2]);
 339 }
 340
 341 static LLVMValueRef split_64bit(struct lp_build_nir_context *bld_base,
 342                                 LLVMValueRef src,
 343                                 bool hi)
 344 {
 345    struct gallivm_state *gallivm = bld_base->base.gallivm;
 346    LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
 347    LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
 348    int len = bld_base->base.type.length * 2;
 349    for (unsigned i = 0; i < bld_base->base.type.length; i++) {
 350       shuffles[i] = lp_build_const_int32(gallivm, i * 2);
 351       shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
 352    }
 353
 354    src = LLVMBuildBitCast(gallivm->builder, src, LLVMVectorType(LLVMInt32TypeInContext(gallivm->context), len), "");
 355    return LLVMBuildShuffleVector(gallivm->builder, src,
 356                                  LLVMGetUndef(LLVMTypeOf(src)),
 357                                  LLVMConstVector(hi ? shuffles2 : shuffles,
 358                                                  bld_base->base.type.length),
 359                                  "");
 360 }
 361
 362 static LLVMValueRef
 363 merge_64bit(struct lp_build_nir_context *bld_base,
 364             LLVMValueRef input,
 365             LLVMValueRef input2)
 366 {
 367    struct gallivm_state *gallivm = bld_base->base.gallivm;
 368    LLVMBuilderRef builder = gallivm->builder;
 369    int i;
 370    LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
 371    int len = bld_base->base.type.length * 2;
 372    assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
 373
 374    for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
 375       shuffles[i] = lp_build_const_int32(gallivm, i / 2);
 376       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
 377    }
 378    return LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
 379 }
 380
 381 static LLVMValueRef do_alu_action(struct lp_build_nir_context *bld_base,
 382                                   nir_op op, unsigned src_bit_size[4], LLVMValueRef src[4])
 383 {
 384    struct gallivm_state *gallivm = bld_base->base.gallivm;
 385    LLVMBuilderRef builder = gallivm->builder;
 386    LLVMValueRef result;
 387    switch (op) {
 388    case nir_op_b2f32:
 389       result = emit_b2f(bld_base, src[0], 32);
 390       break;
 391    case nir_op_b2f64:
 392       result = emit_b2f(bld_base, src[0], 64);
 393       break;
 394    case nir_op_b2i32:
 395       result = emit_b2i(bld_base, src[0], 32);
 396       break;
 397    case nir_op_b2i64:
 398       result = emit_b2i(bld_base, src[0], 64);
 399       break;
 400    case nir_op_b32csel:
 401       result = emit_b32csel(bld_base, src_bit_size, src);
 402       break;
 403    case nir_op_bit_count:
 404       result = lp_build_popcount(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
 405       break;
 406    case nir_op_bitfield_select:
 407       result = lp_build_xor(&bld_base->uint_bld, src[2], lp_build_and(&bld_base->uint_bld, src[0], lp_build_xor(&bld_base->uint_bld, src[1], src[2])));
 408       break;
 409    case nir_op_f2b32:
 410       result = flt_to_bool32(bld_base, src_bit_size[0], src[0]);
 411       break;
 412    case nir_op_f2f32:
 413       result = LLVMBuildFPTrunc(builder, src[0],
 414                                 bld_base->base.vec_type, "");
 415       break;
 416    case nir_op_f2f64:
 417       result = LLVMBuildFPExt(builder, src[0],
 418                               bld_base->dbl_bld.vec_type, "");
 419       break;
 420    case nir_op_f2i32:
 421       result = LLVMBuildFPToSI(builder, src[0], bld_base->base.int_vec_type, "");
 422       break;
 423    case nir_op_f2u32:
 424       result = LLVMBuildFPToUI(builder,
 425                                src[0],
 426                                bld_base->base.int_vec_type, "");
 427       break;
 428    case nir_op_f2i64:
 429       result = LLVMBuildFPToSI(builder,
 430                                src[0],
 431                                bld_base->int64_bld.vec_type, "");
 432       break;
 433    case nir_op_f2u64:
 434       result = LLVMBuildFPToUI(builder,
 435                                src[0],
 436                                bld_base->uint64_bld.vec_type, "");
 437       break;
 438    case nir_op_fabs:
 439       result = lp_build_abs(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 440       break;
 441    case nir_op_fadd:
 442       result = lp_build_add(get_flt_bld(bld_base, src_bit_size[0]),
 443                             src[0], src[1]);
 444       break;
 445    case nir_op_fceil:
 446       result = lp_build_ceil(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 447       break;
 448    case nir_op_fcos:
 449       result = lp_build_cos(&bld_base->base, src[0]);
 450       break;
 451    case nir_op_fddx:
 452       result = lp_build_ddx(&bld_base->base, src[0]);
 453       break;
 454    case nir_op_fddy:
 455       result = lp_build_ddy(&bld_base->base, src[0]);
 456       break;
 457    case nir_op_fdiv:
 458       result = lp_build_div(get_flt_bld(bld_base, src_bit_size[0]),
 459                             src[0], src[1]);
 460       break;
 461    case nir_op_feq32:
 462       result = fcmp32(bld_base, PIPE_FUNC_EQUAL, src_bit_size[0], src);
 463       break;
 464    case nir_op_fexp2:
 465       result = lp_build_exp2(&bld_base->base, src[0]);
 466       break;
 467    case nir_op_ffloor:
 468       result = lp_build_floor(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 469       break;
 470    case nir_op_ffma:
 471       result = lp_build_fmuladd(builder, src[0], src[1], src[2]);
 472       break;
 473    case nir_op_ffract: {
 474       struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
 475       LLVMValueRef tmp = lp_build_floor(flt_bld, src[0]);
 476       result = lp_build_sub(flt_bld, src[0], tmp);
 477       break;
 478    }
 479    case nir_op_fge32:
 480       result = fcmp32(bld_base, PIPE_FUNC_GEQUAL, src_bit_size[0], src);
 481       break;
 482    case nir_op_find_lsb:
 483       result = lp_build_cttz(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
 484       break;
 485    case nir_op_flog2:
 486       result = lp_build_log2_safe(&bld_base->base, src[0]);
 487       break;
 488    case nir_op_flt32:
 489       result = fcmp32(bld_base, PIPE_FUNC_LESS, src_bit_size[0], src);
 490       break;
 491    case nir_op_fmin:
 492       result = lp_build_min(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
 493       break;
 494    case nir_op_fmod: {
 495       struct lp_build_context *flt_bld = get_flt_bld(bld_base, src_bit_size[0]);
 496       result = lp_build_div(flt_bld, src[0], src[1]);
 497       result = lp_build_floor(flt_bld, result);
 498       result = lp_build_mul(flt_bld, src[1], result);
 499       result = lp_build_sub(flt_bld, src[0], result);
 500       break;
 501    }
 502    case nir_op_fmul:
 503       result = lp_build_mul(get_flt_bld(bld_base, src_bit_size[0]),
 504                             src[0], src[1]);
 505       break;
 506    case nir_op_fmax:
 507       result = lp_build_max(get_flt_bld(bld_base, src_bit_size[0]), src[0], src[1]);
 508       break;
 509    case nir_op_fne32:
 510       result = fcmp32(bld_base, PIPE_FUNC_NOTEQUAL, src_bit_size[0], src);
 511       break;
 512    case nir_op_fneg:
 513       result = lp_build_negate(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 514       break;
 515    case nir_op_fpow:
 516       result = lp_build_pow(&bld_base->base, src[0], src[1]);
 517       break;
 518    case nir_op_frcp:
 519       result = lp_build_rcp(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 520       break;
 521    case nir_op_fround_even:
 522       result = lp_build_round(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 523       break;
 524    case nir_op_frsq:
 525       result = lp_build_rsqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 526       break;
 527    case nir_op_fsat:
 528       result = lp_build_clamp_zero_one_nanzero(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 529       break;
 530    case nir_op_fsign:
 531       result = lp_build_sgn(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 532       break;
 533    case nir_op_fsin:
 534       result = lp_build_sin(&bld_base->base, src[0]);
 535       break;
 536    case nir_op_fsqrt:
 537       result = lp_build_sqrt(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 538       break;
 539    case nir_op_ftrunc:
 540       result = lp_build_trunc(get_flt_bld(bld_base, src_bit_size[0]), src[0]);
 541       break;
 542    case nir_op_i2b32:
 543       result = int_to_bool32(bld_base, src_bit_size[0], false, src[0]);
 544       break;
 545    case nir_op_i2f32:
 546       result = lp_build_int_to_float(&bld_base->base, src[0]);
 547       break;
 548    case nir_op_i2f64:
 549       result = lp_build_int_to_float(&bld_base->dbl_bld, src[0]);
 550       break;
 551    case nir_op_i2i32:
 552       result = LLVMBuildTrunc(builder, src[0], bld_base->int_bld.vec_type, "");
 553       break;
 554    case nir_op_i2i64:
 555       result = LLVMBuildSExt(builder, src[0], bld_base->int64_bld.vec_type, "");
 556       break;
 557    case nir_op_iabs:
 558       result = lp_build_abs(&bld_base->int_bld, src[0]);
 559       break;
 560    case nir_op_iadd:
 561       result = lp_build_add(get_int_bld(bld_base, false, src_bit_size[0]),
 562                             src[0], src[1]);
 563       break;
 564    case nir_op_iand:
 565       result = lp_build_and(get_int_bld(bld_base, false, src_bit_size[0]),
 566                             src[0], src[1]);
 567       break;
 568    case nir_op_idiv:
 569       result = lp_build_div(&bld_base->int_bld,
 570                             src[0], src[1]);
 571       break;
 572    case nir_op_ieq32:
 573       result = icmp32(bld_base, PIPE_FUNC_EQUAL, false, src_bit_size[0], src);
 574       break;
 575    case nir_op_ige32:
 576       result = icmp32(bld_base, PIPE_FUNC_GEQUAL, false, src_bit_size[0], src);
 577       break;
 578    case nir_op_ilt32:
 579       result = icmp32(bld_base, PIPE_FUNC_LESS, false, src_bit_size[0], src);
 580       break;
 581    case nir_op_imax:
 582       result = lp_build_max(&bld_base->int_bld, src[0], src[1]);
 583       break;
 584    case nir_op_imin:
 585       result = lp_build_min(&bld_base->int_bld, src[0], src[1]);
 586       break;
 587    case nir_op_imul:
 588       result = lp_build_mul(&bld_base->int_bld,
 589                             src[0], src[1]);
 590       break;
 591    case nir_op_imul_high: {
 592       LLVMValueRef hi_bits;
 593       lp_build_mul_32_lohi(&bld_base->int_bld, src[0], src[1], &hi_bits);
 594       result = hi_bits;
 595       break;
 596    }
 597    case nir_op_ine32:
 598       result = icmp32(bld_base, PIPE_FUNC_NOTEQUAL, false, src_bit_size[0], src);
 599       break;
 600    case nir_op_ineg:
 601       result = lp_build_negate(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
 602       break;
 603    case nir_op_inot:
 604       result = lp_build_not(get_int_bld(bld_base, false, src_bit_size[0]), src[0]);
 605       break;
 606    case nir_op_ior:
 607       result = lp_build_or(get_int_bld(bld_base, false, src_bit_size[0]),
 608                            src[0], src[1]);
 609       break;
 610    case nir_op_ishl:
 611       src[1] = lp_build_and(&bld_base->uint_bld, src[1], lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, (src_bit_size[0] - 1)));
 612       result = lp_build_shl(&bld_base->int_bld, src[0], src[1]);
 613       break;
 614    case nir_op_ishr:
 615       src[1] = lp_build_and(&bld_base->uint_bld, src[1], lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, (src_bit_size[0] - 1)));
 616       result = lp_build_shr(&bld_base->int_bld, src[0], src[1]);
 617       break;
 618    case nir_op_isign:
 619       result = lp_build_sgn(&bld_base->int_bld, src[0]);
 620       break;
 621    case nir_op_ixor:
 622       result = lp_build_xor(get_int_bld(bld_base, false, src_bit_size[0]),
 623                             src[0], src[1]);
 624       break;
 625    case nir_op_mov:
 626       result = src[0];
 627       break;
 628    case nir_op_unpack_64_2x32_split_x:
 629       result = split_64bit(bld_base, src[0], false);
 630       break;
 631    case nir_op_unpack_64_2x32_split_y:
 632       result = split_64bit(bld_base, src[0], true);
 633       break;
 634
 635    case nir_op_pack_64_2x32_split: {
 636       LLVMValueRef tmp = merge_64bit(bld_base, src[0], src[1]);
 637       result = LLVMBuildBitCast(builder, tmp, bld_base->dbl_bld.vec_type, "");
 638       break;
 639    }
 640    case nir_op_u2f32:
 641       result = LLVMBuildUIToFP(builder, src[0], bld_base->base.vec_type, "");
 642       break;
 643    case nir_op_u2f64:
 644       result = LLVMBuildUIToFP(builder, src[0], bld_base->dbl_bld.vec_type, "");
 645       break;
 646    case nir_op_u2u32:
 647       result = LLVMBuildTrunc(builder, src[0], bld_base->uint_bld.vec_type, "");
 648       break;
 649    case nir_op_u2u64:
 650       result = LLVMBuildZExt(builder, src[0], bld_base->uint64_bld.vec_type, "");
 651       break;
 652    case nir_op_udiv:
 653       result = lp_build_div(&bld_base->uint_bld,
 654                             src[0], src[1]);
 655       break;
 656    case nir_op_uge32:
 657       result = icmp32(bld_base, PIPE_FUNC_GEQUAL, true, src_bit_size[0], src);
 658       break;
 659    case nir_op_ult32:
 660       result = icmp32(bld_base, PIPE_FUNC_LESS, true, src_bit_size[0], src);
 661       break;
 662    case nir_op_umax:
 663       result = lp_build_max(&bld_base->uint_bld, src[0], src[1]);
 664       break;
 665    case nir_op_umin:
 666       result = lp_build_min(&bld_base->uint_bld, src[0], src[1]);
 667       break;
 668    case nir_op_umod:
 669       result = lp_build_mod(&bld_base->uint_bld, src[0], src[1]);
 670       break;
 671    case nir_op_umul_high: {
 672       LLVMValueRef hi_bits;
 673       lp_build_mul_32_lohi(&bld_base->uint_bld, src[0], src[1], &hi_bits);
 674       result = hi_bits;
 675       break;
 676    }
 677    case nir_op_ushr:
 678       src[1] = lp_build_and(&bld_base->uint_bld, src[1], lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, (src_bit_size[0] - 1)));
 679       result = lp_build_shr(&bld_base->uint_bld, src[0], src[1]);
 680       break;
 681    default:
 682       assert(0);
 683       break;
 684    }
 685    return result;
 686 }
 687
 688 static void visit_alu(struct lp_build_nir_context *bld_base, const nir_alu_instr *instr)
 689 {
 690    struct gallivm_state *gallivm = bld_base->base.gallivm;
 691    LLVMValueRef src[4];
 692    unsigned src_bit_size[4];
 693    unsigned num_components = nir_dest_num_components(instr->dest.dest);
 694    unsigned src_components;
 695    switch (instr->op) {
 696    case nir_op_vec2:
 697    case nir_op_vec3:
 698    case nir_op_vec4:
 699       src_components = 1;
 700       break;
 701    case nir_op_pack_half_2x16:
 702       src_components = 2;
 703       break;
 704    case nir_op_unpack_half_2x16:
 705       src_components = 1;
 706       break;
 707    case nir_op_cube_face_coord:
 708    case nir_op_cube_face_index:
 709       src_components = 3;
 710       break;
 711    default:
 712       src_components = num_components;
 713       break;
 714    }
 715    for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
 716       src[i] = get_alu_src(bld_base, instr->src[i], src_components);
 717       src_bit_size[i] = nir_src_bit_size(instr->src[i].src);
 718    }
 719
 720    LLVMValueRef result[4];
 721    if (instr->op == nir_op_vec4 || instr->op == nir_op_vec3 || instr->op == nir_op_vec2) {
 722       for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
 723          result[i] = cast_type(bld_base, src[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
 724       }
 725    } else {
 726       for (unsigned c = 0; c < num_components; c++) {
 727          LLVMValueRef src_chan[4];
 728
 729          for (unsigned i = 0; i < nir_op_infos[instr->op].num_inputs; i++) {
 730             if (num_components > 1) {
 731                src_chan[i] = LLVMBuildExtractValue(gallivm->builder,
 732                                                      src[i], c, "");
 733             } else
 734                src_chan[i] = src[i];
 735             src_chan[i] = cast_type(bld_base, src_chan[i], nir_op_infos[instr->op].input_types[i], src_bit_size[i]);
 736          }
 737          result[c] = do_alu_action(bld_base, instr->op, src_bit_size, src_chan);
 738          result[c] = cast_type(bld_base, result[c], nir_op_infos[instr->op].output_type, nir_dest_bit_size(instr->dest.dest));
 739       }
 740    }
 741    assign_alu_dest(bld_base, &instr->dest, result);
 742  }
 743
 744 static void visit_load_const(struct lp_build_nir_context *bld_base,
 745                              const nir_load_const_instr *instr)
 746 {
 747    LLVMValueRef result[4];
 748    struct lp_build_context *int_bld = get_int_bld(bld_base, true, instr->def.bit_size);
 749    for (unsigned i = 0; i < instr->def.num_components; i++)
 750       result[i] = lp_build_const_int_vec(bld_base->base.gallivm, int_bld->type, instr->value[i].u64);
 751    assign_ssa_dest(bld_base, &instr->def, result);
 752 }
 753
 754 static void
 755 get_deref_offset(struct lp_build_nir_context *bld_base, nir_deref_instr *instr,
 756                  bool vs_in, unsigned *vertex_index_out,
 757                  LLVMValueRef *vertex_index_ref,
 758                  unsigned *const_out, LLVMValueRef *indir_out)
 759 {
 760    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
 761    nir_variable *var = nir_deref_instr_get_variable(instr);
 762    nir_deref_path path;
 763    unsigned idx_lvl = 1;
 764
 765    nir_deref_path_init(&path, instr, NULL);
 766
 767    if (vertex_index_out != NULL || vertex_index_ref != NULL) {
 768       if (vertex_index_ref) {
 769          *vertex_index_ref = get_src(bld_base, path.path[idx_lvl]->arr.index);
 770          if (vertex_index_out)
 771             *vertex_index_out = 0;
 772       } else {
 773          *vertex_index_out = nir_src_as_uint(path.path[idx_lvl]->arr.index);
 774       }
 775       ++idx_lvl;
 776    }
 777
 778    uint32_t const_offset = 0;
 779    LLVMValueRef offset = NULL;
 780
 781    if (var->data.compact) {
 782       assert(instr->deref_type == nir_deref_type_array);
 783       const_offset = nir_src_as_uint(instr->arr.index);
 784       goto out;
 785    }
 786
 787    for (; path.path[idx_lvl]; ++idx_lvl) {
 788       const struct glsl_type *parent_type = path.path[idx_lvl - 1]->type;
 789       if (path.path[idx_lvl]->deref_type == nir_deref_type_struct) {
 790          unsigned index = path.path[idx_lvl]->strct.index;
 791
 792          for (unsigned i = 0; i < index; i++) {
 793             const struct glsl_type *ft = glsl_get_struct_field(parent_type, i);
 794             const_offset += glsl_count_attribute_slots(ft, vs_in);
 795          }
 796       } else if(path.path[idx_lvl]->deref_type == nir_deref_type_array) {
 797          unsigned size = glsl_count_attribute_slots(path.path[idx_lvl]->type, vs_in);
 798          if (nir_src_is_const(path.path[idx_lvl]->arr.index)) {
 799            const_offset += nir_src_comp_as_int(path.path[idx_lvl]->arr.index, 0) * size;
 800          } else {
 801            LLVMValueRef idx_src = get_src(bld_base, path.path[idx_lvl]->arr.index);
 802            idx_src = cast_type(bld_base, idx_src, nir_type_uint, 32);
 803            LLVMValueRef array_off = lp_build_mul(&bld_base->uint_bld, lp_build_const_int_vec(bld_base->base.gallivm, bld_base->base.type, size),
 804                                                idx_src);
 805            if (offset)
 806              offset = lp_build_add(&bld_base->uint_bld, offset, array_off);
 807            else
 808              offset = array_off;
 809          }
 810       } else
 811          unreachable("Uhandled deref type in get_deref_instr_offset");
 812    }
 813
 814 out:
 815    nir_deref_path_finish(&path);
 816
 817    if (const_offset && offset)
 818       offset = LLVMBuildAdd(builder, offset,
 819                             lp_build_const_int_vec(bld_base->base.gallivm, bld_base->uint_bld.type, const_offset),
 820                             "");
 821    *const_out = const_offset;
 822    *indir_out = offset;
 823 }
 824
 825 static void visit_load_var(struct lp_build_nir_context *bld_base,
 826                            nir_intrinsic_instr *instr,
 827                            LLVMValueRef result[4])
 828 {
 829    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
 830    nir_variable *var = nir_deref_instr_get_variable(deref);
 831    nir_variable_mode mode = deref->mode;
 832    unsigned const_index;
 833    LLVMValueRef indir_index;
 834    unsigned vertex_index = 0;
 835    unsigned nc = nir_dest_num_components(instr->dest);
 836    unsigned bit_size = nir_dest_bit_size(instr->dest);
 837    if (var) {
 838       bool vs_in = bld_base->shader->info.stage == MESA_SHADER_VERTEX &&
 839          var->data.mode == nir_var_shader_in;
 840       bool gs_in = bld_base->shader->info.stage == MESA_SHADER_GEOMETRY &&
 841          var->data.mode == nir_var_shader_in;
 842       mode = var->data.mode;
 843
 844       get_deref_offset(bld_base, deref, vs_in, gs_in ? &vertex_index : NULL, NULL,
 845                        &const_index, &indir_index);
 846    }
 847    bld_base->load_var(bld_base, mode, nc, bit_size, var, vertex_index, const_index, indir_index, result);
 848 }
 849
 850 static void
 851 visit_store_var(struct lp_build_nir_context *bld_base,
 852                 nir_intrinsic_instr *instr)
 853 {
 854    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
 855    nir_variable *var = nir_deref_instr_get_variable(deref);
 856    nir_variable_mode mode = deref->mode;
 857    int writemask = instr->const_index[0];
 858    unsigned bit_size = nir_src_bit_size(instr->src[1]);
 859    LLVMValueRef src = get_src(bld_base, instr->src[1]);
 860    unsigned const_index = 0;
 861    LLVMValueRef indir_index;
 862    if (var)
 863       get_deref_offset(bld_base, deref, false, NULL, NULL,
 864                        &const_index, &indir_index);
 865    bld_base->store_var(bld_base, mode, bit_size, instr->num_components, writemask, const_index, var, src);
 866 }
 867
 868 static void visit_load_ubo(struct lp_build_nir_context *bld_base,
 869                            nir_intrinsic_instr *instr,
 870                            LLVMValueRef result[4])
 871 {
 872    struct gallivm_state *gallivm = bld_base->base.gallivm;
 873    LLVMBuilderRef builder = gallivm->builder;
 874    LLVMValueRef idx = get_src(bld_base, instr->src[0]);
 875    LLVMValueRef offset = get_src(bld_base, instr->src[1]);
 876
 877    bool offset_is_uniform = nir_src_is_dynamically_uniform(instr->src[1]);
 878    idx = LLVMBuildExtractElement(builder, idx, lp_build_const_int32(gallivm, 0), "");
 879    bld_base->load_ubo(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
 880                       offset_is_uniform, idx, offset, result);
 881 }
 882
 883
 884 static void visit_load_ssbo(struct lp_build_nir_context *bld_base,
 885                            nir_intrinsic_instr *instr,
 886                            LLVMValueRef result[4])
 887 {
 888    LLVMValueRef idx = get_src(bld_base, instr->src[0]);
 889    LLVMValueRef offset = get_src(bld_base, instr->src[1]);
 890    bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
 891                        idx, offset, result);
 892 }
 893
 894 static void visit_store_ssbo(struct lp_build_nir_context *bld_base,
 895                              nir_intrinsic_instr *instr)
 896 {
 897    LLVMValueRef val = get_src(bld_base, instr->src[0]);
 898    LLVMValueRef idx = get_src(bld_base, instr->src[1]);
 899    LLVMValueRef offset = get_src(bld_base, instr->src[2]);
 900    int writemask = instr->const_index[0];
 901    int nc = nir_src_num_components(instr->src[0]);
 902    int bitsize = nir_src_bit_size(instr->src[0]);
 903    bld_base->store_mem(bld_base, writemask, nc, bitsize, idx, offset, val);
 904 }
 905
 906 static void visit_get_buffer_size(struct lp_build_nir_context *bld_base,
 907                                   nir_intrinsic_instr *instr,
 908                                   LLVMValueRef result[4])
 909 {
 910    LLVMValueRef idx = get_src(bld_base, instr->src[0]);
 911    result[0] = bld_base->get_buffer_size(bld_base, idx);
 912 }
 913
 914 static void visit_ssbo_atomic(struct lp_build_nir_context *bld_base,
 915                               nir_intrinsic_instr *instr,
 916                               LLVMValueRef result[4])
 917 {
 918    LLVMValueRef idx = get_src(bld_base, instr->src[0]);
 919    LLVMValueRef offset = get_src(bld_base, instr->src[1]);
 920    LLVMValueRef val = get_src(bld_base, instr->src[2]);
 921    LLVMValueRef val2 = NULL;
 922    if (instr->intrinsic == nir_intrinsic_ssbo_atomic_comp_swap)
 923       val2 = get_src(bld_base, instr->src[3]);
 924
 925    bld_base->atomic_mem(bld_base, instr->intrinsic, idx, offset, val, val2, &result[0]);
 926
 927 }
 928
 929 static void visit_load_image(struct lp_build_nir_context *bld_base,
 930                              nir_intrinsic_instr *instr,
 931                              LLVMValueRef result[4])
 932 {
 933    struct gallivm_state *gallivm = bld_base->base.gallivm;
 934    LLVMBuilderRef builder = gallivm->builder;
 935    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
 936    nir_variable *var = nir_deref_instr_get_variable(deref);
 937    LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
 938    LLVMValueRef coords[5];
 939    struct lp_img_params params;
 940    const struct glsl_type *type = glsl_without_array(var->type);
 941
 942    memset(&params, 0, sizeof(params));
 943    params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
 944    for (unsigned i = 0; i < 4; i++)
 945       coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
 946    if (params.target == PIPE_TEXTURE_1D_ARRAY)
 947       coords[2] = coords[1];
 948
 949    params.coords = coords;
 950    params.outdata = result;
 951    params.img_op = LP_IMG_LOAD;
 952    params.image_index = var->data.binding;
 953    bld_base->image_op(bld_base, &params);
 954 }
 955
 956 static void visit_store_image(struct lp_build_nir_context *bld_base,
 957                               nir_intrinsic_instr *instr)
 958 {
 959    struct gallivm_state *gallivm = bld_base->base.gallivm;
 960    LLVMBuilderRef builder = gallivm->builder;
 961    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
 962    nir_variable *var = nir_deref_instr_get_variable(deref);
 963    LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
 964    LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
 965    LLVMValueRef coords[5];
 966    struct lp_img_params params;
 967    const struct glsl_type *type = glsl_without_array(var->type);
 968
 969    memset(&params, 0, sizeof(params));
 970    params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
 971    for (unsigned i = 0; i < 4; i++)
 972       coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
 973    if (params.target == PIPE_TEXTURE_1D_ARRAY)
 974       coords[2] = coords[1];
 975    params.coords = coords;
 976
 977    for (unsigned i = 0; i < 4; i++) {
 978       params.indata[i] = LLVMBuildExtractValue(builder, in_val, i, "");
 979       params.indata[i] = LLVMBuildBitCast(builder, params.indata[i], bld_base->base.vec_type, "");
 980    }
 981    params.img_op = LP_IMG_STORE;
 982    params.image_index = var->data.binding;
 983
 984    if (params.target == PIPE_TEXTURE_1D_ARRAY)
 985       coords[2] = coords[1];
 986    bld_base->image_op(bld_base, &params);
 987 }
 988
 989 static void visit_atomic_image(struct lp_build_nir_context *bld_base,
 990                                nir_intrinsic_instr *instr,
 991                                LLVMValueRef result[4])
 992 {
 993    struct gallivm_state *gallivm = bld_base->base.gallivm;
 994    LLVMBuilderRef builder = gallivm->builder;
 995    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
 996    nir_variable *var = nir_deref_instr_get_variable(deref);
 997    struct lp_img_params params;
 998    LLVMValueRef coord_val = get_src(bld_base, instr->src[1]);
 999    LLVMValueRef in_val = get_src(bld_base, instr->src[3]);
1000    LLVMValueRef coords[5];
1001    const struct glsl_type *type = glsl_without_array(var->type);
1002
1003    memset(&params, 0, sizeof(params));
1004
1005    switch (instr->intrinsic) {
1006    case nir_intrinsic_image_deref_atomic_add:
1007       params.op = LLVMAtomicRMWBinOpAdd;
1008       break;
1009    case nir_intrinsic_image_deref_atomic_exchange:
1010       params.op = LLVMAtomicRMWBinOpXchg;
1011       break;
1012    case nir_intrinsic_image_deref_atomic_and:
1013       params.op = LLVMAtomicRMWBinOpAnd;
1014       break;
1015    case nir_intrinsic_image_deref_atomic_or:
1016       params.op = LLVMAtomicRMWBinOpOr;
1017       break;
1018    case nir_intrinsic_image_deref_atomic_xor:
1019       params.op = LLVMAtomicRMWBinOpXor;
1020       break;
1021    case nir_intrinsic_image_deref_atomic_umin:
1022       params.op = LLVMAtomicRMWBinOpUMin;
1023       break;
1024    case nir_intrinsic_image_deref_atomic_umax:
1025       params.op = LLVMAtomicRMWBinOpUMax;
1026       break;
1027    case nir_intrinsic_image_deref_atomic_imin:
1028       params.op = LLVMAtomicRMWBinOpMin;
1029       break;
1030    case nir_intrinsic_image_deref_atomic_imax:
1031       params.op = LLVMAtomicRMWBinOpMax;
1032       break;
1033    default:
1034       break;
1035    }
1036
1037    params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(type), glsl_sampler_type_is_array(type));
1038    for (unsigned i = 0; i < 4; i++)
1039       coords[i] = LLVMBuildExtractValue(builder, coord_val, i, "");
1040    if (params.target == PIPE_TEXTURE_1D_ARRAY)
1041       coords[2] = coords[1];
1042    params.coords = coords;
1043    if (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) {
1044       LLVMValueRef cas_val = get_src(bld_base, instr->src[4]);
1045       params.indata[0] = in_val;
1046       params.indata2[0] = cas_val;
1047    } else
1048       params.indata[0] = in_val;
1049
1050    params.outdata = result;
1051    params.img_op = (instr->intrinsic == nir_intrinsic_image_deref_atomic_comp_swap) ? LP_IMG_ATOMIC_CAS : LP_IMG_ATOMIC;
1052    params.image_index = var->data.binding;
1053
1054    bld_base->image_op(bld_base, &params);
1055 }
1056
1057
1058 static void visit_image_size(struct lp_build_nir_context *bld_base,
1059                              nir_intrinsic_instr *instr,
1060                              LLVMValueRef result[4])
1061 {
1062    nir_deref_instr *deref = nir_instr_as_deref(instr->src[0].ssa->parent_instr);
1063    nir_variable *var = nir_deref_instr_get_variable(deref);
1064    struct lp_sampler_size_query_params params = { 0 };
1065    params.texture_unit = var->data.binding;
1066    params.target = glsl_sampler_to_pipe(glsl_get_sampler_dim(var->type), glsl_sampler_type_is_array(var->type));
1067    params.sizes_out = result;
1068
1069    bld_base->image_size(bld_base, &params);
1070 }
1071
1072 static void visit_shared_load(struct lp_build_nir_context *bld_base,
1073                                 nir_intrinsic_instr *instr,
1074                                 LLVMValueRef result[4])
1075 {
1076    LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1077    bld_base->load_mem(bld_base, nir_dest_num_components(instr->dest), nir_dest_bit_size(instr->dest),
1078                       NULL, offset, result);
1079 }
1080
1081 static void visit_shared_store(struct lp_build_nir_context *bld_base,
1082                                nir_intrinsic_instr *instr)
1083 {
1084    LLVMValueRef val = get_src(bld_base, instr->src[0]);
1085    LLVMValueRef offset = get_src(bld_base, instr->src[1]);
1086    int writemask = instr->const_index[1];
1087    int nc = nir_src_num_components(instr->src[0]);
1088    int bitsize = nir_src_bit_size(instr->src[0]);
1089    bld_base->store_mem(bld_base, writemask, nc, bitsize, NULL, offset, val);
1090 }
1091
1092 static void visit_shared_atomic(struct lp_build_nir_context *bld_base,
1093                                 nir_intrinsic_instr *instr,
1094                                 LLVMValueRef result[4])
1095 {
1096    LLVMValueRef offset = get_src(bld_base, instr->src[0]);
1097    LLVMValueRef val = get_src(bld_base, instr->src[1]);
1098    LLVMValueRef val2 = NULL;
1099    if (instr->intrinsic == nir_intrinsic_shared_atomic_comp_swap)
1100       val2 = get_src(bld_base, instr->src[2]);
1101
1102    bld_base->atomic_mem(bld_base, instr->intrinsic, NULL, offset, val, val2, &result[0]);
1103
1104 }
1105
1106 static void visit_barrier(struct lp_build_nir_context *bld_base)
1107 {
1108    bld_base->barrier(bld_base);
1109 }
1110
1111 static void visit_discard(struct lp_build_nir_context *bld_base,
1112                           nir_intrinsic_instr *instr)
1113 {
1114    LLVMValueRef cond = NULL;
1115    if (instr->intrinsic == nir_intrinsic_discard_if) {
1116       cond = get_src(bld_base, instr->src[0]);
1117       cond = cast_type(bld_base, cond, nir_type_int, 32);
1118    }
1119    bld_base->discard(bld_base, cond);
1120 }
1121
1122 static void visit_intrinsic(struct lp_build_nir_context *bld_base,
1123                             nir_intrinsic_instr *instr)
1124 {
1125    LLVMValueRef result[4] = {0};
1126    switch (instr->intrinsic) {
1127    case nir_intrinsic_load_deref:
1128       visit_load_var(bld_base, instr, result);
1129       break;
1130    case nir_intrinsic_store_deref:
1131       visit_store_var(bld_base, instr);
1132       break;
1133    case nir_intrinsic_load_ubo:
1134       visit_load_ubo(bld_base, instr, result);
1135       break;
1136    case nir_intrinsic_load_ssbo:
1137       visit_load_ssbo(bld_base, instr, result);
1138       break;
1139    case nir_intrinsic_store_ssbo:
1140       visit_store_ssbo(bld_base, instr);
1141       break;
1142    case nir_intrinsic_get_buffer_size:
1143       visit_get_buffer_size(bld_base, instr, result);
1144       break;
1145    case nir_intrinsic_load_vertex_id:
1146    case nir_intrinsic_load_primitive_id:
1147    case nir_intrinsic_load_instance_id:
1148    case nir_intrinsic_load_work_group_id:
1149    case nir_intrinsic_load_local_invocation_id:
1150    case nir_intrinsic_load_num_work_groups:
1151    case nir_intrinsic_load_invocation_id:
1152       bld_base->sysval_intrin(bld_base, instr, result);
1153       break;
1154    case nir_intrinsic_discard_if:
1155    case nir_intrinsic_discard:
1156       visit_discard(bld_base, instr);
1157       break;
1158    case nir_intrinsic_emit_vertex:
1159       bld_base->emit_vertex(bld_base, nir_intrinsic_stream_id(instr));
1160       break;
1161    case nir_intrinsic_end_primitive:
1162       bld_base->end_primitive(bld_base, nir_intrinsic_stream_id(instr));
1163       break;
1164    case nir_intrinsic_ssbo_atomic_add:
1165    case nir_intrinsic_ssbo_atomic_imin:
1166    case nir_intrinsic_ssbo_atomic_imax:
1167    case nir_intrinsic_ssbo_atomic_umin:
1168    case nir_intrinsic_ssbo_atomic_umax:
1169    case nir_intrinsic_ssbo_atomic_and:
1170    case nir_intrinsic_ssbo_atomic_or:
1171    case nir_intrinsic_ssbo_atomic_xor:
1172    case nir_intrinsic_ssbo_atomic_exchange:
1173    case nir_intrinsic_ssbo_atomic_comp_swap:
1174       visit_ssbo_atomic(bld_base, instr, result);
1175       break;
1176    case nir_intrinsic_image_deref_load:
1177       visit_load_image(bld_base, instr, result);
1178       break;
1179    case nir_intrinsic_image_deref_store:
1180       visit_store_image(bld_base, instr);
1181       break;
1182    case nir_intrinsic_image_deref_atomic_add:
1183    case nir_intrinsic_image_deref_atomic_imin:
1184    case nir_intrinsic_image_deref_atomic_imax:
1185    case nir_intrinsic_image_deref_atomic_umin:
1186    case nir_intrinsic_image_deref_atomic_umax:
1187    case nir_intrinsic_image_deref_atomic_and:
1188    case nir_intrinsic_image_deref_atomic_or:
1189    case nir_intrinsic_image_deref_atomic_xor:
1190    case nir_intrinsic_image_deref_atomic_exchange:
1191    case nir_intrinsic_image_deref_atomic_comp_swap:
1192       visit_atomic_image(bld_base, instr, result);
1193       break;
1194    case nir_intrinsic_image_deref_size:
1195       visit_image_size(bld_base, instr, result);
1196       break;
1197    case nir_intrinsic_load_shared:
1198       visit_shared_load(bld_base, instr, result);
1199       break;
1200    case nir_intrinsic_store_shared:
1201       visit_shared_store(bld_base, instr);
1202       break;
1203    case nir_intrinsic_shared_atomic_add:
1204    case nir_intrinsic_shared_atomic_imin:
1205    case nir_intrinsic_shared_atomic_umin:
1206    case nir_intrinsic_shared_atomic_imax:
1207    case nir_intrinsic_shared_atomic_umax:
1208    case nir_intrinsic_shared_atomic_and:
1209    case nir_intrinsic_shared_atomic_or:
1210    case nir_intrinsic_shared_atomic_xor:
1211    case nir_intrinsic_shared_atomic_exchange:
1212    case nir_intrinsic_shared_atomic_comp_swap:
1213       visit_shared_atomic(bld_base, instr, result);
1214       break;
1215    case nir_intrinsic_barrier:
1216       visit_barrier(bld_base);
1217       break;
1218    case nir_intrinsic_memory_barrier:
1219       break;
1220    default:
1221       assert(0);
1222       break;
1223    }
1224    if (result[0]) {
1225       assign_dest(bld_base, &instr->dest, result);
1226    }
1227 }
1228
1229 static void visit_txs(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1230 {
1231    struct lp_sampler_size_query_params params;
1232    LLVMValueRef sizes_out[4];
1233    LLVMValueRef explicit_lod = NULL;
1234
1235    for (unsigned i = 0; i < instr->num_srcs; i++) {
1236       switch (instr->src[i].src_type) {
1237       case nir_tex_src_lod:
1238          explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1239          break;
1240       default:
1241          break;
1242       }
1243    }
1244
1245    params.target = glsl_sampler_to_pipe(instr->sampler_dim, instr->is_array);
1246    params.texture_unit = instr->texture_index;
1247    params.explicit_lod = explicit_lod;
1248    params.is_sviewinfo = TRUE;
1249    params.sizes_out = sizes_out;
1250
1251    if (instr->op == nir_texop_query_levels)
1252       params.explicit_lod = bld_base->uint_bld.zero;
1253    bld_base->tex_size(bld_base, &params);
1254    assign_dest(bld_base, &instr->dest, &sizes_out[instr->op == nir_texop_query_levels ? 3 : 0]);
1255 }
1256
1257 static enum lp_sampler_lod_property lp_build_nir_lod_property(struct lp_build_nir_context *bld_base,
1258                                                               nir_src lod_src)
1259 {
1260    enum lp_sampler_lod_property lod_property;
1261
1262    if (nir_src_is_dynamically_uniform(lod_src))
1263       lod_property = LP_SAMPLER_LOD_SCALAR;
1264    else if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1265       if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1266          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1267       else
1268          lod_property = LP_SAMPLER_LOD_PER_QUAD;
1269    }
1270    else
1271       lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1272    return lod_property;
1273 }
1274
1275 static void visit_tex(struct lp_build_nir_context *bld_base, nir_tex_instr *instr)
1276 {
1277    struct gallivm_state *gallivm = bld_base->base.gallivm;
1278    LLVMBuilderRef builder = gallivm->builder;
1279    LLVMValueRef coords[5];
1280    LLVMValueRef offsets[3] = { NULL };
1281    LLVMValueRef explicit_lod = NULL, projector = NULL;
1282    struct lp_sampler_params params;
1283    struct lp_derivatives derivs;
1284    unsigned sample_key = 0;
1285    nir_deref_instr *texture_deref_instr = NULL;
1286    nir_deref_instr *sampler_deref_instr = NULL;
1287    LLVMValueRef texel[4];
1288    unsigned lod_src = 0;
1289    LLVMValueRef coord_undef = LLVMGetUndef(bld_base->base.int_vec_type);
1290
1291    memset(&params, 0, sizeof(params));
1292    enum lp_sampler_lod_property lod_property = LP_SAMPLER_LOD_SCALAR;
1293
1294    if (instr->op == nir_texop_txs || instr->op == nir_texop_query_levels) {
1295       visit_txs(bld_base, instr);
1296       return;
1297    }
1298    if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1299       sample_key |= LP_SAMPLER_OP_FETCH << LP_SAMPLER_OP_TYPE_SHIFT;
1300    else if (instr->op == nir_texop_tg4)
1301       sample_key |= LP_SAMPLER_OP_GATHER << LP_SAMPLER_OP_TYPE_SHIFT;
1302    else if (instr->op == nir_texop_lod)
1303       sample_key |= LP_SAMPLER_OP_LODQ << LP_SAMPLER_OP_TYPE_SHIFT;
1304    for (unsigned i = 0; i < instr->num_srcs; i++) {
1305       switch (instr->src[i].src_type) {
1306       case nir_tex_src_coord: {
1307          LLVMValueRef coord = get_src(bld_base, instr->src[i].src);
1308          if (instr->coord_components == 1)
1309             coords[0] = coord;
1310          else {
1311             for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1312                coords[chan] = LLVMBuildExtractValue(builder, coord,
1313                                                     chan, "");
1314          }
1315          for (unsigned chan = instr->coord_components; chan < 5; chan++)
1316             coords[chan] = coord_undef;
1317
1318          break;
1319       }
1320       case nir_tex_src_texture_deref:
1321          texture_deref_instr = nir_src_as_deref(instr->src[i].src);
1322          break;
1323       case nir_tex_src_sampler_deref:
1324          sampler_deref_instr = nir_src_as_deref(instr->src[i].src);
1325          break;
1326       case nir_tex_src_projector:
1327          projector = lp_build_rcp(&bld_base->base, cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32));
1328          break;
1329       case nir_tex_src_comparator:
1330          sample_key |= LP_SAMPLER_SHADOW;
1331          coords[4] = get_src(bld_base, instr->src[i].src);
1332          coords[4] = cast_type(bld_base, coords[4], nir_type_float, 32);
1333          break;
1334       case nir_tex_src_bias:
1335          sample_key |= LP_SAMPLER_LOD_BIAS << LP_SAMPLER_LOD_CONTROL_SHIFT;
1336          lod_src = i;
1337          explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1338          break;
1339       case nir_tex_src_lod:
1340          sample_key |= LP_SAMPLER_LOD_EXPLICIT << LP_SAMPLER_LOD_CONTROL_SHIFT;
1341          lod_src = i;
1342          if (instr->op == nir_texop_txf)
1343             explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_int, 32);
1344          else
1345             explicit_lod = cast_type(bld_base, get_src(bld_base, instr->src[i].src), nir_type_float, 32);
1346          break;
1347       case nir_tex_src_ddx: {
1348          int deriv_cnt = instr->coord_components;
1349          if (instr->is_array)
1350             deriv_cnt--;
1351          LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1352          if (deriv_cnt == 1)
1353             derivs.ddx[0] = deriv_val;
1354          else
1355             for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1356                derivs.ddx[chan] = LLVMBuildExtractValue(builder, deriv_val,
1357                                                         chan, "");
1358          for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1359             derivs.ddx[chan] = cast_type(bld_base, derivs.ddx[chan], nir_type_float, 32);
1360          break;
1361       }
1362       case nir_tex_src_ddy: {
1363          int deriv_cnt = instr->coord_components;
1364          if (instr->is_array)
1365             deriv_cnt--;
1366          LLVMValueRef deriv_val = get_src(bld_base, instr->src[i].src);
1367          if (deriv_cnt == 1)
1368             derivs.ddy[0] = deriv_val;
1369          else
1370             for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1371                derivs.ddy[chan] = LLVMBuildExtractValue(builder, deriv_val,
1372                                                         chan, "");
1373          for (unsigned chan = 0; chan < deriv_cnt; ++chan)
1374             derivs.ddy[chan] = cast_type(bld_base, derivs.ddy[chan], nir_type_float, 32);
1375          break;
1376       }
1377       case nir_tex_src_offset: {
1378          int offset_cnt = instr->coord_components;
1379          if (instr->is_array)
1380             offset_cnt--;
1381          LLVMValueRef offset_val = get_src(bld_base, instr->src[i].src);
1382          sample_key |= LP_SAMPLER_OFFSETS;
1383          if (offset_cnt == 1)
1384             offsets[0] = offset_val;
1385          else {
1386             for (unsigned chan = 0; chan < offset_cnt; ++chan)
1387                offsets[chan] = LLVMBuildExtractValue(builder, offset_val,
1388                                                      chan, "");
1389          }
1390          break;
1391       }
1392       case nir_tex_src_ms_index:
1393          break;
1394       default:
1395          assert(0);
1396          break;
1397       }
1398    }
1399    if (!sampler_deref_instr)
1400       sampler_deref_instr = texture_deref_instr;
1401
1402    if (explicit_lod)
1403       lod_property = lp_build_nir_lod_property(bld_base, instr->src[lod_src].src);
1404
1405    if (instr->op == nir_texop_tex || instr->op == nir_texop_tg4 || instr->op == nir_texop_txb ||
1406        instr->op == nir_texop_txl || instr->op == nir_texop_txd || instr->op == nir_texop_lod)
1407       for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1408          coords[chan] = cast_type(bld_base, coords[chan], nir_type_float, 32);
1409    else if (instr->op == nir_texop_txf || instr->op == nir_texop_txf_ms)
1410       for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1411          coords[chan] = cast_type(bld_base, coords[chan], nir_type_int, 32);
1412
1413    if (instr->is_array && instr->sampler_dim == GLSL_SAMPLER_DIM_1D) {
1414       /* move layer coord for 1d arrays. */
1415       coords[2] = coords[1];
1416       coords[1] = coord_undef;
1417    }
1418
1419    if (projector) {
1420       for (unsigned chan = 0; chan < instr->coord_components; ++chan)
1421          coords[chan] = lp_build_mul(&bld_base->base, coords[chan], projector);
1422       if (sample_key & LP_SAMPLER_SHADOW)
1423          coords[4] = lp_build_mul(&bld_base->base, coords[4], projector);
1424    }
1425
1426    uint32_t base_index = 0;
1427    if (!texture_deref_instr) {
1428       int samp_src_index = nir_tex_instr_src_index(instr, nir_tex_src_sampler_handle);
1429       if (samp_src_index == -1) {
1430          base_index = instr->sampler_index;
1431       }
1432    }
1433
1434    if (instr->op == nir_texop_txd) {
1435       sample_key |= LP_SAMPLER_LOD_DERIVATIVES << LP_SAMPLER_LOD_CONTROL_SHIFT;
1436       params.derivs = &derivs;
1437       if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1438          if (gallivm_perf & GALLIVM_PERF_NO_QUAD_LOD)
1439             lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1440          else
1441             lod_property = LP_SAMPLER_LOD_PER_QUAD;
1442       } else
1443          lod_property = LP_SAMPLER_LOD_PER_ELEMENT;
1444    }
1445
1446    sample_key |= lod_property << LP_SAMPLER_LOD_PROPERTY_SHIFT;
1447    params.sample_key = sample_key;
1448    params.offsets = offsets;
1449    params.texture_index = base_index;
1450    params.sampler_index = base_index;
1451    params.coords = coords;
1452    params.texel = texel;
1453    params.lod = explicit_lod;
1454    bld_base->tex(bld_base, &params);
1455    assign_dest(bld_base, &instr->dest, texel);
1456 }
1457
1458 static void visit_ssa_undef(struct lp_build_nir_context *bld_base,
1459                             const nir_ssa_undef_instr *instr)
1460 {
1461    unsigned num_components = instr->def.num_components;
1462    LLVMValueRef undef[4];
1463    for (unsigned i = 0; i < num_components; i++)
1464       undef[i] = LLVMGetUndef(bld_base->base.vec_type);
1465    assign_ssa_dest(bld_base, &instr->def, undef);
1466 }
1467
1468 static void visit_jump(struct lp_build_nir_context *bld_base,
1469                        const nir_jump_instr *instr)
1470 {
1471    switch (instr->type) {
1472    case nir_jump_break:
1473       bld_base->break_stmt(bld_base);
1474       break;
1475    case nir_jump_continue:
1476       bld_base->continue_stmt(bld_base);
1477       break;
1478    default:
1479       unreachable("Unknown jump instr\n");
1480    }
1481 }
1482
1483 static void visit_deref(struct lp_build_nir_context *bld_base,
1484                         nir_deref_instr *instr)
1485 {
1486    if (instr->mode != nir_var_mem_shared &&
1487        instr->mode != nir_var_mem_global)
1488       return;
1489    LLVMValueRef result = NULL;
1490    switch(instr->deref_type) {
1491    case nir_deref_type_var: {
1492       struct hash_entry *entry = _mesa_hash_table_search(bld_base->vars, instr->var);
1493       result = entry->data;
1494       break;
1495    }
1496    default:
1497       unreachable("Unhandled deref_instr deref type");
1498    }
1499
1500    assign_ssa(bld_base, instr->dest.ssa.index, result);
1501 }
1502
1503 static void visit_block(struct lp_build_nir_context *bld_base, nir_block *block)
1504 {
1505    nir_foreach_instr(instr, block)
1506    {
1507       switch (instr->type) {
1508       case nir_instr_type_alu:
1509          visit_alu(bld_base, nir_instr_as_alu(instr));
1510          break;
1511       case nir_instr_type_load_const:
1512          visit_load_const(bld_base, nir_instr_as_load_const(instr));
1513          break;
1514       case nir_instr_type_intrinsic:
1515          visit_intrinsic(bld_base, nir_instr_as_intrinsic(instr));
1516          break;
1517       case nir_instr_type_tex:
1518          visit_tex(bld_base, nir_instr_as_tex(instr));
1519          break;
1520       case nir_instr_type_phi:
1521          assert(0);
1522          break;
1523       case nir_instr_type_ssa_undef:
1524          visit_ssa_undef(bld_base, nir_instr_as_ssa_undef(instr));
1525          break;
1526       case nir_instr_type_jump:
1527          visit_jump(bld_base, nir_instr_as_jump(instr));
1528          break;
1529       case nir_instr_type_deref:
1530          visit_deref(bld_base, nir_instr_as_deref(instr));
1531          break;
1532       default:
1533          fprintf(stderr, "Unknown NIR instr type: ");
1534          nir_print_instr(instr, stderr);
1535          fprintf(stderr, "\n");
1536          abort();
1537       }
1538    }
1539 }
1540
1541 static void visit_if(struct lp_build_nir_context *bld_base, nir_if *if_stmt)
1542 {
1543    LLVMValueRef cond = get_src(bld_base, if_stmt->condition);
1544
1545    bld_base->if_cond(bld_base, cond);
1546    visit_cf_list(bld_base, &if_stmt->then_list);
1547
1548    if (!exec_list_is_empty(&if_stmt->else_list)) {
1549       bld_base->else_stmt(bld_base);
1550       visit_cf_list(bld_base, &if_stmt->else_list);
1551    }
1552    bld_base->endif_stmt(bld_base);
1553 }
1554
1555 static void visit_loop(struct lp_build_nir_context *bld_base, nir_loop *loop)
1556 {
1557    bld_base->bgnloop(bld_base);
1558    visit_cf_list(bld_base, &loop->body);
1559    bld_base->endloop(bld_base);
1560 }
1561
1562 static void visit_cf_list(struct lp_build_nir_context *bld_base,
1563                           struct exec_list *list)
1564 {
1565    foreach_list_typed(nir_cf_node, node, node, list)
1566    {
1567       switch (node->type) {
1568       case nir_cf_node_block:
1569          visit_block(bld_base, nir_cf_node_as_block(node));
1570          break;
1571
1572       case nir_cf_node_if:
1573          visit_if(bld_base, nir_cf_node_as_if(node));
1574          break;
1575
1576       case nir_cf_node_loop:
1577          visit_loop(bld_base, nir_cf_node_as_loop(node));
1578          break;
1579
1580       default:
1581          assert(0);
1582       }
1583    }
1584 }
1585
1586 static void
1587 handle_shader_output_decl(struct lp_build_nir_context *bld_base,
1588                           struct nir_shader *nir,
1589                           struct nir_variable *variable)
1590 {
1591    bld_base->emit_var_decl(bld_base, variable);
1592 }
1593
1594 /* vector registers are stored as arrays in LLVM side,
1595    so we can use GEP on them, as to do exec mask stores
1596    we need to operate on a single components.
1597    arrays are:
1598    0.x, 1.x, 2.x, 3.x
1599    0.y, 1.y, 2.y, 3.y
1600    ....
1601 */
1602 static LLVMTypeRef get_register_type(struct lp_build_nir_context *bld_base,
1603                                      nir_register *reg)
1604 {
1605    struct lp_build_context *int_bld = get_int_bld(bld_base, true, reg->bit_size);
1606
1607    LLVMTypeRef type = int_bld->vec_type;
1608    if (reg->num_array_elems)
1609       type = LLVMArrayType(type, reg->num_array_elems);
1610    if (reg->num_components > 1)
1611       type = LLVMArrayType(type, reg->num_components);
1612
1613    return type;
1614 }
1615
1616
1617 bool lp_build_nir_llvm(
1618    struct lp_build_nir_context *bld_base,
1619    struct nir_shader *nir)
1620 {
1621    struct nir_function *func;
1622
1623    nir_convert_from_ssa(nir, true);
1624    nir_lower_locals_to_regs(nir);
1625    nir_remove_dead_derefs(nir);
1626    nir_remove_dead_variables(nir, nir_var_function_temp);
1627
1628    nir_foreach_variable(variable, &nir->outputs)
1629       handle_shader_output_decl(bld_base, nir, variable);
1630
1631    bld_base->regs = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1632                                             _mesa_key_pointer_equal);
1633    bld_base->vars = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
1634                                             _mesa_key_pointer_equal);
1635
1636    func = (struct nir_function *)exec_list_get_head(&nir->functions);
1637
1638    nir_foreach_register(reg, &func->impl->registers) {
1639       LLVMTypeRef type = get_register_type(bld_base, reg);
1640       LLVMValueRef reg_alloc = lp_build_alloca_undef(bld_base->base.gallivm,
1641                                                      type, "reg");
1642       _mesa_hash_table_insert(bld_base->regs, reg, reg_alloc);
1643    }
1644    nir_index_ssa_defs(func->impl);
1645    bld_base->ssa_defs = calloc(func->impl->ssa_alloc, sizeof(LLVMValueRef));
1646    visit_cf_list(bld_base, &func->impl->body);
1647
1648    free(bld_base->ssa_defs);
1649    ralloc_free(bld_base->vars);
1650    ralloc_free(bld_base->regs);
1651    return true;
1652 }
1653
1654 /* do some basic opts to remove some things we don't want to see. */
1655 void lp_build_opt_nir(struct nir_shader *nir)
1656 {
1657    bool progress;
1658    do {
1659       progress = false;
1660       NIR_PASS_V(nir, nir_opt_constant_folding);
1661       NIR_PASS_V(nir, nir_opt_algebraic);
1662    } while (progress);
1663    nir_lower_bool_to_int32(nir);
1664 }