src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2019 Red Hat.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  **************************************************************************/
  25
  26 #include "lp_bld_nir.h"
  27 #include "lp_bld_init.h"
  28 #include "lp_bld_flow.h"
  29 #include "lp_bld_logic.h"
  30 #include "lp_bld_gather.h"
  31 #include "lp_bld_const.h"
  32 #include "lp_bld_struct.h"
  33 #include "lp_bld_arit.h"
  34 #include "lp_bld_bitarit.h"
  35 #include "lp_bld_coro.h"
  36 #include "lp_bld_printf.h"
  37 #include "util/u_math.h"
  38 /*
  39  * combine the execution mask if there is one with the current mask.
  40  */
  41 static LLVMValueRef
  42 mask_vec(struct lp_build_nir_context *bld_base)
  43 {
  44    struct lp_build_nir_soa_context * bld = (struct lp_build_nir_soa_context *)bld_base;
  45    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
  46    struct lp_exec_mask *exec_mask = &bld->exec_mask;
  47    LLVMValueRef bld_mask = bld->mask ? lp_build_mask_value(bld->mask) : NULL;
  48    if (!exec_mask->has_mask) {
  49       return bld_mask;
  50    }
  51    if (!bld_mask)
  52       return exec_mask->exec_mask;
  53    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
  54                        exec_mask->exec_mask, "");
  55 }
  56
  57 static LLVMValueRef
  58 emit_fetch_64bit(
  59    struct lp_build_nir_context * bld_base,
  60    LLVMValueRef input,
  61    LLVMValueRef input2)
  62 {
  63    struct gallivm_state *gallivm = bld_base->base.gallivm;
  64    LLVMBuilderRef builder = gallivm->builder;
  65    LLVMValueRef res;
  66    int i;
  67    LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
  68    int len = bld_base->base.type.length * 2;
  69    assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
  70
  71    for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
  72 #if UTIL_ARCH_LITTLE_ENDIAN
  73       shuffles[i] = lp_build_const_int32(gallivm, i / 2);
  74       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
  75 #else
  76       shuffles[i] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
  77       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2);
  78 #endif
  79    }
  80    res = LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
  81
  82    return LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
  83 }
  84
  85 static void
  86 emit_store_64bit_split(struct lp_build_nir_context *bld_base,
  87                        LLVMValueRef value,
  88                        LLVMValueRef split_values[2])
  89 {
  90    struct gallivm_state *gallivm = bld_base->base.gallivm;
  91    LLVMBuilderRef builder = gallivm->builder;
  92    unsigned i;
  93    LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
  94    LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
  95    int len = bld_base->base.type.length * 2;
  96
  97    value = LLVMBuildBitCast(gallivm->builder, value, LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), len), "");
  98    for (i = 0; i < bld_base->base.type.length; i++) {
  99 #if UTIL_ARCH_LITTLE_ENDIAN
 100       shuffles[i] = lp_build_const_int32(gallivm, i * 2);
 101       shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
 102 #else
 103       shuffles[i] = lp_build_const_int32(gallivm, i * 2 + 1);
 104       shuffles2[i] = lp_build_const_int32(gallivm, i * 2);
 105 #endif
 106    }
 107
 108    split_values[0] = LLVMBuildShuffleVector(builder, value,
 109                                  LLVMGetUndef(LLVMTypeOf(value)),
 110                                  LLVMConstVector(shuffles,
 111                                                  bld_base->base.type.length),
 112                                  "");
 113    split_values[1] = LLVMBuildShuffleVector(builder, value,
 114                                   LLVMGetUndef(LLVMTypeOf(value)),
 115                                   LLVMConstVector(shuffles2,
 116                                                   bld_base->base.type.length),
 117                                   "");
 118 }
 119
 120 static void
 121 emit_store_64bit_chan(struct lp_build_nir_context *bld_base,
 122                       LLVMValueRef chan_ptr,
 123                       LLVMValueRef chan_ptr2,
 124                       LLVMValueRef value)
 125 {
 126    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 127    struct lp_build_context *float_bld = &bld_base->base;
 128    LLVMValueRef split_vals[2];
 129
 130    emit_store_64bit_split(bld_base, value, split_vals);
 131
 132    lp_exec_mask_store(&bld->exec_mask, float_bld, split_vals[0], chan_ptr);
 133    lp_exec_mask_store(&bld->exec_mask, float_bld, split_vals[1], chan_ptr2);
 134 }
 135
 136 static LLVMValueRef
 137 get_soa_array_offsets(struct lp_build_context *uint_bld,
 138                       LLVMValueRef indirect_index,
 139                       int num_components,
 140                       unsigned chan_index,
 141                       bool need_perelement_offset)
 142 {
 143    struct gallivm_state *gallivm = uint_bld->gallivm;
 144    LLVMValueRef chan_vec =
 145       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
 146    LLVMValueRef length_vec =
 147       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
 148    LLVMValueRef index_vec;
 149
 150    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
 151    index_vec = lp_build_mul(uint_bld, indirect_index, lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, num_components));
 152    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
 153    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
 154
 155    if (need_perelement_offset) {
 156       LLVMValueRef pixel_offsets;
 157       unsigned i;
 158      /* build pixel offset vector: {0, 1, 2, 3, ...} */
 159       pixel_offsets = uint_bld->undef;
 160       for (i = 0; i < uint_bld->type.length; i++) {
 161          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
 162          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
 163                                                 ii, ii, "");
 164       }
 165       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
 166    }
 167    return index_vec;
 168 }
 169
 170 static LLVMValueRef
 171 build_gather(struct lp_build_nir_context *bld_base,
 172              struct lp_build_context *bld,
 173              LLVMValueRef base_ptr,
 174              LLVMValueRef indexes,
 175              LLVMValueRef overflow_mask,
 176              LLVMValueRef indexes2)
 177 {
 178    struct gallivm_state *gallivm = bld_base->base.gallivm;
 179    LLVMBuilderRef builder = gallivm->builder;
 180    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 181    LLVMValueRef res;
 182    unsigned i;
 183
 184    if (indexes2)
 185       res = LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2));
 186    else
 187       res = bld->undef;
 188    /*
 189     * overflow_mask is a vector telling us which channels
 190     * in the vector overflowed. We use the overflow behavior for
 191     * constant buffers which is defined as:
 192     * Out of bounds access to constant buffer returns 0 in all
 193     * components. Out of bounds behavior is always with respect
 194     * to the size of the buffer bound at that slot.
 195     */
 196
 197    if (overflow_mask) {
 198       /*
 199        * We avoid per-element control flow here (also due to llvm going crazy,
 200        * though I suspect it's better anyway since overflow is likely rare).
 201        * Note that since we still fetch from buffers even if num_elements was
 202        * zero (in this case we'll fetch from index zero) the jit func callers
 203        * MUST provide valid fake constant buffers of size 4x32 (the values do
 204        * not matter), otherwise we'd still need (not per element though)
 205        * control flow.
 206        */
 207       indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);
 208       if (indexes2)
 209          indexes2 = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes2);
 210    }
 211
 212    /*
 213     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 214     */
 215    for (i = 0; i < bld->type.length * (indexes2 ? 2 : 1); i++) {
 216       LLVMValueRef si, di;
 217       LLVMValueRef index;
 218       LLVMValueRef scalar_ptr, scalar;
 219
 220       di = lp_build_const_int32(gallivm, i);
 221       if (indexes2)
 222          si = lp_build_const_int32(gallivm, i >> 1);
 223       else
 224          si = di;
 225
 226       if (indexes2 && (i & 1)) {
 227          index = LLVMBuildExtractElement(builder,
 228                                          indexes2, si, "");
 229       } else {
 230          index = LLVMBuildExtractElement(builder,
 231                                          indexes, si, "");
 232       }
 233       scalar_ptr = LLVMBuildGEP(builder, base_ptr,
 234                                 &index, 1, "gather_ptr");
 235       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
 236
 237       res = LLVMBuildInsertElement(builder, res, scalar, di, "");
 238    }
 239
 240    if (overflow_mask) {
 241       if (indexes2) {
 242          res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
 243          overflow_mask = LLVMBuildSExt(builder, overflow_mask,
 244                                        bld_base->dbl_bld.int_vec_type, "");
 245          res = lp_build_select(&bld_base->dbl_bld, overflow_mask,
 246                                bld_base->dbl_bld.zero, res);
 247       } else
 248          res = lp_build_select(bld, overflow_mask, bld->zero, res);
 249    }
 250
 251    return res;
 252 }
 253
 254 /**
 255  * Scatter/store vector.
 256  */
 257 static void
 258 emit_mask_scatter(struct lp_build_nir_soa_context *bld,
 259                   LLVMValueRef base_ptr,
 260                   LLVMValueRef indexes,
 261                   LLVMValueRef values,
 262                   struct lp_exec_mask *mask)
 263 {
 264    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
 265    LLVMBuilderRef builder = gallivm->builder;
 266    unsigned i;
 267    LLVMValueRef pred = mask->has_mask ? mask->exec_mask : NULL;
 268
 269    /*
 270     * Loop over elements of index_vec, store scalar value.
 271     */
 272    for (i = 0; i < bld->bld_base.base.type.length; i++) {
 273       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
 274       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
 275       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
 276       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
 277       LLVMValueRef scalar_pred = pred ?
 278          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
 279
 280       if (0)
 281          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
 282                          ii, val, index, scalar_ptr);
 283
 284       if (scalar_pred) {
 285          LLVMValueRef real_val, dst_val;
 286          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
 287          real_val = lp_build_select(&bld->uint_elem_bld, scalar_pred, val, dst_val);
 288          LLVMBuildStore(builder, real_val, scalar_ptr);
 289       }
 290       else {
 291          LLVMBuildStore(builder, val, scalar_ptr);
 292       }
 293    }
 294 }
 295
 296 static void emit_load_var(struct lp_build_nir_context *bld_base,
 297                            nir_variable_mode deref_mode,
 298                            unsigned num_components,
 299                            unsigned bit_size,
 300                            nir_variable *var,
 301                            unsigned vertex_index,
 302                            LLVMValueRef indir_vertex_index,
 303                            unsigned const_index,
 304                            LLVMValueRef indir_index,
 305                            LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
 306 {
 307    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 308    struct gallivm_state *gallivm = bld_base->base.gallivm;
 309    int dmul = bit_size == 64 ? 2 : 1;
 310    unsigned location = var->data.driver_location;
 311    unsigned location_frac = var->data.location_frac;
 312
 313    if (!var->data.compact && !indir_index)
 314       location += const_index;
 315    else if (var->data.compact) {
 316       location += const_index / 4;
 317       location_frac += const_index % 4;
 318       const_index = 0;
 319    }
 320    switch (deref_mode) {
 321    case nir_var_shader_in:
 322       for (unsigned i = 0; i < num_components; i++) {
 323          int idx = (i * dmul) + location_frac;
 324
 325          if (bld->gs_iface) {
 326             LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
 327             LLVMValueRef attrib_index_val = lp_build_const_int32(gallivm, location);
 328             LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
 329             LLVMValueRef result2;
 330
 331             result[i] = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
 332                                                    false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
 333             if (bit_size == 64) {
 334                LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 335                result2 = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
 336                                                     false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
 337                result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 338             }
 339          } else if (bld->tes_iface) {
 340             LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
 341             LLVMValueRef attrib_index_val;
 342             LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
 343             LLVMValueRef result2;
 344
 345             if (indir_index) {
 346                if (var->data.compact) {
 347                   swizzle_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, idx));
 348                   attrib_index_val = lp_build_const_int32(gallivm, location);
 349                } else
 350                   attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
 351             } else
 352                attrib_index_val = lp_build_const_int32(gallivm, location);
 353
 354             if (var->data.patch) {
 355                result[i] = bld->tes_iface->fetch_patch_input(bld->tes_iface, &bld_base->base,
 356                                                              indir_index ? true : false, attrib_index_val, swizzle_index_val);
 357                if (bit_size == 64) {
 358                   LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 359                   result2 = bld->tes_iface->fetch_patch_input(bld->tes_iface, &bld_base->base,
 360                                                               indir_index ? true : false, attrib_index_val, swizzle_index_val);
 361                   result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 362                }
 363             }
 364             else {
 365                result[i] = bld->tes_iface->fetch_vertex_input(bld->tes_iface, &bld_base->base,
 366                                                               indir_vertex_index ? true : false,
 367                                                               indir_vertex_index ? indir_vertex_index : vertex_index_val,
 368                                                               (indir_index && !var->data.compact) ? true : false, attrib_index_val,
 369                                                               (indir_index && var->data.compact) ? true : false, swizzle_index_val);
 370                if (bit_size == 64) {
 371                   LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 372                   result2 = bld->tes_iface->fetch_vertex_input(bld->tes_iface, &bld_base->base,
 373                                                                indir_vertex_index ? true : false,
 374                                                                indir_vertex_index ? indir_vertex_index : vertex_index_val,
 375                                                                indir_index ? true : false, attrib_index_val, false, swizzle_index_val);
 376                   result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 377                }
 378             }
 379          } else if (bld->tcs_iface) {
 380             LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
 381             LLVMValueRef attrib_index_val;
 382             LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
 383
 384             if (indir_index) {
 385                if (var->data.compact) {
 386                   swizzle_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, idx));
 387                   attrib_index_val = lp_build_const_int32(gallivm, location);
 388                } else
 389                   attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
 390             } else
 391                attrib_index_val = lp_build_const_int32(gallivm, location);
 392             result[i] = bld->tcs_iface->emit_fetch_input(bld->tcs_iface, &bld_base->base,
 393                                                          indir_vertex_index ? true : false, indir_vertex_index ? indir_vertex_index : vertex_index_val,
 394                                                          (indir_index && !var->data.compact) ? true : false, attrib_index_val,
 395                                                          (indir_index && var->data.compact) ? true : false, swizzle_index_val);
 396             if (bit_size == 64) {
 397                LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 398                LLVMValueRef result2 = bld->tcs_iface->emit_fetch_input(bld->tcs_iface, &bld_base->base,
 399                                                                        indir_vertex_index ? true : false, indir_vertex_index ? indir_vertex_index : vertex_index_val,
 400                                                                        indir_index ? true : false, attrib_index_val,
 401                                                                        false, swizzle_index_val);
 402                result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 403             }
 404          } else {
 405             if (indir_index) {
 406                LLVMValueRef attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
 407                LLVMValueRef index_vec = get_soa_array_offsets(&bld_base->uint_bld,
 408                                                               attrib_index_val, 4, idx,
 409                                                               TRUE);
 410                LLVMValueRef index_vec2 = NULL;
 411                LLVMTypeRef fptr_type;
 412                LLVMValueRef inputs_array;
 413                fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
 414                inputs_array = LLVMBuildBitCast(gallivm->builder, bld->inputs_array, fptr_type, "");
 415
 416                if (bit_size == 64)
 417                   index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
 418                                                      indir_index, 4, idx + 1, TRUE);
 419
 420                /* Gather values from the input register array */
 421                result[i] = build_gather(bld_base, &bld_base->base, inputs_array, index_vec, NULL, index_vec2);
 422             } else {
 423                if (bld->indirects & nir_var_shader_in) {
 424                   LLVMValueRef lindex = lp_build_const_int32(gallivm,
 425                                                              location * 4 + idx);
 426                   LLVMValueRef input_ptr = lp_build_pointer_get(gallivm->builder,
 427                                                              bld->inputs_array, lindex);
 428                   if (bit_size == 64) {
 429                      LLVMValueRef lindex2 = lp_build_const_int32(gallivm,
 430                                                                  location * 4 + (idx + 1));
 431                      LLVMValueRef input_ptr2 = lp_build_pointer_get(gallivm->builder,
 432                                                                     bld->inputs_array, lindex2);
 433                      result[i] = emit_fetch_64bit(bld_base, input_ptr, input_ptr2);
 434                   } else {
 435                      result[i] = input_ptr;
 436                   }
 437                } else {
 438                   if (bit_size == 64) {
 439                      LLVMValueRef tmp[2];
 440                      tmp[0] = bld->inputs[location][idx];
 441                      tmp[1] = bld->inputs[location][idx + 1];
 442                      result[i] = emit_fetch_64bit(bld_base, tmp[0], tmp[1]);
 443                   } else {
 444                      result[i] = bld->inputs[location][idx];
 445                   }
 446                }
 447             }
 448          }
 449       }
 450       break;
 451    case nir_var_shader_out:
 452       if (bld->fs_iface && bld->fs_iface->fb_fetch) {
 453          bld->fs_iface->fb_fetch(bld->fs_iface, &bld_base->base, var->data.driver_location, result);
 454          return;
 455       }
 456       for (unsigned i = 0; i < num_components; i++) {
 457          int idx = (i * dmul) + location_frac;
 458          if (bld->tcs_iface) {
 459             LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
 460             LLVMValueRef attrib_index_val;
 461             LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
 462
 463             if (indir_index)
 464                attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
 465             else
 466                attrib_index_val = lp_build_const_int32(gallivm, location);
 467
 468             result[i] = bld->tcs_iface->emit_fetch_output(bld->tcs_iface, &bld_base->base,
 469                                                           indir_vertex_index ? true : false, indir_vertex_index ? indir_vertex_index : vertex_index_val,
 470                                                           (indir_index && !var->data.compact) ? true : false, attrib_index_val,
 471                                                           (indir_index && var->data.compact) ? true : false, swizzle_index_val, 0);
 472             if (bit_size == 64) {
 473                LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 474                LLVMValueRef result2 = bld->tcs_iface->emit_fetch_output(bld->tcs_iface, &bld_base->base,
 475                                                                         indir_vertex_index ? true : false, indir_vertex_index ? indir_vertex_index : vertex_index_val,
 476                                                                         indir_index ? true : false, attrib_index_val,
 477                                                                         false, swizzle_index_val, 0);
 478                result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 479             }
 480          }
 481       }
 482       break;
 483    default:
 484       break;
 485    }
 486 }
 487
 488 static void emit_store_chan(struct lp_build_nir_context *bld_base,
 489                             nir_variable_mode deref_mode,
 490                             unsigned bit_size,
 491                             unsigned location, unsigned comp,
 492                             unsigned chan,
 493                             LLVMValueRef dst)
 494 {
 495    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 496    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 497    struct lp_build_context *float_bld = &bld_base->base;
 498
 499    if (bit_size == 64) {
 500       chan *= 2;
 501       chan += comp;
 502       if (chan >= 4) {
 503          chan -= 4;
 504          location++;
 505       }
 506       emit_store_64bit_chan(bld_base, bld->outputs[location][chan],
 507                             bld->outputs[location][chan + 1], dst);
 508    } else {
 509       dst = LLVMBuildBitCast(builder, dst, float_bld->vec_type, "");
 510       lp_exec_mask_store(&bld->exec_mask, float_bld, dst,
 511                          bld->outputs[location][chan + comp]);
 512    }
 513 }
 514
 515 static void emit_store_tcs_chan(struct lp_build_nir_context *bld_base,
 516                                 bool is_compact,
 517                                 unsigned bit_size,
 518                                 unsigned location,
 519                                 unsigned const_index,
 520                                 LLVMValueRef indir_vertex_index,
 521                                 LLVMValueRef indir_index,
 522                                 unsigned comp,
 523                                 unsigned chan,
 524                                 LLVMValueRef chan_val)
 525 {
 526    struct gallivm_state *gallivm = bld_base->base.gallivm;
 527    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 528    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 529    unsigned swizzle = chan;
 530    if (bit_size == 64) {
 531       swizzle *= 2;
 532       swizzle += comp;
 533       if (swizzle >= 4) {
 534          swizzle -= 4;
 535          location++;
 536       }
 537    } else
 538       swizzle += comp;
 539    LLVMValueRef attrib_index_val;
 540    LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, swizzle);
 541
 542    if (indir_index) {
 543       if (is_compact) {
 544          swizzle_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, swizzle));
 545          attrib_index_val = lp_build_const_int32(gallivm, const_index + location);
 546       } else
 547          attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, location));
 548    } else
 549       attrib_index_val = lp_build_const_int32(gallivm, const_index + location);
 550    if (bit_size == 64) {
 551       LLVMValueRef split_vals[2];
 552       LLVMValueRef swizzle_index_val2 = lp_build_const_int32(gallivm, swizzle + 1);
 553       emit_store_64bit_split(bld_base, chan_val, split_vals);
 554       bld->tcs_iface->emit_store_output(bld->tcs_iface, &bld_base->base, 0,
 555                                         indir_vertex_index ? true : false,
 556                                         indir_vertex_index,
 557                                         indir_index ? true : false,
 558                                         attrib_index_val,
 559                                         false, swizzle_index_val,
 560                                         split_vals[0], mask_vec(bld_base));
 561       bld->tcs_iface->emit_store_output(bld->tcs_iface, &bld_base->base, 0,
 562                                         indir_vertex_index ? true : false,
 563                                         indir_vertex_index,
 564                                         indir_index ? true : false,
 565                                         attrib_index_val,
 566                                         false, swizzle_index_val2,
 567                                         split_vals[1], mask_vec(bld_base));
 568    } else {
 569       chan_val = LLVMBuildBitCast(builder, chan_val, bld_base->base.vec_type, "");
 570       bld->tcs_iface->emit_store_output(bld->tcs_iface, &bld_base->base, 0,
 571                                         indir_vertex_index ? true : false,
 572                                         indir_vertex_index,
 573                                         indir_index && !is_compact ? true : false,
 574                                         attrib_index_val,
 575                                         indir_index && is_compact ? true : false,
 576                                         swizzle_index_val,
 577                                         chan_val, mask_vec(bld_base));
 578    }
 579 }
 580
 581 static void emit_store_var(struct lp_build_nir_context *bld_base,
 582                            nir_variable_mode deref_mode,
 583                            unsigned num_components,
 584                            unsigned bit_size,
 585                            nir_variable *var,
 586                            unsigned writemask,
 587                            LLVMValueRef indir_vertex_index,
 588                            unsigned const_index,
 589                            LLVMValueRef indir_index,
 590                            LLVMValueRef dst)
 591 {
 592    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 593    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 594    switch (deref_mode) {
 595    case nir_var_shader_out: {
 596       unsigned location = var->data.driver_location;
 597       unsigned comp = var->data.location_frac;
 598       if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
 599          if (var->data.location == FRAG_RESULT_STENCIL)
 600             comp = 1;
 601          else if (var->data.location == FRAG_RESULT_DEPTH)
 602             comp = 2;
 603       }
 604
 605       if (var->data.compact) {
 606          location += const_index / 4;
 607          comp += const_index % 4;
 608          const_index = 0;
 609       }
 610
 611       for (unsigned chan = 0; chan < num_components; chan++) {
 612          if (writemask & (1u << chan)) {
 613             LLVMValueRef chan_val = (num_components == 1) ? dst : LLVMBuildExtractValue(builder, dst, chan, "");
 614             if (bld->tcs_iface) {
 615                emit_store_tcs_chan(bld_base, var->data.compact, bit_size, location, const_index, indir_vertex_index, indir_index, comp, chan, chan_val);
 616             } else
 617                emit_store_chan(bld_base, deref_mode, bit_size, location + const_index, comp, chan, chan_val);
 618          }
 619       }
 620       break;
 621    }
 622    default:
 623       break;
 624    }
 625 }
 626
 627 static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
 628                                   struct lp_build_context *reg_bld,
 629                                   const nir_reg_src *reg,
 630                                   LLVMValueRef indir_src,
 631                                   LLVMValueRef reg_storage)
 632 {
 633    struct gallivm_state *gallivm = bld_base->base.gallivm;
 634    LLVMBuilderRef builder = gallivm->builder;
 635    int nc = reg->reg->num_components;
 636    LLVMValueRef vals[NIR_MAX_VEC_COMPONENTS] = { NULL };
 637    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 638    if (reg->reg->num_array_elems) {
 639       LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
 640       if (reg->indirect) {
 641          LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
 642          indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
 643          indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
 644       }
 645       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
 646       for (unsigned i = 0; i < nc; i++) {
 647          LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
 648          vals[i] = build_gather(bld_base, reg_bld, reg_storage, indirect_offset, NULL, NULL);
 649       }
 650    } else {
 651       for (unsigned i = 0; i < nc; i++) {
 652          LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
 653                                                                                     lp_build_const_int32(gallivm, i));
 654          vals[i] = LLVMBuildLoad(builder, this_storage, "");
 655       }
 656    }
 657    return nc == 1 ? vals[0] : lp_nir_array_build_gather_values(builder, vals, nc);
 658 }
 659
 660 static void emit_store_reg(struct lp_build_nir_context *bld_base,
 661                            struct lp_build_context *reg_bld,
 662                            const nir_reg_dest *reg,
 663                            unsigned writemask,
 664                            LLVMValueRef indir_src,
 665                            LLVMValueRef reg_storage,
 666                            LLVMValueRef dst[NIR_MAX_VEC_COMPONENTS])
 667 {
 668    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 669    struct gallivm_state *gallivm = bld_base->base.gallivm;
 670    LLVMBuilderRef builder = gallivm->builder;
 671    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 672    int nc = reg->reg->num_components;
 673    if (reg->reg->num_array_elems > 0) {
 674       LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
 675       if (reg->indirect) {
 676          LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
 677          indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
 678          indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
 679       }
 680       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
 681       for (unsigned i = 0; i < nc; i++) {
 682          if (!(writemask & (1 << i)))
 683             continue;
 684          LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
 685          dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
 686          emit_mask_scatter(bld, reg_storage, indirect_offset, dst[i], &bld->exec_mask);
 687       }
 688       return;
 689    }
 690
 691    for (unsigned i = 0; i < nc; i++) {
 692       LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
 693                                                          lp_build_const_int32(gallivm, i));
 694       dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
 695       lp_exec_mask_store(&bld->exec_mask, reg_bld, dst[i], this_storage);
 696    }
 697 }
 698
 699 static void emit_load_kernel_arg(struct lp_build_nir_context *bld_base,
 700                                  unsigned nc,
 701                                  unsigned bit_size,
 702                                  unsigned offset_bit_size,
 703                                  bool offset_is_uniform,
 704                                  LLVMValueRef offset,
 705                                  LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
 706 {
 707    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 708    struct gallivm_state *gallivm = bld_base->base.gallivm;
 709    LLVMBuilderRef builder = gallivm->builder;
 710    struct lp_build_context *bld_broad = get_int_bld(bld_base, true, bit_size);
 711    LLVMValueRef kernel_args_ptr = bld->kernel_args_ptr;
 712    unsigned size_shift = 0;
 713    struct lp_build_context *bld_offset = get_int_bld(bld_base, true, offset_bit_size);
 714    if (bit_size == 16)
 715       size_shift = 1;
 716    else if (bit_size == 32)
 717       size_shift = 2;
 718    else if (bit_size == 64)
 719       size_shift = 3;
 720    if (size_shift)
 721       offset = lp_build_shr(bld_offset, offset, lp_build_const_int_vec(gallivm, bld_offset->type, size_shift));
 722
 723    LLVMTypeRef ptr_type = LLVMPointerType(bld_broad->elem_type, 0);
 724    kernel_args_ptr = LLVMBuildBitCast(builder, kernel_args_ptr, ptr_type, "");
 725
 726    if (offset_is_uniform) {
 727       offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
 728
 729       for (unsigned c = 0; c < nc; c++) {
 730          LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, offset_bit_size == 64 ? lp_build_const_int64(gallivm, c) : lp_build_const_int32(gallivm, c), "");
 731
 732          LLVMValueRef scalar = lp_build_pointer_get(builder, kernel_args_ptr, this_offset);
 733          result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
 734       }
 735    }
 736 }
 737
 738 static LLVMValueRef global_addr_to_ptr(struct gallivm_state *gallivm, LLVMValueRef addr_ptr, unsigned bit_size)
 739 {
 740    LLVMBuilderRef builder = gallivm->builder;
 741    switch (bit_size) {
 742    case 8:
 743       addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0), "");
 744       break;
 745    case 16:
 746       addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt16TypeInContext(gallivm->context), 0), "");
 747       break;
 748    case 32:
 749    default:
 750       addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt32TypeInContext(gallivm->context), 0), "");
 751       break;
 752    case 64:
 753       addr_ptr = LLVMBuildIntToPtr(builder, addr_ptr, LLVMPointerType(LLVMInt64TypeInContext(gallivm->context), 0), "");
 754       break;
 755    }
 756    return addr_ptr;
 757 }
 758
 759 static void emit_load_global(struct lp_build_nir_context *bld_base,
 760                              unsigned nc,
 761                              unsigned bit_size,
 762                              unsigned addr_bit_size,
 763                              LLVMValueRef addr,
 764                              LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
 765 {
 766    struct gallivm_state *gallivm = bld_base->base.gallivm;
 767    LLVMBuilderRef builder = gallivm->builder;
 768    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 769    struct lp_build_context *res_bld;
 770
 771    res_bld = get_int_bld(bld_base, true, bit_size);
 772
 773    for (unsigned c = 0; c < nc; c++) {
 774       LLVMValueRef result = lp_build_alloca(gallivm, res_bld->vec_type, "");
 775
 776       struct lp_build_loop_state loop_state;
 777       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 778
 779       LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
 780                                                       loop_state.counter, "");
 781       addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, bit_size);
 782
 783       LLVMValueRef value_ptr = lp_build_pointer_get(builder, addr_ptr, lp_build_const_int32(gallivm, c));
 784
 785       LLVMValueRef temp_res;
 786       temp_res = LLVMBuildLoad(builder, result, "");
 787       temp_res = LLVMBuildInsertElement(builder, temp_res, value_ptr, loop_state.counter, "");
 788       LLVMBuildStore(builder, temp_res, result);
 789       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 790                              NULL, LLVMIntUGE);
 791       outval[c] = LLVMBuildLoad(builder, result, "");
 792    }
 793 }
 794
 795 static void emit_store_global(struct lp_build_nir_context *bld_base,
 796                               unsigned writemask,
 797                               unsigned nc, unsigned bit_size,
 798                               unsigned addr_bit_size,
 799                               LLVMValueRef addr,
 800                               LLVMValueRef dst)
 801 {
 802    struct gallivm_state *gallivm = bld_base->base.gallivm;
 803    LLVMBuilderRef builder = gallivm->builder;
 804    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 805
 806    for (unsigned c = 0; c < nc; c++) {
 807       if (!(writemask & (1u << c)))
 808          continue;
 809       LLVMValueRef val = (nc == 1) ? dst : LLVMBuildExtractValue(builder, dst, c, "");
 810
 811       LLVMValueRef exec_mask = mask_vec(bld_base);
 812       struct lp_build_loop_state loop_state;
 813       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 814       LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
 815                                                        loop_state.counter, "");
 816
 817       LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
 818                                                       loop_state.counter, "");
 819       addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, bit_size);
 820       switch (bit_size) {
 821       case 32:
 822          value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt32TypeInContext(gallivm->context), "");
 823          break;
 824       case 64:
 825          value_ptr = LLVMBuildBitCast(builder, value_ptr, LLVMInt64TypeInContext(gallivm->context), "");
 826          break;
 827       default:
 828          break;
 829       }
 830       struct lp_build_if_state ifthen;
 831
 832       LLVMValueRef cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
 833       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
 834       lp_build_if(&ifthen, gallivm, cond);
 835       lp_build_pointer_set(builder, addr_ptr, lp_build_const_int32(gallivm, c), value_ptr);
 836       lp_build_endif(&ifthen);
 837       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 838                              NULL, LLVMIntUGE);
 839    }
 840 }
 841
 842 static void emit_atomic_global(struct lp_build_nir_context *bld_base,
 843                                nir_intrinsic_op nir_op,
 844                                unsigned addr_bit_size,
 845                                LLVMValueRef addr,
 846                                LLVMValueRef val, LLVMValueRef val2,
 847                                LLVMValueRef *result)
 848 {
 849    struct gallivm_state *gallivm = bld_base->base.gallivm;
 850    LLVMBuilderRef builder = gallivm->builder;
 851    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 852
 853    LLVMValueRef atom_res = lp_build_alloca(gallivm,
 854                                            uint_bld->vec_type, "");
 855    LLVMValueRef exec_mask = mask_vec(bld_base);
 856    struct lp_build_loop_state loop_state;
 857    lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 858
 859    LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
 860                                                     loop_state.counter, "");
 861
 862    LLVMValueRef addr_ptr = LLVMBuildExtractElement(gallivm->builder, addr,
 863                                                    loop_state.counter, "");
 864    addr_ptr = global_addr_to_ptr(gallivm, addr_ptr, 32);
 865    struct lp_build_if_state ifthen;
 866    LLVMValueRef cond, temp_res;
 867    LLVMValueRef scalar;
 868    cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
 869    cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
 870    lp_build_if(&ifthen, gallivm, cond);
 871
 872    if (nir_op == nir_intrinsic_global_atomic_comp_swap) {
 873       LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, val2,
 874                                                          loop_state.counter, "");
 875       cas_src_ptr = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, uint_bld->elem_type, "");
 876       scalar = LLVMBuildAtomicCmpXchg(builder, addr_ptr, value_ptr,
 877                                       cas_src_ptr,
 878                                       LLVMAtomicOrderingSequentiallyConsistent,
 879                                       LLVMAtomicOrderingSequentiallyConsistent,
 880                                       false);
 881       scalar = LLVMBuildExtractValue(gallivm->builder, scalar, 0, "");
 882    } else {
 883       LLVMAtomicRMWBinOp op;
 884       switch (nir_op) {
 885       case nir_intrinsic_global_atomic_add:
 886          op = LLVMAtomicRMWBinOpAdd;
 887          break;
 888       case nir_intrinsic_global_atomic_exchange:
 889          op = LLVMAtomicRMWBinOpXchg;
 890          break;
 891       case nir_intrinsic_global_atomic_and:
 892          op = LLVMAtomicRMWBinOpAnd;
 893          break;
 894       case nir_intrinsic_global_atomic_or:
 895          op = LLVMAtomicRMWBinOpOr;
 896          break;
 897       case nir_intrinsic_global_atomic_xor:
 898          op = LLVMAtomicRMWBinOpXor;
 899          break;
 900       case nir_intrinsic_global_atomic_umin:
 901          op = LLVMAtomicRMWBinOpUMin;
 902          break;
 903       case nir_intrinsic_global_atomic_umax:
 904          op = LLVMAtomicRMWBinOpUMax;
 905          break;
 906       case nir_intrinsic_global_atomic_imin:
 907          op = LLVMAtomicRMWBinOpMin;
 908          break;
 909       case nir_intrinsic_global_atomic_imax:
 910          op = LLVMAtomicRMWBinOpMax;
 911          break;
 912       default:
 913          unreachable("unknown atomic op");
 914       }
 915
 916       scalar = LLVMBuildAtomicRMW(builder, op,
 917                                   addr_ptr, value_ptr,
 918                                   LLVMAtomicOrderingSequentiallyConsistent,
 919                                   false);
 920    }
 921    temp_res = LLVMBuildLoad(builder, atom_res, "");
 922    temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
 923    LLVMBuildStore(builder, temp_res, atom_res);
 924    lp_build_else(&ifthen);
 925    temp_res = LLVMBuildLoad(builder, atom_res, "");
 926    temp_res = LLVMBuildInsertElement(builder, temp_res, lp_build_const_int32(gallivm, 0), loop_state.counter, "");
 927    LLVMBuildStore(builder, temp_res, atom_res);
 928    lp_build_endif(&ifthen);
 929    lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 930                           NULL, LLVMIntUGE);
 931    *result = LLVMBuildLoad(builder, atom_res, "");
 932 }
 933
 934 static void emit_load_ubo(struct lp_build_nir_context *bld_base,
 935                           unsigned nc,
 936                           unsigned bit_size,
 937                           bool offset_is_uniform,
 938                           LLVMValueRef index,
 939                           LLVMValueRef offset,
 940                           LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
 941 {
 942    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 943    struct gallivm_state *gallivm = bld_base->base.gallivm;
 944    LLVMBuilderRef builder = gallivm->builder;
 945    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 946    struct lp_build_context *bld_broad = bit_size == 64 ? &bld_base->dbl_bld : &bld_base->base;
 947    LLVMValueRef consts_ptr = lp_build_array_get(gallivm, bld->consts_ptr, index);
 948    unsigned size_shift = 0;
 949    if (bit_size == 32)
 950       size_shift = 2;
 951    else if (bit_size == 64)
 952       size_shift = 3;
 953    if (size_shift)
 954       offset = lp_build_shr(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, size_shift));
 955    if (bit_size == 64) {
 956       LLVMTypeRef dptr_type = LLVMPointerType(bld_base->dbl_bld.elem_type, 0);
 957       consts_ptr = LLVMBuildBitCast(builder, consts_ptr, dptr_type, "");
 958    }
 959
 960    if (offset_is_uniform) {
 961       offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
 962
 963       for (unsigned c = 0; c < nc; c++) {
 964          LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, lp_build_const_int32(gallivm, c), "");
 965
 966          LLVMValueRef scalar = lp_build_pointer_get(builder, consts_ptr, this_offset);
 967          result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
 968       }
 969    } else {
 970       LLVMValueRef overflow_mask;
 971       LLVMValueRef num_consts = lp_build_array_get(gallivm, bld->const_sizes_ptr, index);
 972
 973       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
 974       for (unsigned c = 0; c < nc; c++) {
 975          LLVMValueRef this_offset = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
 976          overflow_mask = lp_build_compare(gallivm, uint_bld->type, PIPE_FUNC_GEQUAL,
 977                                           this_offset, num_consts);
 978          result[c] = build_gather(bld_base, bld_broad, consts_ptr, this_offset, overflow_mask, NULL);
 979       }
 980    }
 981 }
 982
 983
 984 static void emit_load_mem(struct lp_build_nir_context *bld_base,
 985                           unsigned nc,
 986                           unsigned bit_size,
 987                           LLVMValueRef index,
 988                           LLVMValueRef offset,
 989                           LLVMValueRef outval[NIR_MAX_VEC_COMPONENTS])
 990 {
 991    struct gallivm_state *gallivm = bld_base->base.gallivm;
 992    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 993    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 994    LLVMValueRef ssbo_ptr = NULL;
 995    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 996    struct lp_build_context *uint64_bld = &bld_base->uint64_bld;
 997    LLVMValueRef ssbo_limit = NULL;
 998
 999    if (index) {
1000       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1001       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
1002       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
1003
1004       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1005    } else
1006       ssbo_ptr = bld->shared_ptr;
1007
1008    offset = LLVMBuildAShr(gallivm->builder, offset, lp_build_const_int_vec(gallivm, uint_bld->type, bit_size == 64 ? 3 : 2), "");
1009    for (unsigned c = 0; c < nc; c++) {
1010       LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
1011       LLVMValueRef exec_mask = mask_vec(bld_base);
1012
1013       if (ssbo_limit) {
1014          LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
1015          exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
1016       }
1017
1018       LLVMValueRef result = lp_build_alloca(gallivm, bit_size == 64 ? uint64_bld->vec_type : uint_bld->vec_type, "");
1019       struct lp_build_loop_state loop_state;
1020       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1021
1022       struct lp_build_if_state ifthen;
1023       LLVMValueRef cond, temp_res;
1024
1025       loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
1026                                            loop_state.counter, "");
1027
1028       cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
1029       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
1030
1031       lp_build_if(&ifthen, gallivm, cond);
1032       LLVMValueRef scalar;
1033       if (bit_size == 64) {
1034          LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(uint64_bld->elem_type, 0), "");
1035          scalar = lp_build_pointer_get(builder, ssbo_ptr2, loop_index);
1036       } else
1037          scalar = lp_build_pointer_get(builder, ssbo_ptr, loop_index);
1038
1039       temp_res = LLVMBuildLoad(builder, result, "");
1040       temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
1041       LLVMBuildStore(builder, temp_res, result);
1042       lp_build_else(&ifthen);
1043       temp_res = LLVMBuildLoad(builder, result, "");
1044       LLVMValueRef zero;
1045       if (bit_size == 64)
1046          zero = LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 0, 0);
1047       else
1048          zero = lp_build_const_int32(gallivm, 0);
1049       temp_res = LLVMBuildInsertElement(builder, temp_res, zero, loop_state.counter, "");
1050       LLVMBuildStore(builder, temp_res, result);
1051       lp_build_endif(&ifthen);
1052       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
1053                                 NULL, LLVMIntUGE);
1054       outval[c] = LLVMBuildLoad(gallivm->builder, result, "");
1055    }
1056 }
1057
1058 static void emit_store_mem(struct lp_build_nir_context *bld_base,
1059                            unsigned writemask,
1060                            unsigned nc,
1061                            unsigned bit_size,
1062                            LLVMValueRef index,
1063                            LLVMValueRef offset,
1064                            LLVMValueRef dst)
1065 {
1066    struct gallivm_state *gallivm = bld_base->base.gallivm;
1067    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1068    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1069    LLVMValueRef ssbo_ptr;
1070    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1071    LLVMValueRef ssbo_limit = NULL;
1072
1073    if (index) {
1074       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1075       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
1076       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
1077       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1078    } else
1079       ssbo_ptr = bld->shared_ptr;
1080
1081    offset = lp_build_shr_imm(uint_bld, offset, bit_size == 64 ? 3 : 2);
1082    for (unsigned c = 0; c < nc; c++) {
1083       if (!(writemask & (1u << c)))
1084          continue;
1085       LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
1086       LLVMValueRef val = (nc == 1) ? dst : LLVMBuildExtractValue(builder, dst, c, "");
1087
1088       LLVMValueRef exec_mask = mask_vec(bld_base);
1089       if (ssbo_limit) {
1090          LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
1091          exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
1092       }
1093
1094       struct lp_build_loop_state loop_state;
1095       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1096       LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
1097                                                        loop_state.counter, "");
1098       if (bit_size == 64)
1099          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, bld_base->uint64_bld.elem_type, "");
1100       else
1101          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
1102       struct lp_build_if_state ifthen;
1103       LLVMValueRef cond;
1104
1105       loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
1106                                            loop_state.counter, "");
1107       cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
1108       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
1109       lp_build_if(&ifthen, gallivm, cond);
1110       if (bit_size == 64) {
1111          LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint64_bld.elem_type, 0), "");
1112          lp_build_pointer_set(builder, ssbo_ptr2, loop_index, value_ptr);
1113       } else
1114          lp_build_pointer_set(builder, ssbo_ptr, loop_index, value_ptr);
1115       lp_build_endif(&ifthen);
1116       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
1117                              NULL, LLVMIntUGE);
1118    }
1119 }
1120
1121 static void emit_atomic_mem(struct lp_build_nir_context *bld_base,
1122                             nir_intrinsic_op nir_op,
1123                             LLVMValueRef index, LLVMValueRef offset,
1124                             LLVMValueRef val, LLVMValueRef val2,
1125                             LLVMValueRef *result)
1126 {
1127    struct gallivm_state *gallivm = bld_base->base.gallivm;
1128    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1129    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1130    LLVMValueRef ssbo_ptr;
1131    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1132    LLVMValueRef ssbo_limit = NULL;
1133
1134    if (index) {
1135       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1136       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, 2), "");
1137       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
1138       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1139    } else
1140       ssbo_ptr = bld->shared_ptr;
1141
1142    offset = lp_build_shr_imm(uint_bld, offset, 2);
1143    LLVMValueRef atom_res = lp_build_alloca(gallivm,
1144                                            uint_bld->vec_type, "");
1145
1146    LLVMValueRef exec_mask = mask_vec(bld_base);
1147    if (ssbo_limit) {
1148       LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, offset, ssbo_limit);
1149       exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
1150    }
1151
1152    struct lp_build_loop_state loop_state;
1153    lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1154
1155    LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
1156                                                     loop_state.counter, "");
1157    value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
1158
1159    offset = LLVMBuildExtractElement(gallivm->builder, offset,
1160                                    loop_state.counter, "");
1161
1162    LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, ssbo_ptr,
1163                                           &offset, 1, "");
1164
1165    struct lp_build_if_state ifthen;
1166    LLVMValueRef cond, temp_res;
1167    LLVMValueRef scalar;
1168    cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
1169    cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
1170    lp_build_if(&ifthen, gallivm, cond);
1171
1172    if (nir_op == nir_intrinsic_ssbo_atomic_comp_swap || nir_op == nir_intrinsic_shared_atomic_comp_swap) {
1173       LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, val2,
1174                                                          loop_state.counter, "");
1175       cas_src_ptr = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, uint_bld->elem_type, "");
1176       scalar = LLVMBuildAtomicCmpXchg(builder, scalar_ptr, value_ptr,
1177                                       cas_src_ptr,
1178                                       LLVMAtomicOrderingSequentiallyConsistent,
1179                                       LLVMAtomicOrderingSequentiallyConsistent,
1180                                       false);
1181       scalar = LLVMBuildExtractValue(gallivm->builder, scalar, 0, "");
1182    } else {
1183       LLVMAtomicRMWBinOp op;
1184
1185       switch (nir_op) {
1186       case nir_intrinsic_shared_atomic_add:
1187       case nir_intrinsic_ssbo_atomic_add:
1188          op = LLVMAtomicRMWBinOpAdd;
1189          break;
1190       case nir_intrinsic_shared_atomic_exchange:
1191       case nir_intrinsic_ssbo_atomic_exchange:
1192          op = LLVMAtomicRMWBinOpXchg;
1193          break;
1194       case nir_intrinsic_shared_atomic_and:
1195       case nir_intrinsic_ssbo_atomic_and:
1196          op = LLVMAtomicRMWBinOpAnd;
1197          break;
1198       case nir_intrinsic_shared_atomic_or:
1199       case nir_intrinsic_ssbo_atomic_or:
1200          op = LLVMAtomicRMWBinOpOr;
1201          break;
1202       case nir_intrinsic_shared_atomic_xor:
1203       case nir_intrinsic_ssbo_atomic_xor:
1204          op = LLVMAtomicRMWBinOpXor;
1205          break;
1206       case nir_intrinsic_shared_atomic_umin:
1207       case nir_intrinsic_ssbo_atomic_umin:
1208          op = LLVMAtomicRMWBinOpUMin;
1209          break;
1210       case nir_intrinsic_shared_atomic_umax:
1211       case nir_intrinsic_ssbo_atomic_umax:
1212          op = LLVMAtomicRMWBinOpUMax;
1213          break;
1214       case nir_intrinsic_ssbo_atomic_imin:
1215       case nir_intrinsic_shared_atomic_imin:
1216          op = LLVMAtomicRMWBinOpMin;
1217          break;
1218       case nir_intrinsic_ssbo_atomic_imax:
1219       case nir_intrinsic_shared_atomic_imax:
1220          op = LLVMAtomicRMWBinOpMax;
1221          break;
1222       default:
1223          unreachable("unknown atomic op");
1224       }
1225       scalar = LLVMBuildAtomicRMW(builder, op,
1226                                   scalar_ptr, value_ptr,
1227                                   LLVMAtomicOrderingSequentiallyConsistent,
1228                                   false);
1229    }
1230    temp_res = LLVMBuildLoad(builder, atom_res, "");
1231    temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
1232    LLVMBuildStore(builder, temp_res, atom_res);
1233    lp_build_else(&ifthen);
1234    temp_res = LLVMBuildLoad(builder, atom_res, "");
1235    temp_res = LLVMBuildInsertElement(builder, temp_res, lp_build_const_int32(gallivm, 0), loop_state.counter, "");
1236    LLVMBuildStore(builder, temp_res, atom_res);
1237    lp_build_endif(&ifthen);
1238
1239    lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
1240                           NULL, LLVMIntUGE);
1241    *result = LLVMBuildLoad(builder, atom_res, "");
1242 }
1243
1244 static void emit_barrier(struct lp_build_nir_context *bld_base)
1245 {
1246    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1247    struct gallivm_state * gallivm = bld_base->base.gallivm;
1248
1249    LLVMBasicBlockRef resume = lp_build_insert_new_block(gallivm, "resume");
1250
1251    lp_build_coro_suspend_switch(gallivm, bld->coro, resume, false);
1252    LLVMPositionBuilderAtEnd(gallivm->builder, resume);
1253 }
1254
1255 static LLVMValueRef emit_get_buffer_size(struct lp_build_nir_context *bld_base,
1256                                          LLVMValueRef index)
1257 {
1258    struct gallivm_state *gallivm = bld_base->base.gallivm;
1259    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1260    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1261    struct lp_build_context *bld_broad = &bld_base->uint_bld;
1262    LLVMValueRef size_ptr = lp_build_array_get(bld_base->base.gallivm, bld->ssbo_sizes_ptr,
1263                                               LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
1264    return lp_build_broadcast_scalar(bld_broad, size_ptr);
1265 }
1266
1267 static void emit_image_op(struct lp_build_nir_context *bld_base,
1268                           struct lp_img_params *params)
1269 {
1270    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1271    struct gallivm_state *gallivm = bld_base->base.gallivm;
1272
1273    params->type = bld_base->base.type;
1274    params->context_ptr = bld->context_ptr;
1275    params->thread_data_ptr = bld->thread_data_ptr;
1276    params->exec_mask = mask_vec(bld_base);
1277
1278    if (params->image_index_offset)
1279       params->image_index_offset = LLVMBuildExtractElement(gallivm->builder, params->image_index_offset,
1280                                                            lp_build_const_int32(gallivm, 0), "");
1281
1282    bld->image->emit_op(bld->image,
1283                        bld->bld_base.base.gallivm,
1284                        params);
1285
1286 }
1287
1288 static void emit_image_size(struct lp_build_nir_context *bld_base,
1289                             struct lp_sampler_size_query_params *params)
1290 {
1291    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1292    struct gallivm_state *gallivm = bld_base->base.gallivm;
1293
1294    params->int_type = bld_base->int_bld.type;
1295    params->context_ptr = bld->context_ptr;
1296
1297    if (params->texture_unit_offset)
1298       params->texture_unit_offset = LLVMBuildExtractElement(gallivm->builder, params->texture_unit_offset,
1299                                                             lp_build_const_int32(gallivm, 0), "");
1300    bld->image->emit_size_query(bld->image,
1301                                bld->bld_base.base.gallivm,
1302                                params);
1303
1304 }
1305
1306 static void init_var_slots(struct lp_build_nir_context *bld_base,
1307                            nir_variable *var, unsigned sc)
1308 {
1309    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1310    unsigned slots = glsl_count_attribute_slots(var->type, false) * 4;
1311
1312    if (!bld->outputs)
1313      return;
1314    for (unsigned comp = sc; comp < slots + sc; comp++) {
1315       unsigned this_loc = var->data.driver_location + (comp / 4);
1316       unsigned this_chan = comp % 4;
1317
1318       if (!bld->outputs[this_loc][this_chan])
1319          bld->outputs[this_loc][this_chan] = lp_build_alloca(bld_base->base.gallivm,
1320                                                              bld_base->base.vec_type, "output");
1321    }
1322 }
1323
1324 static void emit_var_decl(struct lp_build_nir_context *bld_base,
1325                           nir_variable *var)
1326 {
1327    unsigned sc = var->data.location_frac;
1328    switch (var->data.mode) {
1329    case nir_var_shader_out: {
1330       if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
1331          if (var->data.location == FRAG_RESULT_STENCIL)
1332             sc = 1;
1333          else if (var->data.location == FRAG_RESULT_DEPTH)
1334             sc = 2;
1335       }
1336       init_var_slots(bld_base, var, sc);
1337       break;
1338    }
1339    default:
1340       break;
1341    }
1342 }
1343
1344 static void emit_tex(struct lp_build_nir_context *bld_base,
1345                      struct lp_sampler_params *params)
1346 {
1347    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1348    struct gallivm_state *gallivm = bld_base->base.gallivm;
1349
1350    params->type = bld_base->base.type;
1351    params->context_ptr = bld->context_ptr;
1352    params->thread_data_ptr = bld->thread_data_ptr;
1353
1354    if (params->texture_index_offset && bld_base->shader->info.stage != MESA_SHADER_FRAGMENT) {
1355       /* this is horrible but this can be dynamic */
1356       LLVMValueRef coords[5];
1357       LLVMValueRef *orig_texel_ptr;
1358       struct lp_build_context *uint_bld = &bld_base->uint_bld;
1359       LLVMValueRef result[4] = { LLVMGetUndef(bld_base->base.vec_type),
1360                                  LLVMGetUndef(bld_base->base.vec_type),
1361                                  LLVMGetUndef(bld_base->base.vec_type),
1362                                  LLVMGetUndef(bld_base->base.vec_type) };
1363       LLVMValueRef texel[4], orig_offset;
1364       unsigned i;
1365       orig_texel_ptr = params->texel;
1366
1367       for (i = 0; i < 5; i++) {
1368          coords[i] = params->coords[i];
1369       }
1370       orig_offset = params->texture_index_offset;
1371
1372       for (unsigned v = 0; v < uint_bld->type.length; v++) {
1373          LLVMValueRef idx = lp_build_const_int32(gallivm, v);
1374          LLVMValueRef new_coords[5];
1375          for (i = 0; i < 5; i++) {
1376             new_coords[i] = LLVMBuildExtractElement(gallivm->builder,
1377                                                     coords[i], idx, "");
1378          }
1379          params->coords = new_coords;
1380          params->texture_index_offset = LLVMBuildExtractElement(gallivm->builder,
1381                                                                 orig_offset,
1382                                                                 idx, "");
1383          params->type = lp_elem_type(bld_base->base.type);
1384
1385          params->texel = texel;
1386          bld->sampler->emit_tex_sample(bld->sampler,
1387                                        gallivm,
1388                                        params);
1389
1390          for (i = 0; i < 4; i++) {
1391             result[i] = LLVMBuildInsertElement(gallivm->builder, result[i], texel[i], idx, "");
1392          }
1393       }
1394       for (i = 0; i < 4; i++) {
1395          orig_texel_ptr[i] = result[i];
1396       }
1397       return;
1398    }
1399
1400    if (params->texture_index_offset)
1401       params->texture_index_offset = LLVMBuildExtractElement(bld_base->base.gallivm->builder,
1402                                                              params->texture_index_offset,
1403                                                              lp_build_const_int32(bld_base->base.gallivm, 0), "");
1404
1405    params->type = bld_base->base.type;
1406    bld->sampler->emit_tex_sample(bld->sampler,
1407                                  bld->bld_base.base.gallivm,
1408                                  params);
1409 }
1410
1411 static void emit_tex_size(struct lp_build_nir_context *bld_base,
1412                           struct lp_sampler_size_query_params *params)
1413 {
1414    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1415
1416    params->int_type = bld_base->int_bld.type;
1417    params->context_ptr = bld->context_ptr;
1418
1419    if (params->texture_unit_offset)
1420       params->texture_unit_offset = LLVMBuildExtractElement(bld_base->base.gallivm->builder,
1421                                                              params->texture_unit_offset,
1422                                                              lp_build_const_int32(bld_base->base.gallivm, 0), "");
1423    bld->sampler->emit_size_query(bld->sampler,
1424                                  bld->bld_base.base.gallivm,
1425                                  params);
1426 }
1427
1428 static void emit_sysval_intrin(struct lp_build_nir_context *bld_base,
1429                                nir_intrinsic_instr *instr,
1430                                LLVMValueRef result[NIR_MAX_VEC_COMPONENTS])
1431 {
1432    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1433    struct gallivm_state *gallivm = bld_base->base.gallivm;
1434    switch (instr->intrinsic) {
1435    case nir_intrinsic_load_instance_id:
1436       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
1437       break;
1438    case nir_intrinsic_load_base_instance:
1439       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.base_instance);
1440       break;
1441    case nir_intrinsic_load_base_vertex:
1442       result[0] = bld->system_values.basevertex;
1443       break;
1444    case nir_intrinsic_load_vertex_id:
1445       result[0] = bld->system_values.vertex_id;
1446       break;
1447    case nir_intrinsic_load_primitive_id:
1448       result[0] = bld->system_values.prim_id;
1449       break;
1450    case nir_intrinsic_load_work_group_id:
1451       for (unsigned i = 0; i < 3; i++)
1452          result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.block_id, lp_build_const_int32(gallivm, i), ""));
1453       break;
1454    case nir_intrinsic_load_local_invocation_id:
1455       for (unsigned i = 0; i < 3; i++)
1456          result[i] = LLVMBuildExtractValue(gallivm->builder, bld->system_values.thread_id, i, "");
1457       break;
1458    case nir_intrinsic_load_num_work_groups:
1459       for (unsigned i = 0; i < 3; i++)
1460          result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.grid_size, lp_build_const_int32(gallivm, i), ""));
1461       break;
1462    case nir_intrinsic_load_invocation_id:
1463       if (bld_base->shader->info.stage == MESA_SHADER_TESS_CTRL)
1464          result[0] = bld->system_values.invocation_id;
1465       else
1466          result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.invocation_id);
1467       break;
1468    case nir_intrinsic_load_front_face:
1469       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.front_facing);
1470       break;
1471    case nir_intrinsic_load_draw_id:
1472       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.draw_id);
1473       break;
1474    default:
1475       break;
1476    case nir_intrinsic_load_local_group_size:
1477      for (unsigned i = 0; i < 3; i++)
1478        result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.block_size, lp_build_const_int32(gallivm, i), ""));
1479      break;
1480    case nir_intrinsic_load_work_dim:
1481       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.work_dim);
1482       break;
1483    case nir_intrinsic_load_tess_coord:
1484       for (unsigned i = 0; i < 3; i++) {
1485          result[i] = LLVMBuildExtractValue(gallivm->builder, bld->system_values.tess_coord, i, "");
1486       }
1487       break;
1488    case nir_intrinsic_load_tess_level_outer:
1489       for (unsigned i = 0; i < 4; i++)
1490          result[i] = lp_build_broadcast_scalar(&bld_base->base, LLVMBuildExtractValue(gallivm->builder, bld->system_values.tess_outer, i, ""));
1491       break;
1492    case nir_intrinsic_load_tess_level_inner:
1493       for (unsigned i = 0; i < 2; i++)
1494          result[i] = lp_build_broadcast_scalar(&bld_base->base, LLVMBuildExtractValue(gallivm->builder, bld->system_values.tess_inner, i, ""));
1495       break;
1496    case nir_intrinsic_load_patch_vertices_in:
1497       result[0] = bld->system_values.vertices_in;
1498       break;
1499    case nir_intrinsic_load_sample_id:
1500       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.sample_id);
1501       break;
1502    case nir_intrinsic_load_sample_pos:
1503       for (unsigned i = 0; i < 2; i++) {
1504          LLVMValueRef idx = LLVMBuildMul(gallivm->builder, bld->system_values.sample_id, lp_build_const_int32(gallivm, 2), "");
1505          idx = LLVMBuildAdd(gallivm->builder, idx, lp_build_const_int32(gallivm, i), "");
1506          LLVMValueRef val = lp_build_array_get(gallivm, bld->system_values.sample_pos, idx);
1507          result[i] = lp_build_broadcast_scalar(&bld_base->base, val);
1508       }
1509       break;
1510    case nir_intrinsic_load_sample_mask_in:
1511       result[0] = bld->system_values.sample_mask_in;
1512       break;
1513    }
1514 }
1515
1516 static void emit_helper_invocation(struct lp_build_nir_context *bld_base,
1517                                    LLVMValueRef *dst)
1518 {
1519    struct gallivm_state *gallivm = bld_base->base.gallivm;
1520    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1521    *dst = lp_build_cmp(uint_bld, PIPE_FUNC_NOTEQUAL, mask_vec(bld_base), lp_build_const_int_vec(gallivm, uint_bld->type, -1));
1522 }
1523
1524 static void bgnloop(struct lp_build_nir_context *bld_base)
1525 {
1526    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1527    lp_exec_bgnloop(&bld->exec_mask, true);
1528 }
1529
1530 static void endloop(struct lp_build_nir_context *bld_base)
1531 {
1532    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1533    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
1534 }
1535
1536 static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1537 {
1538    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1539    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1540    lp_exec_mask_cond_push(&bld->exec_mask, LLVMBuildBitCast(builder, cond, bld_base->base.int_vec_type, ""));
1541 }
1542
1543 static void else_stmt(struct lp_build_nir_context *bld_base)
1544 {
1545    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1546    lp_exec_mask_cond_invert(&bld->exec_mask);
1547 }
1548
1549 static void endif_stmt(struct lp_build_nir_context *bld_base)
1550 {
1551    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1552    lp_exec_mask_cond_pop(&bld->exec_mask);
1553 }
1554
1555 static void break_stmt(struct lp_build_nir_context *bld_base)
1556 {
1557    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1558
1559    lp_exec_break(&bld->exec_mask, NULL, false);
1560 }
1561
1562 static void continue_stmt(struct lp_build_nir_context *bld_base)
1563 {
1564    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1565    lp_exec_continue(&bld->exec_mask);
1566 }
1567
1568 static void discard(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1569 {
1570    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1571    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1572    LLVMValueRef mask;
1573
1574    if (!cond) {
1575       if (bld->exec_mask.has_mask) {
1576          mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1577       } else {
1578          mask = LLVMConstNull(bld->bld_base.base.int_vec_type);
1579       }
1580    } else {
1581       mask = LLVMBuildNot(builder, cond, "");
1582       if (bld->exec_mask.has_mask) {
1583          LLVMValueRef invmask;
1584          invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1585          mask = LLVMBuildOr(builder, mask, invmask, "");
1586       }
1587    }
1588    lp_build_mask_update(bld->mask, mask);
1589 }
1590
1591 static void
1592 increment_vec_ptr_by_mask(struct lp_build_nir_context * bld_base,
1593                           LLVMValueRef ptr,
1594                           LLVMValueRef mask)
1595 {
1596    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1597    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1598
1599    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
1600
1601    LLVMBuildStore(builder, current_vec, ptr);
1602 }
1603
1604 static void
1605 clear_uint_vec_ptr_from_mask(struct lp_build_nir_context * bld_base,
1606                              LLVMValueRef ptr,
1607                              LLVMValueRef mask)
1608 {
1609    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1610    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1611
1612    current_vec = lp_build_select(&bld_base->uint_bld,
1613                                  mask,
1614                                  bld_base->uint_bld.zero,
1615                                  current_vec);
1616
1617    LLVMBuildStore(builder, current_vec, ptr);
1618 }
1619
1620 static LLVMValueRef
1621 clamp_mask_to_max_output_vertices(struct lp_build_nir_soa_context * bld,
1622                                   LLVMValueRef current_mask_vec,
1623                                   LLVMValueRef total_emitted_vertices_vec)
1624 {
1625    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1626    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
1627    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
1628                                             total_emitted_vertices_vec,
1629                                             bld->max_output_vertices_vec);
1630
1631    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
1632 }
1633
1634 static void emit_vertex(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1635 {
1636    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1637    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1638
1639    if (stream_id >= bld->gs_vertex_streams)
1640       return;
1641    assert(bld->gs_iface->emit_vertex);
1642    LLVMValueRef total_emitted_vertices_vec =
1643       LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr[stream_id], "");
1644    LLVMValueRef mask = mask_vec(bld_base);
1645    mask = clamp_mask_to_max_output_vertices(bld, mask,
1646                                             total_emitted_vertices_vec);
1647    bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base.base,
1648                               bld->outputs,
1649                               total_emitted_vertices_vec,
1650                               mask,
1651                               lp_build_const_int_vec(bld->bld_base.base.gallivm, bld->bld_base.base.type, stream_id));
1652
1653    increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr[stream_id],
1654                              mask);
1655    increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr[stream_id],
1656                              mask);
1657 }
1658
1659 static void
1660 end_primitive_masked(struct lp_build_nir_context * bld_base,
1661                      LLVMValueRef mask, uint32_t stream_id)
1662 {
1663    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1664    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1665
1666    if (stream_id >= bld->gs_vertex_streams)
1667       return;
1668    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1669    LLVMValueRef emitted_vertices_vec =
1670       LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr[stream_id], "");
1671    LLVMValueRef emitted_prims_vec =
1672       LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr[stream_id], "");
1673    LLVMValueRef total_emitted_vertices_vec =
1674       LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr[stream_id], "");
1675
1676    LLVMValueRef emitted_mask = lp_build_cmp(uint_bld,
1677                                             PIPE_FUNC_NOTEQUAL,
1678                                             emitted_vertices_vec,
1679                                             uint_bld->zero);
1680    mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
1681    bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base.base,
1682                                 total_emitted_vertices_vec,
1683                                 emitted_vertices_vec, emitted_prims_vec, mask, stream_id);
1684    increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr[stream_id],
1685                              mask);
1686    clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr[stream_id],
1687                                 mask);
1688 }
1689
1690 static void end_primitive(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1691 {
1692    ASSERTED struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1693
1694    assert(bld->gs_iface->end_primitive);
1695
1696    LLVMValueRef mask = mask_vec(bld_base);
1697    end_primitive_masked(bld_base, mask, stream_id);
1698 }
1699
1700 static void
1701 emit_prologue(struct lp_build_nir_soa_context *bld)
1702 {
1703    struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
1704    if (bld->indirects & nir_var_shader_in && !bld->gs_iface && !bld->tcs_iface && !bld->tes_iface) {
1705       uint32_t num_inputs = util_bitcount64(bld->bld_base.shader->info.inputs_read);
1706       unsigned index, chan;
1707       LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
1708       LLVMValueRef array_size = lp_build_const_int32(gallivm, num_inputs * 4);
1709       bld->inputs_array = lp_build_array_alloca(gallivm,
1710                                                vec_type, array_size,
1711                                                "input_array");
1712
1713       for (index = 0; index < num_inputs; ++index) {
1714          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1715             LLVMValueRef lindex =
1716                lp_build_const_int32(gallivm, index * 4 + chan);
1717             LLVMValueRef input_ptr =
1718                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
1719                             &lindex, 1, "");
1720             LLVMValueRef value = bld->inputs[index][chan];
1721             if (value)
1722                LLVMBuildStore(gallivm->builder, value, input_ptr);
1723          }
1724       }
1725    }
1726 }
1727
1728 static void emit_vote(struct lp_build_nir_context *bld_base, LLVMValueRef src, nir_intrinsic_instr *instr, LLVMValueRef result[4])
1729 {
1730    struct gallivm_state * gallivm = bld_base->base.gallivm;
1731    LLVMBuilderRef builder = gallivm->builder;
1732
1733    LLVMValueRef exec_mask = mask_vec(bld_base);
1734    struct lp_build_loop_state loop_state;
1735
1736    LLVMValueRef outer_cond = LLVMBuildICmp(builder, LLVMIntNE, exec_mask, bld_base->uint_bld.zero, "");
1737
1738    LLVMValueRef res_store = lp_build_alloca(gallivm, bld_base->int_bld.elem_type, "");
1739    LLVMValueRef init_val = NULL;
1740    if (instr->intrinsic == nir_intrinsic_vote_ieq) {
1741       /* for equal we unfortunately have to loop and find the first valid one. */
1742       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1743       LLVMValueRef if_cond = LLVMBuildExtractElement(gallivm->builder, outer_cond, loop_state.counter, "");
1744
1745       struct lp_build_if_state ifthen;
1746       lp_build_if(&ifthen, gallivm, if_cond);
1747       LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, src,
1748                                                        loop_state.counter, "");
1749       LLVMBuildStore(builder, value_ptr, res_store);
1750       lp_build_endif(&ifthen);
1751       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, bld_base->uint_bld.type.length),
1752                              NULL, LLVMIntUGE);
1753       init_val = LLVMBuildLoad(builder, res_store, "");
1754    } else {
1755       LLVMBuildStore(builder, lp_build_const_int32(gallivm, instr->intrinsic == nir_intrinsic_vote_any ? 0 : -1), res_store);
1756    }
1757
1758    LLVMValueRef res;
1759    lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
1760    LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, src,
1761                                                        loop_state.counter, "");
1762    struct lp_build_if_state ifthen;
1763    LLVMValueRef if_cond;
1764    if_cond = LLVMBuildExtractElement(gallivm->builder, outer_cond, loop_state.counter, "");
1765
1766    lp_build_if(&ifthen, gallivm, if_cond);
1767    res = LLVMBuildLoad(builder, res_store, "");
1768
1769    if (instr->intrinsic == nir_intrinsic_vote_ieq) {
1770       LLVMValueRef tmp = LLVMBuildICmp(builder, LLVMIntEQ, init_val, value_ptr, "");
1771       tmp = LLVMBuildSExt(builder, tmp, bld_base->uint_bld.elem_type, "");
1772       res = LLVMBuildOr(builder, res, tmp, "");
1773    } else if (instr->intrinsic == nir_intrinsic_vote_any)
1774       res = LLVMBuildOr(builder, res, value_ptr, "");
1775    else
1776       res = LLVMBuildAnd(builder, res, value_ptr, "");
1777    LLVMBuildStore(builder, res, res_store);
1778    lp_build_endif(&ifthen);
1779    lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, bld_base->uint_bld.type.length),
1780                           NULL, LLVMIntUGE);
1781    result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildLoad(builder, res_store, ""));
1782 }
1783
1784 static void
1785 emit_interp_at(struct lp_build_nir_context *bld_base,
1786                unsigned num_components,
1787                nir_variable *var,
1788                bool centroid,
1789                bool sample,
1790                unsigned const_index,
1791                LLVMValueRef indir_index,
1792                LLVMValueRef offsets[2],
1793                LLVMValueRef dst[4])
1794 {
1795    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1796
1797    for (unsigned i = 0; i < num_components; i++) {
1798       dst[i] = bld->fs_iface->interp_fn(bld->fs_iface, &bld_base->base,
1799                                         const_index + var->data.driver_location, i + var->data.location_frac,
1800                                         centroid, sample, indir_index, offsets);
1801    }
1802 }
1803
1804 void lp_build_nir_soa(struct gallivm_state *gallivm,
1805                       struct nir_shader *shader,
1806                       const struct lp_build_tgsi_params *params,
1807                       LLVMValueRef (*outputs)[4])
1808 {
1809    struct lp_build_nir_soa_context bld;
1810    struct lp_type type = params->type;
1811    struct lp_type res_type;
1812
1813    assert(type.length <= LP_MAX_VECTOR_LENGTH);
1814    memset(&res_type, 0, sizeof res_type);
1815    res_type.width = type.width;
1816    res_type.length = type.length;
1817    res_type.sign = 1;
1818
1819    /* Setup build context */
1820    memset(&bld, 0, sizeof bld);
1821    lp_build_context_init(&bld.bld_base.base, gallivm, type);
1822    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
1823    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
1824    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
1825    lp_build_context_init(&bld.uint_elem_bld, gallivm, lp_elem_type(lp_uint_type(type)));
1826    {
1827       struct lp_type dbl_type;
1828       dbl_type = type;
1829       dbl_type.width *= 2;
1830       lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
1831    }
1832    {
1833       struct lp_type uint64_type;
1834       uint64_type = lp_uint_type(type);
1835       uint64_type.width *= 2;
1836       lp_build_context_init(&bld.bld_base.uint64_bld, gallivm, uint64_type);
1837    }
1838    {
1839       struct lp_type int64_type;
1840       int64_type = lp_int_type(type);
1841       int64_type.width *= 2;
1842       lp_build_context_init(&bld.bld_base.int64_bld, gallivm, int64_type);
1843    }
1844    {
1845       struct lp_type uint16_type;
1846       uint16_type = lp_uint_type(type);
1847       uint16_type.width /= 2;
1848       lp_build_context_init(&bld.bld_base.uint16_bld, gallivm, uint16_type);
1849    }
1850    {
1851       struct lp_type int16_type;
1852       int16_type = lp_int_type(type);
1853       int16_type.width /= 2;
1854       lp_build_context_init(&bld.bld_base.int16_bld, gallivm, int16_type);
1855    }
1856    {
1857       struct lp_type uint8_type;
1858       uint8_type = lp_uint_type(type);
1859       uint8_type.width /= 4;
1860       lp_build_context_init(&bld.bld_base.uint8_bld, gallivm, uint8_type);
1861    }
1862    {
1863       struct lp_type int8_type;
1864       int8_type = lp_int_type(type);
1865       int8_type.width /= 4;
1866       lp_build_context_init(&bld.bld_base.int8_bld, gallivm, int8_type);
1867    }
1868    bld.bld_base.load_var = emit_load_var;
1869    bld.bld_base.store_var = emit_store_var;
1870    bld.bld_base.load_reg = emit_load_reg;
1871    bld.bld_base.store_reg = emit_store_reg;
1872    bld.bld_base.emit_var_decl = emit_var_decl;
1873    bld.bld_base.load_ubo = emit_load_ubo;
1874    bld.bld_base.load_kernel_arg = emit_load_kernel_arg;
1875    bld.bld_base.load_global = emit_load_global;
1876    bld.bld_base.store_global = emit_store_global;
1877    bld.bld_base.atomic_global = emit_atomic_global;
1878    bld.bld_base.tex = emit_tex;
1879    bld.bld_base.tex_size = emit_tex_size;
1880    bld.bld_base.bgnloop = bgnloop;
1881    bld.bld_base.endloop = endloop;
1882    bld.bld_base.if_cond = if_cond;
1883    bld.bld_base.else_stmt = else_stmt;
1884    bld.bld_base.endif_stmt = endif_stmt;
1885    bld.bld_base.break_stmt = break_stmt;
1886    bld.bld_base.continue_stmt = continue_stmt;
1887    bld.bld_base.sysval_intrin = emit_sysval_intrin;
1888    bld.bld_base.discard = discard;
1889    bld.bld_base.emit_vertex = emit_vertex;
1890    bld.bld_base.end_primitive = end_primitive;
1891    bld.bld_base.load_mem = emit_load_mem;
1892    bld.bld_base.store_mem = emit_store_mem;
1893    bld.bld_base.get_buffer_size = emit_get_buffer_size;
1894    bld.bld_base.atomic_mem = emit_atomic_mem;
1895    bld.bld_base.barrier = emit_barrier;
1896    bld.bld_base.image_op = emit_image_op;
1897    bld.bld_base.image_size = emit_image_size;
1898    bld.bld_base.vote = emit_vote;
1899    bld.bld_base.helper_invocation = emit_helper_invocation;
1900    bld.bld_base.interp_at = emit_interp_at;
1901
1902    bld.mask = params->mask;
1903    bld.inputs = params->inputs;
1904    bld.outputs = outputs;
1905    bld.consts_ptr = params->consts_ptr;
1906    bld.const_sizes_ptr = params->const_sizes_ptr;
1907    bld.ssbo_ptr = params->ssbo_ptr;
1908    bld.ssbo_sizes_ptr = params->ssbo_sizes_ptr;
1909    bld.sampler = params->sampler;
1910 //   bld.bld_base.info = params->info;
1911
1912    bld.context_ptr = params->context_ptr;
1913    bld.thread_data_ptr = params->thread_data_ptr;
1914    bld.image = params->image;
1915    bld.shared_ptr = params->shared_ptr;
1916    bld.coro = params->coro;
1917    bld.kernel_args_ptr = params->kernel_args;
1918    bld.indirects = 0;
1919    if (params->info->indirect_files & (1 << TGSI_FILE_INPUT))
1920       bld.indirects |= nir_var_shader_in;
1921
1922    bld.gs_iface = params->gs_iface;
1923    bld.tcs_iface = params->tcs_iface;
1924    bld.tes_iface = params->tes_iface;
1925    bld.fs_iface = params->fs_iface;
1926    if (bld.gs_iface) {
1927       struct lp_build_context *uint_bld = &bld.bld_base.uint_bld;
1928
1929       bld.gs_vertex_streams = params->gs_vertex_streams;
1930       bld.max_output_vertices_vec = lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
1931                                                            shader->info.gs.vertices_out);
1932       for (int i = 0; i < params->gs_vertex_streams; i++) {
1933          bld.emitted_prims_vec_ptr[i] =
1934             lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_prims_ptr");
1935          bld.emitted_vertices_vec_ptr[i] =
1936             lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_vertices_ptr");
1937          bld.total_emitted_vertices_vec_ptr[i] =
1938             lp_build_alloca(gallivm, uint_bld->vec_type, "total_emitted_vertices_ptr");
1939       }
1940    }
1941    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
1942
1943    bld.system_values = *params->system_values;
1944
1945    bld.bld_base.shader = shader;
1946
1947    emit_prologue(&bld);
1948    lp_build_nir_llvm(&bld.bld_base, shader);
1949
1950    if (bld.gs_iface) {
1951       LLVMBuilderRef builder = bld.bld_base.base.gallivm->builder;
1952       LLVMValueRef total_emitted_vertices_vec;
1953       LLVMValueRef emitted_prims_vec;
1954
1955       for (int i = 0; i < params->gs_vertex_streams; i++) {
1956          end_primitive_masked(&bld.bld_base, lp_build_mask_value(bld.mask), i);
1957
1958          total_emitted_vertices_vec =
1959             LLVMBuildLoad(builder, bld.total_emitted_vertices_vec_ptr[i], "");
1960
1961          emitted_prims_vec =
1962             LLVMBuildLoad(builder, bld.emitted_prims_vec_ptr[i], "");
1963          bld.gs_iface->gs_epilogue(bld.gs_iface,
1964                                    total_emitted_vertices_vec,
1965                                    emitted_prims_vec, i);
1966       }
1967    }
1968    lp_exec_mask_fini(&bld.exec_mask);
1969 }