src/gallium/auxiliary/gallivm/lp_bld_nir_soa.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2019 Red Hat.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included
  14  * in all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  17  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  22  * SOFTWARE.
  23  *
  24  **************************************************************************/
  25
  26 #include "lp_bld_nir.h"
  27 #include "lp_bld_init.h"
  28 #include "lp_bld_flow.h"
  29 #include "lp_bld_logic.h"
  30 #include "lp_bld_gather.h"
  31 #include "lp_bld_const.h"
  32 #include "lp_bld_struct.h"
  33 #include "lp_bld_arit.h"
  34 #include "lp_bld_bitarit.h"
  35 #include "lp_bld_coro.h"
  36 #include "lp_bld_printf.h"
  37 #include "util/u_math.h"
  38 /*
  39  * combine the execution mask if there is one with the current mask.
  40  */
  41 static LLVMValueRef
  42 mask_vec(struct lp_build_nir_context *bld_base)
  43 {
  44    struct lp_build_nir_soa_context * bld = (struct lp_build_nir_soa_context *)bld_base;
  45    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
  46    struct lp_exec_mask *exec_mask = &bld->exec_mask;
  47    LLVMValueRef bld_mask = bld->mask ? lp_build_mask_value(bld->mask) : NULL;
  48    if (!exec_mask->has_mask) {
  49       return bld_mask;
  50    }
  51    if (!bld_mask)
  52       return exec_mask->exec_mask;
  53    return LLVMBuildAnd(builder, lp_build_mask_value(bld->mask),
  54                        exec_mask->exec_mask, "");
  55 }
  56
  57 static LLVMValueRef
  58 emit_fetch_64bit(
  59    struct lp_build_nir_context * bld_base,
  60    LLVMValueRef input,
  61    LLVMValueRef input2)
  62 {
  63    struct gallivm_state *gallivm = bld_base->base.gallivm;
  64    LLVMBuilderRef builder = gallivm->builder;
  65    LLVMValueRef res;
  66    int i;
  67    LLVMValueRef shuffles[2 * (LP_MAX_VECTOR_WIDTH/32)];
  68    int len = bld_base->base.type.length * 2;
  69    assert(len <= (2 * (LP_MAX_VECTOR_WIDTH/32)));
  70
  71    for (i = 0; i < bld_base->base.type.length * 2; i+=2) {
  72       shuffles[i] = lp_build_const_int32(gallivm, i / 2);
  73       shuffles[i + 1] = lp_build_const_int32(gallivm, i / 2 + bld_base->base.type.length);
  74    }
  75    res = LLVMBuildShuffleVector(builder, input, input2, LLVMConstVector(shuffles, len), "");
  76
  77    return LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
  78 }
  79
  80 static void
  81 emit_store_64bit_chan(struct lp_build_nir_context *bld_base,
  82                       LLVMValueRef chan_ptr,
  83                       LLVMValueRef chan_ptr2,
  84                       LLVMValueRef value)
  85 {
  86    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
  87    struct gallivm_state *gallivm = bld_base->base.gallivm;
  88    LLVMBuilderRef builder = gallivm->builder;
  89    struct lp_build_context *float_bld = &bld_base->base;
  90    unsigned i;
  91    LLVMValueRef temp, temp2;
  92    LLVMValueRef shuffles[LP_MAX_VECTOR_WIDTH/32];
  93    LLVMValueRef shuffles2[LP_MAX_VECTOR_WIDTH/32];
  94    int len = bld_base->base.type.length * 2;
  95
  96    value = LLVMBuildBitCast(gallivm->builder, value, LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), len), "");
  97    for (i = 0; i < bld_base->base.type.length; i++) {
  98       shuffles[i] = lp_build_const_int32(gallivm, i * 2);
  99       shuffles2[i] = lp_build_const_int32(gallivm, (i * 2) + 1);
 100    }
 101
 102    temp = LLVMBuildShuffleVector(builder, value,
 103                                  LLVMGetUndef(LLVMTypeOf(value)),
 104                                  LLVMConstVector(shuffles,
 105                                                  bld_base->base.type.length),
 106                                  "");
 107    temp2 = LLVMBuildShuffleVector(builder, value,
 108                                   LLVMGetUndef(LLVMTypeOf(value)),
 109                                   LLVMConstVector(shuffles2,
 110                                                   bld_base->base.type.length),
 111                                   "");
 112
 113    lp_exec_mask_store(&bld->exec_mask, float_bld, temp, chan_ptr);
 114    lp_exec_mask_store(&bld->exec_mask, float_bld, temp2, chan_ptr2);
 115 }
 116
 117 static LLVMValueRef
 118 get_soa_array_offsets(struct lp_build_context *uint_bld,
 119                       LLVMValueRef indirect_index,
 120                       int num_components,
 121                       unsigned chan_index,
 122                       bool need_perelement_offset)
 123 {
 124    struct gallivm_state *gallivm = uint_bld->gallivm;
 125    LLVMValueRef chan_vec =
 126       lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, chan_index);
 127    LLVMValueRef length_vec =
 128       lp_build_const_int_vec(gallivm, uint_bld->type, uint_bld->type.length);
 129    LLVMValueRef index_vec;
 130
 131    /* index_vec = (indirect_index * 4 + chan_index) * length + offsets */
 132    index_vec = lp_build_mul(uint_bld, indirect_index, lp_build_const_int_vec(uint_bld->gallivm, uint_bld->type, num_components));
 133    index_vec = lp_build_add(uint_bld, index_vec, chan_vec);
 134    index_vec = lp_build_mul(uint_bld, index_vec, length_vec);
 135
 136    if (need_perelement_offset) {
 137       LLVMValueRef pixel_offsets;
 138       unsigned i;
 139      /* build pixel offset vector: {0, 1, 2, 3, ...} */
 140       pixel_offsets = uint_bld->undef;
 141       for (i = 0; i < uint_bld->type.length; i++) {
 142          LLVMValueRef ii = lp_build_const_int32(gallivm, i);
 143          pixel_offsets = LLVMBuildInsertElement(gallivm->builder, pixel_offsets,
 144                                                 ii, ii, "");
 145       }
 146       index_vec = lp_build_add(uint_bld, index_vec, pixel_offsets);
 147    }
 148    return index_vec;
 149 }
 150
 151 static LLVMValueRef
 152 build_gather(struct lp_build_nir_context *bld_base,
 153              struct lp_build_context *bld,
 154              LLVMValueRef base_ptr,
 155              LLVMValueRef indexes,
 156              LLVMValueRef overflow_mask,
 157              LLVMValueRef indexes2)
 158 {
 159    struct gallivm_state *gallivm = bld_base->base.gallivm;
 160    LLVMBuilderRef builder = gallivm->builder;
 161    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 162    LLVMValueRef res;
 163    unsigned i;
 164
 165    if (indexes2)
 166       res = LLVMGetUndef(LLVMVectorType(LLVMFloatTypeInContext(gallivm->context), bld_base->base.type.length * 2));
 167    else
 168       res = bld->undef;
 169    /*
 170     * overflow_mask is a vector telling us which channels
 171     * in the vector overflowed. We use the overflow behavior for
 172     * constant buffers which is defined as:
 173     * Out of bounds access to constant buffer returns 0 in all
 174     * components. Out of bounds behavior is always with respect
 175     * to the size of the buffer bound at that slot.
 176     */
 177
 178    if (overflow_mask) {
 179       /*
 180        * We avoid per-element control flow here (also due to llvm going crazy,
 181        * though I suspect it's better anyway since overflow is likely rare).
 182        * Note that since we still fetch from buffers even if num_elements was
 183        * zero (in this case we'll fetch from index zero) the jit func callers
 184        * MUST provide valid fake constant buffers of size 4x32 (the values do
 185        * not matter), otherwise we'd still need (not per element though)
 186        * control flow.
 187        */
 188       indexes = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes);
 189       if (indexes2)
 190          indexes2 = lp_build_select(uint_bld, overflow_mask, uint_bld->zero, indexes2);
 191    }
 192
 193    /*
 194     * Loop over elements of index_vec, load scalar value, insert it into 'res'.
 195     */
 196    for (i = 0; i < bld->type.length * (indexes2 ? 2 : 1); i++) {
 197       LLVMValueRef si, di;
 198       LLVMValueRef index;
 199       LLVMValueRef scalar_ptr, scalar;
 200
 201       di = lp_build_const_int32(gallivm, i);
 202       if (indexes2)
 203          si = lp_build_const_int32(gallivm, i >> 1);
 204       else
 205          si = di;
 206
 207       if (indexes2 && (i & 1)) {
 208          index = LLVMBuildExtractElement(builder,
 209                                          indexes2, si, "");
 210       } else {
 211          index = LLVMBuildExtractElement(builder,
 212                                          indexes, si, "");
 213       }
 214       scalar_ptr = LLVMBuildGEP(builder, base_ptr,
 215                                 &index, 1, "gather_ptr");
 216       scalar = LLVMBuildLoad(builder, scalar_ptr, "");
 217
 218       res = LLVMBuildInsertElement(builder, res, scalar, di, "");
 219    }
 220
 221    if (overflow_mask) {
 222       if (indexes2) {
 223          res = LLVMBuildBitCast(builder, res, bld_base->dbl_bld.vec_type, "");
 224          overflow_mask = LLVMBuildSExt(builder, overflow_mask,
 225                                        bld_base->dbl_bld.int_vec_type, "");
 226          res = lp_build_select(&bld_base->dbl_bld, overflow_mask,
 227                                bld_base->dbl_bld.zero, res);
 228       } else
 229          res = lp_build_select(bld, overflow_mask, bld->zero, res);
 230    }
 231
 232    return res;
 233 }
 234
 235 /**
 236  * Scatter/store vector.
 237  */
 238 static void
 239 emit_mask_scatter(struct lp_build_nir_soa_context *bld,
 240                   LLVMValueRef base_ptr,
 241                   LLVMValueRef indexes,
 242                   LLVMValueRef values,
 243                   struct lp_exec_mask *mask)
 244 {
 245    struct gallivm_state *gallivm = bld->bld_base.base.gallivm;
 246    LLVMBuilderRef builder = gallivm->builder;
 247    unsigned i;
 248    LLVMValueRef pred = mask->has_mask ? mask->exec_mask : NULL;
 249
 250    /*
 251     * Loop over elements of index_vec, store scalar value.
 252     */
 253    for (i = 0; i < bld->bld_base.base.type.length; i++) {
 254       LLVMValueRef ii = lp_build_const_int32(gallivm, i);
 255       LLVMValueRef index = LLVMBuildExtractElement(builder, indexes, ii, "");
 256       LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, base_ptr, &index, 1, "scatter_ptr");
 257       LLVMValueRef val = LLVMBuildExtractElement(builder, values, ii, "scatter_val");
 258       LLVMValueRef scalar_pred = pred ?
 259          LLVMBuildExtractElement(builder, pred, ii, "scatter_pred") : NULL;
 260
 261       if (0)
 262          lp_build_printf(gallivm, "scatter %d: val %f at %d %p\n",
 263                          ii, val, index, scalar_ptr);
 264
 265       if (scalar_pred) {
 266          LLVMValueRef real_val, dst_val;
 267          dst_val = LLVMBuildLoad(builder, scalar_ptr, "");
 268          real_val = lp_build_select(&bld->uint_elem_bld, scalar_pred, val, dst_val);
 269          LLVMBuildStore(builder, real_val, scalar_ptr);
 270       }
 271       else {
 272          LLVMBuildStore(builder, val, scalar_ptr);
 273       }
 274    }
 275 }
 276
 277 static void emit_load_var(struct lp_build_nir_context *bld_base,
 278                            nir_variable_mode deref_mode,
 279                            unsigned num_components,
 280                            unsigned bit_size,
 281                            nir_variable *var,
 282                            unsigned vertex_index,
 283                            unsigned const_index,
 284                            LLVMValueRef indir_index,
 285                            LLVMValueRef result[4])
 286 {
 287    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 288    struct gallivm_state *gallivm = bld_base->base.gallivm;
 289    int dmul = bit_size == 64 ? 2 : 1;
 290    switch (deref_mode) {
 291    case nir_var_shader_in: {
 292       for (unsigned i = 0; i < num_components; i++) {
 293          int idx = (i * dmul) + var->data.location_frac;
 294          if (bld->gs_iface) {
 295             LLVMValueRef vertex_index_val = lp_build_const_int32(gallivm, vertex_index);
 296             LLVMValueRef attrib_index_val = lp_build_const_int32(gallivm, const_index + var->data.driver_location);
 297             LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx);
 298             LLVMValueRef result2;
 299             result[i] = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
 300                                                    false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
 301             if (bit_size == 64) {
 302                LLVMValueRef swizzle_index_val = lp_build_const_int32(gallivm, idx + 1);
 303                result2 = bld->gs_iface->fetch_input(bld->gs_iface, &bld_base->base,
 304                                                     false, vertex_index_val, 0, attrib_index_val, swizzle_index_val);
 305                result[i] = emit_fetch_64bit(bld_base, result[i], result2);
 306             }
 307          } else {
 308             if (indir_index) {
 309                LLVMValueRef attrib_index_val = lp_build_add(&bld_base->uint_bld, indir_index, lp_build_const_int_vec(gallivm, bld_base->uint_bld.type, var->data.driver_location));
 310                LLVMValueRef index_vec = get_soa_array_offsets(&bld_base->uint_bld,
 311                                                               attrib_index_val, 4, idx,
 312                                                               TRUE);
 313                LLVMValueRef index_vec2 = NULL;
 314                LLVMTypeRef fptr_type;
 315                LLVMValueRef inputs_array;
 316                fptr_type = LLVMPointerType(LLVMFloatTypeInContext(gallivm->context), 0);
 317                inputs_array = LLVMBuildBitCast(gallivm->builder, bld->inputs_array, fptr_type, "");
 318
 319                if (bit_size == 64)
 320                   index_vec2 = get_soa_array_offsets(&bld_base->uint_bld,
 321                                                      indir_index, 4, idx + 1, TRUE);
 322
 323                /* Gather values from the input register array */
 324                result[i] = build_gather(bld_base, &bld_base->base, inputs_array, index_vec, NULL, index_vec2);
 325             } else {
 326                if (bld->indirects & nir_var_shader_in) {
 327                   LLVMValueRef lindex = lp_build_const_int32(gallivm,
 328                                                              var->data.driver_location * 4 + idx);
 329                   LLVMValueRef input_ptr = lp_build_pointer_get(gallivm->builder,
 330                                                              bld->inputs_array, lindex);
 331                   if (bit_size == 64) {
 332                      LLVMValueRef lindex2 = lp_build_const_int32(gallivm,
 333                                                                  var->data.driver_location * 4 + (idx + 1));
 334                      LLVMValueRef input_ptr2 = lp_build_pointer_get(gallivm->builder,
 335                                                                     bld->inputs_array, lindex2);
 336                      result[i] = emit_fetch_64bit(bld_base, input_ptr, input_ptr2);
 337                   } else {
 338                      result[i] = input_ptr;
 339                   }
 340                } else {
 341                   if (bit_size == 64) {
 342                      LLVMValueRef tmp[2];
 343                      tmp[0] = bld->inputs[var->data.driver_location + const_index][idx];
 344                      tmp[1] = bld->inputs[var->data.driver_location + const_index][idx + 1];
 345                      result[i] = emit_fetch_64bit(bld_base, tmp[0], tmp[1]);
 346                   } else {
 347                      result[i] = bld->inputs[var->data.driver_location + const_index][idx];
 348                   }
 349                }
 350             }
 351          }
 352       }
 353    }
 354    default:
 355       break;
 356    }
 357 }
 358
 359 static void emit_store_chan(struct lp_build_nir_context *bld_base,
 360                             nir_variable_mode deref_mode,
 361                             unsigned bit_size,
 362                             unsigned location, unsigned comp,
 363                             unsigned chan,
 364                             LLVMValueRef dst)
 365 {
 366    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 367    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 368    struct lp_build_context *float_bld = &bld_base->base;
 369
 370    if (bit_size == 64) {
 371       chan *= 2;
 372       chan += comp;
 373       if (chan >= 4) {
 374          chan -= 4;
 375          location++;
 376       }
 377       emit_store_64bit_chan(bld_base, bld->outputs[location][chan],
 378                             bld->outputs[location][chan + 1], dst);
 379    } else {
 380       dst = LLVMBuildBitCast(builder, dst, float_bld->vec_type, "");
 381       lp_exec_mask_store(&bld->exec_mask, float_bld, dst,
 382                          bld->outputs[location][chan + comp]);
 383    }
 384 }
 385
 386 static void emit_store_var(struct lp_build_nir_context *bld_base,
 387                            nir_variable_mode deref_mode,
 388                            unsigned bit_size,
 389                            unsigned num_components,
 390                            unsigned writemask,
 391                            unsigned const_index,
 392                            nir_variable *var, LLVMValueRef dst)
 393 {
 394    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 395    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 396    switch (deref_mode) {
 397    case nir_var_shader_out: {
 398       unsigned location = var->data.driver_location + const_index;
 399       unsigned comp = var->data.location_frac;
 400       if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
 401          if (var->data.location == FRAG_RESULT_STENCIL)
 402             comp = 1;
 403          else if (var->data.location == FRAG_RESULT_DEPTH)
 404             comp = 2;
 405       }
 406       for (unsigned chan = 0; chan < num_components; chan++) {
 407          if (writemask & (1u << chan)) {
 408             LLVMValueRef chan_val = (num_components == 1) ? dst : LLVMBuildExtractValue(builder, dst, chan, "");
 409             emit_store_chan(bld_base, deref_mode, bit_size, location, comp, chan, chan_val);
 410          }
 411       }
 412       break;
 413    }
 414    default:
 415       break;
 416    }
 417 }
 418
 419 static LLVMValueRef emit_load_reg(struct lp_build_nir_context *bld_base,
 420                                   struct lp_build_context *reg_bld,
 421                                   const nir_reg_src *reg,
 422                                   LLVMValueRef indir_src,
 423                                   LLVMValueRef reg_storage)
 424 {
 425    struct gallivm_state *gallivm = bld_base->base.gallivm;
 426    LLVMBuilderRef builder = gallivm->builder;
 427    int nc = reg->reg->num_components;
 428    LLVMValueRef vals[4];
 429    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 430    if (reg->reg->num_array_elems) {
 431       LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
 432       if (reg->indirect) {
 433          LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
 434          indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
 435          indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
 436       }
 437       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
 438       for (unsigned i = 0; i < nc; i++) {
 439          LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
 440          vals[i] = build_gather(bld_base, reg_bld, reg_storage, indirect_offset, NULL, NULL);
 441       }
 442    } else {
 443       for (unsigned i = 0; i < nc; i++) {
 444          LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
 445                                                                                     lp_build_const_int32(gallivm, i));
 446          vals[i] = LLVMBuildLoad(builder, this_storage, "");
 447       }
 448    }
 449    return nc == 1 ? vals[0] : lp_nir_array_build_gather_values(builder, vals, nc);
 450 }
 451
 452 static void emit_store_reg(struct lp_build_nir_context *bld_base,
 453                            struct lp_build_context *reg_bld,
 454                            const nir_reg_dest *reg,
 455                            unsigned writemask,
 456                            LLVMValueRef indir_src,
 457                            LLVMValueRef reg_storage,
 458                            LLVMValueRef dst[4])
 459 {
 460    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 461    struct gallivm_state *gallivm = bld_base->base.gallivm;
 462    LLVMBuilderRef builder = gallivm->builder;
 463    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 464    int nc = reg->reg->num_components;
 465    if (reg->reg->num_array_elems > 0) {
 466       LLVMValueRef indirect_val = lp_build_const_int_vec(gallivm, uint_bld->type, reg->base_offset);
 467       if (reg->indirect) {
 468          LLVMValueRef max_index = lp_build_const_int_vec(gallivm, uint_bld->type, reg->reg->num_array_elems - 1);
 469          indirect_val = LLVMBuildAdd(builder, indirect_val, indir_src, "");
 470          indirect_val = lp_build_min(uint_bld, indirect_val, max_index);
 471       }
 472       reg_storage = LLVMBuildBitCast(builder, reg_storage, LLVMPointerType(reg_bld->elem_type, 0), "");
 473       for (unsigned i = 0; i < nc; i++) {
 474          if (!(writemask & (1 << i)))
 475             continue;
 476          LLVMValueRef indirect_offset = get_soa_array_offsets(uint_bld, indirect_val, nc, i, TRUE);
 477          dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
 478          emit_mask_scatter(bld, reg_storage, indirect_offset, dst[i], &bld->exec_mask);
 479       }
 480       return;
 481    }
 482
 483    for (unsigned i = 0; i < nc; i++) {
 484       LLVMValueRef this_storage = nc == 1 ? reg_storage : lp_build_array_get_ptr(gallivm, reg_storage,
 485                                                          lp_build_const_int32(gallivm, i));
 486       dst[i] = LLVMBuildBitCast(builder, dst[i], reg_bld->vec_type, "");
 487       lp_exec_mask_store(&bld->exec_mask, reg_bld, dst[i], this_storage);
 488    }
 489 }
 490
 491 static void emit_load_kernel_arg(struct lp_build_nir_context *bld_base,
 492                                  unsigned nc,
 493                                  unsigned bit_size,
 494                                  unsigned offset_bit_size,
 495                                  bool offset_is_uniform,
 496                                  LLVMValueRef offset,
 497                                  LLVMValueRef result[4])
 498 {
 499    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 500    struct gallivm_state *gallivm = bld_base->base.gallivm;
 501    LLVMBuilderRef builder = gallivm->builder;
 502    struct lp_build_context *bld_broad = get_int_bld(bld_base, true, bit_size);
 503    LLVMValueRef kernel_args_ptr = bld->kernel_args_ptr;
 504    unsigned size_shift = 0;
 505    struct lp_build_context *bld_offset = get_int_bld(bld_base, true, offset_bit_size);
 506    if (bit_size == 16)
 507       size_shift = 1;
 508    else if (bit_size == 32)
 509       size_shift = 2;
 510    else if (bit_size == 64)
 511       size_shift = 3;
 512    if (size_shift)
 513       offset = lp_build_shr(bld_offset, offset, lp_build_const_int_vec(gallivm, bld_offset->type, size_shift));
 514
 515    LLVMTypeRef ptr_type = LLVMPointerType(bld_broad->elem_type, 0);
 516    kernel_args_ptr = LLVMBuildBitCast(builder, kernel_args_ptr, ptr_type, "");
 517
 518    if (offset_is_uniform) {
 519       offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
 520
 521       for (unsigned c = 0; c < nc; c++) {
 522          LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, offset_bit_size == 64 ? lp_build_const_int64(gallivm, c) : lp_build_const_int32(gallivm, c), "");
 523
 524          LLVMValueRef scalar = lp_build_pointer_get(builder, kernel_args_ptr, this_offset);
 525          result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
 526       }
 527    }
 528 }
 529
 530 static void emit_load_ubo(struct lp_build_nir_context *bld_base,
 531                           unsigned nc,
 532                           unsigned bit_size,
 533                           bool offset_is_uniform,
 534                           LLVMValueRef index,
 535                           LLVMValueRef offset,
 536                           LLVMValueRef result[4])
 537 {
 538    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 539    struct gallivm_state *gallivm = bld_base->base.gallivm;
 540    LLVMBuilderRef builder = gallivm->builder;
 541    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 542    struct lp_build_context *bld_broad = bit_size == 64 ? &bld_base->dbl_bld : &bld_base->base;
 543    LLVMValueRef consts_ptr = lp_build_array_get(gallivm, bld->consts_ptr, index);
 544    unsigned size_shift = 0;
 545    if (bit_size == 32)
 546       size_shift = 2;
 547    else if (bit_size == 64)
 548       size_shift = 3;
 549    if (size_shift)
 550       offset = lp_build_shr(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, size_shift));
 551    if (bit_size == 64) {
 552       LLVMTypeRef dptr_type = LLVMPointerType(bld_base->dbl_bld.elem_type, 0);
 553       consts_ptr = LLVMBuildBitCast(builder, consts_ptr, dptr_type, "");
 554    }
 555
 556    if (offset_is_uniform) {
 557       offset = LLVMBuildExtractElement(builder, offset, lp_build_const_int32(gallivm, 0), "");
 558
 559       for (unsigned c = 0; c < nc; c++) {
 560          LLVMValueRef this_offset = LLVMBuildAdd(builder, offset, lp_build_const_int32(gallivm, c), "");
 561
 562          LLVMValueRef scalar = lp_build_pointer_get(builder, consts_ptr, this_offset);
 563          result[c] = lp_build_broadcast_scalar(bld_broad, scalar);
 564       }
 565    } else {
 566       LLVMValueRef overflow_mask;
 567       LLVMValueRef num_consts = lp_build_array_get(gallivm, bld->const_sizes_ptr, index);
 568
 569       num_consts = LLVMBuildShl(gallivm->builder, num_consts, lp_build_const_int32(gallivm, 4), "");
 570       num_consts = lp_build_broadcast_scalar(uint_bld, num_consts);
 571       for (unsigned c = 0; c < nc; c++) {
 572          LLVMValueRef this_offset = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
 573          overflow_mask = lp_build_compare(gallivm, uint_bld->type, PIPE_FUNC_GEQUAL,
 574                                           this_offset, num_consts);
 575
 576          result[c] = build_gather(bld_base, bld_broad, consts_ptr, this_offset, overflow_mask, NULL);
 577       }
 578    }
 579 }
 580
 581
 582 static void emit_load_mem(struct lp_build_nir_context *bld_base,
 583                           unsigned nc,
 584                           unsigned bit_size,
 585                           LLVMValueRef index,
 586                           LLVMValueRef offset,
 587                           LLVMValueRef outval[4])
 588 {
 589    struct gallivm_state *gallivm = bld_base->base.gallivm;
 590    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 591    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 592    LLVMValueRef ssbo_ptr = NULL;
 593    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 594    struct lp_build_context *uint64_bld = &bld_base->uint64_bld;
 595    LLVMValueRef ssbo_limit = NULL;
 596
 597    if (index) {
 598       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 599       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
 600       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
 601
 602       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 603    } else
 604       ssbo_ptr = bld->shared_ptr;
 605
 606    offset = LLVMBuildAShr(gallivm->builder, offset, lp_build_const_int_vec(gallivm, uint_bld->type, bit_size == 64 ? 3 : 2), "");
 607    for (unsigned c = 0; c < nc; c++) {
 608       LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
 609       LLVMValueRef exec_mask = mask_vec(bld_base);
 610
 611       if (ssbo_limit) {
 612          LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
 613          exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
 614       }
 615
 616       LLVMValueRef result = lp_build_alloca(gallivm, bit_size == 64 ? uint64_bld->vec_type : uint_bld->vec_type, "");
 617       struct lp_build_loop_state loop_state;
 618       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 619
 620       struct lp_build_if_state ifthen;
 621       LLVMValueRef cond, temp_res;
 622
 623       loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
 624                                            loop_state.counter, "");
 625
 626       cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
 627       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
 628
 629       lp_build_if(&ifthen, gallivm, cond);
 630       LLVMValueRef scalar;
 631       if (bit_size == 64) {
 632          LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(uint64_bld->elem_type, 0), "");
 633          scalar = lp_build_pointer_get(builder, ssbo_ptr2, loop_index);
 634       } else
 635          scalar = lp_build_pointer_get(builder, ssbo_ptr, loop_index);
 636
 637       temp_res = LLVMBuildLoad(builder, result, "");
 638       temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
 639       LLVMBuildStore(builder, temp_res, result);
 640       lp_build_else(&ifthen);
 641       temp_res = LLVMBuildLoad(builder, result, "");
 642       LLVMValueRef zero;
 643       if (bit_size == 64)
 644          zero = LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 0, 0);
 645       else
 646          zero = lp_build_const_int32(gallivm, 0);
 647       temp_res = LLVMBuildInsertElement(builder, temp_res, zero, loop_state.counter, "");
 648       LLVMBuildStore(builder, temp_res, result);
 649       lp_build_endif(&ifthen);
 650       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 651                                 NULL, LLVMIntUGE);
 652       outval[c] = LLVMBuildLoad(gallivm->builder, result, "");
 653    }
 654 }
 655
 656 static void emit_store_mem(struct lp_build_nir_context *bld_base,
 657                            unsigned writemask,
 658                            unsigned nc,
 659                            unsigned bit_size,
 660                            LLVMValueRef index,
 661                            LLVMValueRef offset,
 662                            LLVMValueRef dst)
 663 {
 664    struct gallivm_state *gallivm = bld_base->base.gallivm;
 665    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 666    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 667    LLVMValueRef ssbo_ptr;
 668    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 669    LLVMValueRef ssbo_limit = NULL;
 670
 671    if (index) {
 672       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 673       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, bit_size == 64 ? 3 : 2), "");
 674       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
 675       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 676    } else
 677       ssbo_ptr = bld->shared_ptr;
 678
 679    offset = lp_build_shr_imm(uint_bld, offset, bit_size == 64 ? 3 : 2);
 680    for (unsigned c = 0; c < nc; c++) {
 681       if (!(writemask & (1u << c)))
 682          continue;
 683       LLVMValueRef loop_index = lp_build_add(uint_bld, offset, lp_build_const_int_vec(gallivm, uint_bld->type, c));
 684       LLVMValueRef val = (nc == 1) ? dst : LLVMBuildExtractValue(builder, dst, c, "");
 685
 686       LLVMValueRef exec_mask = mask_vec(bld_base);
 687       if (ssbo_limit) {
 688          LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, loop_index, ssbo_limit);
 689          exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
 690       }
 691
 692       struct lp_build_loop_state loop_state;
 693       lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 694       LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
 695                                                        loop_state.counter, "");
 696       if (bit_size == 64)
 697          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, bld_base->uint64_bld.elem_type, "");
 698       else
 699          value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
 700       struct lp_build_if_state ifthen;
 701       LLVMValueRef cond;
 702
 703       loop_index = LLVMBuildExtractElement(gallivm->builder, loop_index,
 704                                            loop_state.counter, "");
 705       cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
 706       cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
 707       lp_build_if(&ifthen, gallivm, cond);
 708       if (bit_size == 64) {
 709          LLVMValueRef ssbo_ptr2 = LLVMBuildBitCast(builder, ssbo_ptr, LLVMPointerType(bld_base->uint64_bld.elem_type, 0), "");
 710          lp_build_pointer_set(builder, ssbo_ptr2, loop_index, value_ptr);
 711       } else
 712          lp_build_pointer_set(builder, ssbo_ptr, loop_index, value_ptr);
 713       lp_build_endif(&ifthen);
 714       lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 715                              NULL, LLVMIntUGE);
 716    }
 717 }
 718
 719 static void emit_atomic_mem(struct lp_build_nir_context *bld_base,
 720                             nir_intrinsic_op nir_op,
 721                             LLVMValueRef index, LLVMValueRef offset,
 722                             LLVMValueRef val, LLVMValueRef val2,
 723                             LLVMValueRef *result)
 724 {
 725    struct gallivm_state *gallivm = bld_base->base.gallivm;
 726    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 727    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 728    LLVMValueRef ssbo_ptr;
 729    struct lp_build_context *uint_bld = &bld_base->uint_bld;
 730    LLVMAtomicRMWBinOp op;
 731    LLVMValueRef ssbo_limit = NULL;
 732
 733    if (index) {
 734       LLVMValueRef ssbo_size_ptr = lp_build_array_get(gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 735       ssbo_limit = LLVMBuildAShr(gallivm->builder, ssbo_size_ptr, lp_build_const_int32(gallivm, 2), "");
 736       ssbo_limit = lp_build_broadcast_scalar(uint_bld, ssbo_limit);
 737       ssbo_ptr = lp_build_array_get(gallivm, bld->ssbo_ptr, LLVMBuildExtractElement(builder, index, lp_build_const_int32(gallivm, 0), ""));
 738    } else
 739       ssbo_ptr = bld->shared_ptr;
 740
 741    switch (nir_op) {
 742    case nir_intrinsic_shared_atomic_add:
 743    case nir_intrinsic_ssbo_atomic_add:
 744       op = LLVMAtomicRMWBinOpAdd;
 745       break;
 746    case nir_intrinsic_shared_atomic_exchange:
 747    case nir_intrinsic_ssbo_atomic_exchange:
 748       op = LLVMAtomicRMWBinOpXchg;
 749       break;
 750    case nir_intrinsic_shared_atomic_and:
 751    case nir_intrinsic_ssbo_atomic_and:
 752       op = LLVMAtomicRMWBinOpAnd;
 753       break;
 754    case nir_intrinsic_shared_atomic_or:
 755    case nir_intrinsic_ssbo_atomic_or:
 756       op = LLVMAtomicRMWBinOpOr;
 757       break;
 758    case nir_intrinsic_shared_atomic_xor:
 759    case nir_intrinsic_ssbo_atomic_xor:
 760       op = LLVMAtomicRMWBinOpXor;
 761       break;
 762    case nir_intrinsic_shared_atomic_umin:
 763    case nir_intrinsic_ssbo_atomic_umin:
 764       op = LLVMAtomicRMWBinOpUMin;
 765       break;
 766    case nir_intrinsic_shared_atomic_umax:
 767    case nir_intrinsic_ssbo_atomic_umax:
 768       op = LLVMAtomicRMWBinOpUMax;
 769       break;
 770    case nir_intrinsic_ssbo_atomic_imin:
 771    case nir_intrinsic_shared_atomic_imin:
 772       op = LLVMAtomicRMWBinOpMin;
 773       break;
 774    case nir_intrinsic_ssbo_atomic_imax:
 775    case nir_intrinsic_shared_atomic_imax:
 776       op = LLVMAtomicRMWBinOpMax;
 777       break;
 778    default:
 779       break;
 780    }
 781
 782    offset = lp_build_shr_imm(uint_bld, offset, 2);
 783    LLVMValueRef atom_res = lp_build_alloca(gallivm,
 784                                            uint_bld->vec_type, "");
 785
 786    LLVMValueRef exec_mask = mask_vec(bld_base);
 787    if (ssbo_limit) {
 788       LLVMValueRef ssbo_oob_cmp = lp_build_cmp(uint_bld, PIPE_FUNC_LESS, offset, ssbo_limit);
 789       exec_mask = LLVMBuildAnd(builder, exec_mask, ssbo_oob_cmp, "");
 790    }
 791
 792    struct lp_build_loop_state loop_state;
 793    lp_build_loop_begin(&loop_state, gallivm, lp_build_const_int32(gallivm, 0));
 794
 795    LLVMValueRef value_ptr = LLVMBuildExtractElement(gallivm->builder, val,
 796                                                     loop_state.counter, "");
 797    value_ptr = LLVMBuildBitCast(gallivm->builder, value_ptr, uint_bld->elem_type, "");
 798
 799    offset = LLVMBuildExtractElement(gallivm->builder, offset,
 800                                    loop_state.counter, "");
 801
 802    LLVMValueRef scalar_ptr = LLVMBuildGEP(builder, ssbo_ptr,
 803                                           &offset, 1, "");
 804
 805    struct lp_build_if_state ifthen;
 806    LLVMValueRef cond, temp_res;
 807    LLVMValueRef scalar;
 808    cond = LLVMBuildICmp(gallivm->builder, LLVMIntNE, exec_mask, uint_bld->zero, "");
 809    cond = LLVMBuildExtractElement(gallivm->builder, cond, loop_state.counter, "");
 810    lp_build_if(&ifthen, gallivm, cond);
 811
 812    if (nir_op == nir_intrinsic_ssbo_atomic_comp_swap || nir_op == nir_intrinsic_shared_atomic_comp_swap) {
 813       LLVMValueRef cas_src_ptr = LLVMBuildExtractElement(gallivm->builder, val2,
 814                                                          loop_state.counter, "");
 815       cas_src_ptr = LLVMBuildBitCast(gallivm->builder, cas_src_ptr, uint_bld->elem_type, "");
 816       scalar = LLVMBuildAtomicCmpXchg(builder, scalar_ptr, value_ptr,
 817                                       cas_src_ptr,
 818                                       LLVMAtomicOrderingSequentiallyConsistent,
 819                                       LLVMAtomicOrderingSequentiallyConsistent,
 820                                       false);
 821       scalar = LLVMBuildExtractValue(gallivm->builder, scalar, 0, "");
 822    } else {
 823       scalar = LLVMBuildAtomicRMW(builder, op,
 824                                   scalar_ptr, value_ptr,
 825                                   LLVMAtomicOrderingSequentiallyConsistent,
 826                                   false);
 827    }
 828    temp_res = LLVMBuildLoad(builder, atom_res, "");
 829    temp_res = LLVMBuildInsertElement(builder, temp_res, scalar, loop_state.counter, "");
 830    LLVMBuildStore(builder, temp_res, atom_res);
 831    lp_build_else(&ifthen);
 832    temp_res = LLVMBuildLoad(builder, atom_res, "");
 833    temp_res = LLVMBuildInsertElement(builder, temp_res, lp_build_const_int32(gallivm, 0), loop_state.counter, "");
 834    LLVMBuildStore(builder, temp_res, atom_res);
 835    lp_build_endif(&ifthen);
 836
 837    lp_build_loop_end_cond(&loop_state, lp_build_const_int32(gallivm, uint_bld->type.length),
 838                           NULL, LLVMIntUGE);
 839    *result = LLVMBuildLoad(builder, atom_res, "");
 840 }
 841
 842 static void emit_barrier(struct lp_build_nir_context *bld_base)
 843 {
 844    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 845    struct gallivm_state * gallivm = bld_base->base.gallivm;
 846
 847    LLVMBasicBlockRef resume = lp_build_insert_new_block(gallivm, "resume");
 848
 849    lp_build_coro_suspend_switch(gallivm, bld->coro, resume, false);
 850    LLVMPositionBuilderAtEnd(gallivm->builder, resume);
 851 }
 852
 853 static LLVMValueRef emit_get_buffer_size(struct lp_build_nir_context *bld_base,
 854                                          LLVMValueRef index)
 855 {
 856    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 857    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
 858    struct lp_build_context *bld_broad = &bld_base->uint_bld;
 859    LLVMValueRef size_ptr = lp_build_array_get(bld_base->base.gallivm, bld->ssbo_sizes_ptr, LLVMBuildExtractElement(builder, index, bld_broad->zero, ""));
 860    return lp_build_broadcast_scalar(bld_broad, size_ptr);
 861 }
 862
 863 static void emit_image_op(struct lp_build_nir_context *bld_base,
 864                           struct lp_img_params *params)
 865 {
 866    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 867    params->type = bld_base->base.type;
 868    params->context_ptr = bld->context_ptr;
 869    params->thread_data_ptr = bld->thread_data_ptr;
 870    params->exec_mask = mask_vec(bld_base);
 871    bld->image->emit_op(bld->image,
 872                        bld->bld_base.base.gallivm,
 873                        params);
 874
 875 }
 876
 877 static void emit_image_size(struct lp_build_nir_context *bld_base,
 878                             struct lp_sampler_size_query_params *params)
 879 {
 880    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 881
 882    params->int_type = bld_base->int_bld.type;
 883    params->context_ptr = bld->context_ptr;
 884
 885    bld->image->emit_size_query(bld->image,
 886                                bld->bld_base.base.gallivm,
 887                                params);
 888
 889 }
 890
 891 static void init_var_slots(struct lp_build_nir_context *bld_base,
 892                            nir_variable *var, unsigned sc)
 893 {
 894    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 895    unsigned slots = glsl_count_attribute_slots(var->type, false) * 4;
 896
 897    for (unsigned comp = sc; comp < slots + sc; comp++) {
 898       unsigned this_loc = var->data.driver_location + (comp / 4);
 899       unsigned this_chan = comp % 4;
 900
 901       if (!bld->outputs[this_loc][this_chan])
 902          bld->outputs[this_loc][this_chan] = lp_build_alloca(bld_base->base.gallivm,
 903                                                              bld_base->base.vec_type, "output");
 904    }
 905 }
 906
 907 static void emit_var_decl(struct lp_build_nir_context *bld_base,
 908                           nir_variable *var)
 909 {
 910    unsigned sc = var->data.location_frac;
 911    switch (var->data.mode) {
 912    case nir_var_shader_out: {
 913       if (bld_base->shader->info.stage == MESA_SHADER_FRAGMENT) {
 914          if (var->data.location == FRAG_RESULT_STENCIL)
 915             sc = 1;
 916          else if (var->data.location == FRAG_RESULT_DEPTH)
 917             sc = 2;
 918       }
 919       init_var_slots(bld_base, var, sc);
 920       break;
 921    }
 922    default:
 923       break;
 924    }
 925 }
 926
 927 static void emit_tex(struct lp_build_nir_context *bld_base,
 928                      struct lp_sampler_params *params)
 929 {
 930    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 931
 932    params->type = bld_base->base.type;
 933    params->context_ptr = bld->context_ptr;
 934    params->thread_data_ptr = bld->thread_data_ptr;
 935
 936    bld->sampler->emit_tex_sample(bld->sampler,
 937                                  bld->bld_base.base.gallivm,
 938                                  params);
 939 }
 940
 941 static void emit_tex_size(struct lp_build_nir_context *bld_base,
 942                           struct lp_sampler_size_query_params *params)
 943 {
 944    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 945
 946    params->int_type = bld_base->int_bld.type;
 947    params->context_ptr = bld->context_ptr;
 948
 949    bld->sampler->emit_size_query(bld->sampler,
 950                                  bld->bld_base.base.gallivm,
 951                                  params);
 952 }
 953
 954 static void emit_sysval_intrin(struct lp_build_nir_context *bld_base,
 955                                nir_intrinsic_instr *instr,
 956                                LLVMValueRef result[4])
 957 {
 958    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
 959    struct gallivm_state *gallivm = bld_base->base.gallivm;
 960    switch (instr->intrinsic) {
 961    case nir_intrinsic_load_instance_id:
 962       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.instance_id);
 963       break;
 964    case nir_intrinsic_load_base_instance:
 965       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.base_instance);
 966       break;
 967    case nir_intrinsic_load_base_vertex:
 968       result[0] = bld->system_values.basevertex;
 969       break;
 970    case nir_intrinsic_load_vertex_id:
 971       result[0] = bld->system_values.vertex_id;
 972       break;
 973    case nir_intrinsic_load_primitive_id:
 974       result[0] = bld->system_values.prim_id;
 975       break;
 976    case nir_intrinsic_load_work_group_id:
 977       for (unsigned i = 0; i < 3; i++)
 978          result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.block_id, lp_build_const_int32(gallivm, i), ""));
 979       break;
 980    case nir_intrinsic_load_local_invocation_id:
 981       for (unsigned i = 0; i < 3; i++)
 982          result[i] = LLVMBuildExtractValue(gallivm->builder, bld->system_values.thread_id, i, "");
 983       break;
 984    case nir_intrinsic_load_num_work_groups:
 985       for (unsigned i = 0; i < 3; i++)
 986          result[i] = lp_build_broadcast_scalar(&bld_base->uint_bld, LLVMBuildExtractElement(gallivm->builder, bld->system_values.grid_size, lp_build_const_int32(gallivm, i), ""));
 987       break;
 988    case nir_intrinsic_load_invocation_id:
 989       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.invocation_id);
 990       break;
 991    case nir_intrinsic_load_front_face:
 992       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.front_facing);
 993       break;
 994    case nir_intrinsic_load_draw_id:
 995       result[0] = lp_build_broadcast_scalar(&bld_base->uint_bld, bld->system_values.draw_id);
 996       break;
 997    default:
 998       break;
 999    }
1000 }
1001
1002 static void bgnloop(struct lp_build_nir_context *bld_base)
1003 {
1004    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1005    lp_exec_bgnloop(&bld->exec_mask, true);
1006 }
1007
1008 static void endloop(struct lp_build_nir_context *bld_base)
1009 {
1010    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1011    lp_exec_endloop(bld_base->base.gallivm, &bld->exec_mask);
1012 }
1013
1014 static void if_cond(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1015 {
1016    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1017    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1018    lp_exec_mask_cond_push(&bld->exec_mask, LLVMBuildBitCast(builder, cond, bld_base->base.int_vec_type, ""));
1019 }
1020
1021 static void else_stmt(struct lp_build_nir_context *bld_base)
1022 {
1023    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1024    lp_exec_mask_cond_invert(&bld->exec_mask);
1025 }
1026
1027 static void endif_stmt(struct lp_build_nir_context *bld_base)
1028 {
1029    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1030    lp_exec_mask_cond_pop(&bld->exec_mask);
1031 }
1032
1033 static void break_stmt(struct lp_build_nir_context *bld_base)
1034 {
1035    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1036
1037    lp_exec_break(&bld->exec_mask, NULL, false);
1038 }
1039
1040 static void continue_stmt(struct lp_build_nir_context *bld_base)
1041 {
1042    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1043    lp_exec_continue(&bld->exec_mask);
1044 }
1045
1046 static void discard(struct lp_build_nir_context *bld_base, LLVMValueRef cond)
1047 {
1048    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1049    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1050    LLVMValueRef mask;
1051
1052    if (!cond) {
1053       if (bld->exec_mask.has_mask) {
1054          mask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1055       } else {
1056          mask = LLVMConstNull(bld->bld_base.base.int_vec_type);
1057       }
1058    } else {
1059       mask = LLVMBuildNot(builder, cond, "");
1060       if (bld->exec_mask.has_mask) {
1061          LLVMValueRef invmask;
1062          invmask = LLVMBuildNot(builder, bld->exec_mask.exec_mask, "kilp");
1063          mask = LLVMBuildOr(builder, mask, invmask, "");
1064       }
1065    }
1066    lp_build_mask_update(bld->mask, mask);
1067 }
1068
1069 static void
1070 increment_vec_ptr_by_mask(struct lp_build_nir_context * bld_base,
1071                           LLVMValueRef ptr,
1072                           LLVMValueRef mask)
1073 {
1074    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1075    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1076
1077    current_vec = LLVMBuildSub(builder, current_vec, mask, "");
1078
1079    LLVMBuildStore(builder, current_vec, ptr);
1080 }
1081
1082 static void
1083 clear_uint_vec_ptr_from_mask(struct lp_build_nir_context * bld_base,
1084                              LLVMValueRef ptr,
1085                              LLVMValueRef mask)
1086 {
1087    LLVMBuilderRef builder = bld_base->base.gallivm->builder;
1088    LLVMValueRef current_vec = LLVMBuildLoad(builder, ptr, "");
1089
1090    current_vec = lp_build_select(&bld_base->uint_bld,
1091                                  mask,
1092                                  bld_base->uint_bld.zero,
1093                                  current_vec);
1094
1095    LLVMBuildStore(builder, current_vec, ptr);
1096 }
1097
1098 static LLVMValueRef
1099 clamp_mask_to_max_output_vertices(struct lp_build_nir_soa_context * bld,
1100                                   LLVMValueRef current_mask_vec,
1101                                   LLVMValueRef total_emitted_vertices_vec)
1102 {
1103    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1104    struct lp_build_context *int_bld = &bld->bld_base.int_bld;
1105    LLVMValueRef max_mask = lp_build_cmp(int_bld, PIPE_FUNC_LESS,
1106                                             total_emitted_vertices_vec,
1107                                             bld->max_output_vertices_vec);
1108
1109    return LLVMBuildAnd(builder, current_mask_vec, max_mask, "");
1110 }
1111
1112 static void emit_vertex(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1113 {
1114    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1115    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1116
1117    assert(bld->gs_iface->emit_vertex);
1118    LLVMValueRef total_emitted_vertices_vec =
1119       LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
1120    LLVMValueRef mask = mask_vec(bld_base);
1121    mask = clamp_mask_to_max_output_vertices(bld, mask,
1122                                             total_emitted_vertices_vec);
1123    bld->gs_iface->emit_vertex(bld->gs_iface, &bld->bld_base.base,
1124                               bld->outputs,
1125                               total_emitted_vertices_vec,
1126                               lp_build_const_int_vec(bld->bld_base.base.gallivm, bld->bld_base.base.type, stream_id));
1127
1128    increment_vec_ptr_by_mask(bld_base, bld->emitted_vertices_vec_ptr,
1129                              mask);
1130    increment_vec_ptr_by_mask(bld_base, bld->total_emitted_vertices_vec_ptr,
1131                              mask);
1132 }
1133
1134 static void
1135 end_primitive_masked(struct lp_build_nir_context * bld_base,
1136                      LLVMValueRef mask)
1137 {
1138    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1139    LLVMBuilderRef builder = bld->bld_base.base.gallivm->builder;
1140
1141    struct lp_build_context *uint_bld = &bld_base->uint_bld;
1142    LLVMValueRef emitted_vertices_vec =
1143       LLVMBuildLoad(builder, bld->emitted_vertices_vec_ptr, "");
1144    LLVMValueRef emitted_prims_vec =
1145       LLVMBuildLoad(builder, bld->emitted_prims_vec_ptr, "");
1146    LLVMValueRef total_emitted_vertices_vec =
1147       LLVMBuildLoad(builder, bld->total_emitted_vertices_vec_ptr, "");
1148
1149    LLVMValueRef emitted_mask = lp_build_cmp(uint_bld,
1150                                             PIPE_FUNC_NOTEQUAL,
1151                                             emitted_vertices_vec,
1152                                             uint_bld->zero);
1153    mask = LLVMBuildAnd(builder, mask, emitted_mask, "");
1154    bld->gs_iface->end_primitive(bld->gs_iface, &bld->bld_base.base,
1155                                 total_emitted_vertices_vec,
1156                                 emitted_vertices_vec, emitted_prims_vec, mask_vec(bld_base));
1157    increment_vec_ptr_by_mask(bld_base, bld->emitted_prims_vec_ptr,
1158                              mask);
1159    clear_uint_vec_ptr_from_mask(bld_base, bld->emitted_vertices_vec_ptr,
1160                                 mask);
1161 }
1162
1163 static void end_primitive(struct lp_build_nir_context *bld_base, uint32_t stream_id)
1164 {
1165    struct lp_build_nir_soa_context *bld = (struct lp_build_nir_soa_context *)bld_base;
1166
1167    assert(bld->gs_iface->end_primitive);
1168
1169    LLVMValueRef mask = mask_vec(bld_base);
1170    end_primitive_masked(bld_base, mask);
1171 }
1172
1173 static void
1174 emit_prologue(struct lp_build_nir_soa_context *bld)
1175 {
1176    struct gallivm_state * gallivm = bld->bld_base.base.gallivm;
1177    if (bld->indirects & nir_var_shader_in && !bld->gs_iface) {
1178       uint32_t num_inputs = util_bitcount64(bld->bld_base.shader->info.inputs_read);
1179       unsigned index, chan;
1180       LLVMTypeRef vec_type = bld->bld_base.base.vec_type;
1181       LLVMValueRef array_size = lp_build_const_int32(gallivm, num_inputs * 4);
1182       bld->inputs_array = lp_build_array_alloca(gallivm,
1183                                                vec_type, array_size,
1184                                                "input_array");
1185
1186       for (index = 0; index < num_inputs; ++index) {
1187          for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
1188             LLVMValueRef lindex =
1189                lp_build_const_int32(gallivm, index * 4 + chan);
1190             LLVMValueRef input_ptr =
1191                LLVMBuildGEP(gallivm->builder, bld->inputs_array,
1192                             &lindex, 1, "");
1193             LLVMValueRef value = bld->inputs[index][chan];
1194             if (value)
1195                LLVMBuildStore(gallivm->builder, value, input_ptr);
1196          }
1197       }
1198    }
1199 }
1200
1201 void lp_build_nir_soa(struct gallivm_state *gallivm,
1202                       struct nir_shader *shader,
1203                       const struct lp_build_tgsi_params *params,
1204                       LLVMValueRef (*outputs)[4])
1205 {
1206    struct lp_build_nir_soa_context bld;
1207    struct lp_type type = params->type;
1208    struct lp_type res_type;
1209
1210    assert(type.length <= LP_MAX_VECTOR_LENGTH);
1211    memset(&res_type, 0, sizeof res_type);
1212    res_type.width = type.width;
1213    res_type.length = type.length;
1214    res_type.sign = 1;
1215
1216    /* Setup build context */
1217    memset(&bld, 0, sizeof bld);
1218    lp_build_context_init(&bld.bld_base.base, gallivm, type);
1219    lp_build_context_init(&bld.bld_base.uint_bld, gallivm, lp_uint_type(type));
1220    lp_build_context_init(&bld.bld_base.int_bld, gallivm, lp_int_type(type));
1221    lp_build_context_init(&bld.elem_bld, gallivm, lp_elem_type(type));
1222    lp_build_context_init(&bld.uint_elem_bld, gallivm, lp_elem_type(lp_uint_type(type)));
1223    {
1224       struct lp_type dbl_type;
1225       dbl_type = type;
1226       dbl_type.width *= 2;
1227       lp_build_context_init(&bld.bld_base.dbl_bld, gallivm, dbl_type);
1228    }
1229    {
1230       struct lp_type uint64_type;
1231       uint64_type = lp_uint_type(type);
1232       uint64_type.width *= 2;
1233       lp_build_context_init(&bld.bld_base.uint64_bld, gallivm, uint64_type);
1234    }
1235    {
1236       struct lp_type int64_type;
1237       int64_type = lp_int_type(type);
1238       int64_type.width *= 2;
1239       lp_build_context_init(&bld.bld_base.int64_bld, gallivm, int64_type);
1240    }
1241    bld.bld_base.load_var = emit_load_var;
1242    bld.bld_base.store_var = emit_store_var;
1243    bld.bld_base.load_reg = emit_load_reg;
1244    bld.bld_base.store_reg = emit_store_reg;
1245    bld.bld_base.emit_var_decl = emit_var_decl;
1246    bld.bld_base.load_ubo = emit_load_ubo;
1247    bld.bld_base.load_kernel_arg = emit_load_kernel_arg;
1248    bld.bld_base.tex = emit_tex;
1249    bld.bld_base.tex_size = emit_tex_size;
1250    bld.bld_base.bgnloop = bgnloop;
1251    bld.bld_base.endloop = endloop;
1252    bld.bld_base.if_cond = if_cond;
1253    bld.bld_base.else_stmt = else_stmt;
1254    bld.bld_base.endif_stmt = endif_stmt;
1255    bld.bld_base.break_stmt = break_stmt;
1256    bld.bld_base.continue_stmt = continue_stmt;
1257    bld.bld_base.sysval_intrin = emit_sysval_intrin;
1258    bld.bld_base.discard = discard;
1259    bld.bld_base.emit_vertex = emit_vertex;
1260    bld.bld_base.end_primitive = end_primitive;
1261    bld.bld_base.load_mem = emit_load_mem;
1262    bld.bld_base.store_mem = emit_store_mem;
1263    bld.bld_base.get_buffer_size = emit_get_buffer_size;
1264    bld.bld_base.atomic_mem = emit_atomic_mem;
1265    bld.bld_base.barrier = emit_barrier;
1266    bld.bld_base.image_op = emit_image_op;
1267    bld.bld_base.image_size = emit_image_size;
1268
1269    bld.mask = params->mask;
1270    bld.inputs = params->inputs;
1271    bld.outputs = outputs;
1272    bld.consts_ptr = params->consts_ptr;
1273    bld.const_sizes_ptr = params->const_sizes_ptr;
1274    bld.ssbo_ptr = params->ssbo_ptr;
1275    bld.ssbo_sizes_ptr = params->ssbo_sizes_ptr;
1276    bld.sampler = params->sampler;
1277 //   bld.bld_base.info = params->info;
1278
1279    bld.context_ptr = params->context_ptr;
1280    bld.thread_data_ptr = params->thread_data_ptr;
1281    bld.image = params->image;
1282    bld.shared_ptr = params->shared_ptr;
1283    bld.coro = params->coro;
1284    bld.kernel_args_ptr = params->kernel_args;
1285    bld.indirects = 0;
1286    if (params->info->indirect_files & (1 << TGSI_FILE_INPUT))
1287       bld.indirects |= nir_var_shader_in;
1288
1289    bld.gs_iface = params->gs_iface;
1290    if (bld.gs_iface) {
1291       struct lp_build_context *uint_bld = &bld.bld_base.uint_bld;
1292
1293       bld.max_output_vertices_vec = lp_build_const_int_vec(gallivm, bld.bld_base.int_bld.type,
1294                                                            shader->info.gs.vertices_out);
1295       bld.emitted_prims_vec_ptr =
1296          lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_prims_ptr");
1297       bld.emitted_vertices_vec_ptr =
1298          lp_build_alloca(gallivm, uint_bld->vec_type, "emitted_vertices_ptr");
1299       bld.total_emitted_vertices_vec_ptr =
1300          lp_build_alloca(gallivm, uint_bld->vec_type, "total_emitted_vertices_ptr");
1301    }
1302    lp_exec_mask_init(&bld.exec_mask, &bld.bld_base.int_bld);
1303
1304    bld.system_values = *params->system_values;
1305
1306    bld.bld_base.shader = shader;
1307
1308    emit_prologue(&bld);
1309    lp_build_nir_llvm(&bld.bld_base, shader);
1310
1311    if (bld.gs_iface) {
1312       LLVMBuilderRef builder = bld.bld_base.base.gallivm->builder;
1313       LLVMValueRef total_emitted_vertices_vec;
1314       LLVMValueRef emitted_prims_vec;
1315       end_primitive_masked(&bld.bld_base, lp_build_mask_value(bld.mask));
1316       total_emitted_vertices_vec =
1317          LLVMBuildLoad(builder, bld.total_emitted_vertices_vec_ptr, "");
1318       emitted_prims_vec =
1319          LLVMBuildLoad(builder, bld.emitted_prims_vec_ptr, "");
1320
1321       bld.gs_iface->gs_epilogue(bld.gs_iface,
1322                                  total_emitted_vertices_vec,
1323                                  emitted_prims_vec);
1324    }
1325    lp_exec_mask_fini(&bld.exec_mask);
1326 }