gallium: fix windows build from params change.
[mesa.git] / src / gallium / auxiliary / draw / draw_llvm.c
index 414f2dc022df9fd23019d9c2077667a7965a9588..d4c89a851461823dbae5be20e9a2a05a9be2b0da 100644 (file)
@@ -234,6 +234,10 @@ create_jit_context_type(struct gallivm_state *gallivm,
                                  PIPE_MAX_SHADER_SAMPLER_VIEWS); /* textures */
    elem_types[5] = LLVMArrayType(sampler_type,
                                  PIPE_MAX_SAMPLERS); /* samplers */
+   elem_types[6] = LLVMArrayType(LLVMPointerType(int_type, 0), /* vs_ssbo */
+                                 LP_MAX_TGSI_SHADER_BUFFERS);
+   elem_types[7] = LLVMArrayType(int_type, /* num_vs_ssbos */
+                                 LP_MAX_TGSI_SHADER_BUFFERS);
    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
                                           ARRAY_SIZE(elem_types), 0);
 
@@ -252,6 +256,10 @@ create_jit_context_type(struct gallivm_state *gallivm,
    LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, samplers,
                           target, context_type,
                           DRAW_JIT_CTX_SAMPLERS);
+   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, vs_ssbos,
+                          target, context_type, DRAW_JIT_CTX_SSBOS);
+   LP_CHECK_MEMBER_OFFSET(struct draw_jit_context, num_vs_ssbos,
+                          target, context_type, DRAW_JIT_CTX_NUM_SSBOS);
    LP_CHECK_STRUCT_SIZE(struct draw_jit_context,
                         target, context_type);
 
@@ -293,6 +301,10 @@ create_gs_jit_context_type(struct gallivm_state *gallivm,
    elem_types[8] = LLVMPointerType(LLVMVectorType(int_type,
                                                   vector_length), 0);
 
+   elem_types[9] = LLVMArrayType(LLVMPointerType(int_type, 0), /* ssbos */
+                                 LP_MAX_TGSI_SHADER_BUFFERS);
+   elem_types[10] = LLVMArrayType(int_type, /* num_ssbos */
+                                 LP_MAX_TGSI_SHADER_BUFFERS);
    context_type = LLVMStructTypeInContext(gallivm->context, elem_types,
                                           ARRAY_SIZE(elem_types), 0);
 
@@ -320,6 +332,10 @@ create_gs_jit_context_type(struct gallivm_state *gallivm,
    LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, emitted_prims,
                           target, context_type,
                           DRAW_GS_JIT_CTX_EMITTED_PRIMS);
+   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, ssbos,
+                          target, context_type, DRAW_GS_JIT_CTX_SSBOS);
+   LP_CHECK_MEMBER_OFFSET(struct draw_gs_jit_context, num_ssbos,
+                          target, context_type, DRAW_GS_JIT_CTX_NUM_SSBOS);
    LP_CHECK_STRUCT_SIZE(struct draw_gs_jit_context,
                         target, context_type);
 
@@ -352,9 +368,9 @@ create_jit_vertex_buffer_type(struct gallivm_state *gallivm,
    LLVMTypeRef elem_types[4];
    LLVMTypeRef vb_type;
 
-   elem_types[0] =
-   elem_types[1] = LLVMInt32TypeInContext(gallivm->context);
-   elem_types[2] =
+   elem_types[0] = LLVMInt16TypeInContext(gallivm->context);
+   elem_types[1] = LLVMInt8TypeInContext(gallivm->context);
+   elem_types[2] = LLVMInt32TypeInContext(gallivm->context);
    elem_types[3] = LLVMPointerType(LLVMInt8TypeInContext(gallivm->context), 0);
 
    vb_type = LLVMStructTypeInContext(gallivm->context, elem_types,
@@ -363,8 +379,12 @@ create_jit_vertex_buffer_type(struct gallivm_state *gallivm,
    (void) target; /* silence unused var warning for non-debug build */
    LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, stride,
                           target, vb_type, 0);
-   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
+   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, is_user_buffer,
                           target, vb_type, 1);
+   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer_offset,
+                          target, vb_type, 2);
+   LP_CHECK_MEMBER_OFFSET(struct pipe_vertex_buffer, buffer.resource,
+                          target, vb_type, 3);
 
    LP_CHECK_STRUCT_SIZE(struct pipe_vertex_buffer, target, vb_type);
 
@@ -383,7 +403,7 @@ create_jit_vertex_header(struct gallivm_state *gallivm, int data_elems)
    LLVMTypeRef vertex_header;
    char struct_name[24];
 
-   util_snprintf(struct_name, 23, "vertex_header%d", data_elems);
+   snprintf(struct_name, 23, "vertex_header%d", data_elems);
 
    elem_types[DRAW_JIT_VERTEX_VERTEX_ID]  = LLVMIntTypeInContext(gallivm->context, 32);
    elem_types[DRAW_JIT_VERTEX_CLIP_POS]  = LLVMArrayType(LLVMFloatTypeInContext(gallivm->context), 4);
@@ -558,8 +578,8 @@ draw_llvm_create_variant(struct draw_llvm *llvm,
    variant->llvm = llvm;
    variant->shader = shader;
 
-   util_snprintf(module_name, sizeof(module_name), "draw_llvm_vs_variant%u",
-                 variant->shader->variants_cached);
+   snprintf(module_name, sizeof(module_name), "draw_llvm_vs_variant%u",
+            variant->shader->variants_cached);
 
    variant->gallivm = gallivm_create(module_name, llvm->context);
 
@@ -602,8 +622,9 @@ generate_vs(struct draw_llvm_variant *variant,
             const LLVMValueRef (*inputs)[TGSI_NUM_CHANNELS],
             const struct lp_bld_tgsi_system_values *system_values,
             LLVMValueRef context_ptr,
-            struct lp_build_sampler_soa *draw_sampler,
-            boolean clamp_vertex_color)
+            const struct lp_build_sampler_soa *draw_sampler,
+            boolean clamp_vertex_color,
+            struct lp_build_mask_context *bld_mask)
 {
    struct draw_llvm *llvm = variant->llvm;
    const struct tgsi_token *tokens = llvm->draw->vs.vertex_shader->state.tokens;
@@ -611,21 +632,30 @@ generate_vs(struct draw_llvm_variant *variant,
       draw_jit_context_vs_constants(variant->gallivm, context_ptr);
    LLVMValueRef num_consts_ptr =
       draw_jit_context_num_vs_constants(variant->gallivm, context_ptr);
+   LLVMValueRef ssbos_ptr =
+      draw_jit_context_vs_ssbos(variant->gallivm, context_ptr);
+   LLVMValueRef num_ssbos_ptr =
+      draw_jit_context_num_vs_ssbos(variant->gallivm, context_ptr);
+
+   struct lp_build_tgsi_params params;
+   memset(&params, 0, sizeof(params));
+
+   params.type = vs_type;
+   params.mask = bld_mask;
+   params.consts_ptr = consts_ptr;
+   params.const_sizes_ptr = num_consts_ptr;
+   params.system_values = system_values;
+   params.inputs = inputs;
+   params.context_ptr = context_ptr;
+   params.sampler = draw_sampler;
+   params.info = &llvm->draw->vs.vertex_shader->info;
+   params.ssbo_ptr = ssbos_ptr;
+   params.ssbo_sizes_ptr = num_ssbos_ptr;
 
    lp_build_tgsi_soa(variant->gallivm,
                      tokens,
-                     vs_type,
-                     NULL /*struct lp_build_mask_context *mask*/,
-                     consts_ptr,
-                     num_consts_ptr,
-                     system_values,
-                     inputs,
-                     outputs,
-                     context_ptr,
-                     NULL,
-                     draw_sampler,
-                     &llvm->draw->vs.vertex_shader->info,
-                     NULL);
+                     &params,
+                     outputs);
 
    {
       LLVMValueRef out;
@@ -669,18 +699,17 @@ fetch_instanced(struct gallivm_state *gallivm,
    LLVMValueRef zero = LLVMConstNull(i32_t);
    LLVMBuilderRef builder = gallivm->builder;
    LLVMValueRef stride, buffer_overflowed, aos, index_valid;
-   LLVMValueRef ofbit = NULL;
    unsigned i;
 
    aosf_t = lp_build_vec_type(gallivm, lp_float32_vec4_type());
    aosi_t = lp_build_vec_type(gallivm, lp_int32_vec4_type());
 
-   stride = lp_build_umul_overflow(gallivm, vb_stride, index, &ofbit);
+   /* This mul can overflow. Wraparound is ok. */
+   stride = LLVMBuildMul(builder, vb_stride, index, "");
 
    buffer_overflowed = LLVMBuildICmp(builder, LLVMIntUGE,
                                      stride, buffer_size_adj,
                                      "buffer_overflowed");
-   buffer_overflowed = LLVMBuildOr(builder, buffer_overflowed, ofbit, "");
 
    if (0) {
       lp_build_print_value(gallivm, "   instance index = ", index);
@@ -713,39 +742,6 @@ fetch_instanced(struct gallivm_state *gallivm,
 }
 
 
-static void
-convert_to_soa(struct gallivm_state *gallivm,
-               LLVMValueRef src_aos[LP_MAX_VECTOR_WIDTH / 32],
-               LLVMValueRef dst_soa[TGSI_NUM_CHANNELS],
-               const struct lp_type soa_type)
-{
-   unsigned j, k;
-   struct lp_type aos_channel_type = soa_type;
-
-   LLVMValueRef aos_channels[TGSI_NUM_CHANNELS];
-   unsigned pixels_per_channel = soa_type.length / TGSI_NUM_CHANNELS;
-
-   debug_assert(TGSI_NUM_CHANNELS == 4);
-   debug_assert((soa_type.length % TGSI_NUM_CHANNELS) == 0);
-
-   aos_channel_type.length >>= 1;
-
-   for (j = 0; j < TGSI_NUM_CHANNELS; ++j) {
-      LLVMValueRef channel[LP_MAX_VECTOR_LENGTH] = { 0 };
-
-      assert(pixels_per_channel <= LP_MAX_VECTOR_LENGTH);
-
-      for (k = 0; k < pixels_per_channel; ++k) {
-         channel[k] = src_aos[j + TGSI_NUM_CHANNELS * k];
-      }
-
-      aos_channels[j] = lp_build_concat(gallivm, channel, aos_channel_type, pixels_per_channel);
-   }
-
-   lp_build_transpose_aos(gallivm, soa_type, aos_channels, dst_soa);
-}
-
-
 static void
 fetch_vector(struct gallivm_state *gallivm,
              const struct util_format_description *format_desc,
@@ -756,11 +752,10 @@ fetch_vector(struct gallivm_state *gallivm,
              LLVMValueRef *inputs,
              LLVMValueRef indices)
 {
-   LLVMValueRef zero = LLVMConstNull(LLVMInt32TypeInContext(gallivm->context));
    LLVMBuilderRef builder = gallivm->builder;
    struct lp_build_context blduivec;
-   LLVMValueRef offset, tmp, valid_mask;
-   LLVMValueRef aos_fetch[LP_MAX_VECTOR_WIDTH / 32];
+   struct lp_type fetch_type = vs_type;
+   LLVMValueRef offset, valid_mask;
    unsigned i;
 
    lp_build_context_init(&blduivec, gallivm, lp_uint_type(vs_type));
@@ -768,24 +763,11 @@ fetch_vector(struct gallivm_state *gallivm,
    vb_stride = lp_build_broadcast_scalar(&blduivec, vb_stride);
    buffer_size_adj = lp_build_broadcast_scalar(&blduivec, buffer_size_adj);
 
-   /*
-    * Sort of interestingly, with interleaved attribs, llvm 3.7+ will
-    * recognize these calculations to be constant with different attribs
-    * (the different offset has been added to map_ptr).
-    * llvm 3.3, however, will not (I can't get llvm 3.4-3.6 to link...)
-    *
-    * XXX: could actually avoid this altogether (replacing by simple
-    * non-widening mul) by precalculating the max index instead outside
-    * the loop (at the cost of one scalar udiv per vertex element).
-    */
-   offset = lp_build_mul_32_lohi_cpu(&blduivec, vb_stride, indices, &tmp);
+   /* This mul can overflow. Wraparound is ok. */
+   offset = lp_build_mul(&blduivec, vb_stride, indices);
 
    valid_mask = lp_build_compare(gallivm, blduivec.type,
-                                 PIPE_FUNC_EQUAL, tmp, blduivec.zero);
-
-   tmp = lp_build_compare(gallivm, blduivec.type,
-                          PIPE_FUNC_LESS, offset, buffer_size_adj);
-   valid_mask = LLVMBuildAnd(builder, tmp, valid_mask, "");
+                                 PIPE_FUNC_LESS, offset, buffer_size_adj);
 
    /* not valid elements use offset 0 */
    offset = LLVMBuildAnd(builder, offset, valid_mask, "");
@@ -797,28 +779,37 @@ fetch_vector(struct gallivm_state *gallivm,
    }
 
    /*
-    * Note: we probably really want to use SoA fetch, not AoS one (albeit
-    * for most formats it will amount to the same as this isn't very
-    * optimized). But looks dangerous since it assumes alignment.
+    * Unlike fetch_instanced, use SoA fetch instead of multiple AoS fetches.
+    * This should always produce better code.
     */
-   for (i = 0; i < vs_type.length; i++) {
-      LLVMValueRef offset1, elem;
-      elem = lp_build_const_int32(gallivm, i);
-      offset1 = LLVMBuildExtractElement(builder, offset, elem, "");
 
-      aos_fetch[i] = lp_build_fetch_rgba_aos(gallivm, format_desc,
-                                             lp_float32_vec4_type(),
-                                             FALSE, map_ptr, offset1,
-                                             zero, zero, NULL);
+   /* The type handling is annoying here... */
+   if (format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB &&
+       format_desc->channel[0].pure_integer) {
+      if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_SIGNED) {
+         fetch_type = lp_type_int_vec(vs_type.width, vs_type.width * vs_type.length);
+      }
+      else if (format_desc->channel[0].type == UTIL_FORMAT_TYPE_UNSIGNED) {
+         fetch_type = lp_type_uint_vec(vs_type.width, vs_type.width * vs_type.length);
+      }
    }
-   convert_to_soa(gallivm, aos_fetch, inputs, vs_type);
+
+   lp_build_fetch_rgba_soa(gallivm, format_desc,
+                           fetch_type, FALSE, map_ptr, offset,
+                           blduivec.zero, blduivec.zero,
+                           NULL, inputs);
 
+   for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
+      inputs[i] = LLVMBuildBitCast(builder, inputs[i],
+                                   lp_build_vec_type(gallivm, vs_type), "");
+   }
+
+   /* out-of-bound fetches return all zeros */
    for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
       inputs[i] = LLVMBuildBitCast(builder, inputs[i], blduivec.vec_type, "");
       inputs[i] = LLVMBuildAnd(builder, inputs[i], valid_mask, "");
       inputs[i] = LLVMBuildBitCast(builder, inputs[i],
                                    lp_build_vec_type(gallivm, vs_type), "");
-
    }
 }
 
@@ -1566,10 +1557,10 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
    LLVMBuilderRef builder;
    char func_name[64];
    struct lp_type vs_type;
-   LLVMValueRef count, fetch_elts, start_or_maxelt, start;
+   LLVMValueRef count, fetch_elts, start_or_maxelt;
    LLVMValueRef vertex_id_offset, start_instance;
    LLVMValueRef stride, step, io_itr;
-   LLVMValueRef ind_vec, ind_vec_store, have_elts, fetch_max, tmp;
+   LLVMValueRef ind_vec, start_vec, have_elts, fetch_max, tmp;
    LLVMValueRef io_ptr, vbuffers_ptr, vb_ptr;
    LLVMValueRef vb_stride[PIPE_MAX_ATTRIBS];
    LLVMValueRef map_ptr[PIPE_MAX_ATTRIBS];
@@ -1580,7 +1571,7 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
    struct draw_context *draw = llvm->draw;
    const struct tgsi_shader_info *vs_info = &draw->vs.vertex_shader->info;
    unsigned i, j;
-   struct lp_build_context bld, bldivec, blduivec;
+   struct lp_build_context bld, blduivec;
    struct lp_build_loop_state lp_loop;
    struct lp_build_if_state if_ctx;
    const int vector_length = lp_native_vector_width / 32;
@@ -1609,8 +1600,8 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
 
    memset(&system_values, 0, sizeof(system_values));
 
-   util_snprintf(func_name, sizeof(func_name), "draw_llvm_vs_variant%u",
-                 variant->shader->variants_cached);
+   snprintf(func_name, sizeof(func_name), "draw_llvm_vs_variant%u",
+            variant->shader->variants_cached);
 
    i = 0;
    arg_types[i++] = get_context_ptr_type(variant);       /* context */
@@ -1640,6 +1631,11 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
    io_ptr                    = LLVMGetParam(variant_func, 1);
    vbuffers_ptr              = LLVMGetParam(variant_func, 2);
    count                     = LLVMGetParam(variant_func, 3);
+   /*
+    * XXX: the maxelt part is unused. Not really useful, since we cannot
+    * get index buffer overflows due to vsplit (which provides its own
+    * elts buffer, with a different size than what's passed in here).
+    */
    start_or_maxelt           = LLVMGetParam(variant_func, 4);
    /*
     * XXX: stride is actually unused. The stride we use is strictly calculated
@@ -1682,7 +1678,6 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
    vs_type.length = vector_length;
 
    lp_build_context_init(&bld, gallivm, lp_type_uint(32));
-   lp_build_context_init(&bldivec, gallivm, lp_int_type(vs_type));
    lp_build_context_init(&blduivec, gallivm, lp_uint_type(vs_type));
 
    /* hold temporary "bool" clipmask */
@@ -1705,30 +1700,15 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
       ind_vec = LLVMBuildInsertElement(builder, ind_vec, index, index, "");
    }
 
-   fetch_max = lp_build_alloca(gallivm, int32_type, "fetch_max");
-   ind_vec_store = lp_build_alloca(gallivm, bldivec.vec_type, "ind_vec");
-
    have_elts = LLVMBuildICmp(builder, LLVMIntNE,
                              LLVMConstPointerNull(arg_types[10]), fetch_elts, "");
-   lp_build_if(&if_ctx, gallivm, have_elts);
-   {
-      LLVMBuildStore(builder, ind_vec, ind_vec_store);
-      LLVMBuildStore(builder, count, fetch_max);
-   }
-   lp_build_else(&if_ctx);
-   {
-      tmp = lp_build_add(&bld, count, start_or_maxelt);
-      LLVMBuildStore(builder, tmp, fetch_max);
-      start = lp_build_broadcast_scalar(&bldivec, start_or_maxelt);
-      tmp = lp_build_add(&bldivec, start, ind_vec);
-      LLVMBuildStore(builder, tmp, ind_vec_store);
-   }
-   lp_build_endif(&if_ctx);
 
-   fetch_max = LLVMBuildLoad(builder, fetch_max, "");
-   fetch_max = LLVMBuildSub(builder, fetch_max, bld.one, "fetch_max");
-   fetch_max = lp_build_broadcast_scalar(&bldivec, fetch_max);
-   ind_vec = LLVMBuildLoad(builder, ind_vec_store, "");
+   fetch_max = LLVMBuildSub(builder, count, bld.one, "fetch_max");
+   fetch_max = lp_build_broadcast_scalar(&blduivec, fetch_max);
+   /*
+    * Only needed for non-indexed path.
+    */
+   start_vec = lp_build_broadcast_scalar(&blduivec, start_or_maxelt);
 
    /*
     * Pre-calculate everything which is constant per shader invocation.
@@ -1749,6 +1729,8 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
          vbuffer_ptr = LLVMBuildGEP(builder, vbuffers_ptr, &vb_index, 1, "");
          vb_info = LLVMBuildGEP(builder, vb_ptr, &vb_index, 1, "");
          vb_stride[j] = draw_jit_vbuffer_stride(gallivm, vb_info);
+         vb_stride[j] = LLVMBuildZExt(gallivm->builder, vb_stride[j],
+                                      LLVMInt32TypeInContext(context), "");
          vb_buffer_offset = draw_jit_vbuffer_offset(gallivm, vb_info);
          map_ptr[j] = draw_jit_dvbuffer_map(gallivm, vbuffer_ptr);
          buffer_size = draw_jit_dvbuffer_size(gallivm, vbuffer_ptr);
@@ -1757,9 +1739,12 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
          /*
           * We'll set buffer_size_adj to zero if we have of, so it will
           * always overflow later automatically without having to keep ofbit.
+          * Overflows (with normal wraparound) doing the actual offset
+          * calculation should be ok, just not for the buffer size calc.
+          * It would also be possible to detect such overflows and return
+          * zeros if that happens, but this would be more complex.
           */
-         buf_offset = lp_build_uadd_overflow(gallivm, vb_buffer_offset,
-                                             src_offset, &ofbit);
+         buf_offset = lp_build_add(&bld, vb_buffer_offset, src_offset);
          tmp = lp_build_sub(&bld, bsize, bld.one);
          buffer_size_adj[j] = lp_build_usub_overflow(gallivm, buffer_size, tmp,
                                                      &ofbit);
@@ -1842,22 +1827,19 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
       true_index_array = lp_build_broadcast_scalar(&blduivec, lp_loop.counter);
       true_index_array = LLVMBuildAdd(builder, true_index_array, ind_vec, "");
 
+      LLVMValueRef exec_mask = lp_build_cmp(&blduivec, PIPE_FUNC_LEQUAL, true_index_array, fetch_max);
       /*
-       * XXX: This code is really fishy. We are required to use a int min
-       * here, not uint. The reason is that for some non-indexed draws, we
-       * might get something like MAX_UINT - 3 as start value (due to start
-       * vertex). So, the first 3 elements in the vector are huge, and
-       * limiting them to fetch_max is incorrect. By using int min, we'll
-       * pick that huge value - we rely on this creating an overflow (which
-       * is guaranteed) in the stride mul later (using (signed) cmp and
-       * incorporating the result into ofmask would also work).
-       * For the later elements, this just wraps around the indices, which
-       * is apparently ok...
+       * Limit indices to fetch_max, otherwise might try to access indices
+       * beyond index buffer (or rather vsplit elt buffer) size.
+       * Could probably safely (?) skip this for non-indexed draws and
+       * simplify things minimally (by removing it could combine the ind_vec
+       * and start_vec adds). I think the only effect for non-indexed draws will
+       * be that for the invalid elements they will be all fetched from the
+       * same location as the last valid one, but noone should really care.
        */
-      true_index_array = lp_build_min(&bldivec, true_index_array, fetch_max);
+      true_index_array = lp_build_min(&blduivec, true_index_array, fetch_max);
 
-      index_store = lp_build_alloca_undef(gallivm, bldivec.vec_type, "index_store");
-      LLVMBuildStore(builder, true_index_array, index_store);
+      index_store = lp_build_alloca_undef(gallivm, blduivec.vec_type, "index_store");
 
       lp_build_if(&if_ctx, gallivm, have_elts);
       {
@@ -1875,22 +1857,27 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
           * not being zero will get a different fetch index than the valid
           * index 0. So, just rely on vsplit code preventing out-of-bounds
           * fetches. This is also why it's safe to do elts fetch even if there
-          * was no index buffer bound - the real buffer is never seen here.
+          * was no index buffer bound - the real buffer is never seen here, at
+          * least not if there are index buffer overflows...
           */
 
          /*
           * XXX should not have to do this, as scale can be handled
           * natively by loads (hits asserts though).
           */
-         true_index_array = lp_build_shl_imm(&blduivec, true_index_array, 2);
+         tmp = lp_build_shl_imm(&blduivec, true_index_array, 2);
          fetch_elts = LLVMBuildBitCast(builder, fetch_elts,
                                        LLVMPointerType(LLVMInt8TypeInContext(context),
                                                        0), "");
-         true_index_array = lp_build_gather(gallivm, vs_type.length,
-                                            32, 32, TRUE,
-                                            fetch_elts, true_index_array,
-                                            FALSE);
-         LLVMBuildStore(builder, true_index_array, index_store);
+         tmp = lp_build_gather(gallivm, vs_type.length,
+                               32, bld.type, TRUE,
+                               fetch_elts, tmp, FALSE);
+         LLVMBuildStore(builder, tmp, index_store);
+      }
+      lp_build_else(&if_ctx);
+      {
+         tmp = LLVMBuildAdd(builder, true_index_array, start_vec, "");
+         LLVMBuildStore(builder, tmp, index_store);
       }
       lp_build_endif(&if_ctx);
 
@@ -1920,6 +1907,9 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
          }
       }
 
+      struct lp_build_mask_context mask;
+
+      lp_build_mask_begin(&mask, gallivm, vs_type, exec_mask);
       /* In the paths with elts vertex id has to be unaffected by the
        * index bias and because indices inside our elements array have
        * already had index bias applied we need to subtract it here to
@@ -1946,8 +1936,10 @@ draw_llvm_generate(struct draw_llvm *llvm, struct draw_llvm_variant *variant)
                   &system_values,
                   context_ptr,
                   sampler,
-                  key->clamp_vertex_color);
+                  key->clamp_vertex_color,
+                  &mask);
 
+      lp_build_mask_end(&mask);
       if (pos != -1 && cv != -1) {
          /* store original positions in clip before further manipulation */
          store_clip(gallivm, vs_type, io, outputs, pos);
@@ -2112,7 +2104,7 @@ draw_llvm_dump_variant_key(struct draw_llvm_variant_key *key)
 
 void
 draw_llvm_set_mapped_texture(struct draw_context *draw,
-                             unsigned shader_stage,
+                             enum pipe_shader_type shader_stage,
                              unsigned sview_idx,
                              uint32_t width, uint32_t height, uint32_t depth,
                              uint32_t first_level, uint32_t last_level,
@@ -2157,7 +2149,7 @@ draw_llvm_set_mapped_texture(struct draw_context *draw,
 
 void
 draw_llvm_set_sampler_state(struct draw_context *draw, 
-                            unsigned shader_type)
+                            enum pipe_shader_type shader_type)
 {
    unsigned i;
 
@@ -2196,6 +2188,11 @@ draw_llvm_destroy_variant(struct draw_llvm_variant *variant)
 {
    struct draw_llvm *llvm = variant->llvm;
 
+   if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
+      debug_printf("Deleting VS variant: %u vs variants,\t%u total variants\n",
+                    variant->shader->variants_cached, llvm->nr_variants);
+   }
+
    gallivm_destroy(variant->gallivm);
 
    remove_from_list(&variant->list_item_local);
@@ -2282,6 +2279,7 @@ draw_gs_llvm_generate(struct draw_llvm *llvm,
    struct draw_gs_llvm_iface gs_iface;
    const struct tgsi_token *tokens = variant->shader->base.state.tokens;
    LLVMValueRef consts_ptr, num_consts_ptr;
+   LLVMValueRef ssbos_ptr, num_ssbos_ptr;
    LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
    struct lp_build_mask_context mask;
    const struct tgsi_shader_info *gs_info = &variant->shader->base.info;
@@ -2289,8 +2287,8 @@ draw_gs_llvm_generate(struct draw_llvm *llvm,
 
    memset(&system_values, 0, sizeof(system_values));
 
-   util_snprintf(func_name, sizeof(func_name), "draw_llvm_gs_variant%u",
-                 variant->shader->variants_cached);
+   snprintf(func_name, sizeof(func_name), "draw_llvm_gs_variant%u",
+            variant->shader->variants_cached);
 
    assert(variant->vertex_header_ptr_type);
 
@@ -2363,6 +2361,10 @@ draw_gs_llvm_generate(struct draw_llvm *llvm,
    num_consts_ptr =
       draw_gs_jit_context_num_constants(variant->gallivm, context_ptr);
 
+   ssbos_ptr = draw_gs_jit_context_ssbos(variant->gallivm, context_ptr);
+   num_ssbos_ptr =
+      draw_gs_jit_context_num_ssbos(variant->gallivm, context_ptr);
+
    /* code generated texture sampling */
    sampler = draw_llvm_sampler_soa_create(variant->key.samplers);
 
@@ -2378,20 +2380,25 @@ draw_gs_llvm_generate(struct draw_llvm *llvm,
       draw_gs_llvm_dump_variant_key(&variant->key);
    }
 
+   struct lp_build_tgsi_params params;
+   memset(&params, 0, sizeof(params));
+
+   params.type = gs_type;
+   params.mask = &mask;
+   params.consts_ptr = consts_ptr;
+   params.const_sizes_ptr = num_consts_ptr;
+   params.system_values = &system_values;
+   params.context_ptr = context_ptr;
+   params.sampler = sampler;
+   params.info = &llvm->draw->gs.geometry_shader->info;
+   params.gs_iface = (const struct lp_build_tgsi_gs_iface *)&gs_iface;
+   params.ssbo_ptr = ssbos_ptr;
+   params.ssbo_sizes_ptr = num_ssbos_ptr;
+
    lp_build_tgsi_soa(variant->gallivm,
                      tokens,
-                     gs_type,
-                     &mask,
-                     consts_ptr,
-                     num_consts_ptr,
-                     &system_values,
-                     NULL,
-                     outputs,
-                     context_ptr,
-                     NULL,
-                     sampler,
-                     &llvm->draw->gs.geometry_shader->info,
-                     (const struct lp_build_tgsi_gs_iface *)&gs_iface);
+                     &params,
+                     outputs);
 
    sampler->destroy(sampler);
 
@@ -2423,8 +2430,8 @@ draw_gs_llvm_create_variant(struct draw_llvm *llvm,
    variant->llvm = llvm;
    variant->shader = shader;
 
-   util_snprintf(module_name, sizeof(module_name), "draw_llvm_gs_variant%u",
-                 variant->shader->variants_cached);
+   snprintf(module_name, sizeof(module_name), "draw_llvm_gs_variant%u",
+            variant->shader->variants_cached);
 
    variant->gallivm = gallivm_create(module_name, llvm->context);
 
@@ -2458,6 +2465,11 @@ draw_gs_llvm_destroy_variant(struct draw_gs_llvm_variant *variant)
 {
    struct draw_llvm *llvm = variant->llvm;
 
+   if (gallivm_debug & (GALLIVM_DEBUG_TGSI | GALLIVM_DEBUG_IR)) {
+      debug_printf("Deleting GS variant: %u gs variants,\t%u total variants\n",
+                    variant->shader->variants_cached, llvm->nr_gs_variants);
+   }
+
    gallivm_destroy(variant->gallivm);
 
    remove_from_list(&variant->list_item_local);