X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fauxiliary%2Fdraw%2Fdraw_gs.c;h=2f94eaeda4fa574c7f09eb5cf54a296629ffced5;hb=f01f754ca13373d62f5f4ba5ff76d83aa4eac62b;hp=25c117bafae12ed39fffff697cc8ae36dce03887;hpb=6b35c2b110be0d9ae6a292250fdcbab11a9190a5;p=mesa.git

diff --git a/src/gallium/auxiliary/draw/draw_gs.c b/src/gallium/auxiliary/draw/draw_gs.c
index 25c117bafae..2f94eaeda4f 100644
--- a/src/gallium/auxiliary/draw/draw_gs.c
+++ b/src/gallium/auxiliary/draw/draw_gs.c
@@ -29,6 +29,9 @@
 
 #include "draw_private.h"
 #include "draw_context.h"
+#ifdef HAVE_LLVM
+#include "draw_llvm.h"
+#endif
 
 #include "tgsi/tgsi_parse.h"
 #include "tgsi/tgsi_exec.h"
@@ -42,120 +45,43 @@
 /* fixme: move it from here */
 #define MAX_PRIMITIVES 64
 
-boolean
-draw_gs_init( struct draw_context *draw )
-{
-   draw->gs.tgsi.machine = tgsi_exec_machine_create();
-   if (!draw->gs.tgsi.machine)
-      return FALSE;
-
-   draw->gs.tgsi.machine->Primitives = align_malloc(
-      MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
-   if (!draw->gs.tgsi.machine->Primitives)
-      return FALSE;
-   memset(draw->gs.tgsi.machine->Primitives, 0,
-          MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
-
-   return TRUE;
-}
-
-void draw_gs_destroy( struct draw_context *draw )
-{
-   if (!draw->gs.tgsi.machine)
-      return;
-
-   align_free(draw->gs.tgsi.machine->Primitives);
-
-   tgsi_exec_machine_destroy(draw->gs.tgsi.machine);
-}
-
-struct draw_geometry_shader *
-draw_create_geometry_shader(struct draw_context *draw,
-                            const struct pipe_shader_state *state)
+static INLINE int
+draw_gs_get_input_index(int semantic, int index,
+                        const struct tgsi_shader_info *input_info)
 {
-   struct draw_geometry_shader *gs;
-   unsigned i;
-
-   gs = CALLOC_STRUCT(draw_geometry_shader);
-
-   if (!gs)
-      return NULL;
-
-   gs->draw = draw;
-   gs->state = *state;
-   gs->state.tokens = tgsi_dup_tokens(state->tokens);
-   if (!gs->state.tokens) {
-      FREE(gs);
-      return NULL;
+   int i;
+   const ubyte *input_semantic_names = input_info->output_semantic_name;
+   const ubyte *input_semantic_indices = input_info->output_semantic_index;
+   for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
+      if (input_semantic_names[i] == semantic &&
+          input_semantic_indices[i] == index)
+         return i;
    }
-
-   tgsi_scan_shader(state->tokens, &gs->info);
-
-   /* setup the defaults */
-   gs->input_primitive = PIPE_PRIM_TRIANGLES;
-   gs->output_primitive = PIPE_PRIM_TRIANGLE_STRIP;
-   gs->max_output_vertices = 32;
-
-   for (i = 0; i < gs->info.num_properties; ++i) {
-      if (gs->info.properties[i].name ==
-          TGSI_PROPERTY_GS_INPUT_PRIM)
-         gs->input_primitive = gs->info.properties[i].data[0];
-      else if (gs->info.properties[i].name ==
-               TGSI_PROPERTY_GS_OUTPUT_PRIM)
-         gs->output_primitive = gs->info.properties[i].data[0];
-      else if (gs->info.properties[i].name ==
-               TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES)
-         gs->max_output_vertices = gs->info.properties[i].data[0];
-   }
-
-   gs->machine = draw->gs.tgsi.machine;
-
-   if (gs)
-   {
-      uint i;
-      for (i = 0; i < gs->info.num_outputs; i++) {
-         if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
-             gs->info.output_semantic_index[i] == 0)
-            gs->position_output = i;
-      }
-   }
-
-   return gs;
+   debug_assert(0);
+   return -1;
 }
 
-void draw_bind_geometry_shader(struct draw_context *draw,
-                               struct draw_geometry_shader *dgs)
-{
-   draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
-
-   if (dgs) {
-      draw->gs.geometry_shader = dgs;
-      draw->gs.num_gs_outputs = dgs->info.num_outputs;
-      draw->gs.position_output = dgs->position_output;
-      draw_geometry_shader_prepare(dgs, draw);
-   }
-   else {
-      draw->gs.geometry_shader = NULL;
-      draw->gs.num_gs_outputs = 0;
-   }
-}
-
-void draw_delete_geometry_shader(struct draw_context *draw,
-                                 struct draw_geometry_shader *dgs)
+/**
+ * We execute geometry shaders in the SOA mode, so ideally we want to
+ * flush when the number of currently fetched primitives is equal to
+ * the number of elements in the SOA vector. This ensures that the
+ * throughput is optimized for the given vector instrunction set.
+ */
+static INLINE boolean
+draw_gs_should_flush(struct draw_geometry_shader *shader)
 {
-   FREE(dgs->primitive_lengths);
-   FREE((void*) dgs->state.tokens);
-   FREE(dgs);
+   return (shader->fetched_prim_count == shader->vector_length);
 }
 
 /*#define DEBUG_OUTPUTS 1*/
-static INLINE void
-draw_geometry_fetch_outputs(struct draw_geometry_shader *shader,
-                            unsigned num_primitives,
-                            float (**p_output)[4])
+static void
+tgsi_fetch_gs_outputs(struct draw_geometry_shader *shader,
+                      unsigned num_primitives,
+                      float (**p_output)[4])
 {
    struct tgsi_exec_machine *machine = shader->machine;
    unsigned prim_idx, j, slot;
+   unsigned current_idx = 0;
    float (*output)[4];
 
    output = *p_output;
@@ -168,9 +94,8 @@ draw_geometry_fetch_outputs(struct draw_geometry_shader *shader,
       shader->primitive_lengths[prim_idx +   shader->emitted_primitives] =
          machine->Primitives[prim_idx];
       shader->emitted_vertices += num_verts_per_prim;
-      for (j = 0; j < num_verts_per_prim; j++) {
-         int idx = (prim_idx * num_verts_per_prim + j) *
-                   shader->info.num_outputs;
+      for (j = 0; j < num_verts_per_prim; j++, current_idx++) {
+         int idx = current_idx * shader->info.num_outputs;
 #ifdef DEBUG_OUTPUTS
          debug_printf("%d) Output vert:\n", idx / shader->info.num_outputs);
 #endif
@@ -192,11 +117,11 @@ draw_geometry_fetch_outputs(struct draw_geometry_shader *shader,
       }
    }
    *p_output = output;
-         shader->emitted_primitives += num_primitives;
+   shader->emitted_primitives += num_primitives;
 }
 
 /*#define DEBUG_INPUTS 1*/
-static void draw_fetch_gs_input(struct draw_geometry_shader *shader,
+static void tgsi_fetch_gs_input(struct draw_geometry_shader *shader,
                                 unsigned *indices,
                                 unsigned num_vertices,
                                 unsigned prim_idx)
@@ -228,11 +153,13 @@ static void draw_fetch_gs_input(struct draw_geometry_shader *shader,
             machine->Inputs[idx].xyzw[3].f[prim_idx] =
                (float)shader->in_prim_idx;
          } else {
+            vs_slot = draw_gs_get_input_index(
+                        shader->info.input_semantic_name[slot],
+                        shader->info.input_semantic_index[slot],
+                        shader->input_info);
 #if DEBUG_INPUTS
             debug_printf("\tSlot = %d, vs_slot = %d, idx = %d:\n",
                          slot, vs_slot, idx);
-#endif
-#if 1
             assert(!util_is_inf_or_nan(input[vs_slot][0]));
             assert(!util_is_inf_or_nan(input[vs_slot][1]));
             assert(!util_is_inf_or_nan(input[vs_slot][2]));
@@ -255,14 +182,20 @@ static void draw_fetch_gs_input(struct draw_geometry_shader *shader,
    }
 }
 
-static void gs_flush(struct draw_geometry_shader *shader,
-                     unsigned input_primitives)
+static void tgsi_gs_prepare(struct draw_geometry_shader *shader,
+                            const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                            const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
 {
-   unsigned out_prim_count;
    struct tgsi_exec_machine *machine = shader->machine;
 
-   debug_assert(input_primitives > 0 &&
-                input_primitives < 4);
+   tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
+                                  constants, constants_size);
+}
+
+static unsigned tgsi_gs_run(struct draw_geometry_shader *shader,
+                            unsigned input_primitives)
+{
+   struct tgsi_exec_machine *machine = shader->machine;
 
    tgsi_set_exec_mask(machine,
                       1,
@@ -273,16 +206,195 @@ static void gs_flush(struct draw_geometry_shader *shader,
    /* run interpreter */
    tgsi_exec_machine_run(machine);
 
-   out_prim_count =
+   return
       machine->Temps[TGSI_EXEC_TEMP_PRIMITIVE_I].xyzw[TGSI_EXEC_TEMP_PRIMITIVE_C].u[0];
+}
+
+#ifdef HAVE_LLVM
+
+static void
+llvm_fetch_gs_input(struct draw_geometry_shader *shader,
+                    unsigned *indices,
+                    unsigned num_vertices,
+                    unsigned prim_idx)
+{
+   unsigned slot, vs_slot, i;
+   unsigned input_vertex_stride = shader->input_vertex_stride;
+   const float (*input_ptr)[4];
+   float (*input_data)[6][PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS][TGSI_NUM_CHANNELS] = &shader->gs_input->data;
+
+   shader->llvm_prim_ids[shader->fetched_prim_count] =
+      shader->in_prim_idx;
+
+   input_ptr = shader->input;
+
+   for (i = 0; i < num_vertices; ++i) {
+      const float (*input)[4];
+#if DEBUG_INPUTS
+      debug_printf("%d) vertex index = %d (prim idx = %d)\n",
+                   i, indices[i], prim_idx);
+#endif
+      input = (const float (*)[4])(
+         (const char *)input_ptr + (indices[i] * input_vertex_stride));
+      for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; ++slot) {
+         if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
+            /* skip. we handle system values through gallivm */
+         } else {
+            vs_slot = draw_gs_get_input_index(
+                        shader->info.input_semantic_name[slot],
+                        shader->info.input_semantic_index[slot],
+                        shader->input_info);
+#if DEBUG_INPUTS
+            debug_printf("\tSlot = %d, vs_slot = %d, i = %d:\n",
+                         slot, vs_slot, i);
+            assert(!util_is_inf_or_nan(input[vs_slot][0]));
+            assert(!util_is_inf_or_nan(input[vs_slot][1]));
+            assert(!util_is_inf_or_nan(input[vs_slot][2]));
+            assert(!util_is_inf_or_nan(input[vs_slot][3]));
+#endif
+            (*input_data)[i][slot][0][prim_idx] = input[vs_slot][0];
+            (*input_data)[i][slot][1][prim_idx] = input[vs_slot][1];
+            (*input_data)[i][slot][2][prim_idx] = input[vs_slot][2];
+            (*input_data)[i][slot][3][prim_idx] = input[vs_slot][3];
+#if DEBUG_INPUTS
+            debug_printf("\t\t%f %f %f %f\n",
+                         (*input_data)[i][slot][0][prim_idx],
+                         (*input_data)[i][slot][1][prim_idx],
+                         (*input_data)[i][slot][2][prim_idx],
+                         (*input_data)[i][slot][3][prim_idx]);
+#endif
+            ++vs_slot;
+         }
+      }
+   }
+}
+
+static void
+llvm_fetch_gs_outputs(struct draw_geometry_shader *shader,
+                      unsigned num_primitives,
+                      float (**p_output)[4])
+{
+   int total_verts = 0;
+   int vertex_count = 0;
+   int total_prims = 0;
+   int max_prims_per_invocation = 0;
+   char *output_ptr = (char*)shader->gs_output;
+   int i, j, prim_idx;
+   unsigned next_prim_boundary = shader->primitive_boundary;
+
+   for (i = 0; i < shader->vector_length; ++i) {
+      int prims = shader->llvm_emitted_primitives[i];
+      total_prims += prims;
+      max_prims_per_invocation = MAX2(max_prims_per_invocation, prims);
+   }
+   for (i = 0; i < shader->vector_length; ++i) {
+      total_verts += shader->llvm_emitted_vertices[i];
+   }
+
+   output_ptr += shader->emitted_vertices * shader->vertex_size;
+   for (i = 0; i < shader->vector_length - 1; ++i) {
+      int current_verts = shader->llvm_emitted_vertices[i];
+      int next_verts = shader->llvm_emitted_vertices[i + 1];
+#if 0
+      int j; 
+      for (j = 0; j < current_verts; ++j) {
+         struct vertex_header *vh = (struct vertex_header *)
+            (output_ptr + shader->vertex_size * (i * next_prim_boundary + j));
+         debug_printf("--- %d) [%f, %f, %f, %f]\n", j + vertex_count,
+                      vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
+         
+      }
+#endif
+      debug_assert(current_verts <= shader->max_output_vertices);
+      debug_assert(next_verts <= shader->max_output_vertices);
+      if (next_verts) {
+         memmove(output_ptr + (vertex_count + current_verts) * shader->vertex_size,
+                 output_ptr + ((i + 1) * next_prim_boundary) * shader->vertex_size,
+                 shader->vertex_size * next_verts);
+      }
+      vertex_count += current_verts;
+   }
+
+#if 0
+   {
+      int i;
+      for (i = 0; i < total_verts; ++i) {
+         struct vertex_header *vh = (struct vertex_header *)(output_ptr + shader->vertex_size * i);
+         debug_printf("%d) [%f, %f, %f, %f]\n", i,
+                      vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
+         
+      }
+   }
+#endif
+
+   prim_idx = 0;
+   for (i = 0; i < shader->vector_length; ++i) {
+      int num_prims = shader->llvm_emitted_primitives[i];
+      for (j = 0; j < num_prims; ++j) {
+         int prim_length =
+            shader->llvm_prim_lengths[j][i];
+         shader->primitive_lengths[shader->emitted_primitives + prim_idx] =
+            prim_length;
+         ++prim_idx;
+      }
+   }
+
+   shader->emitted_primitives += total_prims;
+   shader->emitted_vertices += total_verts;
+}
+
+static void
+llvm_gs_prepare(struct draw_geometry_shader *shader,
+                const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
+{
+}
+
+static unsigned
+llvm_gs_run(struct draw_geometry_shader *shader,
+            unsigned input_primitives)
+{
+   unsigned ret;
+   char *input = (char*)shader->gs_output;
+
+   input += (shader->emitted_vertices * shader->vertex_size);
+
+   ret = shader->current_variant->jit_func(
+      shader->jit_context, shader->gs_input->data,
+      (struct vertex_header*)input,
+      input_primitives,
+      shader->draw->instance_id,
+      shader->llvm_prim_ids);
+
+   return ret;
+}
+
+#endif
+
+static void gs_flush(struct draw_geometry_shader *shader)
+{
+   unsigned out_prim_count;
+
+   unsigned input_primitives = shader->fetched_prim_count;
+
+   if (shader->draw->collect_statistics) {
+      shader->draw->statistics.gs_invocations += input_primitives;
+   }
+
+   debug_assert(input_primitives > 0 &&
+                input_primitives <= 4);
+
+   out_prim_count = shader->run(shader, input_primitives);
+   shader->fetch_outputs(shader, out_prim_count,
+                         &shader->tmp_output);
 
 #if 0
    debug_printf("PRIM emitted prims = %d (verts=%d), cur prim count = %d\n",
                 shader->emitted_primitives, shader->emitted_vertices,
                 out_prim_count);
 #endif
-   draw_geometry_fetch_outputs(shader, out_prim_count,
-                               &shader->tmp_output);
+
+   shader->fetched_prim_count = 0;
 }
 
 static void gs_point(struct draw_geometry_shader *shader,
@@ -292,10 +404,13 @@ static void gs_point(struct draw_geometry_shader *shader,
 
    indices[0] = idx;
 
-   draw_fetch_gs_input(shader, indices, 1, 0);
+   shader->fetch_inputs(shader, indices, 1,
+                        shader->fetched_prim_count);
    ++shader->in_prim_idx;
+   ++shader->fetched_prim_count;
 
-   gs_flush(shader, 1);
+   if (draw_gs_should_flush(shader))
+      gs_flush(shader);
 }
 
 static void gs_line(struct draw_geometry_shader *shader,
@@ -306,10 +421,13 @@ static void gs_line(struct draw_geometry_shader *shader,
    indices[0] = i0;
    indices[1] = i1;
 
-   draw_fetch_gs_input(shader, indices, 2, 0);
+   shader->fetch_inputs(shader, indices, 2,
+                        shader->fetched_prim_count);
    ++shader->in_prim_idx;
-
-   gs_flush(shader, 1);
+   ++shader->fetched_prim_count;
+   
+   if (draw_gs_should_flush(shader))   
+      gs_flush(shader);
 }
 
 static void gs_line_adj(struct draw_geometry_shader *shader,
@@ -322,10 +440,13 @@ static void gs_line_adj(struct draw_geometry_shader *shader,
    indices[2] = i2;
    indices[3] = i3;
 
-   draw_fetch_gs_input(shader, indices, 4, 0);
+   shader->fetch_inputs(shader, indices, 4,
+                        shader->fetched_prim_count);
    ++shader->in_prim_idx;
+   ++shader->fetched_prim_count;
 
-   gs_flush(shader, 1);
+   if (draw_gs_should_flush(shader))
+      gs_flush(shader);
 }
 
 static void gs_tri(struct draw_geometry_shader *shader,
@@ -337,10 +458,13 @@ static void gs_tri(struct draw_geometry_shader *shader,
    indices[1] = i1;
    indices[2] = i2;
 
-   draw_fetch_gs_input(shader, indices, 3, 0);
+   shader->fetch_inputs(shader, indices, 3,
+                        shader->fetched_prim_count);
    ++shader->in_prim_idx;
+   ++shader->fetched_prim_count;
 
-   gs_flush(shader, 1);
+   if (draw_gs_should_flush(shader))
+      gs_flush(shader);
 }
 
 static void gs_tri_adj(struct draw_geometry_shader *shader,
@@ -356,10 +480,13 @@ static void gs_tri_adj(struct draw_geometry_shader *shader,
    indices[4] = i4;
    indices[5] = i5;
 
-   draw_fetch_gs_input(shader, indices, 6, 0);
+   shader->fetch_inputs(shader, indices, 6,
+                        shader->fetched_prim_count);
    ++shader->in_prim_idx;
+   ++shader->fetched_prim_count;
 
-   gs_flush(shader, 1);
+   if (draw_gs_should_flush(shader))
+      gs_flush(shader);
 }
 
 #define FUNC         gs_run
@@ -374,37 +501,48 @@ static void gs_tri_adj(struct draw_geometry_shader *shader,
 
 
 /**
- * Execute geometry shader using TGSI interpreter.
+ * Execute geometry shader.
  */
 int draw_geometry_shader_run(struct draw_geometry_shader *shader,
-                             const void *constants[PIPE_MAX_CONSTANT_BUFFERS], 
-                             const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS], 
+                             const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
+                             const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS],
                              const struct draw_vertex_info *input_verts,
                              const struct draw_prim_info *input_prim,
+                             const struct tgsi_shader_info *input_info,
                              struct draw_vertex_info *output_verts,
                              struct draw_prim_info *output_prims )
 {
    const float (*input)[4] = (const float (*)[4])input_verts->verts->data;
    unsigned input_stride = input_verts->vertex_size;
-   unsigned vertex_size = input_verts->vertex_size;
-   struct tgsi_exec_machine *machine = shader->machine;
+   unsigned num_outputs = shader->info.num_outputs;
+   unsigned vertex_size = sizeof(struct vertex_header) + num_outputs * 4 * sizeof(float);
    unsigned num_input_verts = input_prim->linear ?
-                              input_verts->count :
-                              input_prim->count;
+      input_verts->count :
+      input_prim->count;
    unsigned num_in_primitives =
-      MAX2(u_gs_prims_for_vertices(input_prim->prim, num_input_verts),
-           u_gs_prims_for_vertices(shader->input_primitive, num_input_verts));
-   unsigned max_out_prims = u_gs_prims_for_vertices(shader->output_primitive,
-                                                    shader->max_output_vertices)
-                            * num_in_primitives;
-
-   output_verts->vertex_size = input_verts->vertex_size;
-   output_verts->stride = input_verts->vertex_size;
+      align(
+         MAX2(u_decomposed_prims_for_vertices(input_prim->prim,
+                                              num_input_verts),
+              u_decomposed_prims_for_vertices(shader->input_primitive,
+                                              num_input_verts)),
+         shader->vector_length);
+   unsigned max_out_prims =
+      u_decomposed_prims_for_vertices(shader->output_primitive,
+                                      shader->max_output_vertices)
+      * num_in_primitives;
+
+   //Assume at least one primitive
+   max_out_prims = MAX2(max_out_prims, 1);
+
+
+   output_verts->vertex_size = vertex_size;
+   output_verts->stride = output_verts->vertex_size;
+   /* we allocate exactly one extra vertex per primitive to allow the GS to emit
+    * overflown vertices into some area where they won't harm anyone */
    output_verts->verts =
-      (struct vertex_header *)MALLOC(input_verts->vertex_size *
-                                     num_in_primitives *
-                                     shader->max_output_vertices);
-
+      (struct vertex_header *)MALLOC(output_verts->vertex_size *
+                                     max_out_prims *
+                                     shader->primitive_boundary);
 
 #if 0
    debug_printf("%s count = %d (in prims # = %d)\n",
@@ -423,13 +561,42 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
    shader->vertex_size = vertex_size;
    shader->tmp_output = (float (*)[4])output_verts->verts->data;
    shader->in_prim_idx = 0;
+   shader->fetched_prim_count = 0;
    shader->input_vertex_stride = input_stride;
    shader->input = input;
+   shader->input_info = input_info;
    FREE(shader->primitive_lengths);
    shader->primitive_lengths = MALLOC(max_out_prims * sizeof(unsigned));
 
-   tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
-                                  constants, constants_size);
+
+#ifdef HAVE_LLVM
+   if (draw_get_option_use_llvm()) {
+      shader->gs_output = output_verts->verts;
+      if (max_out_prims > shader->max_out_prims) {
+         unsigned i;
+         if (shader->llvm_prim_lengths) {
+            for (i = 0; i < shader->max_out_prims; ++i) {
+               align_free(shader->llvm_prim_lengths[i]);
+            }
+            FREE(shader->llvm_prim_lengths);
+         }
+
+         shader->llvm_prim_lengths = MALLOC(max_out_prims * sizeof(unsigned*));
+         for (i = 0; i < max_out_prims; ++i) {
+            int vector_size = shader->vector_length * sizeof(unsigned);
+            shader->llvm_prim_lengths[i] =
+               align_malloc(vector_size, vector_size);
+         }
+
+         shader->max_out_prims = max_out_prims;
+      }
+      shader->jit_context->prim_lengths = shader->llvm_prim_lengths;
+      shader->jit_context->emitted_vertices = shader->llvm_emitted_vertices;
+      shader->jit_context->emitted_prims = shader->llvm_emitted_primitives;
+   }
+#endif
+
+   shader->prepare(shader, constants, constants_size);
 
    if (input_prim->linear)
       gs_run(shader, input_prim, input_verts,
@@ -438,6 +605,15 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
       gs_run_elts(shader, input_prim, input_verts,
                   output_prims, output_verts);
 
+   /* Flush the remaining primitives. Will happen if
+    * num_input_primitives % 4 != 0
+    */
+   if (shader->fetched_prim_count > 0) {
+      gs_flush(shader);
+   }
+
+   debug_assert(shader->fetched_prim_count == 0);
+
    /* Update prim_info:
     */
    output_prims->linear = TRUE;
@@ -450,6 +626,15 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
    output_prims->primitive_count = shader->emitted_primitives;
    output_verts->count = shader->emitted_vertices;
 
+   if (shader->draw->collect_statistics) {
+      unsigned i;
+      for (i = 0; i < shader->emitted_primitives; ++i) {
+         shader->draw->statistics.gs_primitives +=
+            u_decomposed_prims_for_vertices(shader->output_primitive,
+                                            shader->primitive_lengths[i]);
+      }
+   }
+
 #if 0
    debug_printf("GS finished, prims = %d, verts = %d\n",
                 output_prims->primitive_count,
@@ -468,3 +653,219 @@ void draw_geometry_shader_prepare(struct draw_geometry_shader *shader,
                                     draw->gs.tgsi.sampler);
    }
 }
+
+
+boolean
+draw_gs_init( struct draw_context *draw )
+{
+   draw->gs.tgsi.machine = tgsi_exec_machine_create();
+   if (!draw->gs.tgsi.machine)
+      return FALSE;
+
+   draw->gs.tgsi.machine->Primitives = align_malloc(
+      MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
+   if (!draw->gs.tgsi.machine->Primitives)
+      return FALSE;
+   memset(draw->gs.tgsi.machine->Primitives, 0,
+          MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
+
+   return TRUE;
+}
+
+void draw_gs_destroy( struct draw_context *draw )
+{
+   if (draw->gs.tgsi.machine) {
+      align_free(draw->gs.tgsi.machine->Primitives);
+      tgsi_exec_machine_destroy(draw->gs.tgsi.machine);
+   }
+}
+
+struct draw_geometry_shader *
+draw_create_geometry_shader(struct draw_context *draw,
+                            const struct pipe_shader_state *state)
+{
+#ifdef HAVE_LLVM
+   boolean use_llvm = draw_get_option_use_llvm();
+   struct llvm_geometry_shader *llvm_gs;
+#endif
+   struct draw_geometry_shader *gs;
+   unsigned i;
+
+#ifdef HAVE_LLVM
+   if (use_llvm) {
+      llvm_gs = CALLOC_STRUCT(llvm_geometry_shader);
+
+      if (llvm_gs == NULL)
+         return NULL;
+
+      gs = &llvm_gs->base;
+
+      make_empty_list(&llvm_gs->variants);
+   } else
+#endif
+   {
+      gs = CALLOC_STRUCT(draw_geometry_shader);
+   }
+
+   if (!gs)
+      return NULL;
+
+   gs->draw = draw;
+   gs->state = *state;
+   gs->state.tokens = tgsi_dup_tokens(state->tokens);
+   if (!gs->state.tokens) {
+      FREE(gs);
+      return NULL;
+   }
+
+   tgsi_scan_shader(state->tokens, &gs->info);
+
+   /* setup the defaults */
+   gs->input_primitive = PIPE_PRIM_TRIANGLES;
+   gs->output_primitive = PIPE_PRIM_TRIANGLE_STRIP;
+   gs->max_output_vertices = 32;
+   gs->max_out_prims = 0;
+
+#ifdef HAVE_LLVM
+   if (use_llvm) {
+      /* TODO: change the input array to handle the following
+         vector length, instead of the currently hardcoded
+         TGSI_NUM_CHANNELS
+      gs->vector_length = lp_native_vector_width / 32;*/
+      gs->vector_length = TGSI_NUM_CHANNELS;
+   } else
+#endif
+   {
+      gs->vector_length = 1;
+   }
+
+   for (i = 0; i < gs->info.num_properties; ++i) {
+      if (gs->info.properties[i].name ==
+          TGSI_PROPERTY_GS_INPUT_PRIM)
+         gs->input_primitive = gs->info.properties[i].data[0];
+      else if (gs->info.properties[i].name ==
+               TGSI_PROPERTY_GS_OUTPUT_PRIM)
+         gs->output_primitive = gs->info.properties[i].data[0];
+      else if (gs->info.properties[i].name ==
+               TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES)
+         gs->max_output_vertices = gs->info.properties[i].data[0];
+   }
+   /* Primitive boundary is bigger than max_output_vertices by one, because
+    * the specification says that the geometry shader should exit if the 
+    * number of emitted vertices is bigger or equal to max_output_vertices and
+    * we can't do that because we're running in the SoA mode, which means that
+    * our storing routines will keep getting called on channels that have
+    * overflown.
+    * So we need some scratch area where we can keep writing the overflown 
+    * vertices without overwriting anything important or crashing.
+    */
+   gs->primitive_boundary = gs->max_output_vertices + 1;
+
+   for (i = 0; i < gs->info.num_outputs; i++) {
+      if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
+          gs->info.output_semantic_index[i] == 0)
+         gs->position_output = i;
+   }
+
+   gs->machine = draw->gs.tgsi.machine;
+
+#ifdef HAVE_LLVM
+   if (use_llvm) {
+      int vector_size = gs->vector_length * sizeof(float);
+      gs->gs_input = align_malloc(sizeof(struct draw_gs_inputs), 16);
+      memset(gs->gs_input, 0, sizeof(struct draw_gs_inputs));
+      gs->llvm_prim_lengths = 0;
+
+      gs->llvm_emitted_primitives = align_malloc(vector_size, vector_size);
+      gs->llvm_emitted_vertices = align_malloc(vector_size, vector_size);
+      gs->llvm_prim_ids = align_malloc(vector_size, vector_size);
+
+      gs->fetch_outputs = llvm_fetch_gs_outputs;
+      gs->fetch_inputs = llvm_fetch_gs_input;
+      gs->prepare = llvm_gs_prepare;
+      gs->run = llvm_gs_run;
+
+      gs->jit_context = &draw->llvm->gs_jit_context;
+
+
+      llvm_gs->variant_key_size =
+         draw_gs_llvm_variant_key_size(
+            MAX2(gs->info.file_max[TGSI_FILE_SAMPLER]+1,
+                 gs->info.file_max[TGSI_FILE_SAMPLER_VIEW]+1));
+   } else
+#endif
+   {
+      gs->fetch_outputs = tgsi_fetch_gs_outputs;
+      gs->fetch_inputs = tgsi_fetch_gs_input;
+      gs->prepare = tgsi_gs_prepare;
+      gs->run = tgsi_gs_run;
+   }
+
+   return gs;
+}
+
+void draw_bind_geometry_shader(struct draw_context *draw,
+                               struct draw_geometry_shader *dgs)
+{
+   draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
+
+   if (dgs) {
+      draw->gs.geometry_shader = dgs;
+      draw->gs.num_gs_outputs = dgs->info.num_outputs;
+      draw->gs.position_output = dgs->position_output;
+      draw_geometry_shader_prepare(dgs, draw);
+   }
+   else {
+      draw->gs.geometry_shader = NULL;
+      draw->gs.num_gs_outputs = 0;
+   }
+}
+
+void draw_delete_geometry_shader(struct draw_context *draw,
+                                 struct draw_geometry_shader *dgs)
+{
+   if (!dgs) {
+      return;
+   }
+#ifdef HAVE_LLVM
+   if (draw_get_option_use_llvm()) {
+      struct llvm_geometry_shader *shader = llvm_geometry_shader(dgs);
+      struct draw_gs_llvm_variant_list_item *li;
+
+      li = first_elem(&shader->variants);
+      while(!at_end(&shader->variants, li)) {
+         struct draw_gs_llvm_variant_list_item *next = next_elem(li);
+         draw_gs_llvm_destroy_variant(li->base);
+         li = next;
+      }
+
+      assert(shader->variants_cached == 0);
+
+      if (dgs->llvm_prim_lengths) {
+         unsigned i;
+         for (i = 0; i < dgs->max_out_prims; ++i) {
+            align_free(dgs->llvm_prim_lengths[i]);
+         }
+         FREE(dgs->llvm_prim_lengths);
+      }
+      align_free(dgs->llvm_emitted_primitives);
+      align_free(dgs->llvm_emitted_vertices);
+      align_free(dgs->llvm_prim_ids);
+
+      align_free(dgs->gs_input);
+   }
+#endif
+
+   FREE(dgs->primitive_lengths);
+   FREE((void*) dgs->state.tokens);
+   FREE(dgs);
+}
+
+
+#ifdef HAVE_LLVM
+void draw_gs_set_current_variant(struct draw_geometry_shader *shader,
+                                 struct draw_gs_llvm_variant *variant)
+{
+   shader->current_variant = variant;
+}
+#endif