draw/gs: reduce the size of the gs output buffer

We used to overallocate the output buffer sometimes running out
of memory with applications rendering large geometries. The actual
maximum number of vertices out is simply the maximum number of
primitives in (number of gs invocations) multiplied by the maximum
number of output vertices per gs input primitive (i.e. gs invocation).

Signed-off-by: Zack Rusin <zackr@vmware.com>
Reviewed-by: Jose Fonseca <jfonseca@vmware.com>
Reviewed-by: Roland Scheidegger <sroland@vmware.com>

diff --git a/src/gallium/auxiliary/draw/draw_gs.c b/src/gallium/auxiliary/draw/draw_gs.c
index 97e8a90f284f8cfb174bf0dd15cdd95162e0223f..7de5e0308ec6cbc2fd26af865821c78b251206bd 100644 (file)
--- a/src/gallium/auxiliary/draw/draw_gs.c
+++ b/src/gallium/auxiliary/draw/draw_gs.c
@@ -552,6 +552,10 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
       u_decomposed_prims_for_vertices(shader->output_primitive,
                                       shader->max_output_vertices)
       * num_in_primitives;
+   /* we allocate exactly one extra vertex per primitive to allow the GS to emit
+    * overflown vertices into some area where they won't harm anyone */
+   unsigned total_verts_per_buffer = shader->primitive_boundary *
+      num_in_primitives;
 
    //Assume at least one primitive
    max_out_prims = MAX2(max_out_prims, 1);
@@ -559,23 +563,25 @@ int draw_geometry_shader_run(struct draw_geometry_shader *shader,
 
    output_verts->vertex_size = vertex_size;
    output_verts->stride = output_verts->vertex_size;
-   /* we allocate exactly one extra vertex per primitive to allow the GS to emit
-    * overflown vertices into some area where they won't harm anyone */
    output_verts->verts =
       (struct vertex_header *)MALLOC(output_verts->vertex_size *
-                                     max_out_prims *
-                                     shader->primitive_boundary);
+                                     total_verts_per_buffer);
+   debug_assert(output_verts->verts);
 
 #if 0
    debug_printf("%s count = %d (in prims # = %d)\n",
                 __FUNCTION__, num_input_verts, num_in_primitives);
    debug_printf("\tlinear = %d, prim_info->count = %d\n",
                 input_prim->linear, input_prim->count);
-   debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s, max out = %d\n",
+   debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s\n"
                 u_prim_name(input_prim->prim),
                 u_prim_name(shader->input_primitive),
-                u_prim_name(shader->output_primitive),
-                shader->max_output_vertices);
+                u_prim_name(shader->output_primitive));
+   debug_printf("\tmaxv  = %d, maxp = %d, primitive_boundary = %d, "
+                "vertex_size = %d, tverts = %d\n",
+                shader->max_output_vertices, max_out_prims,
+                shader->primitive_boundary, output_verts->vertex_size,
+                total_verts_per_buffer);
 #endif
 
    shader->emitted_vertices = 0;