src/gallium/auxiliary/draw/draw_gs.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2009 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "draw_gs.h"
  29
  30 #include "draw_private.h"
  31 #include "draw_context.h"
  32 #ifdef HAVE_LLVM
  33 #include "draw_llvm.h"
  34 #endif
  35
  36 #include "tgsi/tgsi_parse.h"
  37 #include "tgsi/tgsi_exec.h"
  38
  39 #include "pipe/p_shader_tokens.h"
  40
  41 #include "util/u_math.h"
  42 #include "util/u_memory.h"
  43 #include "util/u_prim.h"
  44
  45 /* fixme: move it from here */
  46 #define MAX_PRIMITIVES 64
  47
  48 static INLINE int
  49 draw_gs_get_input_index(int semantic, int index,
  50                         const struct tgsi_shader_info *input_info)
  51 {
  52    int i;
  53    const ubyte *input_semantic_names = input_info->output_semantic_name;
  54    const ubyte *input_semantic_indices = input_info->output_semantic_index;
  55    for (i = 0; i < PIPE_MAX_SHADER_OUTPUTS; i++) {
  56       if (input_semantic_names[i] == semantic &&
  57           input_semantic_indices[i] == index)
  58          return i;
  59    }
  60    debug_assert(0);
  61    return -1;
  62 }
  63
  64 /**
  65  * We execute geometry shaders in the SOA mode, so ideally we want to
  66  * flush when the number of currently fetched primitives is equal to
  67  * the number of elements in the SOA vector. This ensures that the
  68  * throughput is optimized for the given vector instrunction set.
  69  */
  70 static INLINE boolean
  71 draw_gs_should_flush(struct draw_geometry_shader *shader)
  72 {
  73    return (shader->fetched_prim_count == shader->vector_length);
  74 }
  75
  76 /*#define DEBUG_OUTPUTS 1*/
  77 static void
  78 tgsi_fetch_gs_outputs(struct draw_geometry_shader *shader,
  79                       unsigned num_primitives,
  80                       float (**p_output)[4])
  81 {
  82    struct tgsi_exec_machine *machine = shader->machine;
  83    unsigned prim_idx, j, slot;
  84    unsigned current_idx = 0;
  85    float (*output)[4];
  86
  87    output = *p_output;
  88
  89    /* Unswizzle all output results.
  90     */
  91
  92    for (prim_idx = 0; prim_idx < num_primitives; ++prim_idx) {
  93       unsigned num_verts_per_prim = machine->Primitives[prim_idx];
  94       shader->primitive_lengths[prim_idx +   shader->emitted_primitives] =
  95          machine->Primitives[prim_idx];
  96       shader->emitted_vertices += num_verts_per_prim;
  97       for (j = 0; j < num_verts_per_prim; j++, current_idx++) {
  98          int idx = current_idx * shader->info.num_outputs;
  99 #ifdef DEBUG_OUTPUTS
 100          debug_printf("%d) Output vert:\n", idx / shader->info.num_outputs);
 101 #endif
 102          for (slot = 0; slot < shader->info.num_outputs; slot++) {
 103             output[slot][0] = machine->Outputs[idx + slot].xyzw[0].f[0];
 104             output[slot][1] = machine->Outputs[idx + slot].xyzw[1].f[0];
 105             output[slot][2] = machine->Outputs[idx + slot].xyzw[2].f[0];
 106             output[slot][3] = machine->Outputs[idx + slot].xyzw[3].f[0];
 107 #ifdef DEBUG_OUTPUTS
 108             debug_printf("\t%d: %f %f %f %f\n", slot,
 109                          output[slot][0],
 110                          output[slot][1],
 111                          output[slot][2],
 112                          output[slot][3]);
 113 #endif
 114             debug_assert(!util_is_inf_or_nan(output[slot][0]));
 115          }
 116          output = (float (*)[4])((char *)output + shader->vertex_size);
 117       }
 118    }
 119    *p_output = output;
 120    shader->emitted_primitives += num_primitives;
 121 }
 122
 123 /*#define DEBUG_INPUTS 1*/
 124 static void tgsi_fetch_gs_input(struct draw_geometry_shader *shader,
 125                                 unsigned *indices,
 126                                 unsigned num_vertices,
 127                                 unsigned prim_idx)
 128 {
 129    struct tgsi_exec_machine *machine = shader->machine;
 130    unsigned slot, vs_slot, i;
 131    unsigned input_vertex_stride = shader->input_vertex_stride;
 132    const float (*input_ptr)[4];
 133
 134    input_ptr = shader->input;
 135
 136    for (i = 0; i < num_vertices; ++i) {
 137       const float (*input)[4];
 138 #if DEBUG_INPUTS
 139       debug_printf("%d) vertex index = %d (prim idx = %d)\n",
 140                    i, indices[i], prim_idx);
 141 #endif
 142       input = (const float (*)[4])(
 143          (const char *)input_ptr + (indices[i] * input_vertex_stride));
 144       for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; ++slot) {
 145          unsigned idx = i * TGSI_EXEC_MAX_INPUT_ATTRIBS + slot;
 146          if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
 147             machine->Inputs[idx].xyzw[0].f[prim_idx] =
 148                (float)shader->in_prim_idx;
 149             machine->Inputs[idx].xyzw[1].f[prim_idx] =
 150                (float)shader->in_prim_idx;
 151             machine->Inputs[idx].xyzw[2].f[prim_idx] =
 152                (float)shader->in_prim_idx;
 153             machine->Inputs[idx].xyzw[3].f[prim_idx] =
 154                (float)shader->in_prim_idx;
 155          } else {
 156             vs_slot = draw_gs_get_input_index(
 157                         shader->info.input_semantic_name[slot],
 158                         shader->info.input_semantic_index[slot],
 159                         shader->input_info);
 160 #if DEBUG_INPUTS
 161             debug_printf("\tSlot = %d, vs_slot = %d, idx = %d:\n",
 162                          slot, vs_slot, idx);
 163             assert(!util_is_inf_or_nan(input[vs_slot][0]));
 164             assert(!util_is_inf_or_nan(input[vs_slot][1]));
 165             assert(!util_is_inf_or_nan(input[vs_slot][2]));
 166             assert(!util_is_inf_or_nan(input[vs_slot][3]));
 167 #endif
 168             machine->Inputs[idx].xyzw[0].f[prim_idx] = input[vs_slot][0];
 169             machine->Inputs[idx].xyzw[1].f[prim_idx] = input[vs_slot][1];
 170             machine->Inputs[idx].xyzw[2].f[prim_idx] = input[vs_slot][2];
 171             machine->Inputs[idx].xyzw[3].f[prim_idx] = input[vs_slot][3];
 172 #if DEBUG_INPUTS
 173             debug_printf("\t\t%f %f %f %f\n",
 174                          machine->Inputs[idx].xyzw[0].f[prim_idx],
 175                          machine->Inputs[idx].xyzw[1].f[prim_idx],
 176                          machine->Inputs[idx].xyzw[2].f[prim_idx],
 177                          machine->Inputs[idx].xyzw[3].f[prim_idx]);
 178 #endif
 179             ++vs_slot;
 180          }
 181       }
 182    }
 183 }
 184
 185 static void tgsi_gs_prepare(struct draw_geometry_shader *shader,
 186                             const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
 187                             const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
 188 {
 189    struct tgsi_exec_machine *machine = shader->machine;
 190
 191    tgsi_exec_set_constant_buffers(machine, PIPE_MAX_CONSTANT_BUFFERS,
 192                                   constants, constants_size);
 193 }
 194
 195 static unsigned tgsi_gs_run(struct draw_geometry_shader *shader,
 196                             unsigned input_primitives)
 197 {
 198    struct tgsi_exec_machine *machine = shader->machine;
 199
 200    tgsi_set_exec_mask(machine,
 201                       1,
 202                       input_primitives > 1,
 203                       input_primitives > 2,
 204                       input_primitives > 3);
 205
 206    /* run interpreter */
 207    tgsi_exec_machine_run(machine);
 208
 209    return
 210       machine->Temps[TGSI_EXEC_TEMP_PRIMITIVE_I].xyzw[TGSI_EXEC_TEMP_PRIMITIVE_C].u[0];
 211 }
 212
 213 #ifdef HAVE_LLVM
 214
 215 static void
 216 llvm_fetch_gs_input(struct draw_geometry_shader *shader,
 217                     unsigned *indices,
 218                     unsigned num_vertices,
 219                     unsigned prim_idx)
 220 {
 221    unsigned slot, vs_slot, i;
 222    unsigned input_vertex_stride = shader->input_vertex_stride;
 223    const float (*input_ptr)[4];
 224    float (*input_data)[6][PIPE_MAX_SHADER_INPUTS][TGSI_NUM_CHANNELS][TGSI_NUM_CHANNELS] = &shader->gs_input->data;
 225
 226    shader->llvm_prim_ids[shader->fetched_prim_count] =
 227       shader->in_prim_idx;
 228
 229    input_ptr = shader->input;
 230
 231    for (i = 0; i < num_vertices; ++i) {
 232       const float (*input)[4];
 233 #if DEBUG_INPUTS
 234       debug_printf("%d) vertex index = %d (prim idx = %d)\n",
 235                    i, indices[i], prim_idx);
 236 #endif
 237       input = (const float (*)[4])(
 238          (const char *)input_ptr + (indices[i] * input_vertex_stride));
 239       for (slot = 0, vs_slot = 0; slot < shader->info.num_inputs; ++slot) {
 240          if (shader->info.input_semantic_name[slot] == TGSI_SEMANTIC_PRIMID) {
 241             /* skip. we handle system values through gallivm */
 242          } else {
 243             vs_slot = draw_gs_get_input_index(
 244                         shader->info.input_semantic_name[slot],
 245                         shader->info.input_semantic_index[slot],
 246                         shader->input_info);
 247 #if DEBUG_INPUTS
 248             debug_printf("\tSlot = %d, vs_slot = %d, i = %d:\n",
 249                          slot, vs_slot, i);
 250             assert(!util_is_inf_or_nan(input[vs_slot][0]));
 251             assert(!util_is_inf_or_nan(input[vs_slot][1]));
 252             assert(!util_is_inf_or_nan(input[vs_slot][2]));
 253             assert(!util_is_inf_or_nan(input[vs_slot][3]));
 254 #endif
 255             (*input_data)[i][slot][0][prim_idx] = input[vs_slot][0];
 256             (*input_data)[i][slot][1][prim_idx] = input[vs_slot][1];
 257             (*input_data)[i][slot][2][prim_idx] = input[vs_slot][2];
 258             (*input_data)[i][slot][3][prim_idx] = input[vs_slot][3];
 259 #if DEBUG_INPUTS
 260             debug_printf("\t\t%f %f %f %f\n",
 261                          (*input_data)[i][slot][0][prim_idx],
 262                          (*input_data)[i][slot][1][prim_idx],
 263                          (*input_data)[i][slot][2][prim_idx],
 264                          (*input_data)[i][slot][3][prim_idx]);
 265 #endif
 266             ++vs_slot;
 267          }
 268       }
 269    }
 270 }
 271
 272 static void
 273 llvm_fetch_gs_outputs(struct draw_geometry_shader *shader,
 274                       unsigned num_primitives,
 275                       float (**p_output)[4])
 276 {
 277    int total_verts = 0;
 278    int vertex_count = 0;
 279    int total_prims = 0;
 280    int max_prims_per_invocation = 0;
 281    char *output_ptr = (char*)shader->gs_output;
 282    int i, j, prim_idx;
 283    unsigned next_prim_boundary = shader->primitive_boundary;
 284
 285    for (i = 0; i < shader->vector_length; ++i) {
 286       int prims = shader->llvm_emitted_primitives[i];
 287       total_prims += prims;
 288       max_prims_per_invocation = MAX2(max_prims_per_invocation, prims);
 289    }
 290    for (i = 0; i < shader->vector_length; ++i) {
 291       total_verts += shader->llvm_emitted_vertices[i];
 292    }
 293
 294    output_ptr += shader->emitted_vertices * shader->vertex_size;
 295    for (i = 0; i < shader->vector_length - 1; ++i) {
 296       int current_verts = shader->llvm_emitted_vertices[i];
 297       int next_verts = shader->llvm_emitted_vertices[i + 1];
 298 #if 0
 299       int j;
 300       for (j = 0; j < current_verts; ++j) {
 301          struct vertex_header *vh = (struct vertex_header *)
 302             (output_ptr + shader->vertex_size * (i * next_prim_boundary + j));
 303          debug_printf("--- %d) [%f, %f, %f, %f]\n", j + vertex_count,
 304                       vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
 305
 306       }
 307 #endif
 308       debug_assert(current_verts <= shader->max_output_vertices);
 309       debug_assert(next_verts <= shader->max_output_vertices);
 310       if (next_verts) {
 311          memmove(output_ptr + (vertex_count + current_verts) * shader->vertex_size,
 312                  output_ptr + ((i + 1) * next_prim_boundary) * shader->vertex_size,
 313                  shader->vertex_size * next_verts);
 314       }
 315       vertex_count += current_verts;
 316    }
 317
 318 #if 0
 319    {
 320       int i;
 321       for (i = 0; i < total_verts; ++i) {
 322          struct vertex_header *vh = (struct vertex_header *)(output_ptr + shader->vertex_size * i);
 323          debug_printf("%d) [%f, %f, %f, %f]\n", i,
 324                       vh->data[0][0], vh->data[0][1], vh->data[0][2], vh->data[0][3]);
 325
 326       }
 327    }
 328 #endif
 329
 330    prim_idx = 0;
 331    for (i = 0; i < shader->vector_length; ++i) {
 332       int num_prims = shader->llvm_emitted_primitives[i];
 333       for (j = 0; j < num_prims; ++j) {
 334          int prim_length =
 335             shader->llvm_prim_lengths[j][i];
 336          shader->primitive_lengths[shader->emitted_primitives + prim_idx] =
 337             prim_length;
 338          ++prim_idx;
 339       }
 340    }
 341
 342    shader->emitted_primitives += total_prims;
 343    shader->emitted_vertices += total_verts;
 344 }
 345
 346 static void
 347 llvm_gs_prepare(struct draw_geometry_shader *shader,
 348                 const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
 349                 const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS])
 350 {
 351 }
 352
 353 static unsigned
 354 llvm_gs_run(struct draw_geometry_shader *shader,
 355             unsigned input_primitives)
 356 {
 357    unsigned ret;
 358    char *input = (char*)shader->gs_output;
 359
 360    input += (shader->emitted_vertices * shader->vertex_size);
 361
 362    ret = shader->current_variant->jit_func(
 363       shader->jit_context, shader->gs_input->data,
 364       (struct vertex_header*)input,
 365       input_primitives,
 366       shader->draw->instance_id,
 367       shader->llvm_prim_ids);
 368
 369    return ret;
 370 }
 371
 372 #endif
 373
 374 static void gs_flush(struct draw_geometry_shader *shader)
 375 {
 376    unsigned out_prim_count;
 377
 378    unsigned input_primitives = shader->fetched_prim_count;
 379
 380    if (shader->draw->collect_statistics) {
 381       shader->draw->statistics.gs_invocations += input_primitives;
 382    }
 383
 384    debug_assert(input_primitives > 0 &&
 385                 input_primitives <= 4);
 386
 387    out_prim_count = shader->run(shader, input_primitives);
 388    shader->fetch_outputs(shader, out_prim_count,
 389                          &shader->tmp_output);
 390
 391 #if 0
 392    debug_printf("PRIM emitted prims = %d (verts=%d), cur prim count = %d\n",
 393                 shader->emitted_primitives, shader->emitted_vertices,
 394                 out_prim_count);
 395 #endif
 396
 397    shader->fetched_prim_count = 0;
 398 }
 399
 400 static void gs_point(struct draw_geometry_shader *shader,
 401                      int idx)
 402 {
 403    unsigned indices[1];
 404
 405    indices[0] = idx;
 406
 407    shader->fetch_inputs(shader, indices, 1,
 408                         shader->fetched_prim_count);
 409    ++shader->in_prim_idx;
 410    ++shader->fetched_prim_count;
 411
 412    if (draw_gs_should_flush(shader))
 413       gs_flush(shader);
 414 }
 415
 416 static void gs_line(struct draw_geometry_shader *shader,
 417                     int i0, int i1)
 418 {
 419    unsigned indices[2];
 420
 421    indices[0] = i0;
 422    indices[1] = i1;
 423
 424    shader->fetch_inputs(shader, indices, 2,
 425                         shader->fetched_prim_count);
 426    ++shader->in_prim_idx;
 427    ++shader->fetched_prim_count;
 428
 429    if (draw_gs_should_flush(shader))
 430       gs_flush(shader);
 431 }
 432
 433 static void gs_line_adj(struct draw_geometry_shader *shader,
 434                         int i0, int i1, int i2, int i3)
 435 {
 436    unsigned indices[4];
 437
 438    indices[0] = i0;
 439    indices[1] = i1;
 440    indices[2] = i2;
 441    indices[3] = i3;
 442
 443    shader->fetch_inputs(shader, indices, 4,
 444                         shader->fetched_prim_count);
 445    ++shader->in_prim_idx;
 446    ++shader->fetched_prim_count;
 447
 448    if (draw_gs_should_flush(shader))
 449       gs_flush(shader);
 450 }
 451
 452 static void gs_tri(struct draw_geometry_shader *shader,
 453                    int i0, int i1, int i2)
 454 {
 455    unsigned indices[3];
 456
 457    indices[0] = i0;
 458    indices[1] = i1;
 459    indices[2] = i2;
 460
 461    shader->fetch_inputs(shader, indices, 3,
 462                         shader->fetched_prim_count);
 463    ++shader->in_prim_idx;
 464    ++shader->fetched_prim_count;
 465
 466    if (draw_gs_should_flush(shader))
 467       gs_flush(shader);
 468 }
 469
 470 static void gs_tri_adj(struct draw_geometry_shader *shader,
 471                        int i0, int i1, int i2,
 472                        int i3, int i4, int i5)
 473 {
 474    unsigned indices[6];
 475
 476    indices[0] = i0;
 477    indices[1] = i1;
 478    indices[2] = i2;
 479    indices[3] = i3;
 480    indices[4] = i4;
 481    indices[5] = i5;
 482
 483    shader->fetch_inputs(shader, indices, 6,
 484                         shader->fetched_prim_count);
 485    ++shader->in_prim_idx;
 486    ++shader->fetched_prim_count;
 487
 488    if (draw_gs_should_flush(shader))
 489       gs_flush(shader);
 490 }
 491
 492 #define FUNC         gs_run
 493 #define GET_ELT(idx) (idx)
 494 #include "draw_gs_tmp.h"
 495
 496
 497 #define FUNC         gs_run_elts
 498 #define LOCAL_VARS   const ushort *elts = input_prims->elts;
 499 #define GET_ELT(idx) (elts[idx])
 500 #include "draw_gs_tmp.h"
 501
 502
 503 /**
 504  * Execute geometry shader.
 505  */
 506 int draw_geometry_shader_run(struct draw_geometry_shader *shader,
 507                              const void *constants[PIPE_MAX_CONSTANT_BUFFERS],
 508                              const unsigned constants_size[PIPE_MAX_CONSTANT_BUFFERS],
 509                              const struct draw_vertex_info *input_verts,
 510                              const struct draw_prim_info *input_prim,
 511                              const struct tgsi_shader_info *input_info,
 512                              struct draw_vertex_info *output_verts,
 513                              struct draw_prim_info *output_prims )
 514 {
 515    const float (*input)[4] = (const float (*)[4])input_verts->verts->data;
 516    unsigned input_stride = input_verts->vertex_size;
 517    unsigned num_outputs = shader->info.num_outputs;
 518    unsigned vertex_size = sizeof(struct vertex_header) + num_outputs * 4 * sizeof(float);
 519    unsigned num_input_verts = input_prim->linear ?
 520       input_verts->count :
 521       input_prim->count;
 522    unsigned num_in_primitives =
 523       align(
 524          MAX2(u_decomposed_prims_for_vertices(input_prim->prim,
 525                                               num_input_verts),
 526               u_decomposed_prims_for_vertices(shader->input_primitive,
 527                                               num_input_verts)),
 528          shader->vector_length);
 529    unsigned max_out_prims =
 530       u_decomposed_prims_for_vertices(shader->output_primitive,
 531                                       shader->max_output_vertices)
 532       * num_in_primitives;
 533
 534    //Assume at least one primitive
 535    max_out_prims = MAX2(max_out_prims, 1);
 536
 537
 538    output_verts->vertex_size = vertex_size;
 539    output_verts->stride = output_verts->vertex_size;
 540    /* we allocate exactly one extra vertex per primitive to allow the GS to emit
 541     * overflown vertices into some area where they won't harm anyone */
 542    output_verts->verts =
 543       (struct vertex_header *)MALLOC(output_verts->vertex_size *
 544                                      max_out_prims *
 545                                      shader->primitive_boundary);
 546
 547 #if 0
 548    debug_printf("%s count = %d (in prims # = %d)\n",
 549                 __FUNCTION__, num_input_verts, num_in_primitives);
 550    debug_printf("\tlinear = %d, prim_info->count = %d\n",
 551                 input_prim->linear, input_prim->count);
 552    debug_printf("\tprim pipe = %s, shader in = %s, shader out = %s, max out = %d\n",
 553                 u_prim_name(input_prim->prim),
 554                 u_prim_name(shader->input_primitive),
 555                 u_prim_name(shader->output_primitive),
 556                 shader->max_output_vertices);
 557 #endif
 558
 559    shader->emitted_vertices = 0;
 560    shader->emitted_primitives = 0;
 561    shader->vertex_size = vertex_size;
 562    shader->tmp_output = (float (*)[4])output_verts->verts->data;
 563    shader->fetched_prim_count = 0;
 564    shader->input_vertex_stride = input_stride;
 565    shader->input = input;
 566    shader->input_info = input_info;
 567    FREE(shader->primitive_lengths);
 568    shader->primitive_lengths = MALLOC(max_out_prims * sizeof(unsigned));
 569
 570
 571 #ifdef HAVE_LLVM
 572    if (draw_get_option_use_llvm()) {
 573       shader->gs_output = output_verts->verts;
 574       if (max_out_prims > shader->max_out_prims) {
 575          unsigned i;
 576          if (shader->llvm_prim_lengths) {
 577             for (i = 0; i < shader->max_out_prims; ++i) {
 578                align_free(shader->llvm_prim_lengths[i]);
 579             }
 580             FREE(shader->llvm_prim_lengths);
 581          }
 582
 583          shader->llvm_prim_lengths = MALLOC(max_out_prims * sizeof(unsigned*));
 584          for (i = 0; i < max_out_prims; ++i) {
 585             int vector_size = shader->vector_length * sizeof(unsigned);
 586             shader->llvm_prim_lengths[i] =
 587                align_malloc(vector_size, vector_size);
 588          }
 589
 590          shader->max_out_prims = max_out_prims;
 591       }
 592       shader->jit_context->prim_lengths = shader->llvm_prim_lengths;
 593       shader->jit_context->emitted_vertices = shader->llvm_emitted_vertices;
 594       shader->jit_context->emitted_prims = shader->llvm_emitted_primitives;
 595    }
 596 #endif
 597
 598    shader->prepare(shader, constants, constants_size);
 599
 600    if (input_prim->linear)
 601       gs_run(shader, input_prim, input_verts,
 602              output_prims, output_verts);
 603    else
 604       gs_run_elts(shader, input_prim, input_verts,
 605                   output_prims, output_verts);
 606
 607    /* Flush the remaining primitives. Will happen if
 608     * num_input_primitives % 4 != 0
 609     */
 610    if (shader->fetched_prim_count > 0) {
 611       gs_flush(shader);
 612    }
 613
 614    debug_assert(shader->fetched_prim_count == 0);
 615
 616    /* Update prim_info:
 617     */
 618    output_prims->linear = TRUE;
 619    output_prims->elts = NULL;
 620    output_prims->start = 0;
 621    output_prims->count = shader->emitted_vertices;
 622    output_prims->prim = shader->output_primitive;
 623    output_prims->flags = 0x0;
 624    output_prims->primitive_lengths = shader->primitive_lengths;
 625    output_prims->primitive_count = shader->emitted_primitives;
 626    output_verts->count = shader->emitted_vertices;
 627
 628    if (shader->draw->collect_statistics) {
 629       unsigned i;
 630       for (i = 0; i < shader->emitted_primitives; ++i) {
 631          shader->draw->statistics.gs_primitives +=
 632             u_decomposed_prims_for_vertices(shader->output_primitive,
 633                                             shader->primitive_lengths[i]);
 634       }
 635    }
 636
 637 #if 0
 638    debug_printf("GS finished, prims = %d, verts = %d\n",
 639                 output_prims->primitive_count,
 640                 output_verts->count);
 641 #endif
 642
 643    return shader->emitted_vertices;
 644 }
 645
 646 void draw_geometry_shader_prepare(struct draw_geometry_shader *shader,
 647                                   struct draw_context *draw)
 648 {
 649    if (shader && shader->machine->Tokens != shader->state.tokens) {
 650       tgsi_exec_machine_bind_shader(shader->machine,
 651                                     shader->state.tokens,
 652                                     draw->gs.tgsi.sampler);
 653    }
 654 }
 655
 656
 657 boolean
 658 draw_gs_init( struct draw_context *draw )
 659 {
 660    draw->gs.tgsi.machine = tgsi_exec_machine_create();
 661    if (!draw->gs.tgsi.machine)
 662       return FALSE;
 663
 664    draw->gs.tgsi.machine->Primitives = align_malloc(
 665       MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector), 16);
 666    if (!draw->gs.tgsi.machine->Primitives)
 667       return FALSE;
 668    memset(draw->gs.tgsi.machine->Primitives, 0,
 669           MAX_PRIMITIVES * sizeof(struct tgsi_exec_vector));
 670
 671    return TRUE;
 672 }
 673
 674 void draw_gs_destroy( struct draw_context *draw )
 675 {
 676    if (draw->gs.tgsi.machine) {
 677       align_free(draw->gs.tgsi.machine->Primitives);
 678       tgsi_exec_machine_destroy(draw->gs.tgsi.machine);
 679    }
 680 }
 681
 682 struct draw_geometry_shader *
 683 draw_create_geometry_shader(struct draw_context *draw,
 684                             const struct pipe_shader_state *state)
 685 {
 686 #ifdef HAVE_LLVM
 687    boolean use_llvm = draw_get_option_use_llvm();
 688    struct llvm_geometry_shader *llvm_gs;
 689 #endif
 690    struct draw_geometry_shader *gs;
 691    unsigned i;
 692
 693 #ifdef HAVE_LLVM
 694    if (use_llvm) {
 695       llvm_gs = CALLOC_STRUCT(llvm_geometry_shader);
 696
 697       if (llvm_gs == NULL)
 698          return NULL;
 699
 700       gs = &llvm_gs->base;
 701
 702       make_empty_list(&llvm_gs->variants);
 703    } else
 704 #endif
 705    {
 706       gs = CALLOC_STRUCT(draw_geometry_shader);
 707    }
 708
 709    if (!gs)
 710       return NULL;
 711
 712    gs->draw = draw;
 713    gs->state = *state;
 714    gs->state.tokens = tgsi_dup_tokens(state->tokens);
 715    if (!gs->state.tokens) {
 716       FREE(gs);
 717       return NULL;
 718    }
 719
 720    tgsi_scan_shader(state->tokens, &gs->info);
 721
 722    /* setup the defaults */
 723    gs->input_primitive = PIPE_PRIM_TRIANGLES;
 724    gs->output_primitive = PIPE_PRIM_TRIANGLE_STRIP;
 725    gs->max_output_vertices = 32;
 726    gs->max_out_prims = 0;
 727
 728 #ifdef HAVE_LLVM
 729    if (use_llvm) {
 730       /* TODO: change the input array to handle the following
 731          vector length, instead of the currently hardcoded
 732          TGSI_NUM_CHANNELS
 733       gs->vector_length = lp_native_vector_width / 32;*/
 734       gs->vector_length = TGSI_NUM_CHANNELS;
 735    } else
 736 #endif
 737    {
 738       gs->vector_length = 1;
 739    }
 740
 741    for (i = 0; i < gs->info.num_properties; ++i) {
 742       if (gs->info.properties[i].name ==
 743           TGSI_PROPERTY_GS_INPUT_PRIM)
 744          gs->input_primitive = gs->info.properties[i].data[0];
 745       else if (gs->info.properties[i].name ==
 746                TGSI_PROPERTY_GS_OUTPUT_PRIM)
 747          gs->output_primitive = gs->info.properties[i].data[0];
 748       else if (gs->info.properties[i].name ==
 749                TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES)
 750          gs->max_output_vertices = gs->info.properties[i].data[0];
 751    }
 752    /* Primitive boundary is bigger than max_output_vertices by one, because
 753     * the specification says that the geometry shader should exit if the
 754     * number of emitted vertices is bigger or equal to max_output_vertices and
 755     * we can't do that because we're running in the SoA mode, which means that
 756     * our storing routines will keep getting called on channels that have
 757     * overflown.
 758     * So we need some scratch area where we can keep writing the overflown
 759     * vertices without overwriting anything important or crashing.
 760     */
 761    gs->primitive_boundary = gs->max_output_vertices + 1;
 762
 763    for (i = 0; i < gs->info.num_outputs; i++) {
 764       if (gs->info.output_semantic_name[i] == TGSI_SEMANTIC_POSITION &&
 765           gs->info.output_semantic_index[i] == 0)
 766          gs->position_output = i;
 767    }
 768
 769    gs->machine = draw->gs.tgsi.machine;
 770
 771 #ifdef HAVE_LLVM
 772    if (use_llvm) {
 773       int vector_size = gs->vector_length * sizeof(float);
 774       gs->gs_input = align_malloc(sizeof(struct draw_gs_inputs), 16);
 775       memset(gs->gs_input, 0, sizeof(struct draw_gs_inputs));
 776       gs->llvm_prim_lengths = 0;
 777
 778       gs->llvm_emitted_primitives = align_malloc(vector_size, vector_size);
 779       gs->llvm_emitted_vertices = align_malloc(vector_size, vector_size);
 780       gs->llvm_prim_ids = align_malloc(vector_size, vector_size);
 781
 782       gs->fetch_outputs = llvm_fetch_gs_outputs;
 783       gs->fetch_inputs = llvm_fetch_gs_input;
 784       gs->prepare = llvm_gs_prepare;
 785       gs->run = llvm_gs_run;
 786
 787       gs->jit_context = &draw->llvm->gs_jit_context;
 788
 789
 790       llvm_gs->variant_key_size =
 791          draw_gs_llvm_variant_key_size(
 792             MAX2(gs->info.file_max[TGSI_FILE_SAMPLER]+1,
 793                  gs->info.file_max[TGSI_FILE_SAMPLER_VIEW]+1));
 794    } else
 795 #endif
 796    {
 797       gs->fetch_outputs = tgsi_fetch_gs_outputs;
 798       gs->fetch_inputs = tgsi_fetch_gs_input;
 799       gs->prepare = tgsi_gs_prepare;
 800       gs->run = tgsi_gs_run;
 801    }
 802
 803    return gs;
 804 }
 805
 806 void draw_bind_geometry_shader(struct draw_context *draw,
 807                                struct draw_geometry_shader *dgs)
 808 {
 809    draw_do_flush(draw, DRAW_FLUSH_STATE_CHANGE);
 810
 811    if (dgs) {
 812       draw->gs.geometry_shader = dgs;
 813       draw->gs.num_gs_outputs = dgs->info.num_outputs;
 814       draw->gs.position_output = dgs->position_output;
 815       draw_geometry_shader_prepare(dgs, draw);
 816    }
 817    else {
 818       draw->gs.geometry_shader = NULL;
 819       draw->gs.num_gs_outputs = 0;
 820    }
 821 }
 822
 823 void draw_delete_geometry_shader(struct draw_context *draw,
 824                                  struct draw_geometry_shader *dgs)
 825 {
 826    if (!dgs) {
 827       return;
 828    }
 829 #ifdef HAVE_LLVM
 830    if (draw_get_option_use_llvm()) {
 831       struct llvm_geometry_shader *shader = llvm_geometry_shader(dgs);
 832       struct draw_gs_llvm_variant_list_item *li;
 833
 834       li = first_elem(&shader->variants);
 835       while(!at_end(&shader->variants, li)) {
 836          struct draw_gs_llvm_variant_list_item *next = next_elem(li);
 837          draw_gs_llvm_destroy_variant(li->base);
 838          li = next;
 839       }
 840
 841       assert(shader->variants_cached == 0);
 842
 843       if (dgs->llvm_prim_lengths) {
 844          unsigned i;
 845          for (i = 0; i < dgs->max_out_prims; ++i) {
 846             align_free(dgs->llvm_prim_lengths[i]);
 847          }
 848          FREE(dgs->llvm_prim_lengths);
 849       }
 850       align_free(dgs->llvm_emitted_primitives);
 851       align_free(dgs->llvm_emitted_vertices);
 852       align_free(dgs->llvm_prim_ids);
 853
 854       align_free(dgs->gs_input);
 855    }
 856 #endif
 857
 858    FREE(dgs->primitive_lengths);
 859    FREE((void*) dgs->state.tokens);
 860    FREE(dgs);
 861 }
 862
 863
 864 #ifdef HAVE_LLVM
 865 void draw_gs_set_current_variant(struct draw_geometry_shader *shader,
 866                                  struct draw_gs_llvm_variant *variant)
 867 {
 868    shader->current_variant = variant;
 869 }
 870 #endif
 871
 872 /*
 873  * Called at the very begin of the draw call with a new instance
 874  * Used to reset state that should persist between primitive restart.
 875  */
 876 void
 877 draw_geometry_shader_new_instance(struct draw_geometry_shader *gs)
 878 {
 879    if (!gs)
 880       return;
 881
 882    gs->in_prim_idx = 0;
 883 }