src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "util/u_math.h"
  29 #include "util/u_memory.h"
  30 #include "util/u_prim.h"
  31 #include "draw/draw_context.h"
  32 #include "draw/draw_gs.h"
  33 #include "draw/draw_vbuf.h"
  34 #include "draw/draw_vertex.h"
  35 #include "draw/draw_pt.h"
  36 #include "draw/draw_prim_assembler.h"
  37 #include "draw/draw_vs.h"
  38 #include "draw/draw_llvm.h"
  39 #include "gallivm/lp_bld_init.h"
  40 #include "gallivm/lp_bld_debug.h"
  41
  42
  43 struct llvm_middle_end {
  44    struct draw_pt_middle_end base;
  45    struct draw_context *draw;
  46
  47    struct pt_emit *emit;
  48    struct pt_so_emit *so_emit;
  49    struct pt_fetch *fetch;
  50    struct pt_post_vs *post_vs;
  51
  52
  53    unsigned vertex_data_offset;
  54    unsigned vertex_size;
  55    unsigned input_prim;
  56    unsigned opt;
  57
  58    struct draw_llvm *llvm;
  59    struct draw_llvm_variant *current_variant;
  60 };
  61
  62
  63 /** cast wrapper */
  64 static inline struct llvm_middle_end *
  65 llvm_middle_end(struct draw_pt_middle_end *middle)
  66 {
  67    return (struct llvm_middle_end *) middle;
  68 }
  69
  70
  71 static void
  72 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  73 {
  74    struct draw_context *draw = fpme->draw;
  75    struct draw_llvm *llvm = fpme->llvm;
  76    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  77    struct draw_gs_llvm_variant_key *key;
  78    struct draw_gs_llvm_variant *variant = NULL;
  79    struct draw_gs_llvm_variant_list_item *li;
  80    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  81    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  82    unsigned i;
  83
  84    key = draw_gs_llvm_make_variant_key(llvm, store);
  85
  86    /* Search shader's list of variants for the key */
  87    li = first_elem(&shader->variants);
  88    while (!at_end(&shader->variants, li)) {
  89       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  90          variant = li->base;
  91          break;
  92       }
  93       li = next_elem(li);
  94    }
  95
  96    if (variant) {
  97       /* found the variant, move to head of global list (for LRU) */
  98       move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
  99    }
 100    else {
 101       /* Need to create new variant */
 102
 103       /* First check if we've created too many variants.  If so, free
 104        * 3.125% of the LRU to avoid using too much memory.
 105        */
 106       if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
 107          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 108             debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
 109                       shader->variants_cached, llvm->nr_gs_variants);
 110          }
 111
 112          /*
 113           * XXX: should we flush here ?
 114           */
 115          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 116             struct draw_gs_llvm_variant_list_item *item;
 117             if (is_empty_list(&llvm->gs_variants_list)) {
 118                break;
 119             }
 120             item = last_elem(&llvm->gs_variants_list);
 121             assert(item);
 122             assert(item->base);
 123             draw_gs_llvm_destroy_variant(item->base);
 124          }
 125       }
 126
 127       variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);
 128
 129       if (variant) {
 130          insert_at_head(&shader->variants, &variant->list_item_local);
 131          insert_at_head(&llvm->gs_variants_list,
 132                         &variant->list_item_global);
 133          llvm->nr_gs_variants++;
 134          shader->variants_cached++;
 135       }
 136    }
 137
 138    gs->current_variant = variant;
 139 }
 140
 141 /**
 142  * Prepare/validate middle part of the vertex pipeline.
 143  * NOTE: if you change this function, also look at the non-LLVM
 144  * function fetch_pipeline_prepare() for similar changes.
 145  */
 146 static void
 147 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
 148                          unsigned in_prim,
 149                          unsigned opt,
 150                          unsigned *max_vertices )
 151 {
 152    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 153    struct draw_context *draw = fpme->draw;
 154    struct draw_llvm *llvm = fpme->llvm;
 155    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
 156    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
 157    const unsigned out_prim = gs ? gs->output_primitive :
 158       u_assembled_prim(in_prim);
 159    unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
 160                          out_prim == PIPE_PRIM_POINTS;
 161    unsigned nr;
 162
 163    fpme->input_prim = in_prim;
 164    fpme->opt = opt;
 165
 166    draw_pt_post_vs_prepare( fpme->post_vs,
 167                             draw->clip_xy,
 168                             draw->clip_z,
 169                             draw->clip_user,
 170                             point_clip ? draw->guard_band_points_xy :
 171                                          draw->guard_band_xy,
 172                             draw->bypass_viewport,
 173                             draw->rasterizer->clip_halfz,
 174                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
 175
 176    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
 177
 178    if (!(opt & PT_PIPELINE)) {
 179       draw_pt_emit_prepare( fpme->emit, out_prim,
 180                             max_vertices );
 181
 182       *max_vertices = MAX2( *max_vertices, 4096 );
 183    }
 184    else {
 185       /* limit max fetches by limiting max_vertices */
 186       *max_vertices = 4096;
 187    }
 188
 189    /* Get the number of float[4] attributes per vertex.
 190     * Note: this must be done after draw_pt_emit_prepare() since that
 191     * can effect the vertex size.
 192     */
 193    nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
 194
 195    /* Always leave room for the vertex header whether we need it or
 196     * not.  It's hard to get rid of it in particular because of the
 197     * viewport code in draw_pt_post_vs.c.
 198     */
 199    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
 200
 201    /* return even number */
 202    *max_vertices = *max_vertices & ~1;
 203
 204    /* Find/create the vertex shader variant */
 205    {
 206       struct draw_llvm_variant_key *key;
 207       struct draw_llvm_variant *variant = NULL;
 208       struct draw_llvm_variant_list_item *li;
 209       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
 210       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
 211       unsigned i;
 212
 213       key = draw_llvm_make_variant_key(llvm, store);
 214
 215       /* Search shader's list of variants for the key */
 216       li = first_elem(&shader->variants);
 217       while (!at_end(&shader->variants, li)) {
 218          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 219             variant = li->base;
 220             break;
 221          }
 222          li = next_elem(li);
 223       }
 224
 225       if (variant) {
 226          /* found the variant, move to head of global list (for LRU) */
 227          move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
 228       }
 229       else {
 230          /* Need to create new variant */
 231
 232          /* First check if we've created too many variants.  If so, free
 233           * 3.125% of the LRU to avoid using too much memory.
 234           */
 235          if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
 236             if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 237                debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
 238                          shader->variants_cached, llvm->nr_variants);
 239             }
 240
 241             /*
 242              * XXX: should we flush here ?
 243              */
 244             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 245                struct draw_llvm_variant_list_item *item;
 246                if (is_empty_list(&llvm->vs_variants_list)) {
 247                   break;
 248                }
 249                item = last_elem(&llvm->vs_variants_list);
 250                assert(item);
 251                assert(item->base);
 252                draw_llvm_destroy_variant(item->base);
 253             }
 254          }
 255
 256          variant = draw_llvm_create_variant(llvm, nr, key);
 257
 258          if (variant) {
 259             insert_at_head(&shader->variants, &variant->list_item_local);
 260             insert_at_head(&llvm->vs_variants_list,
 261                            &variant->list_item_global);
 262             llvm->nr_variants++;
 263             shader->variants_cached++;
 264          }
 265       }
 266
 267       fpme->current_variant = variant;
 268    }
 269
 270    if (gs) {
 271       llvm_middle_end_prepare_gs(fpme);
 272    }
 273 }
 274
 275
 276 /**
 277  * Bind/update constant buffer pointers, clip planes and viewport dims.
 278  * These are "light weight" parameters which aren't baked into the
 279  * generated code.  Updating these items is much cheaper than revalidating
 280  * and rebuilding the generated pipeline code.
 281  */
 282 static void
 283 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
 284 {
 285    static const float fake_const_buf[4];
 286    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 287    struct draw_context *draw = fpme->draw;
 288    struct draw_llvm *llvm = fpme->llvm;
 289    unsigned i;
 290
 291    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
 292       /*
 293        * There could be a potential issue with rounding this up, as the
 294        * shader expects 16-byte allocations, the fix is likely to move
 295        * to LOAD intrinsic in the future and remove the vec4 constraint.
 296        */
 297       int num_consts =
 298          DIV_ROUND_UP(draw->pt.user.vs_constants_size[i], (sizeof(float) * 4));
 299       llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
 300       llvm->jit_context.num_vs_constants[i] = num_consts;
 301       if (num_consts == 0) {
 302          llvm->jit_context.vs_constants[i] = fake_const_buf;
 303       }
 304    }
 305    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
 306       int num_ssbos = draw->pt.user.vs_ssbos_size[i];
 307       llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
 308       llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
 309       if (num_ssbos == 0) {
 310          llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
 311       }
 312    }
 313
 314    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
 315       int num_consts =
 316          DIV_ROUND_UP(draw->pt.user.gs_constants_size[i], (sizeof(float) * 4));
 317       llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
 318       llvm->gs_jit_context.num_constants[i] = num_consts;
 319       if (num_consts == 0) {
 320          llvm->gs_jit_context.constants[i] = fake_const_buf;
 321       }
 322    }
 323    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
 324       int num_ssbos = draw->pt.user.gs_ssbos_size[i];
 325       llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
 326       llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
 327       if (num_ssbos == 0) {
 328          llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 329       }
 330    }
 331
 332    llvm->jit_context.planes =
 333       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 334    llvm->gs_jit_context.planes =
 335       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 336
 337    llvm->jit_context.viewports = draw->viewports;
 338    llvm->gs_jit_context.viewports = draw->viewports;
 339 }
 340
 341
 342 static void
 343 pipeline(struct llvm_middle_end *llvm,
 344          const struct draw_vertex_info *vert_info,
 345          const struct draw_prim_info *prim_info)
 346 {
 347    if (prim_info->linear)
 348       draw_pipeline_run_linear( llvm->draw,
 349                                 vert_info,
 350                                 prim_info);
 351    else
 352       draw_pipeline_run( llvm->draw,
 353                          vert_info,
 354                          prim_info );
 355 }
 356
 357
 358 static void
 359 emit(struct pt_emit *emit,
 360      const struct draw_vertex_info *vert_info,
 361      const struct draw_prim_info *prim_info)
 362 {
 363    if (prim_info->linear) {
 364       draw_pt_emit_linear(emit, vert_info, prim_info);
 365    }
 366    else {
 367       draw_pt_emit(emit, vert_info, prim_info);
 368    }
 369 }
 370
 371
 372 static void
 373 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
 374                       const struct draw_fetch_info *fetch_info,
 375                       const struct draw_prim_info *in_prim_info)
 376 {
 377    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 378    struct draw_context *draw = fpme->draw;
 379    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
 380    struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
 381    struct draw_vertex_info llvm_vert_info;
 382    struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
 383    struct draw_vertex_info *vert_info;
 384    struct draw_prim_info ia_prim_info;
 385    struct draw_vertex_info ia_vert_info;
 386    const struct draw_prim_info *prim_info = in_prim_info;
 387    boolean free_prim_info = FALSE;
 388    unsigned opt = fpme->opt;
 389    boolean clipped = 0;
 390    unsigned start_or_maxelt, vid_base;
 391    const unsigned *elts;
 392
 393    assert(fetch_info->count > 0);
 394    llvm_vert_info.count = fetch_info->count;
 395    llvm_vert_info.vertex_size = fpme->vertex_size;
 396    llvm_vert_info.stride = fpme->vertex_size;
 397    llvm_vert_info.verts = (struct vertex_header *)
 398       MALLOC(fpme->vertex_size *
 399              align(fetch_info->count, lp_native_vector_width / 32));
 400    if (!llvm_vert_info.verts) {
 401       assert(0);
 402       return;
 403    }
 404
 405    if (draw->collect_statistics) {
 406       draw->statistics.ia_vertices += prim_info->count;
 407       draw->statistics.ia_primitives +=
 408          u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
 409       draw->statistics.vs_invocations += fetch_info->count;
 410    }
 411
 412    if (fetch_info->linear) {
 413       start_or_maxelt = fetch_info->start;
 414       vid_base = draw->start_index;
 415       elts = NULL;
 416    }
 417    else {
 418       start_or_maxelt = draw->pt.user.eltMax;
 419       vid_base = draw->pt.user.eltBias;
 420       elts = fetch_info->elts;
 421    }
 422    clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
 423                                              llvm_vert_info.verts,
 424                                              draw->pt.user.vbuffer,
 425                                              fetch_info->count,
 426                                              start_or_maxelt,
 427                                              fpme->vertex_size,
 428                                              draw->pt.vertex_buffer,
 429                                              draw->instance_id,
 430                                              vid_base,
 431                                              draw->start_instance,
 432                                              elts);
 433
 434    /* Finished with fetch and vs:
 435     */
 436    fetch_info = NULL;
 437    vert_info = &llvm_vert_info;
 438
 439    if ((opt & PT_SHADE) && gshader) {
 440       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 441       draw_geometry_shader_run(gshader,
 442                                draw->pt.user.gs_constants,
 443                                draw->pt.user.gs_constants_size,
 444                                vert_info,
 445                                prim_info,
 446                                &vshader->info,
 447                                gs_vert_info,
 448                                gs_prim_info);
 449
 450       FREE(vert_info->verts);
 451       vert_info = &gs_vert_info[0];
 452       prim_info = &gs_prim_info[0];
 453       /*
 454        * pt emit can only handle ushort number of vertices (see
 455        * render->allocate_vertices).
 456        * vsplit guarantees there's never more than 4096, however GS can
 457        * easily blow this up (by a factor of 256 (or even 1024) max).
 458        */
 459       if (vert_info->count > 65535) {
 460          opt |= PT_PIPELINE;
 461       }
 462    } else {
 463       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
 464          draw_prim_assembler_run(draw, prim_info, vert_info,
 465                                  &ia_prim_info, &ia_vert_info);
 466
 467          if (ia_vert_info.count) {
 468             FREE(vert_info->verts);
 469             vert_info = &ia_vert_info;
 470             prim_info = &ia_prim_info;
 471             free_prim_info = TRUE;
 472          }
 473       }
 474    }
 475    if (prim_info->count == 0) {
 476       debug_printf("GS/IA didn't emit any vertices!\n");
 477
 478       FREE(vert_info->verts);
 479       if (free_prim_info) {
 480          FREE(prim_info->primitive_lengths);
 481       }
 482       return;
 483    }
 484
 485    /* stream output needs to be done before clipping */
 486    draw_pt_so_emit( fpme->so_emit, 1, vert_info, prim_info );
 487
 488    draw_stats_clipper_primitives(draw, prim_info);
 489
 490    /*
 491     * if there's no position, need to stop now, or the latter stages
 492     * will try to access non-existent position output.
 493     */
 494    if (draw_current_shader_position_output(draw) != -1) {
 495       if ((opt & PT_SHADE) && (gshader ||
 496                                draw->vs.vertex_shader->info.writes_viewport_index)) {
 497          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
 498       }
 499       /* "clipped" also includes non-one edgeflag */
 500       if (clipped) {
 501          opt |= PT_PIPELINE;
 502       }
 503
 504       /* Do we need to run the pipeline? Now will come here if clipped
 505        */
 506       if (opt & PT_PIPELINE) {
 507          pipeline( fpme, vert_info, prim_info );
 508       }
 509       else {
 510          emit( fpme->emit, vert_info, prim_info );
 511       }
 512    }
 513    FREE(vert_info->verts);
 514    if (free_prim_info) {
 515       FREE(prim_info->primitive_lengths);
 516    }
 517 }
 518
 519
 520 static inline unsigned
 521 prim_type(unsigned prim, unsigned flags)
 522 {
 523    if (flags & DRAW_LINE_LOOP_AS_STRIP)
 524       return PIPE_PRIM_LINE_STRIP;
 525    else
 526       return prim;
 527 }
 528
 529
 530 static void
 531 llvm_middle_end_run(struct draw_pt_middle_end *middle,
 532                     const unsigned *fetch_elts,
 533                     unsigned fetch_count,
 534                     const ushort *draw_elts,
 535                     unsigned draw_count,
 536                     unsigned prim_flags)
 537 {
 538    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 539    struct draw_fetch_info fetch_info;
 540    struct draw_prim_info prim_info;
 541
 542    fetch_info.linear = FALSE;
 543    fetch_info.start = 0;
 544    fetch_info.elts = fetch_elts;
 545    fetch_info.count = fetch_count;
 546
 547    prim_info.linear = FALSE;
 548    prim_info.start = 0;
 549    prim_info.count = draw_count;
 550    prim_info.elts = draw_elts;
 551    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 552    prim_info.flags = prim_flags;
 553    prim_info.primitive_count = 1;
 554    prim_info.primitive_lengths = &draw_count;
 555
 556    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 557 }
 558
 559
 560 static void
 561 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
 562                            unsigned start,
 563                            unsigned count,
 564                            unsigned prim_flags)
 565 {
 566    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 567    struct draw_fetch_info fetch_info;
 568    struct draw_prim_info prim_info;
 569
 570    fetch_info.linear = TRUE;
 571    fetch_info.start = start;
 572    fetch_info.count = count;
 573    fetch_info.elts = NULL;
 574
 575    prim_info.linear = TRUE;
 576    prim_info.start = 0;
 577    prim_info.count = count;
 578    prim_info.elts = NULL;
 579    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 580    prim_info.flags = prim_flags;
 581    prim_info.primitive_count = 1;
 582    prim_info.primitive_lengths = &count;
 583
 584    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 585 }
 586
 587
 588 static boolean
 589 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
 590                                 unsigned start,
 591                                 unsigned count,
 592                                 const ushort *draw_elts,
 593                                 unsigned draw_count,
 594                                 unsigned prim_flags)
 595 {
 596    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 597    struct draw_fetch_info fetch_info;
 598    struct draw_prim_info prim_info;
 599
 600    fetch_info.linear = TRUE;
 601    fetch_info.start = start;
 602    fetch_info.count = count;
 603    fetch_info.elts = NULL;
 604
 605    prim_info.linear = FALSE;
 606    prim_info.start = 0;
 607    prim_info.count = draw_count;
 608    prim_info.elts = draw_elts;
 609    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 610    prim_info.flags = prim_flags;
 611    prim_info.primitive_count = 1;
 612    prim_info.primitive_lengths = &draw_count;
 613
 614    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 615
 616    return TRUE;
 617 }
 618
 619
 620 static void
 621 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
 622 {
 623    /* nothing to do */
 624 }
 625
 626
 627 static void
 628 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
 629 {
 630    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 631
 632    if (fpme->fetch)
 633       draw_pt_fetch_destroy( fpme->fetch );
 634
 635    if (fpme->emit)
 636       draw_pt_emit_destroy( fpme->emit );
 637
 638    if (fpme->so_emit)
 639       draw_pt_so_emit_destroy( fpme->so_emit );
 640
 641    if (fpme->post_vs)
 642       draw_pt_post_vs_destroy( fpme->post_vs );
 643
 644    FREE(middle);
 645 }
 646
 647
 648 struct draw_pt_middle_end *
 649 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
 650 {
 651    struct llvm_middle_end *fpme = 0;
 652
 653    if (!draw->llvm)
 654       return NULL;
 655
 656    fpme = CALLOC_STRUCT( llvm_middle_end );
 657    if (!fpme)
 658       goto fail;
 659
 660    fpme->base.prepare         = llvm_middle_end_prepare;
 661    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
 662    fpme->base.run             = llvm_middle_end_run;
 663    fpme->base.run_linear      = llvm_middle_end_linear_run;
 664    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
 665    fpme->base.finish          = llvm_middle_end_finish;
 666    fpme->base.destroy         = llvm_middle_end_destroy;
 667
 668    fpme->draw = draw;
 669
 670    fpme->fetch = draw_pt_fetch_create( draw );
 671    if (!fpme->fetch)
 672       goto fail;
 673
 674    fpme->post_vs = draw_pt_post_vs_create( draw );
 675    if (!fpme->post_vs)
 676       goto fail;
 677
 678    fpme->emit = draw_pt_emit_create( draw );
 679    if (!fpme->emit)
 680       goto fail;
 681
 682    fpme->so_emit = draw_pt_so_emit_create( draw );
 683    if (!fpme->so_emit)
 684       goto fail;
 685
 686    fpme->llvm = draw->llvm;
 687    if (!fpme->llvm)
 688       goto fail;
 689
 690    fpme->current_variant = NULL;
 691
 692    return &fpme->base;
 693
 694  fail:
 695    if (fpme)
 696       llvm_middle_end_destroy( &fpme->base );
 697
 698    return NULL;
 699 }