src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "util/u_math.h"
  29 #include "util/u_memory.h"
  30 #include "util/u_prim.h"
  31 #include "draw/draw_context.h"
  32 #include "draw/draw_gs.h"
  33 #include "draw/draw_tess.h"
  34 #include "draw/draw_vbuf.h"
  35 #include "draw/draw_vertex.h"
  36 #include "draw/draw_pt.h"
  37 #include "draw/draw_prim_assembler.h"
  38 #include "draw/draw_vs.h"
  39 #include "draw/draw_llvm.h"
  40 #include "gallivm/lp_bld_init.h"
  41 #include "gallivm/lp_bld_debug.h"
  42
  43
  44 struct llvm_middle_end {
  45    struct draw_pt_middle_end base;
  46    struct draw_context *draw;
  47
  48    struct pt_emit *emit;
  49    struct pt_so_emit *so_emit;
  50    struct pt_fetch *fetch;
  51    struct pt_post_vs *post_vs;
  52
  53
  54    unsigned vertex_data_offset;
  55    unsigned vertex_size;
  56    unsigned input_prim;
  57    unsigned opt;
  58
  59    struct draw_llvm *llvm;
  60    struct draw_llvm_variant *current_variant;
  61 };
  62
  63
  64 /** cast wrapper */
  65 static inline struct llvm_middle_end *
  66 llvm_middle_end(struct draw_pt_middle_end *middle)
  67 {
  68    return (struct llvm_middle_end *) middle;
  69 }
  70
  71
  72 static void
  73 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  74 {
  75    struct draw_context *draw = fpme->draw;
  76    struct draw_llvm *llvm = fpme->llvm;
  77    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  78    struct draw_gs_llvm_variant_key *key;
  79    struct draw_gs_llvm_variant *variant = NULL;
  80    struct draw_gs_llvm_variant_list_item *li;
  81    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  82    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  83    unsigned i;
  84
  85    key = draw_gs_llvm_make_variant_key(llvm, store);
  86
  87    /* Search shader's list of variants for the key */
  88    li = first_elem(&shader->variants);
  89    while (!at_end(&shader->variants, li)) {
  90       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  91          variant = li->base;
  92          break;
  93       }
  94       li = next_elem(li);
  95    }
  96
  97    if (variant) {
  98       /* found the variant, move to head of global list (for LRU) */
  99       move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
 100    }
 101    else {
 102       /* Need to create new variant */
 103
 104       /* First check if we've created too many variants.  If so, free
 105        * 3.125% of the LRU to avoid using too much memory.
 106        */
 107       if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
 108          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 109             debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
 110                       shader->variants_cached, llvm->nr_gs_variants);
 111          }
 112
 113          /*
 114           * XXX: should we flush here ?
 115           */
 116          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 117             struct draw_gs_llvm_variant_list_item *item;
 118             if (is_empty_list(&llvm->gs_variants_list)) {
 119                break;
 120             }
 121             item = last_elem(&llvm->gs_variants_list);
 122             assert(item);
 123             assert(item->base);
 124             draw_gs_llvm_destroy_variant(item->base);
 125          }
 126       }
 127
 128       variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);
 129
 130       if (variant) {
 131          insert_at_head(&shader->variants, &variant->list_item_local);
 132          insert_at_head(&llvm->gs_variants_list,
 133                         &variant->list_item_global);
 134          llvm->nr_gs_variants++;
 135          shader->variants_cached++;
 136       }
 137    }
 138
 139    gs->current_variant = variant;
 140 }
 141
 142 static void
 143 llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
 144 {
 145    struct draw_context *draw = fpme->draw;
 146    struct draw_llvm *llvm = fpme->llvm;
 147    struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
 148    struct draw_tcs_llvm_variant_key *key;
 149    struct draw_tcs_llvm_variant *variant = NULL;
 150    struct draw_tcs_llvm_variant_list_item *li;
 151    struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
 152    char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
 153    unsigned i;
 154
 155    key = draw_tcs_llvm_make_variant_key(llvm, store);
 156
 157    /* Search shader's list of variants for the key */
 158    li = first_elem(&shader->variants);
 159    while (!at_end(&shader->variants, li)) {
 160       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 161          variant = li->base;
 162          break;
 163       }
 164       li = next_elem(li);
 165    }
 166
 167    if (variant) {
 168       /* found the variant, move to head of global list (for LRU) */
 169       move_to_head(&llvm->tcs_variants_list, &variant->list_item_global);
 170    }
 171    else {
 172       /* Need to create new variant */
 173
 174       /* First check if we've created too many variants.  If so, free
 175        * 3.125% of the LRU to avoid using too much memory.
 176        */
 177       if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
 178          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 179             debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
 180                       shader->variants_cached, llvm->nr_tcs_variants);
 181          }
 182
 183          /*
 184           * XXX: should we flush here ?
 185           */
 186          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 187             struct draw_tcs_llvm_variant_list_item *item;
 188             if (is_empty_list(&llvm->tcs_variants_list)) {
 189                break;
 190             }
 191             item = last_elem(&llvm->tcs_variants_list);
 192             assert(item);
 193             assert(item->base);
 194             draw_tcs_llvm_destroy_variant(item->base);
 195          }
 196       }
 197
 198       variant = draw_tcs_llvm_create_variant(llvm, 0, key);
 199
 200       if (variant) {
 201          insert_at_head(&shader->variants, &variant->list_item_local);
 202          insert_at_head(&llvm->tcs_variants_list,
 203                         &variant->list_item_global);
 204          llvm->nr_tcs_variants++;
 205          shader->variants_cached++;
 206       }
 207    }
 208
 209    tcs->current_variant = variant;
 210 }
 211
 212 static void
 213 llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
 214 {
 215    struct draw_context *draw = fpme->draw;
 216    struct draw_llvm *llvm = fpme->llvm;
 217    struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
 218    struct draw_tes_llvm_variant_key *key;
 219    struct draw_tes_llvm_variant *variant = NULL;
 220    struct draw_tes_llvm_variant_list_item *li;
 221    struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
 222    char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
 223    unsigned i;
 224
 225    key = draw_tes_llvm_make_variant_key(llvm, store);
 226
 227    /* Search shader's list of variants for the key */
 228    li = first_elem(&shader->variants);
 229    while (!at_end(&shader->variants, li)) {
 230       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 231          variant = li->base;
 232          break;
 233       }
 234       li = next_elem(li);
 235    }
 236
 237    if (variant) {
 238       /* found the variant, move to head of global list (for LRU) */
 239       move_to_head(&llvm->tes_variants_list, &variant->list_item_global);
 240    }
 241    else {
 242       /* Need to create new variant */
 243
 244       /* First check if we've created too many variants.  If so, free
 245        * 3.125% of the LRU to avoid using too much memory.
 246        */
 247       if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
 248          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 249             debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
 250                       shader->variants_cached, llvm->nr_tes_variants);
 251          }
 252
 253          /*
 254           * XXX: should we flush here ?
 255           */
 256          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 257             struct draw_tes_llvm_variant_list_item *item;
 258             if (is_empty_list(&llvm->tes_variants_list)) {
 259                break;
 260             }
 261             item = last_elem(&llvm->tes_variants_list);
 262             assert(item);
 263             assert(item->base);
 264             draw_tes_llvm_destroy_variant(item->base);
 265          }
 266       }
 267
 268       variant = draw_tes_llvm_create_variant(llvm, tes->info.num_outputs, key);
 269
 270       if (variant) {
 271          insert_at_head(&shader->variants, &variant->list_item_local);
 272          insert_at_head(&llvm->tes_variants_list,
 273                         &variant->list_item_global);
 274          llvm->nr_tes_variants++;
 275          shader->variants_cached++;
 276       }
 277    }
 278
 279    tes->current_variant = variant;
 280 }
 281
 282 /**
 283  * Prepare/validate middle part of the vertex pipeline.
 284  * NOTE: if you change this function, also look at the non-LLVM
 285  * function fetch_pipeline_prepare() for similar changes.
 286  */
 287 static void
 288 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
 289                          unsigned in_prim,
 290                          unsigned opt,
 291                          unsigned *max_vertices )
 292 {
 293    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 294    struct draw_context *draw = fpme->draw;
 295    struct draw_llvm *llvm = fpme->llvm;
 296    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
 297    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
 298    struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
 299    struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
 300    const unsigned out_prim = gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
 301       u_assembled_prim(in_prim);
 302    unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
 303                          out_prim == PIPE_PRIM_POINTS;
 304    unsigned nr;
 305
 306    fpme->input_prim = in_prim;
 307    fpme->opt = opt;
 308
 309    draw_pt_post_vs_prepare( fpme->post_vs,
 310                             draw->clip_xy,
 311                             draw->clip_z,
 312                             draw->clip_user,
 313                             point_clip ? draw->guard_band_points_xy :
 314                                          draw->guard_band_xy,
 315                             draw->bypass_viewport,
 316                             draw->rasterizer->clip_halfz,
 317                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
 318
 319    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
 320
 321    if (!(opt & PT_PIPELINE)) {
 322       draw_pt_emit_prepare( fpme->emit, out_prim,
 323                             max_vertices );
 324
 325       *max_vertices = MAX2( *max_vertices, 4096 );
 326    }
 327    else {
 328       /* limit max fetches by limiting max_vertices */
 329       *max_vertices = 4096;
 330    }
 331
 332    /* Get the number of float[4] attributes per vertex.
 333     * Note: this must be done after draw_pt_emit_prepare() since that
 334     * can effect the vertex size.
 335     */
 336    nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
 337
 338    /* Always leave room for the vertex header whether we need it or
 339     * not.  It's hard to get rid of it in particular because of the
 340     * viewport code in draw_pt_post_vs.c.
 341     */
 342    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
 343
 344    /* return even number */
 345    *max_vertices = *max_vertices & ~1;
 346
 347    /* Find/create the vertex shader variant */
 348    {
 349       struct draw_llvm_variant_key *key;
 350       struct draw_llvm_variant *variant = NULL;
 351       struct draw_llvm_variant_list_item *li;
 352       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
 353       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
 354       unsigned i;
 355
 356       key = draw_llvm_make_variant_key(llvm, store);
 357
 358       /* Search shader's list of variants for the key */
 359       li = first_elem(&shader->variants);
 360       while (!at_end(&shader->variants, li)) {
 361          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 362             variant = li->base;
 363             break;
 364          }
 365          li = next_elem(li);
 366       }
 367
 368       if (variant) {
 369          /* found the variant, move to head of global list (for LRU) */
 370          move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
 371       }
 372       else {
 373          /* Need to create new variant */
 374
 375          /* First check if we've created too many variants.  If so, free
 376           * 3.125% of the LRU to avoid using too much memory.
 377           */
 378          if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
 379             if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 380                debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
 381                          shader->variants_cached, llvm->nr_variants);
 382             }
 383
 384             /*
 385              * XXX: should we flush here ?
 386              */
 387             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 388                struct draw_llvm_variant_list_item *item;
 389                if (is_empty_list(&llvm->vs_variants_list)) {
 390                   break;
 391                }
 392                item = last_elem(&llvm->vs_variants_list);
 393                assert(item);
 394                assert(item->base);
 395                draw_llvm_destroy_variant(item->base);
 396             }
 397          }
 398
 399          variant = draw_llvm_create_variant(llvm, nr, key);
 400
 401          if (variant) {
 402             insert_at_head(&shader->variants, &variant->list_item_local);
 403             insert_at_head(&llvm->vs_variants_list,
 404                            &variant->list_item_global);
 405             llvm->nr_variants++;
 406             shader->variants_cached++;
 407          }
 408       }
 409
 410       fpme->current_variant = variant;
 411    }
 412
 413    if (gs) {
 414       llvm_middle_end_prepare_gs(fpme);
 415    }
 416    if (tcs) {
 417       llvm_middle_end_prepare_tcs(fpme);
 418    }
 419    if (tes) {
 420       llvm_middle_end_prepare_tes(fpme);
 421    }
 422 }
 423
 424
 425 /**
 426  * Bind/update constant buffer pointers, clip planes and viewport dims.
 427  * These are "light weight" parameters which aren't baked into the
 428  * generated code.  Updating these items is much cheaper than revalidating
 429  * and rebuilding the generated pipeline code.
 430  */
 431 static void
 432 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
 433 {
 434    static const float fake_const_buf[4];
 435    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 436    struct draw_context *draw = fpme->draw;
 437    struct draw_llvm *llvm = fpme->llvm;
 438    unsigned i;
 439
 440    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
 441       /*
 442        * There could be a potential issue with rounding this up, as the
 443        * shader expects 16-byte allocations, the fix is likely to move
 444        * to LOAD intrinsic in the future and remove the vec4 constraint.
 445        */
 446       int num_consts =
 447          DIV_ROUND_UP(draw->pt.user.vs_constants_size[i], (sizeof(float) * 4));
 448       llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
 449       llvm->jit_context.num_vs_constants[i] = num_consts;
 450       if (num_consts == 0) {
 451          llvm->jit_context.vs_constants[i] = fake_const_buf;
 452       }
 453    }
 454    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
 455       int num_ssbos = draw->pt.user.vs_ssbos_size[i];
 456       llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
 457       llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
 458       if (num_ssbos == 0) {
 459          llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
 460       }
 461    }
 462
 463    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
 464       int num_consts =
 465          DIV_ROUND_UP(draw->pt.user.gs_constants_size[i], (sizeof(float) * 4));
 466       llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
 467       llvm->gs_jit_context.num_constants[i] = num_consts;
 468       if (num_consts == 0) {
 469          llvm->gs_jit_context.constants[i] = fake_const_buf;
 470       }
 471    }
 472    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
 473       int num_ssbos = draw->pt.user.gs_ssbos_size[i];
 474       llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
 475       llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
 476       if (num_ssbos == 0) {
 477          llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 478       }
 479    }
 480
 481    for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.constants); ++i) {
 482       int num_consts =
 483          DIV_ROUND_UP(draw->pt.user.tcs_constants_size[i], (sizeof(float) * 4));
 484       llvm->tcs_jit_context.constants[i] = draw->pt.user.tcs_constants[i];
 485       llvm->tcs_jit_context.num_constants[i] = num_consts;
 486       if (num_consts == 0) {
 487          llvm->tcs_jit_context.constants[i] = fake_const_buf;
 488       }
 489    }
 490    for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.ssbos); ++i) {
 491       int num_ssbos = draw->pt.user.tcs_ssbos_size[i];
 492       llvm->tcs_jit_context.ssbos[i] = draw->pt.user.tcs_ssbos[i];
 493       llvm->tcs_jit_context.num_ssbos[i] = num_ssbos;
 494       if (num_ssbos == 0) {
 495          llvm->tcs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 496       }
 497    }
 498
 499    for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.constants); ++i) {
 500       int num_consts =
 501          DIV_ROUND_UP(draw->pt.user.tes_constants_size[i], (sizeof(float) * 4));
 502       llvm->tes_jit_context.constants[i] = draw->pt.user.tes_constants[i];
 503       llvm->tes_jit_context.num_constants[i] = num_consts;
 504       if (num_consts == 0) {
 505          llvm->tes_jit_context.constants[i] = fake_const_buf;
 506       }
 507    }
 508    for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.ssbos); ++i) {
 509       int num_ssbos = draw->pt.user.tes_ssbos_size[i];
 510       llvm->tes_jit_context.ssbos[i] = draw->pt.user.tes_ssbos[i];
 511       llvm->tes_jit_context.num_ssbos[i] = num_ssbos;
 512       if (num_ssbos == 0) {
 513          llvm->tes_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 514       }
 515    }
 516
 517    llvm->jit_context.planes =
 518       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 519    llvm->gs_jit_context.planes =
 520       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 521
 522    llvm->jit_context.viewports = draw->viewports;
 523    llvm->gs_jit_context.viewports = draw->viewports;
 524 }
 525
 526
 527 static void
 528 pipeline(struct llvm_middle_end *llvm,
 529          const struct draw_vertex_info *vert_info,
 530          const struct draw_prim_info *prim_info)
 531 {
 532    if (prim_info->linear)
 533       draw_pipeline_run_linear( llvm->draw,
 534                                 vert_info,
 535                                 prim_info);
 536    else
 537       draw_pipeline_run( llvm->draw,
 538                          vert_info,
 539                          prim_info );
 540 }
 541
 542
 543 static void
 544 emit(struct pt_emit *emit,
 545      const struct draw_vertex_info *vert_info,
 546      const struct draw_prim_info *prim_info)
 547 {
 548    if (prim_info->linear) {
 549       draw_pt_emit_linear(emit, vert_info, prim_info);
 550    }
 551    else {
 552       draw_pt_emit(emit, vert_info, prim_info);
 553    }
 554 }
 555
 556
 557 static void
 558 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
 559                       const struct draw_fetch_info *fetch_info,
 560                       const struct draw_prim_info *in_prim_info)
 561 {
 562    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 563    struct draw_context *draw = fpme->draw;
 564    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
 565    struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
 566    struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
 567    struct draw_prim_info tcs_prim_info;
 568    struct draw_prim_info tes_prim_info;
 569    struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
 570    struct draw_vertex_info llvm_vert_info;
 571    struct draw_vertex_info tcs_vert_info;
 572    struct draw_vertex_info tes_vert_info;
 573    struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
 574    struct draw_vertex_info *vert_info;
 575    struct draw_prim_info ia_prim_info;
 576    struct draw_vertex_info ia_vert_info;
 577    const struct draw_prim_info *prim_info = in_prim_info;
 578    boolean free_prim_info = FALSE;
 579    unsigned opt = fpme->opt;
 580    boolean clipped = 0;
 581    unsigned start_or_maxelt, vid_base;
 582    const unsigned *elts;
 583    ushort *tes_elts_out = NULL;
 584
 585    assert(fetch_info->count > 0);
 586    llvm_vert_info.count = fetch_info->count;
 587    llvm_vert_info.vertex_size = fpme->vertex_size;
 588    llvm_vert_info.stride = fpme->vertex_size;
 589    llvm_vert_info.verts = (struct vertex_header *)
 590       MALLOC(fpme->vertex_size *
 591              align(fetch_info->count, lp_native_vector_width / 32));
 592    if (!llvm_vert_info.verts) {
 593       assert(0);
 594       return;
 595    }
 596
 597    if (draw->collect_statistics) {
 598       draw->statistics.ia_vertices += prim_info->count;
 599       if (prim_info->prim == PIPE_PRIM_PATCHES)
 600          draw->statistics.ia_primitives += prim_info->count / draw->pt.vertices_per_patch;
 601       else
 602          draw->statistics.ia_primitives +=
 603             u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
 604       draw->statistics.vs_invocations += fetch_info->count;
 605    }
 606
 607    if (fetch_info->linear) {
 608       start_or_maxelt = fetch_info->start;
 609       vid_base = draw->start_index;
 610       elts = NULL;
 611    }
 612    else {
 613       start_or_maxelt = draw->pt.user.eltMax;
 614       vid_base = draw->pt.user.eltBias;
 615       elts = fetch_info->elts;
 616    }
 617    clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
 618                                              llvm_vert_info.verts,
 619                                              draw->pt.user.vbuffer,
 620                                              fetch_info->count,
 621                                              start_or_maxelt,
 622                                              fpme->vertex_size,
 623                                              draw->pt.vertex_buffer,
 624                                              draw->instance_id,
 625                                              vid_base,
 626                                              draw->start_instance,
 627                                              elts, draw->pt.user.drawid);
 628
 629    /* Finished with fetch and vs:
 630     */
 631    fetch_info = NULL;
 632    vert_info = &llvm_vert_info;
 633
 634    if (opt & PT_SHADE) {
 635       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 636       if (tcs_shader) {
 637          draw_tess_ctrl_shader_run(tcs_shader,
 638                                    draw->pt.user.tcs_constants,
 639                                    draw->pt.user.tcs_constants_size,
 640                                    vert_info,
 641                                    prim_info,
 642                                    &vshader->info,
 643                                    &tcs_vert_info,
 644                                    &tcs_prim_info);
 645          FREE(vert_info->verts);
 646          vert_info = &tcs_vert_info;
 647          prim_info = &tcs_prim_info;
 648
 649       } else if (tes_shader) {
 650          unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
 651          tcs_prim_info = *prim_info;
 652          tcs_prim_info.primitive_count = num_prims;
 653          prim_info = &tcs_prim_info;
 654       }
 655
 656       if (tes_shader) {
 657          draw_tess_eval_shader_run(tes_shader,
 658                                    draw->pt.user.tes_constants,
 659                                    draw->pt.user.tes_constants_size,
 660                                    tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
 661                                    vert_info,
 662                                    prim_info,
 663                                    tcs_shader ? &tcs_shader->info : &vshader->info,
 664                                    &tes_vert_info,
 665                                    &tes_prim_info, &tes_elts_out);
 666
 667          FREE(vert_info->verts);
 668          vert_info = &tes_vert_info;
 669          prim_info = &tes_prim_info;
 670          free_prim_info = TRUE;
 671
 672          /*
 673           * pt emit can only handle ushort number of vertices (see
 674           * render->allocate_vertices).
 675           * vsplit guarantees there's never more than 4096, however GS can
 676           * easily blow this up (by a factor of 256 (or even 1024) max).
 677           */
 678          if (vert_info->count > 65535) {
 679             opt |= PT_PIPELINE;
 680          }
 681       }
 682    }
 683
 684    if ((opt & PT_SHADE) && gshader) {
 685       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 686       draw_geometry_shader_run(gshader,
 687                                draw->pt.user.gs_constants,
 688                                draw->pt.user.gs_constants_size,
 689                                vert_info,
 690                                prim_info,
 691                                tes_shader ? &tes_shader->info : &vshader->info,
 692                                gs_vert_info,
 693                                gs_prim_info);
 694
 695       FREE(vert_info->verts);
 696       if (free_prim_info) {
 697          FREE(prim_info->primitive_lengths);
 698          FREE(tes_elts_out);
 699       }
 700       vert_info = &gs_vert_info[0];
 701       prim_info = &gs_prim_info[0];
 702       free_prim_info = FALSE;
 703       /*
 704        * pt emit can only handle ushort number of vertices (see
 705        * render->allocate_vertices).
 706        * vsplit guarantees there's never more than 4096, however GS can
 707        * easily blow this up (by a factor of 256 (or even 1024) max).
 708        */
 709       if (vert_info->count > 65535) {
 710          opt |= PT_PIPELINE;
 711       }
 712    } else {
 713       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
 714          draw_prim_assembler_run(draw, prim_info, vert_info,
 715                                  &ia_prim_info, &ia_vert_info);
 716
 717          if (ia_vert_info.count) {
 718             FREE(vert_info->verts);
 719             vert_info = &ia_vert_info;
 720             prim_info = &ia_prim_info;
 721             free_prim_info = TRUE;
 722          }
 723       }
 724    }
 725    if (prim_info->count == 0) {
 726       debug_printf("GS/IA didn't emit any vertices!\n");
 727
 728       FREE(vert_info->verts);
 729       if (free_prim_info) {
 730          FREE(tes_elts_out);
 731          FREE(prim_info->primitive_lengths);
 732       }
 733       return;
 734    }
 735
 736    /* stream output needs to be done before clipping */
 737    draw_pt_so_emit( fpme->so_emit, gshader ? gshader->num_vertex_streams : 1, vert_info, prim_info );
 738
 739    draw_stats_clipper_primitives(draw, prim_info);
 740
 741    /*
 742     * if there's no position, need to stop now, or the latter stages
 743     * will try to access non-existent position output.
 744     */
 745    if (draw_current_shader_position_output(draw) != -1) {
 746       if ((opt & PT_SHADE) && (gshader || tes_shader ||
 747                                draw->vs.vertex_shader->info.writes_viewport_index)) {
 748          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
 749       }
 750       /* "clipped" also includes non-one edgeflag */
 751       if (clipped) {
 752          opt |= PT_PIPELINE;
 753       }
 754
 755       /* Do we need to run the pipeline? Now will come here if clipped
 756        */
 757       if (opt & PT_PIPELINE) {
 758          pipeline( fpme, vert_info, prim_info );
 759       }
 760       else {
 761          emit( fpme->emit, vert_info, prim_info );
 762       }
 763    }
 764    FREE(vert_info->verts);
 765    if (free_prim_info) {
 766       FREE(tes_elts_out);
 767       FREE(prim_info->primitive_lengths);
 768    }
 769 }
 770
 771
 772 static inline unsigned
 773 prim_type(unsigned prim, unsigned flags)
 774 {
 775    if (flags & DRAW_LINE_LOOP_AS_STRIP)
 776       return PIPE_PRIM_LINE_STRIP;
 777    else
 778       return prim;
 779 }
 780
 781
 782 static void
 783 llvm_middle_end_run(struct draw_pt_middle_end *middle,
 784                     const unsigned *fetch_elts,
 785                     unsigned fetch_count,
 786                     const ushort *draw_elts,
 787                     unsigned draw_count,
 788                     unsigned prim_flags)
 789 {
 790    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 791    struct draw_fetch_info fetch_info;
 792    struct draw_prim_info prim_info;
 793
 794    fetch_info.linear = FALSE;
 795    fetch_info.start = 0;
 796    fetch_info.elts = fetch_elts;
 797    fetch_info.count = fetch_count;
 798
 799    prim_info.linear = FALSE;
 800    prim_info.start = 0;
 801    prim_info.count = draw_count;
 802    prim_info.elts = draw_elts;
 803    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 804    prim_info.flags = prim_flags;
 805    prim_info.primitive_count = 1;
 806    prim_info.primitive_lengths = &draw_count;
 807
 808    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 809 }
 810
 811
 812 static void
 813 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
 814                            unsigned start,
 815                            unsigned count,
 816                            unsigned prim_flags)
 817 {
 818    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 819    struct draw_fetch_info fetch_info;
 820    struct draw_prim_info prim_info;
 821
 822    fetch_info.linear = TRUE;
 823    fetch_info.start = start;
 824    fetch_info.count = count;
 825    fetch_info.elts = NULL;
 826
 827    prim_info.linear = TRUE;
 828    prim_info.start = 0;
 829    prim_info.count = count;
 830    prim_info.elts = NULL;
 831    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 832    prim_info.flags = prim_flags;
 833    prim_info.primitive_count = 1;
 834    prim_info.primitive_lengths = &count;
 835
 836    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 837 }
 838
 839
 840 static boolean
 841 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
 842                                 unsigned start,
 843                                 unsigned count,
 844                                 const ushort *draw_elts,
 845                                 unsigned draw_count,
 846                                 unsigned prim_flags)
 847 {
 848    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 849    struct draw_fetch_info fetch_info;
 850    struct draw_prim_info prim_info;
 851
 852    fetch_info.linear = TRUE;
 853    fetch_info.start = start;
 854    fetch_info.count = count;
 855    fetch_info.elts = NULL;
 856
 857    prim_info.linear = FALSE;
 858    prim_info.start = 0;
 859    prim_info.count = draw_count;
 860    prim_info.elts = draw_elts;
 861    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 862    prim_info.flags = prim_flags;
 863    prim_info.primitive_count = 1;
 864    prim_info.primitive_lengths = &draw_count;
 865
 866    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 867
 868    return TRUE;
 869 }
 870
 871
 872 static void
 873 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
 874 {
 875    /* nothing to do */
 876 }
 877
 878
 879 static void
 880 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
 881 {
 882    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 883
 884    if (fpme->fetch)
 885       draw_pt_fetch_destroy( fpme->fetch );
 886
 887    if (fpme->emit)
 888       draw_pt_emit_destroy( fpme->emit );
 889
 890    if (fpme->so_emit)
 891       draw_pt_so_emit_destroy( fpme->so_emit );
 892
 893    if (fpme->post_vs)
 894       draw_pt_post_vs_destroy( fpme->post_vs );
 895
 896    FREE(middle);
 897 }
 898
 899
 900 struct draw_pt_middle_end *
 901 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
 902 {
 903    struct llvm_middle_end *fpme = 0;
 904
 905    if (!draw->llvm)
 906       return NULL;
 907
 908    fpme = CALLOC_STRUCT( llvm_middle_end );
 909    if (!fpme)
 910       goto fail;
 911
 912    fpme->base.prepare         = llvm_middle_end_prepare;
 913    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
 914    fpme->base.run             = llvm_middle_end_run;
 915    fpme->base.run_linear      = llvm_middle_end_linear_run;
 916    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
 917    fpme->base.finish          = llvm_middle_end_finish;
 918    fpme->base.destroy         = llvm_middle_end_destroy;
 919
 920    fpme->draw = draw;
 921
 922    fpme->fetch = draw_pt_fetch_create( draw );
 923    if (!fpme->fetch)
 924       goto fail;
 925
 926    fpme->post_vs = draw_pt_post_vs_create( draw );
 927    if (!fpme->post_vs)
 928       goto fail;
 929
 930    fpme->emit = draw_pt_emit_create( draw );
 931    if (!fpme->emit)
 932       goto fail;
 933
 934    fpme->so_emit = draw_pt_so_emit_create( draw );
 935    if (!fpme->so_emit)
 936       goto fail;
 937
 938    fpme->llvm = draw->llvm;
 939    if (!fpme->llvm)
 940       goto fail;
 941
 942    fpme->current_variant = NULL;
 943
 944    return &fpme->base;
 945
 946  fail:
 947    if (fpme)
 948       llvm_middle_end_destroy( &fpme->base );
 949
 950    return NULL;
 951 }