src/gallium/auxiliary/draw/draw_pt_fetch_shade_pipeline_llvm.c

   1 /**************************************************************************
   2  *
   3  * Copyright 2010 VMware, Inc.
   4  * All Rights Reserved.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the
   8  * "Software"), to deal in the Software without restriction, including
   9  * without limitation the rights to use, copy, modify, merge, publish,
  10  * distribute, sub license, and/or sell copies of the Software, and to
  11  * permit persons to whom the Software is furnished to do so, subject to
  12  * the following conditions:
  13  *
  14  * The above copyright notice and this permission notice (including the
  15  * next paragraph) shall be included in all copies or substantial portions
  16  * of the Software.
  17  *
  18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
  19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
  21  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
  22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  25  *
  26  **************************************************************************/
  27
  28 #include "util/u_math.h"
  29 #include "util/u_memory.h"
  30 #include "util/u_prim.h"
  31 #include "draw/draw_context.h"
  32 #include "draw/draw_gs.h"
  33 #include "draw/draw_tess.h"
  34 #include "draw/draw_vbuf.h"
  35 #include "draw/draw_vertex.h"
  36 #include "draw/draw_pt.h"
  37 #include "draw/draw_prim_assembler.h"
  38 #include "draw/draw_vs.h"
  39 #include "draw/draw_llvm.h"
  40 #include "gallivm/lp_bld_init.h"
  41 #include "gallivm/lp_bld_debug.h"
  42
  43
  44 struct llvm_middle_end {
  45    struct draw_pt_middle_end base;
  46    struct draw_context *draw;
  47
  48    struct pt_emit *emit;
  49    struct pt_so_emit *so_emit;
  50    struct pt_fetch *fetch;
  51    struct pt_post_vs *post_vs;
  52
  53
  54    unsigned vertex_data_offset;
  55    unsigned vertex_size;
  56    unsigned input_prim;
  57    unsigned opt;
  58
  59    struct draw_llvm *llvm;
  60    struct draw_llvm_variant *current_variant;
  61 };
  62
  63
  64 /** cast wrapper */
  65 static inline struct llvm_middle_end *
  66 llvm_middle_end(struct draw_pt_middle_end *middle)
  67 {
  68    return (struct llvm_middle_end *) middle;
  69 }
  70
  71
  72 static void
  73 llvm_middle_end_prepare_gs(struct llvm_middle_end *fpme)
  74 {
  75    struct draw_context *draw = fpme->draw;
  76    struct draw_llvm *llvm = fpme->llvm;
  77    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
  78    struct draw_gs_llvm_variant_key *key;
  79    struct draw_gs_llvm_variant *variant = NULL;
  80    struct draw_gs_llvm_variant_list_item *li;
  81    struct llvm_geometry_shader *shader = llvm_geometry_shader(gs);
  82    char store[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE];
  83    unsigned i;
  84
  85    key = draw_gs_llvm_make_variant_key(llvm, store);
  86
  87    /* Search shader's list of variants for the key */
  88    li = first_elem(&shader->variants);
  89    while (!at_end(&shader->variants, li)) {
  90       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
  91          variant = li->base;
  92          break;
  93       }
  94       li = next_elem(li);
  95    }
  96
  97    if (variant) {
  98       /* found the variant, move to head of global list (for LRU) */
  99       move_to_head(&llvm->gs_variants_list, &variant->list_item_global);
 100    }
 101    else {
 102       /* Need to create new variant */
 103
 104       /* First check if we've created too many variants.  If so, free
 105        * 3.125% of the LRU to avoid using too much memory.
 106        */
 107       if (llvm->nr_gs_variants >= DRAW_MAX_SHADER_VARIANTS) {
 108          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 109             debug_printf("Evicting GS: %u gs variants,\t%u total variants\n",
 110                       shader->variants_cached, llvm->nr_gs_variants);
 111          }
 112
 113          /*
 114           * XXX: should we flush here ?
 115           */
 116          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 117             struct draw_gs_llvm_variant_list_item *item;
 118             if (is_empty_list(&llvm->gs_variants_list)) {
 119                break;
 120             }
 121             item = last_elem(&llvm->gs_variants_list);
 122             assert(item);
 123             assert(item->base);
 124             draw_gs_llvm_destroy_variant(item->base);
 125          }
 126       }
 127
 128       variant = draw_gs_llvm_create_variant(llvm, gs->info.num_outputs, key);
 129
 130       if (variant) {
 131          insert_at_head(&shader->variants, &variant->list_item_local);
 132          insert_at_head(&llvm->gs_variants_list,
 133                         &variant->list_item_global);
 134          llvm->nr_gs_variants++;
 135          shader->variants_cached++;
 136       }
 137    }
 138
 139    gs->current_variant = variant;
 140 }
 141
 142 static void
 143 llvm_middle_end_prepare_tcs(struct llvm_middle_end *fpme)
 144 {
 145    struct draw_context *draw = fpme->draw;
 146    struct draw_llvm *llvm = fpme->llvm;
 147    struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
 148    struct draw_tcs_llvm_variant_key *key;
 149    struct draw_tcs_llvm_variant *variant = NULL;
 150    struct draw_tcs_llvm_variant_list_item *li;
 151    struct llvm_tess_ctrl_shader *shader = llvm_tess_ctrl_shader(tcs);
 152    char store[DRAW_TCS_LLVM_MAX_VARIANT_KEY_SIZE];
 153    unsigned i;
 154
 155    key = draw_tcs_llvm_make_variant_key(llvm, store);
 156
 157    /* Search shader's list of variants for the key */
 158    li = first_elem(&shader->variants);
 159    while (!at_end(&shader->variants, li)) {
 160       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 161          variant = li->base;
 162          break;
 163       }
 164       li = next_elem(li);
 165    }
 166
 167    if (variant) {
 168       /* found the variant, move to head of global list (for LRU) */
 169       move_to_head(&llvm->tcs_variants_list, &variant->list_item_global);
 170    }
 171    else {
 172       /* Need to create new variant */
 173
 174       /* First check if we've created too many variants.  If so, free
 175        * 3.125% of the LRU to avoid using too much memory.
 176        */
 177       if (llvm->nr_tcs_variants >= DRAW_MAX_SHADER_VARIANTS) {
 178          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 179             debug_printf("Evicting TCS: %u tcs variants,\t%u total variants\n",
 180                       shader->variants_cached, llvm->nr_tcs_variants);
 181          }
 182
 183          /*
 184           * XXX: should we flush here ?
 185           */
 186          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 187             struct draw_tcs_llvm_variant_list_item *item;
 188             if (is_empty_list(&llvm->tcs_variants_list)) {
 189                break;
 190             }
 191             item = last_elem(&llvm->tcs_variants_list);
 192             assert(item);
 193             assert(item->base);
 194             draw_tcs_llvm_destroy_variant(item->base);
 195          }
 196       }
 197
 198       variant = draw_tcs_llvm_create_variant(llvm, 0, key);
 199
 200       if (variant) {
 201          insert_at_head(&shader->variants, &variant->list_item_local);
 202          insert_at_head(&llvm->tcs_variants_list,
 203                         &variant->list_item_global);
 204          llvm->nr_tcs_variants++;
 205          shader->variants_cached++;
 206       }
 207    }
 208
 209    tcs->current_variant = variant;
 210 }
 211
 212 static void
 213 llvm_middle_end_prepare_tes(struct llvm_middle_end *fpme)
 214 {
 215    struct draw_context *draw = fpme->draw;
 216    struct draw_llvm *llvm = fpme->llvm;
 217    struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
 218    struct draw_tes_llvm_variant_key *key;
 219    struct draw_tes_llvm_variant *variant = NULL;
 220    struct draw_tes_llvm_variant_list_item *li;
 221    struct llvm_tess_eval_shader *shader = llvm_tess_eval_shader(tes);
 222    char store[DRAW_TES_LLVM_MAX_VARIANT_KEY_SIZE];
 223    unsigned i;
 224
 225    key = draw_tes_llvm_make_variant_key(llvm, store);
 226
 227    /* Search shader's list of variants for the key */
 228    li = first_elem(&shader->variants);
 229    while (!at_end(&shader->variants, li)) {
 230       if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 231          variant = li->base;
 232          break;
 233       }
 234       li = next_elem(li);
 235    }
 236
 237    if (variant) {
 238       /* found the variant, move to head of global list (for LRU) */
 239       move_to_head(&llvm->tes_variants_list, &variant->list_item_global);
 240    }
 241    else {
 242       /* Need to create new variant */
 243
 244       /* First check if we've created too many variants.  If so, free
 245        * 3.125% of the LRU to avoid using too much memory.
 246        */
 247       if (llvm->nr_tes_variants >= DRAW_MAX_SHADER_VARIANTS) {
 248          if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 249             debug_printf("Evicting TES: %u tes variants,\t%u total variants\n",
 250                       shader->variants_cached, llvm->nr_tes_variants);
 251          }
 252
 253          /*
 254           * XXX: should we flush here ?
 255           */
 256          for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 257             struct draw_tes_llvm_variant_list_item *item;
 258             if (is_empty_list(&llvm->tes_variants_list)) {
 259                break;
 260             }
 261             item = last_elem(&llvm->tes_variants_list);
 262             assert(item);
 263             assert(item->base);
 264             draw_tes_llvm_destroy_variant(item->base);
 265          }
 266       }
 267
 268       variant = draw_tes_llvm_create_variant(llvm, tes->info.num_outputs, key);
 269
 270       if (variant) {
 271          insert_at_head(&shader->variants, &variant->list_item_local);
 272          insert_at_head(&llvm->tes_variants_list,
 273                         &variant->list_item_global);
 274          llvm->nr_tes_variants++;
 275          shader->variants_cached++;
 276       }
 277    }
 278
 279    tes->current_variant = variant;
 280 }
 281
 282 /**
 283  * Prepare/validate middle part of the vertex pipeline.
 284  * NOTE: if you change this function, also look at the non-LLVM
 285  * function fetch_pipeline_prepare() for similar changes.
 286  */
 287 static void
 288 llvm_middle_end_prepare( struct draw_pt_middle_end *middle,
 289                          unsigned in_prim,
 290                          unsigned opt,
 291                          unsigned *max_vertices )
 292 {
 293    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 294    struct draw_context *draw = fpme->draw;
 295    struct draw_llvm *llvm = fpme->llvm;
 296    struct draw_vertex_shader *vs = draw->vs.vertex_shader;
 297    struct draw_geometry_shader *gs = draw->gs.geometry_shader;
 298    struct draw_tess_ctrl_shader *tcs = draw->tcs.tess_ctrl_shader;
 299    struct draw_tess_eval_shader *tes = draw->tes.tess_eval_shader;
 300    const unsigned out_prim = gs ? gs->output_primitive : tes ? get_tes_output_prim(tes) :
 301       u_assembled_prim(in_prim);
 302    unsigned point_clip = draw->rasterizer->fill_front == PIPE_POLYGON_MODE_POINT ||
 303                          out_prim == PIPE_PRIM_POINTS;
 304    unsigned nr;
 305
 306    fpme->input_prim = in_prim;
 307    fpme->opt = opt;
 308
 309    draw_pt_post_vs_prepare( fpme->post_vs,
 310                             draw->clip_xy,
 311                             draw->clip_z,
 312                             draw->clip_user,
 313                             point_clip ? draw->guard_band_points_xy :
 314                                          draw->guard_band_xy,
 315                             draw->bypass_viewport,
 316                             draw->rasterizer->clip_halfz,
 317                             (draw->vs.edgeflag_output ? TRUE : FALSE) );
 318
 319    draw_pt_so_emit_prepare( fpme->so_emit, gs == NULL );
 320
 321    if (!(opt & PT_PIPELINE)) {
 322       draw_pt_emit_prepare( fpme->emit, out_prim,
 323                             max_vertices );
 324
 325       *max_vertices = MAX2( *max_vertices, 4096 );
 326    }
 327    else {
 328       /* limit max fetches by limiting max_vertices */
 329       *max_vertices = 4096;
 330    }
 331
 332    /* Get the number of float[4] attributes per vertex.
 333     * Note: this must be done after draw_pt_emit_prepare() since that
 334     * can effect the vertex size.
 335     */
 336    nr = MAX2(vs->info.num_inputs, draw_total_vs_outputs(draw));
 337
 338    /* Always leave room for the vertex header whether we need it or
 339     * not.  It's hard to get rid of it in particular because of the
 340     * viewport code in draw_pt_post_vs.c.
 341     */
 342    fpme->vertex_size = sizeof(struct vertex_header) + nr * 4 * sizeof(float);
 343
 344    /* return even number */
 345    *max_vertices = *max_vertices & ~1;
 346
 347    /* Find/create the vertex shader variant */
 348    {
 349       struct draw_llvm_variant_key *key;
 350       struct draw_llvm_variant *variant = NULL;
 351       struct draw_llvm_variant_list_item *li;
 352       struct llvm_vertex_shader *shader = llvm_vertex_shader(vs);
 353       char store[DRAW_LLVM_MAX_VARIANT_KEY_SIZE];
 354       unsigned i;
 355
 356       key = draw_llvm_make_variant_key(llvm, store);
 357
 358       /* Search shader's list of variants for the key */
 359       li = first_elem(&shader->variants);
 360       while (!at_end(&shader->variants, li)) {
 361          if (memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
 362             variant = li->base;
 363             break;
 364          }
 365          li = next_elem(li);
 366       }
 367
 368       if (variant) {
 369          /* found the variant, move to head of global list (for LRU) */
 370          move_to_head(&llvm->vs_variants_list, &variant->list_item_global);
 371       }
 372       else {
 373          /* Need to create new variant */
 374
 375          /* First check if we've created too many variants.  If so, free
 376           * 3.125% of the LRU to avoid using too much memory.
 377           */
 378          if (llvm->nr_variants >= DRAW_MAX_SHADER_VARIANTS) {
 379             if (gallivm_debug & GALLIVM_DEBUG_PERF) {
 380                debug_printf("Evicting VS: %u vs variants,\t%u total variants\n",
 381                          shader->variants_cached, llvm->nr_variants);
 382             }
 383
 384             /*
 385              * XXX: should we flush here ?
 386              */
 387             for (i = 0; i < DRAW_MAX_SHADER_VARIANTS / 32; i++) {
 388                struct draw_llvm_variant_list_item *item;
 389                if (is_empty_list(&llvm->vs_variants_list)) {
 390                   break;
 391                }
 392                item = last_elem(&llvm->vs_variants_list);
 393                assert(item);
 394                assert(item->base);
 395                draw_llvm_destroy_variant(item->base);
 396             }
 397          }
 398
 399          variant = draw_llvm_create_variant(llvm, nr, key);
 400
 401          if (variant) {
 402             insert_at_head(&shader->variants, &variant->list_item_local);
 403             insert_at_head(&llvm->vs_variants_list,
 404                            &variant->list_item_global);
 405             llvm->nr_variants++;
 406             shader->variants_cached++;
 407          }
 408       }
 409
 410       fpme->current_variant = variant;
 411    }
 412
 413    if (gs) {
 414       llvm_middle_end_prepare_gs(fpme);
 415    }
 416    if (tcs) {
 417       llvm_middle_end_prepare_tcs(fpme);
 418    }
 419    if (tes) {
 420       llvm_middle_end_prepare_tes(fpme);
 421    }
 422 }
 423
 424 static unsigned
 425 get_num_consts_robust(struct draw_context *draw, unsigned *sizes, unsigned idx)
 426 {
 427    unsigned const_bytes = sizes[idx];
 428
 429    if (const_bytes < sizeof(float))
 430       return 0;
 431
 432    return DIV_ROUND_UP(const_bytes, draw->constant_buffer_stride);
 433 }
 434
 435 /**
 436  * Bind/update constant buffer pointers, clip planes and viewport dims.
 437  * These are "light weight" parameters which aren't baked into the
 438  * generated code.  Updating these items is much cheaper than revalidating
 439  * and rebuilding the generated pipeline code.
 440  */
 441 static void
 442 llvm_middle_end_bind_parameters(struct draw_pt_middle_end *middle)
 443 {
 444    static const float fake_const_buf[4];
 445    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 446    struct draw_context *draw = fpme->draw;
 447    struct draw_llvm *llvm = fpme->llvm;
 448    unsigned i;
 449
 450    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_constants); ++i) {
 451       /*
 452        * There could be a potential issue with rounding this up, as the
 453        * shader expects 16-byte allocations, the fix is likely to move
 454        * to LOAD intrinsic in the future and remove the vec4 constraint.
 455        */
 456       int num_consts = get_num_consts_robust(draw, draw->pt.user.vs_constants_size, i);
 457       llvm->jit_context.vs_constants[i] = draw->pt.user.vs_constants[i];
 458       llvm->jit_context.num_vs_constants[i] = num_consts;
 459       if (num_consts == 0) {
 460          llvm->jit_context.vs_constants[i] = fake_const_buf;
 461       }
 462    }
 463    for (i = 0; i < ARRAY_SIZE(llvm->jit_context.vs_ssbos); ++i) {
 464       int num_ssbos = draw->pt.user.vs_ssbos_size[i];
 465       llvm->jit_context.vs_ssbos[i] = draw->pt.user.vs_ssbos[i];
 466       llvm->jit_context.num_vs_ssbos[i] = num_ssbos;
 467       if (num_ssbos == 0) {
 468          llvm->jit_context.vs_ssbos[i] = (const uint32_t *)fake_const_buf;
 469       }
 470    }
 471
 472    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.constants); ++i) {
 473       int num_consts = get_num_consts_robust(draw, draw->pt.user.gs_constants_size, i);
 474       llvm->gs_jit_context.constants[i] = draw->pt.user.gs_constants[i];
 475       llvm->gs_jit_context.num_constants[i] = num_consts;
 476       if (num_consts == 0) {
 477          llvm->gs_jit_context.constants[i] = fake_const_buf;
 478       }
 479    }
 480    for (i = 0; i < ARRAY_SIZE(llvm->gs_jit_context.ssbos); ++i) {
 481       int num_ssbos = draw->pt.user.gs_ssbos_size[i];
 482       llvm->gs_jit_context.ssbos[i] = draw->pt.user.gs_ssbos[i];
 483       llvm->gs_jit_context.num_ssbos[i] = num_ssbos;
 484       if (num_ssbos == 0) {
 485          llvm->gs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 486       }
 487    }
 488
 489    for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.constants); ++i) {
 490       int num_consts = get_num_consts_robust(draw, draw->pt.user.tcs_constants_size, i);
 491       llvm->tcs_jit_context.constants[i] = draw->pt.user.tcs_constants[i];
 492       llvm->tcs_jit_context.num_constants[i] = num_consts;
 493       if (num_consts == 0) {
 494          llvm->tcs_jit_context.constants[i] = fake_const_buf;
 495       }
 496    }
 497    for (i = 0; i < ARRAY_SIZE(llvm->tcs_jit_context.ssbos); ++i) {
 498       int num_ssbos = draw->pt.user.tcs_ssbos_size[i];
 499       llvm->tcs_jit_context.ssbos[i] = draw->pt.user.tcs_ssbos[i];
 500       llvm->tcs_jit_context.num_ssbos[i] = num_ssbos;
 501       if (num_ssbos == 0) {
 502          llvm->tcs_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 503       }
 504    }
 505
 506    for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.constants); ++i) {
 507       int num_consts = get_num_consts_robust(draw, draw->pt.user.tes_constants_size, i);
 508       llvm->tes_jit_context.constants[i] = draw->pt.user.tes_constants[i];
 509       llvm->tes_jit_context.num_constants[i] = num_consts;
 510       if (num_consts == 0) {
 511          llvm->tes_jit_context.constants[i] = fake_const_buf;
 512       }
 513    }
 514    for (i = 0; i < ARRAY_SIZE(llvm->tes_jit_context.ssbos); ++i) {
 515       int num_ssbos = draw->pt.user.tes_ssbos_size[i];
 516       llvm->tes_jit_context.ssbos[i] = draw->pt.user.tes_ssbos[i];
 517       llvm->tes_jit_context.num_ssbos[i] = num_ssbos;
 518       if (num_ssbos == 0) {
 519          llvm->tes_jit_context.ssbos[i] = (const uint32_t *)fake_const_buf;
 520       }
 521    }
 522
 523    llvm->jit_context.planes =
 524       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 525    llvm->gs_jit_context.planes =
 526       (float (*)[DRAW_TOTAL_CLIP_PLANES][4]) draw->pt.user.planes[0];
 527
 528    llvm->jit_context.viewports = draw->viewports;
 529    llvm->gs_jit_context.viewports = draw->viewports;
 530 }
 531
 532
 533 static void
 534 pipeline(struct llvm_middle_end *llvm,
 535          const struct draw_vertex_info *vert_info,
 536          const struct draw_prim_info *prim_info)
 537 {
 538    if (prim_info->linear)
 539       draw_pipeline_run_linear( llvm->draw,
 540                                 vert_info,
 541                                 prim_info);
 542    else
 543       draw_pipeline_run( llvm->draw,
 544                          vert_info,
 545                          prim_info );
 546 }
 547
 548
 549 static void
 550 emit(struct pt_emit *emit,
 551      const struct draw_vertex_info *vert_info,
 552      const struct draw_prim_info *prim_info)
 553 {
 554    if (prim_info->linear) {
 555       draw_pt_emit_linear(emit, vert_info, prim_info);
 556    }
 557    else {
 558       draw_pt_emit(emit, vert_info, prim_info);
 559    }
 560 }
 561
 562
 563 static void
 564 llvm_pipeline_generic(struct draw_pt_middle_end *middle,
 565                       const struct draw_fetch_info *fetch_info,
 566                       const struct draw_prim_info *in_prim_info)
 567 {
 568    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 569    struct draw_context *draw = fpme->draw;
 570    struct draw_geometry_shader *gshader = draw->gs.geometry_shader;
 571    struct draw_tess_ctrl_shader *tcs_shader = draw->tcs.tess_ctrl_shader;
 572    struct draw_tess_eval_shader *tes_shader = draw->tes.tess_eval_shader;
 573    struct draw_prim_info tcs_prim_info;
 574    struct draw_prim_info tes_prim_info;
 575    struct draw_prim_info gs_prim_info[TGSI_MAX_VERTEX_STREAMS];
 576    struct draw_vertex_info llvm_vert_info;
 577    struct draw_vertex_info tcs_vert_info;
 578    struct draw_vertex_info tes_vert_info;
 579    struct draw_vertex_info gs_vert_info[TGSI_MAX_VERTEX_STREAMS];
 580    struct draw_vertex_info *vert_info;
 581    struct draw_prim_info ia_prim_info;
 582    struct draw_vertex_info ia_vert_info;
 583    const struct draw_prim_info *prim_info = in_prim_info;
 584    boolean free_prim_info = FALSE;
 585    unsigned opt = fpme->opt;
 586    boolean clipped = 0;
 587    unsigned start_or_maxelt, vid_base;
 588    const unsigned *elts;
 589    ushort *tes_elts_out = NULL;
 590
 591    memset(&gs_vert_info, 0, sizeof(struct draw_vertex_info) * TGSI_MAX_VERTEX_STREAMS);
 592    assert(fetch_info->count > 0);
 593    llvm_vert_info.count = fetch_info->count;
 594    llvm_vert_info.vertex_size = fpme->vertex_size;
 595    llvm_vert_info.stride = fpme->vertex_size;
 596    llvm_vert_info.verts = (struct vertex_header *)
 597       MALLOC(fpme->vertex_size *
 598              align(fetch_info->count, lp_native_vector_width / 32));
 599    if (!llvm_vert_info.verts) {
 600       assert(0);
 601       return;
 602    }
 603
 604    if (draw->collect_statistics) {
 605       draw->statistics.ia_vertices += prim_info->count;
 606       if (prim_info->prim == PIPE_PRIM_PATCHES)
 607          draw->statistics.ia_primitives += prim_info->count / draw->pt.vertices_per_patch;
 608       else
 609          draw->statistics.ia_primitives +=
 610             u_decomposed_prims_for_vertices(prim_info->prim, prim_info->count);
 611       draw->statistics.vs_invocations += fetch_info->count;
 612    }
 613
 614    if (fetch_info->linear) {
 615       start_or_maxelt = fetch_info->start;
 616       vid_base = draw->start_index;
 617       elts = NULL;
 618    }
 619    else {
 620       start_or_maxelt = draw->pt.user.eltMax;
 621       vid_base = draw->pt.user.eltBias;
 622       elts = fetch_info->elts;
 623    }
 624    clipped = fpme->current_variant->jit_func(&fpme->llvm->jit_context,
 625                                              llvm_vert_info.verts,
 626                                              draw->pt.user.vbuffer,
 627                                              fetch_info->count,
 628                                              start_or_maxelt,
 629                                              fpme->vertex_size,
 630                                              draw->pt.vertex_buffer,
 631                                              draw->instance_id,
 632                                              vid_base,
 633                                              draw->start_instance,
 634                                              elts, draw->pt.user.drawid);
 635
 636    /* Finished with fetch and vs:
 637     */
 638    fetch_info = NULL;
 639    vert_info = &llvm_vert_info;
 640
 641    if (opt & PT_SHADE) {
 642       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 643       if (tcs_shader) {
 644          draw_tess_ctrl_shader_run(tcs_shader,
 645                                    draw->pt.user.tcs_constants,
 646                                    draw->pt.user.tcs_constants_size,
 647                                    vert_info,
 648                                    prim_info,
 649                                    &vshader->info,
 650                                    &tcs_vert_info,
 651                                    &tcs_prim_info);
 652          FREE(vert_info->verts);
 653          vert_info = &tcs_vert_info;
 654          prim_info = &tcs_prim_info;
 655
 656       } else if (tes_shader) {
 657          unsigned num_prims = prim_info->count / draw->pt.vertices_per_patch;
 658          tcs_prim_info = *prim_info;
 659          tcs_prim_info.primitive_count = num_prims;
 660          prim_info = &tcs_prim_info;
 661       }
 662
 663       if (tes_shader) {
 664          draw_tess_eval_shader_run(tes_shader,
 665                                    draw->pt.user.tes_constants,
 666                                    draw->pt.user.tes_constants_size,
 667                                    tcs_shader ? tcs_shader->vertices_out : draw->pt.vertices_per_patch,
 668                                    vert_info,
 669                                    prim_info,
 670                                    tcs_shader ? &tcs_shader->info : &vshader->info,
 671                                    &tes_vert_info,
 672                                    &tes_prim_info, &tes_elts_out);
 673
 674          FREE(vert_info->verts);
 675          vert_info = &tes_vert_info;
 676          prim_info = &tes_prim_info;
 677          free_prim_info = TRUE;
 678
 679          /*
 680           * pt emit can only handle ushort number of vertices (see
 681           * render->allocate_vertices).
 682           * vsplit guarantees there's never more than 4096, however GS can
 683           * easily blow this up (by a factor of 256 (or even 1024) max).
 684           */
 685          if (vert_info->count > 65535) {
 686             opt |= PT_PIPELINE;
 687          }
 688       }
 689    }
 690
 691    if ((opt & PT_SHADE) && gshader) {
 692       struct draw_vertex_shader *vshader = draw->vs.vertex_shader;
 693       draw_geometry_shader_run(gshader,
 694                                draw->pt.user.gs_constants,
 695                                draw->pt.user.gs_constants_size,
 696                                vert_info,
 697                                prim_info,
 698                                tes_shader ? &tes_shader->info : &vshader->info,
 699                                gs_vert_info,
 700                                gs_prim_info);
 701
 702       FREE(vert_info->verts);
 703       if (free_prim_info) {
 704          FREE(prim_info->primitive_lengths);
 705          FREE(tes_elts_out);
 706       }
 707       vert_info = &gs_vert_info[0];
 708       prim_info = &gs_prim_info[0];
 709       free_prim_info = FALSE;
 710       /*
 711        * pt emit can only handle ushort number of vertices (see
 712        * render->allocate_vertices).
 713        * vsplit guarantees there's never more than 4096, however GS can
 714        * easily blow this up (by a factor of 256 (or even 1024) max).
 715        */
 716       if (vert_info->count > 65535) {
 717          opt |= PT_PIPELINE;
 718       }
 719    } else {
 720       if (draw_prim_assembler_is_required(draw, prim_info, vert_info)) {
 721          draw_prim_assembler_run(draw, prim_info, vert_info,
 722                                  &ia_prim_info, &ia_vert_info);
 723
 724          if (ia_vert_info.count) {
 725             FREE(vert_info->verts);
 726             vert_info = &ia_vert_info;
 727             prim_info = &ia_prim_info;
 728             free_prim_info = TRUE;
 729          }
 730       }
 731    }
 732
 733    /* stream output needs to be done before clipping */
 734    draw_pt_so_emit( fpme->so_emit, gshader ? gshader->num_vertex_streams : 1, vert_info, prim_info );
 735
 736    if (prim_info->count == 0) {
 737       debug_printf("GS/IA didn't emit any vertices!\n");
 738       goto out;
 739    }
 740
 741    draw_stats_clipper_primitives(draw, prim_info);
 742
 743    /*
 744     * if there's no position, need to stop now, or the latter stages
 745     * will try to access non-existent position output.
 746     */
 747    if (draw_current_shader_position_output(draw) != -1) {
 748       if ((opt & PT_SHADE) && (gshader || tes_shader ||
 749                                draw->vs.vertex_shader->info.writes_viewport_index)) {
 750          clipped = draw_pt_post_vs_run( fpme->post_vs, vert_info, prim_info );
 751       }
 752       /* "clipped" also includes non-one edgeflag */
 753       if (clipped) {
 754          opt |= PT_PIPELINE;
 755       }
 756
 757       /* Do we need to run the pipeline? Now will come here if clipped
 758        */
 759       if (opt & PT_PIPELINE) {
 760          pipeline( fpme, vert_info, prim_info );
 761       }
 762       else {
 763          emit( fpme->emit, vert_info, prim_info );
 764       }
 765    }
 766 out:
 767    FREE(vert_info->verts);
 768    if (gshader && gshader->num_vertex_streams > 1)
 769      for (unsigned i = 1; i < gshader->num_vertex_streams; i++)
 770        FREE(gs_vert_info[i].verts);
 771
 772    if (free_prim_info) {
 773       FREE(tes_elts_out);
 774       FREE(prim_info->primitive_lengths);
 775    }
 776 }
 777
 778
 779 static inline unsigned
 780 prim_type(unsigned prim, unsigned flags)
 781 {
 782    if (flags & DRAW_LINE_LOOP_AS_STRIP)
 783       return PIPE_PRIM_LINE_STRIP;
 784    else
 785       return prim;
 786 }
 787
 788
 789 static void
 790 llvm_middle_end_run(struct draw_pt_middle_end *middle,
 791                     const unsigned *fetch_elts,
 792                     unsigned fetch_count,
 793                     const ushort *draw_elts,
 794                     unsigned draw_count,
 795                     unsigned prim_flags)
 796 {
 797    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 798    struct draw_fetch_info fetch_info;
 799    struct draw_prim_info prim_info;
 800
 801    fetch_info.linear = FALSE;
 802    fetch_info.start = 0;
 803    fetch_info.elts = fetch_elts;
 804    fetch_info.count = fetch_count;
 805
 806    prim_info.linear = FALSE;
 807    prim_info.start = 0;
 808    prim_info.count = draw_count;
 809    prim_info.elts = draw_elts;
 810    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 811    prim_info.flags = prim_flags;
 812    prim_info.primitive_count = 1;
 813    prim_info.primitive_lengths = &draw_count;
 814
 815    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 816 }
 817
 818
 819 static void
 820 llvm_middle_end_linear_run(struct draw_pt_middle_end *middle,
 821                            unsigned start,
 822                            unsigned count,
 823                            unsigned prim_flags)
 824 {
 825    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 826    struct draw_fetch_info fetch_info;
 827    struct draw_prim_info prim_info;
 828
 829    fetch_info.linear = TRUE;
 830    fetch_info.start = start;
 831    fetch_info.count = count;
 832    fetch_info.elts = NULL;
 833
 834    prim_info.linear = TRUE;
 835    prim_info.start = 0;
 836    prim_info.count = count;
 837    prim_info.elts = NULL;
 838    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 839    prim_info.flags = prim_flags;
 840    prim_info.primitive_count = 1;
 841    prim_info.primitive_lengths = &count;
 842
 843    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 844 }
 845
 846
 847 static boolean
 848 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end *middle,
 849                                 unsigned start,
 850                                 unsigned count,
 851                                 const ushort *draw_elts,
 852                                 unsigned draw_count,
 853                                 unsigned prim_flags)
 854 {
 855    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 856    struct draw_fetch_info fetch_info;
 857    struct draw_prim_info prim_info;
 858
 859    fetch_info.linear = TRUE;
 860    fetch_info.start = start;
 861    fetch_info.count = count;
 862    fetch_info.elts = NULL;
 863
 864    prim_info.linear = FALSE;
 865    prim_info.start = 0;
 866    prim_info.count = draw_count;
 867    prim_info.elts = draw_elts;
 868    prim_info.prim = prim_type(fpme->input_prim, prim_flags);
 869    prim_info.flags = prim_flags;
 870    prim_info.primitive_count = 1;
 871    prim_info.primitive_lengths = &draw_count;
 872
 873    llvm_pipeline_generic( middle, &fetch_info, &prim_info );
 874
 875    return TRUE;
 876 }
 877
 878
 879 static void
 880 llvm_middle_end_finish(struct draw_pt_middle_end *middle)
 881 {
 882    /* nothing to do */
 883 }
 884
 885
 886 static void
 887 llvm_middle_end_destroy(struct draw_pt_middle_end *middle)
 888 {
 889    struct llvm_middle_end *fpme = llvm_middle_end(middle);
 890
 891    if (fpme->fetch)
 892       draw_pt_fetch_destroy( fpme->fetch );
 893
 894    if (fpme->emit)
 895       draw_pt_emit_destroy( fpme->emit );
 896
 897    if (fpme->so_emit)
 898       draw_pt_so_emit_destroy( fpme->so_emit );
 899
 900    if (fpme->post_vs)
 901       draw_pt_post_vs_destroy( fpme->post_vs );
 902
 903    FREE(middle);
 904 }
 905
 906
 907 struct draw_pt_middle_end *
 908 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context *draw)
 909 {
 910    struct llvm_middle_end *fpme = 0;
 911
 912    if (!draw->llvm)
 913       return NULL;
 914
 915    fpme = CALLOC_STRUCT( llvm_middle_end );
 916    if (!fpme)
 917       goto fail;
 918
 919    fpme->base.prepare         = llvm_middle_end_prepare;
 920    fpme->base.bind_parameters = llvm_middle_end_bind_parameters;
 921    fpme->base.run             = llvm_middle_end_run;
 922    fpme->base.run_linear      = llvm_middle_end_linear_run;
 923    fpme->base.run_linear_elts = llvm_middle_end_linear_run_elts;
 924    fpme->base.finish          = llvm_middle_end_finish;
 925    fpme->base.destroy         = llvm_middle_end_destroy;
 926
 927    fpme->draw = draw;
 928
 929    fpme->fetch = draw_pt_fetch_create( draw );
 930    if (!fpme->fetch)
 931       goto fail;
 932
 933    fpme->post_vs = draw_pt_post_vs_create( draw );
 934    if (!fpme->post_vs)
 935       goto fail;
 936
 937    fpme->emit = draw_pt_emit_create( draw );
 938    if (!fpme->emit)
 939       goto fail;
 940
 941    fpme->so_emit = draw_pt_so_emit_create( draw );
 942    if (!fpme->so_emit)
 943       goto fail;
 944
 945    fpme->llvm = draw->llvm;
 946    if (!fpme->llvm)
 947       goto fail;
 948
 949    fpme->current_variant = NULL;
 950
 951    return &fpme->base;
 952
 953  fail:
 954    if (fpme)
 955       llvm_middle_end_destroy( &fpme->base );
 956
 957    return NULL;
 958 }