1 /**************************************************************************
3 * Copyright 2010 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
28 #include "util/u_math.h"
29 #include "util/u_memory.h"
30 #include "util/u_prim.h"
31 #include "draw/draw_context.h"
32 #include "draw/draw_gs.h"
33 #include "draw/draw_vbuf.h"
34 #include "draw/draw_vertex.h"
35 #include "draw/draw_pt.h"
36 #include "draw/draw_prim_assembler.h"
37 #include "draw/draw_vs.h"
38 #include "draw/draw_llvm.h"
39 #include "gallivm/lp_bld_init.h"
42 struct llvm_middle_end
{
43 struct draw_pt_middle_end base
;
44 struct draw_context
*draw
;
47 struct pt_so_emit
*so_emit
;
48 struct pt_fetch
*fetch
;
49 struct pt_post_vs
*post_vs
;
52 unsigned vertex_data_offset
;
57 struct draw_llvm
*llvm
;
58 struct draw_llvm_variant
*current_variant
;
63 static INLINE
struct llvm_middle_end
*
64 llvm_middle_end(struct draw_pt_middle_end
*middle
)
66 return (struct llvm_middle_end
*) middle
;
71 llvm_middle_end_prepare_gs(struct llvm_middle_end
*fpme
)
73 struct draw_context
*draw
= fpme
->draw
;
74 struct draw_geometry_shader
*gs
= draw
->gs
.geometry_shader
;
75 struct draw_gs_llvm_variant_key
*key
;
76 struct draw_gs_llvm_variant
*variant
= NULL
;
77 struct draw_gs_llvm_variant_list_item
*li
;
78 struct llvm_geometry_shader
*shader
= llvm_geometry_shader(gs
);
79 char store
[DRAW_GS_LLVM_MAX_VARIANT_KEY_SIZE
];
82 key
= draw_gs_llvm_make_variant_key(fpme
->llvm
, store
);
84 /* Search shader's list of variants for the key */
85 li
= first_elem(&shader
->variants
);
86 while (!at_end(&shader
->variants
, li
)) {
87 if (memcmp(&li
->base
->key
, key
, shader
->variant_key_size
) == 0) {
95 /* found the variant, move to head of global list (for LRU) */
96 move_to_head(&fpme
->llvm
->gs_variants_list
,
97 &variant
->list_item_global
);
100 /* Need to create new variant */
102 /* First check if we've created too many variants. If so, free
103 * 25% of the LRU to avoid using too much memory.
105 if (fpme
->llvm
->nr_gs_variants
>= DRAW_MAX_SHADER_VARIANTS
) {
107 * XXX: should we flush here ?
109 for (i
= 0; i
< DRAW_MAX_SHADER_VARIANTS
/ 4; i
++) {
110 struct draw_gs_llvm_variant_list_item
*item
;
111 if (is_empty_list(&fpme
->llvm
->gs_variants_list
)) {
114 item
= last_elem(&fpme
->llvm
->gs_variants_list
);
117 draw_gs_llvm_destroy_variant(item
->base
);
121 variant
= draw_gs_llvm_create_variant(fpme
->llvm
, gs
->info
.num_outputs
, key
);
124 insert_at_head(&shader
->variants
, &variant
->list_item_local
);
125 insert_at_head(&fpme
->llvm
->gs_variants_list
,
126 &variant
->list_item_global
);
127 fpme
->llvm
->nr_gs_variants
++;
128 shader
->variants_cached
++;
132 gs
->current_variant
= variant
;
136 * Prepare/validate middle part of the vertex pipeline.
137 * NOTE: if you change this function, also look at the non-LLVM
138 * function fetch_pipeline_prepare() for similar changes.
141 llvm_middle_end_prepare( struct draw_pt_middle_end
*middle
,
144 unsigned *max_vertices
)
146 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
147 struct draw_context
*draw
= fpme
->draw
;
148 struct draw_vertex_shader
*vs
= draw
->vs
.vertex_shader
;
149 struct draw_geometry_shader
*gs
= draw
->gs
.geometry_shader
;
150 const unsigned out_prim
= gs
? gs
->output_primitive
:
151 u_assembled_prim(in_prim
);
152 unsigned point_clip
= draw
->rasterizer
->fill_front
== PIPE_POLYGON_MODE_POINT
||
153 out_prim
== PIPE_PRIM_POINTS
;
156 fpme
->input_prim
= in_prim
;
159 draw_pt_post_vs_prepare( fpme
->post_vs
,
163 point_clip
? draw
->guard_band_points_xy
:
165 draw
->identity_viewport
,
166 draw
->rasterizer
->clip_halfz
,
167 (draw
->vs
.edgeflag_output
? TRUE
: FALSE
) );
169 draw_pt_so_emit_prepare( fpme
->so_emit
, gs
== NULL
);
171 if (!(opt
& PT_PIPELINE
)) {
172 draw_pt_emit_prepare( fpme
->emit
,
176 *max_vertices
= MAX2( *max_vertices
, 4096 );
179 /* limit max fetches by limiting max_vertices */
180 *max_vertices
= 4096;
183 /* Get the number of float[4] attributes per vertex.
184 * Note: this must be done after draw_pt_emit_prepare() since that
185 * can effect the vertex size.
187 nr
= MAX2(vs
->info
.num_inputs
, draw_total_vs_outputs(draw
));
189 /* Always leave room for the vertex header whether we need it or
190 * not. It's hard to get rid of it in particular because of the
191 * viewport code in draw_pt_post_vs.c.
193 fpme
->vertex_size
= sizeof(struct vertex_header
) + nr
* 4 * sizeof(float);
195 /* return even number */
196 *max_vertices
= *max_vertices
& ~1;
198 /* Find/create the vertex shader variant */
200 struct draw_llvm_variant_key
*key
;
201 struct draw_llvm_variant
*variant
= NULL
;
202 struct draw_llvm_variant_list_item
*li
;
203 struct llvm_vertex_shader
*shader
= llvm_vertex_shader(vs
);
204 char store
[DRAW_LLVM_MAX_VARIANT_KEY_SIZE
];
207 key
= draw_llvm_make_variant_key(fpme
->llvm
, store
);
209 /* Search shader's list of variants for the key */
210 li
= first_elem(&shader
->variants
);
211 while (!at_end(&shader
->variants
, li
)) {
212 if (memcmp(&li
->base
->key
, key
, shader
->variant_key_size
) == 0) {
220 /* found the variant, move to head of global list (for LRU) */
221 move_to_head(&fpme
->llvm
->vs_variants_list
,
222 &variant
->list_item_global
);
225 /* Need to create new variant */
227 /* First check if we've created too many variants. If so, free
228 * 25% of the LRU to avoid using too much memory.
230 if (fpme
->llvm
->nr_variants
>= DRAW_MAX_SHADER_VARIANTS
) {
232 * XXX: should we flush here ?
234 for (i
= 0; i
< DRAW_MAX_SHADER_VARIANTS
/ 4; i
++) {
235 struct draw_llvm_variant_list_item
*item
;
236 if (is_empty_list(&fpme
->llvm
->vs_variants_list
)) {
239 item
= last_elem(&fpme
->llvm
->vs_variants_list
);
242 draw_llvm_destroy_variant(item
->base
);
246 variant
= draw_llvm_create_variant(fpme
->llvm
, nr
, key
);
249 insert_at_head(&shader
->variants
, &variant
->list_item_local
);
250 insert_at_head(&fpme
->llvm
->vs_variants_list
,
251 &variant
->list_item_global
);
252 fpme
->llvm
->nr_variants
++;
253 shader
->variants_cached
++;
257 fpme
->current_variant
= variant
;
261 llvm_middle_end_prepare_gs(fpme
);
267 * Bind/update constant buffer pointers, clip planes and viewport dims.
268 * These are "light weight" parameters which aren't baked into the
269 * generated code. Updating these items is much cheaper than revalidating
270 * and rebuilding the generated pipeline code.
273 llvm_middle_end_bind_parameters(struct draw_pt_middle_end
*middle
)
275 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
276 struct draw_context
*draw
= fpme
->draw
;
279 for (i
= 0; i
< Elements(fpme
->llvm
->jit_context
.vs_constants
); ++i
) {
281 draw
->pt
.user
.vs_constants_size
[i
] / (sizeof(float) * 4);
282 fpme
->llvm
->jit_context
.vs_constants
[i
] = draw
->pt
.user
.vs_constants
[i
];
283 fpme
->llvm
->jit_context
.num_vs_constants
[i
] = num_consts
;
285 for (i
= 0; i
< Elements(fpme
->llvm
->gs_jit_context
.constants
); ++i
) {
287 draw
->pt
.user
.gs_constants_size
[i
] / (sizeof(float) * 4);
288 fpme
->llvm
->gs_jit_context
.constants
[i
] = draw
->pt
.user
.gs_constants
[i
];
289 fpme
->llvm
->gs_jit_context
.num_constants
[i
] = num_consts
;
292 fpme
->llvm
->jit_context
.planes
=
293 (float (*)[DRAW_TOTAL_CLIP_PLANES
][4]) draw
->pt
.user
.planes
[0];
294 fpme
->llvm
->gs_jit_context
.planes
=
295 (float (*)[DRAW_TOTAL_CLIP_PLANES
][4]) draw
->pt
.user
.planes
[0];
297 fpme
->llvm
->jit_context
.viewport
= (float *) draw
->viewports
[0].scale
;
298 fpme
->llvm
->gs_jit_context
.viewport
= (float *) draw
->viewports
[0].scale
;
303 pipeline(struct llvm_middle_end
*llvm
,
304 const struct draw_vertex_info
*vert_info
,
305 const struct draw_prim_info
*prim_info
)
307 if (prim_info
->linear
)
308 draw_pipeline_run_linear( llvm
->draw
,
312 draw_pipeline_run( llvm
->draw
,
319 emit(struct pt_emit
*emit
,
320 const struct draw_vertex_info
*vert_info
,
321 const struct draw_prim_info
*prim_info
)
323 if (prim_info
->linear
) {
324 draw_pt_emit_linear(emit
, vert_info
, prim_info
);
327 draw_pt_emit(emit
, vert_info
, prim_info
);
333 llvm_pipeline_generic(struct draw_pt_middle_end
*middle
,
334 const struct draw_fetch_info
*fetch_info
,
335 const struct draw_prim_info
*in_prim_info
)
337 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
338 struct draw_context
*draw
= fpme
->draw
;
339 struct draw_geometry_shader
*gshader
= draw
->gs
.geometry_shader
;
340 struct draw_prim_info gs_prim_info
;
341 struct draw_vertex_info llvm_vert_info
;
342 struct draw_vertex_info gs_vert_info
;
343 struct draw_vertex_info
*vert_info
;
344 struct draw_prim_info ia_prim_info
;
345 struct draw_vertex_info ia_vert_info
;
346 const struct draw_prim_info
*prim_info
= in_prim_info
;
347 boolean free_prim_info
= FALSE
;
348 unsigned opt
= fpme
->opt
;
349 unsigned clipped
= 0;
351 llvm_vert_info
.count
= fetch_info
->count
;
352 llvm_vert_info
.vertex_size
= fpme
->vertex_size
;
353 llvm_vert_info
.stride
= fpme
->vertex_size
;
354 llvm_vert_info
.verts
= (struct vertex_header
*)
355 MALLOC(fpme
->vertex_size
*
356 align(fetch_info
->count
, lp_native_vector_width
/ 32));
357 if (!llvm_vert_info
.verts
) {
362 if (draw
->collect_statistics
) {
363 draw
->statistics
.ia_vertices
+= prim_info
->count
;
364 draw
->statistics
.ia_primitives
+=
365 u_decomposed_prims_for_vertices(prim_info
->prim
, prim_info
->count
);
366 draw
->statistics
.vs_invocations
+= fetch_info
->count
;
369 if (fetch_info
->linear
)
370 clipped
= fpme
->current_variant
->jit_func( &fpme
->llvm
->jit_context
,
371 llvm_vert_info
.verts
,
372 draw
->pt
.user
.vbuffer
,
376 draw
->pt
.vertex_buffer
,
379 draw
->start_instance
);
381 clipped
= fpme
->current_variant
->jit_func_elts( &fpme
->llvm
->jit_context
,
382 llvm_vert_info
.verts
,
383 draw
->pt
.user
.vbuffer
,
385 draw
->pt
.user
.eltMax
,
388 draw
->pt
.vertex_buffer
,
390 draw
->pt
.user
.eltBias
,
391 draw
->start_instance
);
393 /* Finished with fetch and vs:
396 vert_info
= &llvm_vert_info
;
398 if ((opt
& PT_SHADE
) && gshader
) {
399 struct draw_vertex_shader
*vshader
= draw
->vs
.vertex_shader
;
400 draw_geometry_shader_run(gshader
,
401 draw
->pt
.user
.gs_constants
,
402 draw
->pt
.user
.gs_constants_size
,
409 FREE(vert_info
->verts
);
410 vert_info
= &gs_vert_info
;
411 prim_info
= &gs_prim_info
;
413 if (draw_prim_assembler_is_required(draw
, prim_info
, vert_info
)) {
414 draw_prim_assembler_run(draw
, prim_info
, vert_info
,
415 &ia_prim_info
, &ia_vert_info
);
417 if (ia_vert_info
.count
) {
418 FREE(vert_info
->verts
);
419 vert_info
= &ia_vert_info
;
420 prim_info
= &ia_prim_info
;
421 free_prim_info
= TRUE
;
425 if (prim_info
->count
== 0) {
426 debug_printf("GS/IA didn't emit any vertices!\n");
428 FREE(vert_info
->verts
);
429 if (free_prim_info
) {
430 FREE(prim_info
->primitive_lengths
);
435 /* stream output needs to be done before clipping */
436 draw_pt_so_emit( fpme
->so_emit
, vert_info
, prim_info
);
438 draw_stats_clipper_primitives(draw
, prim_info
);
441 * if there's no position, need to stop now, or the latter stages
442 * will try to access non-existent position output.
444 if (draw_current_shader_position_output(draw
) != -1) {
445 if ((opt
& PT_SHADE
) && gshader
) {
446 clipped
= draw_pt_post_vs_run( fpme
->post_vs
, vert_info
, prim_info
);
452 /* Do we need to run the pipeline? Now will come here if clipped
454 if (opt
& PT_PIPELINE
) {
455 pipeline( fpme
, vert_info
, prim_info
);
458 emit( fpme
->emit
, vert_info
, prim_info
);
461 FREE(vert_info
->verts
);
462 if (free_prim_info
) {
463 FREE(prim_info
->primitive_lengths
);
469 llvm_middle_end_run(struct draw_pt_middle_end
*middle
,
470 const unsigned *fetch_elts
,
471 unsigned fetch_count
,
472 const ushort
*draw_elts
,
476 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
477 struct draw_fetch_info fetch_info
;
478 struct draw_prim_info prim_info
;
480 fetch_info
.linear
= FALSE
;
481 fetch_info
.start
= 0;
482 fetch_info
.elts
= fetch_elts
;
483 fetch_info
.count
= fetch_count
;
485 prim_info
.linear
= FALSE
;
487 prim_info
.count
= draw_count
;
488 prim_info
.elts
= draw_elts
;
489 prim_info
.prim
= fpme
->input_prim
;
490 prim_info
.flags
= prim_flags
;
491 prim_info
.primitive_count
= 1;
492 prim_info
.primitive_lengths
= &draw_count
;
494 llvm_pipeline_generic( middle
, &fetch_info
, &prim_info
);
499 llvm_middle_end_linear_run(struct draw_pt_middle_end
*middle
,
504 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
505 struct draw_fetch_info fetch_info
;
506 struct draw_prim_info prim_info
;
508 fetch_info
.linear
= TRUE
;
509 fetch_info
.start
= start
;
510 fetch_info
.count
= count
;
511 fetch_info
.elts
= NULL
;
513 prim_info
.linear
= TRUE
;
515 prim_info
.count
= count
;
516 prim_info
.elts
= NULL
;
517 prim_info
.prim
= fpme
->input_prim
;
518 prim_info
.flags
= prim_flags
;
519 prim_info
.primitive_count
= 1;
520 prim_info
.primitive_lengths
= &count
;
522 llvm_pipeline_generic( middle
, &fetch_info
, &prim_info
);
527 llvm_middle_end_linear_run_elts(struct draw_pt_middle_end
*middle
,
530 const ushort
*draw_elts
,
534 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
535 struct draw_fetch_info fetch_info
;
536 struct draw_prim_info prim_info
;
538 fetch_info
.linear
= TRUE
;
539 fetch_info
.start
= start
;
540 fetch_info
.count
= count
;
541 fetch_info
.elts
= NULL
;
543 prim_info
.linear
= FALSE
;
545 prim_info
.count
= draw_count
;
546 prim_info
.elts
= draw_elts
;
547 prim_info
.prim
= fpme
->input_prim
;
548 prim_info
.flags
= prim_flags
;
549 prim_info
.primitive_count
= 1;
550 prim_info
.primitive_lengths
= &draw_count
;
552 llvm_pipeline_generic( middle
, &fetch_info
, &prim_info
);
559 llvm_middle_end_finish(struct draw_pt_middle_end
*middle
)
566 llvm_middle_end_destroy(struct draw_pt_middle_end
*middle
)
568 struct llvm_middle_end
*fpme
= llvm_middle_end(middle
);
571 draw_pt_fetch_destroy( fpme
->fetch
);
574 draw_pt_emit_destroy( fpme
->emit
);
577 draw_pt_so_emit_destroy( fpme
->so_emit
);
580 draw_pt_post_vs_destroy( fpme
->post_vs
);
586 struct draw_pt_middle_end
*
587 draw_pt_fetch_pipeline_or_emit_llvm(struct draw_context
*draw
)
589 struct llvm_middle_end
*fpme
= 0;
594 fpme
= CALLOC_STRUCT( llvm_middle_end
);
598 fpme
->base
.prepare
= llvm_middle_end_prepare
;
599 fpme
->base
.bind_parameters
= llvm_middle_end_bind_parameters
;
600 fpme
->base
.run
= llvm_middle_end_run
;
601 fpme
->base
.run_linear
= llvm_middle_end_linear_run
;
602 fpme
->base
.run_linear_elts
= llvm_middle_end_linear_run_elts
;
603 fpme
->base
.finish
= llvm_middle_end_finish
;
604 fpme
->base
.destroy
= llvm_middle_end_destroy
;
608 fpme
->fetch
= draw_pt_fetch_create( draw
);
612 fpme
->post_vs
= draw_pt_post_vs_create( draw
);
616 fpme
->emit
= draw_pt_emit_create( draw
);
620 fpme
->so_emit
= draw_pt_so_emit_create( draw
);
624 fpme
->llvm
= draw
->llvm
;
628 fpme
->current_variant
= NULL
;
634 llvm_middle_end_destroy( &fpme
->base
);