projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
llvmpipe/fs: hook up the interpolation APIs.
[mesa.git] / src / gallium / drivers / llvmpipe / lp_state_fs.c
1 /**************************************************************************
2 *
3 * Copyright 2009 VMware, Inc.
4 * Copyright 2007 VMware, Inc.
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
20 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
22 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
23 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
24 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
25 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 /**
30 * @file
31 * Code generate the whole fragment pipeline.
32 *
33 * The fragment pipeline consists of the following stages:
34 * - early depth test
35 * - fragment shader
36 * - alpha test
37 * - depth/stencil test
38 * - blending
39 *
40 * This file has only the glue to assemble the fragment pipeline. The actual
41 * plumbing of converting Gallium state into LLVM IR is done elsewhere, in the
42 * lp_bld_*.[ch] files, and in a complete generic and reusable way. Here we
43 * muster the LLVM JIT execution engine to create a function that follows an
44 * established binary interface and that can be called from C directly.
45 *
46 * A big source of complexity here is that we often want to run different
47 * stages with different precisions and data types and precisions. For example,
48 * the fragment shader needs typically to be done in floats, but the
49 * depth/stencil test and blending is better done in the type that most closely
50 * matches the depth/stencil and color buffer respectively.
51 *
52 * Since the width of a SIMD vector register stays the same regardless of the
53 * element type, different types imply different number of elements, so we must
54 * code generate more instances of the stages with larger types to be able to
55 * feed/consume the stages with smaller types.
56 *
57 * @author Jose Fonseca <jfonseca@vmware.com>
58 */
59
60 #include <limits.h>
61 #include "pipe/p_defines.h"
62 #include "util/u_inlines.h"
63 #include "util/u_memory.h"
64 #include "util/u_pointer.h"
65 #include "util/format/u_format.h"
66 #include "util/u_dump.h"
67 #include "util/u_string.h"
68 #include "util/simple_list.h"
69 #include "util/u_dual_blend.h"
70 #include "util/os_time.h"
71 #include "pipe/p_shader_tokens.h"
72 #include "draw/draw_context.h"
73 #include "tgsi/tgsi_dump.h"
74 #include "tgsi/tgsi_scan.h"
75 #include "tgsi/tgsi_parse.h"
76 #include "gallivm/lp_bld_type.h"
77 #include "gallivm/lp_bld_const.h"
78 #include "gallivm/lp_bld_conv.h"
79 #include "gallivm/lp_bld_init.h"
80 #include "gallivm/lp_bld_intr.h"
81 #include "gallivm/lp_bld_logic.h"
82 #include "gallivm/lp_bld_tgsi.h"
83 #include "gallivm/lp_bld_nir.h"
84 #include "gallivm/lp_bld_swizzle.h"
85 #include "gallivm/lp_bld_flow.h"
86 #include "gallivm/lp_bld_debug.h"
87 #include "gallivm/lp_bld_arit.h"
88 #include "gallivm/lp_bld_bitarit.h"
89 #include "gallivm/lp_bld_pack.h"
90 #include "gallivm/lp_bld_format.h"
91 #include "gallivm/lp_bld_quad.h"
92
93 #include "lp_bld_alpha.h"
94 #include "lp_bld_blend.h"
95 #include "lp_bld_depth.h"
96 #include "lp_bld_interp.h"
97 #include "lp_context.h"
98 #include "lp_debug.h"
99 #include "lp_perf.h"
100 #include "lp_setup.h"
101 #include "lp_state.h"
102 #include "lp_tex_sample.h"
103 #include "lp_flush.h"
104 #include "lp_state_fs.h"
105 #include "lp_rast.h"
106 #include "nir/nir_to_tgsi_info.h"
107
108 /** Fragment shader number (for debugging) */
109 static unsigned fs_no = 0;
110
111
112 /**
113 * Expand the relevant bits of mask_input to a n*4-dword mask for the
114 * n*four pixels in n 2x2 quads. This will set the n*four elements of the
115 * quad mask vector to 0 or ~0.
116 * Grouping is 01, 23 for 2 quad mode hence only 0 and 2 are valid
117 * quad arguments with fs length 8.
118 *
119 * \param first_quad which quad(s) of the quad group to test, in [0,3]
120 * \param mask_input bitwise mask for the whole 4x4 stamp
121 */
122 static LLVMValueRef
123 generate_quad_mask(struct gallivm_state *gallivm,
124 struct lp_type fs_type,
125 unsigned first_quad,
126 unsigned sample,
127 LLVMValueRef mask_input) /* int64 */
128 {
129 LLVMBuilderRef builder = gallivm->builder;
130 struct lp_type mask_type;
131 LLVMTypeRef i32t = LLVMInt32TypeInContext(gallivm->context);
132 LLVMValueRef bits[16];
133 LLVMValueRef mask, bits_vec;
134 int shift, i;
135
136 /*
137 * XXX: We'll need a different path for 16 x u8
138 */
139 assert(fs_type.width == 32);
140 assert(fs_type.length <= ARRAY_SIZE(bits));
141 mask_type = lp_int_type(fs_type);
142
143 /*
144 * mask_input >>= (quad * 4)
145 */
146 switch (first_quad) {
147 case 0:
148 shift = 0;
149 break;
150 case 1:
151 assert(fs_type.length == 4);
152 shift = 2;
153 break;
154 case 2:
155 shift = 8;
156 break;
157 case 3:
158 assert(fs_type.length == 4);
159 shift = 10;
160 break;
161 default:
162 assert(0);
163 shift = 0;
164 }
165
166 mask_input = LLVMBuildLShr(builder, mask_input, lp_build_const_int64(gallivm, 16 * sample), "");
167 mask_input = LLVMBuildTrunc(builder, mask_input,
168 i32t, "");
169 mask_input = LLVMBuildAnd(builder, mask_input, lp_build_const_int32(gallivm, 0xffff), "");
170
171 mask_input = LLVMBuildLShr(builder,
172 mask_input,
173 LLVMConstInt(i32t, shift, 0),
174 "");
175
176 /*
177 * mask = { mask_input & (1 << i), for i in [0,3] }
178 */
179 mask = lp_build_broadcast(gallivm,
180 lp_build_vec_type(gallivm, mask_type),
181 mask_input);
182
183 for (i = 0; i < fs_type.length / 4; i++) {
184 unsigned j = 2 * (i % 2) + (i / 2) * 8;
185 bits[4*i + 0] = LLVMConstInt(i32t, 1ULL << (j + 0), 0);
186 bits[4*i + 1] = LLVMConstInt(i32t, 1ULL << (j + 1), 0);
187 bits[4*i + 2] = LLVMConstInt(i32t, 1ULL << (j + 4), 0);
188 bits[4*i + 3] = LLVMConstInt(i32t, 1ULL << (j + 5), 0);
189 }
190 bits_vec = LLVMConstVector(bits, fs_type.length);
191 mask = LLVMBuildAnd(builder, mask, bits_vec, "");
192
193 /*
194 * mask = mask == bits ? ~0 : 0
195 */
196 mask = lp_build_compare(gallivm,
197 mask_type, PIPE_FUNC_EQUAL,
198 mask, bits_vec);
199
200 return mask;
201 }
202
203
204 #define EARLY_DEPTH_TEST 0x1
205 #define LATE_DEPTH_TEST 0x2
206 #define EARLY_DEPTH_WRITE 0x4
207 #define LATE_DEPTH_WRITE 0x8
208
209 static int
210 find_output_by_semantic( const struct tgsi_shader_info *info,
211 unsigned semantic,
212 unsigned index )
213 {
214 int i;
215
216 for (i = 0; i < info->num_outputs; i++)
217 if (info->output_semantic_name[i] == semantic &&
218 info->output_semantic_index[i] == index)
219 return i;
220
221 return -1;
222 }
223
224
225 /**
226 * Fetch the specified lp_jit_viewport structure for a given viewport_index.
227 */
228 static LLVMValueRef
229 lp_llvm_viewport(LLVMValueRef context_ptr,
230 struct gallivm_state *gallivm,
231 LLVMValueRef viewport_index)
232 {
233 LLVMBuilderRef builder = gallivm->builder;
234 LLVMValueRef ptr;
235 LLVMValueRef res;
236 struct lp_type viewport_type =
237 lp_type_float_vec(32, 32 * LP_JIT_VIEWPORT_NUM_FIELDS);
238
239 ptr = lp_jit_context_viewports(gallivm, context_ptr);
240 ptr = LLVMBuildPointerCast(builder, ptr,
241 LLVMPointerType(lp_build_vec_type(gallivm, viewport_type), 0), "");
242
243 res = lp_build_pointer_get(builder, ptr, viewport_index);
244
245 return res;
246 }
247
248
249 static LLVMValueRef
250 lp_build_depth_clamp(struct gallivm_state *gallivm,
251 LLVMBuilderRef builder,
252 struct lp_type type,
253 LLVMValueRef context_ptr,
254 LLVMValueRef thread_data_ptr,
255 LLVMValueRef z)
256 {
257 LLVMValueRef viewport, min_depth, max_depth;
258 LLVMValueRef viewport_index;
259 struct lp_build_context f32_bld;
260
261 assert(type.floating);
262 lp_build_context_init(&f32_bld, gallivm, type);
263
264 /*
265 * Assumes clamping of the viewport index will occur in setup/gs. Value
266 * is passed through the rasterization stage via lp_rast_shader_inputs.
267 *
268 * See: draw_clamp_viewport_idx and lp_clamp_viewport_idx for clamping
269 * semantics.
270 */
271 viewport_index = lp_jit_thread_data_raster_state_viewport_index(gallivm,
272 thread_data_ptr);
273
274 /*
275 * Load the min and max depth from the lp_jit_context.viewports
276 * array of lp_jit_viewport structures.
277 */
278 viewport = lp_llvm_viewport(context_ptr, gallivm, viewport_index);
279
280 /* viewports[viewport_index].min_depth */
281 min_depth = LLVMBuildExtractElement(builder, viewport,
282 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MIN_DEPTH), "");
283 min_depth = lp_build_broadcast_scalar(&f32_bld, min_depth);
284
285 /* viewports[viewport_index].max_depth */
286 max_depth = LLVMBuildExtractElement(builder, viewport,
287 lp_build_const_int32(gallivm, LP_JIT_VIEWPORT_MAX_DEPTH), "");
288 max_depth = lp_build_broadcast_scalar(&f32_bld, max_depth);
289
290 /*
291 * Clamp to the min and max depth values for the given viewport.
292 */
293 return lp_build_clamp(&f32_bld, z, min_depth, max_depth);
294 }
295
296 static void
297 lp_build_sample_alpha_to_coverage(struct gallivm_state *gallivm,
298 struct lp_type type,
299 unsigned coverage_samples,
300 LLVMValueRef num_loop,
301 LLVMValueRef loop_counter,
302 LLVMValueRef coverage_mask_store,
303 LLVMValueRef alpha)
304 {
305 struct lp_build_context bld;
306 LLVMBuilderRef builder = gallivm->builder;
307 float step = 1.0 / coverage_samples;
308
309 lp_build_context_init(&bld, gallivm, type);
310 for (unsigned s = 0; s < coverage_samples; s++) {
311 LLVMValueRef alpha_ref_value = lp_build_const_vec(gallivm, type, step * s);
312 LLVMValueRef test = lp_build_cmp(&bld, PIPE_FUNC_GREATER, alpha, alpha_ref_value);
313
314 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, lp_build_const_int32(gallivm, s), num_loop, "");
315 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_counter, "");
316 LLVMValueRef s_mask_ptr = LLVMBuildGEP(builder, coverage_mask_store, &s_mask_idx, 1, "");
317 LLVMValueRef s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
318 s_mask = LLVMBuildAnd(builder, s_mask, test, "");
319 LLVMBuildStore(builder, s_mask, s_mask_ptr);
320 }
321 };
322
323 struct lp_build_fs_llvm_iface {
324 struct lp_build_fs_iface base;
325 struct lp_build_interp_soa_context *interp;
326 struct lp_build_for_loop_state *loop_state;
327 LLVMValueRef mask_store;
328 };
329
330 static LLVMValueRef fs_interp(const struct lp_build_fs_iface *iface,
331 struct lp_build_context *bld,
332 unsigned attrib, unsigned chan,
333 bool centroid, bool sample,
334 LLVMValueRef attrib_indir,
335 LLVMValueRef offsets[2])
336 {
337 struct lp_build_fs_llvm_iface *fs_iface = (struct lp_build_fs_llvm_iface *)iface;
338 struct lp_build_interp_soa_context *interp = fs_iface->interp;
339 unsigned loc = TGSI_INTERPOLATE_LOC_CENTER;
340 if (centroid)
341 loc = TGSI_INTERPOLATE_LOC_CENTROID;
342 if (sample)
343 loc = TGSI_INTERPOLATE_LOC_SAMPLE;
344
345 return lp_build_interp_soa(interp, bld->gallivm, fs_iface->loop_state->counter,
346 fs_iface->mask_store,
347 attrib, chan, loc, attrib_indir, offsets);
348 }
349
350 /**
351 * Generate the fragment shader, depth/stencil test, and alpha tests.
352 */
353 static void
354 generate_fs_loop(struct gallivm_state *gallivm,
355 struct lp_fragment_shader *shader,
356 const struct lp_fragment_shader_variant_key *key,
357 LLVMBuilderRef builder,
358 struct lp_type type,
359 LLVMValueRef context_ptr,
360 LLVMValueRef sample_pos_array,
361 LLVMValueRef num_loop,
362 struct lp_build_interp_soa_context *interp,
363 const struct lp_build_sampler_soa *sampler,
364 const struct lp_build_image_soa *image,
365 LLVMValueRef mask_store,
366 LLVMValueRef (*out_color)[4],
367 LLVMValueRef depth_base_ptr,
368 LLVMValueRef depth_stride,
369 LLVMValueRef depth_sample_stride,
370 LLVMValueRef facing,
371 LLVMValueRef thread_data_ptr)
372 {
373 const struct util_format_description *zs_format_desc = NULL;
374 const struct tgsi_token *tokens = shader->base.tokens;
375 struct lp_type int_type = lp_int_type(type);
376 LLVMTypeRef vec_type, int_vec_type;
377 LLVMValueRef mask_ptr = NULL, mask_val = NULL;
378 LLVMValueRef consts_ptr, num_consts_ptr;
379 LLVMValueRef ssbo_ptr, num_ssbo_ptr;
380 LLVMValueRef z;
381 LLVMValueRef z_value, s_value;
382 LLVMValueRef z_fb, s_fb;
383 LLVMValueRef depth_ptr;
384 LLVMValueRef stencil_refs[2];
385 LLVMValueRef outputs[PIPE_MAX_SHADER_OUTPUTS][TGSI_NUM_CHANNELS];
386 LLVMValueRef zs_samples = lp_build_const_int32(gallivm, key->zsbuf_nr_samples);
387 struct lp_build_for_loop_state loop_state, sample_loop_state;
388 struct lp_build_mask_context mask;
389 /*
390 * TODO: figure out if simple_shader optimization is really worthwile to
391 * keep. Disabled because it may hide some real bugs in the (depth/stencil)
392 * code since tests tend to take another codepath than real shaders.
393 */
394 boolean simple_shader = (shader->info.base.file_count[TGSI_FILE_SAMPLER] == 0 &&
395 shader->info.base.num_inputs < 3 &&
396 shader->info.base.num_instructions < 8) && 0;
397 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
398 util_blend_state_is_dual(&key->blend, 0);
399 unsigned attrib;
400 unsigned chan;
401 unsigned cbuf;
402 unsigned depth_mode;
403
404 struct lp_bld_tgsi_system_values system_values;
405
406 memset(&system_values, 0, sizeof(system_values));
407
408 /* truncate then sign extend. */
409 system_values.front_facing = LLVMBuildTrunc(gallivm->builder, facing, LLVMInt1TypeInContext(gallivm->context), "");
410 system_values.front_facing = LLVMBuildSExt(gallivm->builder, system_values.front_facing, LLVMInt32TypeInContext(gallivm->context), "");
411
412 if (key->depth.enabled ||
413 key->stencil[0].enabled) {
414
415 zs_format_desc = util_format_description(key->zsbuf_format);
416 assert(zs_format_desc);
417
418 if (shader->info.base.properties[TGSI_PROPERTY_FS_EARLY_DEPTH_STENCIL])
419 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE;
420 else if (!shader->info.base.writes_z && !shader->info.base.writes_stencil) {
421 if (shader->info.base.writes_memory)
422 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
423 else if (key->alpha.enabled ||
424 key->blend.alpha_to_coverage ||
425 shader->info.base.uses_kill ||
426 shader->info.base.writes_samplemask) {
427 /* With alpha test and kill, can do the depth test early
428 * and hopefully eliminate some quads. But need to do a
429 * special deferred depth write once the final mask value
430 * is known. This only works though if there's either no
431 * stencil test or the stencil value isn't written.
432 */
433 if (key->stencil[0].enabled && (key->stencil[0].writemask ||
434 (key->stencil[1].enabled &&
435 key->stencil[1].writemask)))
436 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
437 else
438 depth_mode = EARLY_DEPTH_TEST | LATE_DEPTH_WRITE;
439 }
440 else
441 depth_mode = EARLY_DEPTH_TEST | EARLY_DEPTH_WRITE;
442 }
443 else {
444 depth_mode = LATE_DEPTH_TEST | LATE_DEPTH_WRITE;
445 }
446
447 if (!(key->depth.enabled && key->depth.writemask) &&
448 !(key->stencil[0].enabled && (key->stencil[0].writemask ||
449 (key->stencil[1].enabled &&
450 key->stencil[1].writemask))))
451 depth_mode &= ~(LATE_DEPTH_WRITE | EARLY_DEPTH_WRITE);
452 }
453 else {
454 depth_mode = 0;
455 }
456
457 vec_type = lp_build_vec_type(gallivm, type);
458 int_vec_type = lp_build_vec_type(gallivm, int_type);
459
460 stencil_refs[0] = lp_jit_context_stencil_ref_front_value(gallivm, context_ptr);
461 stencil_refs[1] = lp_jit_context_stencil_ref_back_value(gallivm, context_ptr);
462 /* convert scalar stencil refs into vectors */
463 stencil_refs[0] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[0]);
464 stencil_refs[1] = lp_build_broadcast(gallivm, int_vec_type, stencil_refs[1]);
465
466 consts_ptr = lp_jit_context_constants(gallivm, context_ptr);
467 num_consts_ptr = lp_jit_context_num_constants(gallivm, context_ptr);
468
469 ssbo_ptr = lp_jit_context_ssbos(gallivm, context_ptr);
470 num_ssbo_ptr = lp_jit_context_num_ssbos(gallivm, context_ptr);
471
472 memset(outputs, 0, sizeof outputs);
473
474 /* Allocate color storage for each fragment sample */
475 LLVMValueRef color_store_size = num_loop;
476 if (key->min_samples > 1)
477 color_store_size = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, key->min_samples), "");
478
479 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
480 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
481 out_color[cbuf][chan] = lp_build_array_alloca(gallivm,
482 lp_build_vec_type(gallivm,
483 type),
484 color_store_size, "color");
485 }
486 }
487 if (dual_source_blend) {
488 assert(key->nr_cbufs <= 1);
489 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
490 out_color[1][chan] = lp_build_array_alloca(gallivm,
491 lp_build_vec_type(gallivm,
492 type),
493 color_store_size, "color1");
494 }
495 }
496
497 lp_build_for_loop_begin(&loop_state, gallivm,
498 lp_build_const_int32(gallivm, 0),
499 LLVMIntULT,
500 num_loop,
501 lp_build_const_int32(gallivm, 1));
502
503 if (key->multisample) {
504 /* create shader execution mask by combining all sample masks. */
505 for (unsigned s = 0; s < key->coverage_samples; s++) {
506 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, num_loop, lp_build_const_int32(gallivm, s), "");
507 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
508 LLVMValueRef s_mask = lp_build_pointer_get(builder, mask_store, s_mask_idx);
509 if (s == 0)
510 mask_val = s_mask;
511 else
512 mask_val = LLVMBuildOr(builder, s_mask, mask_val, "");
513 }
514 } else {
515 mask_ptr = LLVMBuildGEP(builder, mask_store,
516 &loop_state.counter, 1, "mask_ptr");
517 mask_val = LLVMBuildLoad(builder, mask_ptr, "");
518 }
519
520 /* 'mask' will control execution based on quad's pixel alive/killed state */
521 lp_build_mask_begin(&mask, gallivm, type, mask_val);
522
523 if (!(depth_mode & EARLY_DEPTH_TEST) && !simple_shader)
524 lp_build_mask_check(&mask);
525
526 /* Create storage for recombining sample masks after early Z pass. */
527 LLVMValueRef s_mask_or = lp_build_alloca(gallivm, lp_build_int_vec_type(gallivm, type), "cov_mask_early_depth");
528 LLVMBuildStore(builder, LLVMConstNull(lp_build_int_vec_type(gallivm, type)), s_mask_or);
529
530 LLVMValueRef s_mask = NULL, s_mask_ptr = NULL;
531 LLVMValueRef z_sample_value_store = NULL, s_sample_value_store = NULL;
532 LLVMValueRef z_fb_store = NULL, s_fb_store = NULL;
533 LLVMTypeRef z_type = NULL, z_fb_type = NULL;
534
535 /* Run early depth once per sample */
536 if (key->multisample) {
537
538 if (zs_format_desc) {
539 struct lp_type zs_type = lp_depth_type(zs_format_desc, type.length);
540 struct lp_type z_type = zs_type;
541 struct lp_type s_type = zs_type;
542 if (zs_format_desc->block.bits < type.width)
543 z_type.width = type.width;
544 else if (zs_format_desc->block.bits > 32) {
545 z_type.width = z_type.width / 2;
546 s_type.width = s_type.width / 2;
547 s_type.floating = 0;
548 }
549 z_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
550 zs_samples, "z_sample_store");
551 s_sample_value_store = lp_build_array_alloca(gallivm, lp_build_int_vec_type(gallivm, type),
552 zs_samples, "s_sample_store");
553 z_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, z_type),
554 zs_samples, "z_fb_store");
555 s_fb_store = lp_build_array_alloca(gallivm, lp_build_vec_type(gallivm, s_type),
556 zs_samples, "s_fb_store");
557 }
558 lp_build_for_loop_begin(&sample_loop_state, gallivm,
559 lp_build_const_int32(gallivm, 0),
560 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
561 lp_build_const_int32(gallivm, 1));
562
563 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
564 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
565 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
566
567 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
568 s_mask = LLVMBuildAnd(builder, s_mask, mask_val, "");
569 }
570
571
572 /* for multisample Z needs to be interpolated at sample points for testing. */
573 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, key->multisample ? sample_loop_state.counter : NULL);
574 z = interp->pos[2];
575
576 depth_ptr = depth_base_ptr;
577 if (key->multisample) {
578 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
579 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
580 }
581
582 if (depth_mode & EARLY_DEPTH_TEST) {
583 /*
584 * Clamp according to ARB_depth_clamp semantics.
585 */
586 if (key->depth_clamp) {
587 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
588 thread_data_ptr, z);
589 }
590 lp_build_depth_stencil_load_swizzled(gallivm, type,
591 zs_format_desc, key->resource_1d,
592 depth_ptr, depth_stride,
593 &z_fb, &s_fb, loop_state.counter);
594 lp_build_depth_stencil_test(gallivm,
595 &key->depth,
596 key->stencil,
597 type,
598 zs_format_desc,
599 key->multisample ? NULL : &mask,
600 &s_mask,
601 stencil_refs,
602 z, z_fb, s_fb,
603 facing,
604 &z_value, &s_value,
605 !simple_shader && !key->multisample);
606
607 if (depth_mode & EARLY_DEPTH_WRITE) {
608 lp_build_depth_stencil_write_swizzled(gallivm, type,
609 zs_format_desc, key->resource_1d,
610 NULL, NULL, NULL, loop_state.counter,
611 depth_ptr, depth_stride,
612 z_value, s_value);
613 }
614 /*
615 * Note mask check if stencil is enabled must be after ds write not after
616 * stencil test otherwise new stencil values may not get written if all
617 * fragments got killed by depth/stencil test.
618 */
619 if (!simple_shader && key->stencil[0].enabled && !key->multisample)
620 lp_build_mask_check(&mask);
621
622 if (key->multisample) {
623 z_fb_type = LLVMTypeOf(z_fb);
624 z_type = LLVMTypeOf(z_value);
625 lp_build_pointer_set(builder, z_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, z_value, lp_build_int_vec_type(gallivm, type), ""));
626 lp_build_pointer_set(builder, s_sample_value_store, sample_loop_state.counter, LLVMBuildBitCast(builder, s_value, lp_build_int_vec_type(gallivm, type), ""));
627 lp_build_pointer_set(builder, z_fb_store, sample_loop_state.counter, z_fb);
628 lp_build_pointer_set(builder, s_fb_store, sample_loop_state.counter, s_fb);
629 }
630 }
631
632 if (key->multisample) {
633 /*
634 * Store the post-early Z coverage mask.
635 * Recombine the resulting coverage masks post early Z into the fragment
636 * shader execution mask.
637 */
638 LLVMValueRef tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
639 tmp_s_mask_or = LLVMBuildOr(builder, tmp_s_mask_or, s_mask, "");
640 LLVMBuildStore(builder, tmp_s_mask_or, s_mask_or);
641
642 LLVMBuildStore(builder, s_mask, s_mask_ptr);
643
644 lp_build_for_loop_end(&sample_loop_state);
645
646 /* recombined all the coverage masks in the shader exec mask. */
647 tmp_s_mask_or = LLVMBuildLoad(builder, s_mask_or, "");
648 lp_build_mask_update(&mask, tmp_s_mask_or);
649
650 if (key->min_samples == 1) {
651 /* for multisample Z needs to be re interpolated at pixel center */
652 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, NULL);
653 lp_build_mask_update(&mask, tmp_s_mask_or);
654 }
655 }
656
657 LLVMValueRef out_sample_mask_storage = NULL;
658 if (shader->info.base.writes_samplemask) {
659 out_sample_mask_storage = lp_build_alloca(gallivm, int_vec_type, "write_mask");
660 if (key->min_samples > 1)
661 LLVMBuildStore(builder, LLVMConstNull(int_vec_type), out_sample_mask_storage);
662 }
663
664 if (key->multisample && key->min_samples > 1) {
665 lp_build_for_loop_begin(&sample_loop_state, gallivm,
666 lp_build_const_int32(gallivm, 0),
667 LLVMIntULT,
668 lp_build_const_int32(gallivm, key->min_samples),
669 lp_build_const_int32(gallivm, 1));
670
671 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
672 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
673 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
674 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
675 lp_build_mask_force(&mask, s_mask);
676 lp_build_interp_soa_update_pos_dyn(interp, gallivm, loop_state.counter, sample_loop_state.counter);
677 system_values.sample_id = sample_loop_state.counter;
678 } else
679 system_values.sample_id = lp_build_const_int32(gallivm, 0);
680
681 system_values.sample_pos = sample_pos_array;
682
683 lp_build_interp_soa_update_inputs_dyn(interp, gallivm, loop_state.counter, mask_store, sample_loop_state.counter);
684
685 struct lp_build_fs_llvm_iface fs_iface = {
686 .base.interp_fn = fs_interp,
687 .interp = interp,
688 .loop_state = &loop_state,
689 .mask_store = mask_store,
690 };
691
692 struct lp_build_tgsi_params params;
693 memset(&params, 0, sizeof(params));
694
695 params.type = type;
696 params.mask = &mask;
697 params.fs_iface = &fs_iface.base;
698 params.consts_ptr = consts_ptr;
699 params.const_sizes_ptr = num_consts_ptr;
700 params.system_values = &system_values;
701 params.inputs = interp->inputs;
702 params.context_ptr = context_ptr;
703 params.thread_data_ptr = thread_data_ptr;
704 params.sampler = sampler;
705 params.info = &shader->info.base;
706 params.ssbo_ptr = ssbo_ptr;
707 params.ssbo_sizes_ptr = num_ssbo_ptr;
708 params.image = image;
709
710 /* Build the actual shader */
711 if (shader->base.type == PIPE_SHADER_IR_TGSI)
712 lp_build_tgsi_soa(gallivm, tokens, &params,
713 outputs);
714 else
715 lp_build_nir_soa(gallivm, shader->base.ir.nir, &params,
716 outputs);
717
718 /* Alpha test */
719 if (key->alpha.enabled) {
720 int color0 = find_output_by_semantic(&shader->info.base,
721 TGSI_SEMANTIC_COLOR,
722 0);
723
724 if (color0 != -1 && outputs[color0][3]) {
725 const struct util_format_description *cbuf_format_desc;
726 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
727 LLVMValueRef alpha_ref_value;
728
729 alpha_ref_value = lp_jit_context_alpha_ref_value(gallivm, context_ptr);
730 alpha_ref_value = lp_build_broadcast(gallivm, vec_type, alpha_ref_value);
731
732 cbuf_format_desc = util_format_description(key->cbuf_format[0]);
733
734 lp_build_alpha_test(gallivm, key->alpha.func, type, cbuf_format_desc,
735 &mask, alpha, alpha_ref_value,
736 (depth_mode & LATE_DEPTH_TEST) != 0);
737 }
738 }
739
740 /* Emulate Alpha to Coverage with Alpha test */
741 if (key->blend.alpha_to_coverage) {
742 int color0 = find_output_by_semantic(&shader->info.base,
743 TGSI_SEMANTIC_COLOR,
744 0);
745
746 if (color0 != -1 && outputs[color0][3]) {
747 LLVMValueRef alpha = LLVMBuildLoad(builder, outputs[color0][3], "alpha");
748
749 if (!key->multisample) {
750 lp_build_alpha_to_coverage(gallivm, type,
751 &mask, alpha,
752 (depth_mode & LATE_DEPTH_TEST) != 0);
753 } else {
754 lp_build_sample_alpha_to_coverage(gallivm, type, key->coverage_samples, num_loop,
755 loop_state.counter,
756 mask_store, alpha);
757 }
758 }
759 }
760 if (key->blend.alpha_to_one && key->multisample) {
761 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib) {
762 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
763 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
764 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
765 if (outputs[cbuf][3]) {
766 LLVMBuildStore(builder, lp_build_const_vec(gallivm, type, 1.0), outputs[cbuf][3]);
767 }
768 }
769 }
770 if (shader->info.base.writes_samplemask) {
771 LLVMValueRef output_smask = NULL;
772 int smaski = find_output_by_semantic(&shader->info.base,
773 TGSI_SEMANTIC_SAMPLEMASK,
774 0);
775 struct lp_build_context smask_bld;
776 lp_build_context_init(&smask_bld, gallivm, int_type);
777
778 assert(smaski >= 0);
779 output_smask = LLVMBuildLoad(builder, outputs[smaski][0], "smask");
780 output_smask = LLVMBuildBitCast(builder, output_smask, smask_bld.vec_type, "");
781
782 if (key->min_samples > 1) {
783 /* only the bit corresponding to this sample is to be used. */
784 LLVMValueRef tmp_mask = LLVMBuildLoad(builder, out_sample_mask_storage, "tmp_mask");
785 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
786 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, lp_build_broadcast(gallivm, int_vec_type, out_smask_idx), "");
787 output_smask = LLVMBuildOr(builder, tmp_mask, smask_bit, "");
788 }
789
790 LLVMBuildStore(builder, output_smask, out_sample_mask_storage);
791 }
792
793 /* Color write - per fragment sample */
794 for (attrib = 0; attrib < shader->info.base.num_outputs; ++attrib)
795 {
796 unsigned cbuf = shader->info.base.output_semantic_index[attrib];
797 if ((shader->info.base.output_semantic_name[attrib] == TGSI_SEMANTIC_COLOR) &&
798 ((cbuf < key->nr_cbufs) || (cbuf == 1 && dual_source_blend)))
799 {
800 for(chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
801 if(outputs[attrib][chan]) {
802 /* XXX: just initialize outputs to point at colors[] and
803 * skip this.
804 */
805 LLVMValueRef out = LLVMBuildLoad(builder, outputs[attrib][chan], "");
806 LLVMValueRef color_ptr;
807 LLVMValueRef color_idx = loop_state.counter;
808 if (key->min_samples > 1)
809 color_idx = LLVMBuildAdd(builder, color_idx,
810 LLVMBuildMul(builder, sample_loop_state.counter, num_loop, ""), "");
811 color_ptr = LLVMBuildGEP(builder, out_color[cbuf][chan],
812 &color_idx, 1, "");
813 lp_build_name(out, "color%u.%c", attrib, "rgba"[chan]);
814 LLVMBuildStore(builder, out, color_ptr);
815 }
816 }
817 }
818 }
819
820 if (key->multisample && key->min_samples > 1) {
821 LLVMBuildStore(builder, lp_build_mask_value(&mask), s_mask_ptr);
822 lp_build_for_loop_end(&sample_loop_state);
823 }
824
825 if (key->multisample) {
826 /* execute depth test for each sample */
827 lp_build_for_loop_begin(&sample_loop_state, gallivm,
828 lp_build_const_int32(gallivm, 0),
829 LLVMIntULT, lp_build_const_int32(gallivm, key->coverage_samples),
830 lp_build_const_int32(gallivm, 1));
831
832 /* load the per-sample coverage mask */
833 LLVMValueRef s_mask_idx = LLVMBuildMul(builder, sample_loop_state.counter, num_loop, "");
834 s_mask_idx = LLVMBuildAdd(builder, s_mask_idx, loop_state.counter, "");
835 s_mask_ptr = LLVMBuildGEP(builder, mask_store, &s_mask_idx, 1, "");
836
837 /* combine the execution mask post fragment shader with the coverage mask. */
838 s_mask = LLVMBuildLoad(builder, s_mask_ptr, "");
839 if (key->min_samples == 1)
840 s_mask = LLVMBuildAnd(builder, s_mask, lp_build_mask_value(&mask), "");
841
842 /* if the shader writes sample mask use that */
843 if (shader->info.base.writes_samplemask) {
844 LLVMValueRef out_smask_idx = LLVMBuildShl(builder, lp_build_const_int32(gallivm, 1), sample_loop_state.counter, "");
845 out_smask_idx = lp_build_broadcast(gallivm, int_vec_type, out_smask_idx);
846 LLVMValueRef output_smask = LLVMBuildLoad(builder, out_sample_mask_storage, "");
847 LLVMValueRef smask_bit = LLVMBuildAnd(builder, output_smask, out_smask_idx, "");
848 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int_vec(gallivm, int_type, 0), "");
849 smask_bit = LLVMBuildSExt(builder, cmp, int_vec_type, "");
850
851 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
852 }
853 }
854
855 depth_ptr = depth_base_ptr;
856 if (key->multisample) {
857 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_loop_state.counter, depth_sample_stride, "");
858 depth_ptr = LLVMBuildGEP(builder, depth_ptr, &sample_offset, 1, "");
859 }
860
861 /* Late Z test */
862 if (depth_mode & LATE_DEPTH_TEST) {
863 int pos0 = find_output_by_semantic(&shader->info.base,
864 TGSI_SEMANTIC_POSITION,
865 0);
866 int s_out = find_output_by_semantic(&shader->info.base,
867 TGSI_SEMANTIC_STENCIL,
868 0);
869 if (pos0 != -1 && outputs[pos0][2]) {
870 z = LLVMBuildLoad(builder, outputs[pos0][2], "output.z");
871 }
872 /*
873 * Clamp according to ARB_depth_clamp semantics.
874 */
875 if (key->depth_clamp) {
876 z = lp_build_depth_clamp(gallivm, builder, type, context_ptr,
877 thread_data_ptr, z);
878 }
879
880 if (s_out != -1 && outputs[s_out][1]) {
881 /* there's only one value, and spec says to discard additional bits */
882 LLVMValueRef s_max_mask = lp_build_const_int_vec(gallivm, int_type, 255);
883 stencil_refs[0] = LLVMBuildLoad(builder, outputs[s_out][1], "output.s");
884 stencil_refs[0] = LLVMBuildBitCast(builder, stencil_refs[0], int_vec_type, "");
885 stencil_refs[0] = LLVMBuildAnd(builder, stencil_refs[0], s_max_mask, "");
886 stencil_refs[1] = stencil_refs[0];
887 }
888
889 lp_build_depth_stencil_load_swizzled(gallivm, type,
890 zs_format_desc, key->resource_1d,
891 depth_ptr, depth_stride,
892 &z_fb, &s_fb, loop_state.counter);
893
894 lp_build_depth_stencil_test(gallivm,
895 &key->depth,
896 key->stencil,
897 type,
898 zs_format_desc,
899 key->multisample ? NULL : &mask,
900 &s_mask,
901 stencil_refs,
902 z, z_fb, s_fb,
903 facing,
904 &z_value, &s_value,
905 !simple_shader);
906 /* Late Z write */
907 if (depth_mode & LATE_DEPTH_WRITE) {
908 lp_build_depth_stencil_write_swizzled(gallivm, type,
909 zs_format_desc, key->resource_1d,
910 NULL, NULL, NULL, loop_state.counter,
911 depth_ptr, depth_stride,
912 z_value, s_value);
913 }
914 }
915 else if ((depth_mode & EARLY_DEPTH_TEST) &&
916 (depth_mode & LATE_DEPTH_WRITE))
917 {
918 /* Need to apply a reduced mask to the depth write. Reload the
919 * depth value, update from zs_value with the new mask value and
920 * write that out.
921 */
922 if (key->multisample) {
923 z_value = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_sample_value_store, sample_loop_state.counter), z_type, "");;
924 s_value = lp_build_pointer_get(builder, s_sample_value_store, sample_loop_state.counter);
925 z_fb = LLVMBuildBitCast(builder, lp_build_pointer_get(builder, z_fb_store, sample_loop_state.counter), z_fb_type, "");
926 s_fb = lp_build_pointer_get(builder, s_fb_store, sample_loop_state.counter);
927 }
928 lp_build_depth_stencil_write_swizzled(gallivm, type,
929 zs_format_desc, key->resource_1d,
930 key->multisample ? s_mask : lp_build_mask_value(&mask), z_fb, s_fb, loop_state.counter,
931 depth_ptr, depth_stride,
932 z_value, s_value);
933 }
934
935 if (key->occlusion_count) {
936 LLVMValueRef counter = lp_jit_thread_data_counter(gallivm, thread_data_ptr);
937 lp_build_name(counter, "counter");
938
939 lp_build_occlusion_count(gallivm, type,
940 key->multisample ? s_mask : lp_build_mask_value(&mask), counter);
941 }
942
943 if (key->multisample) {
944 /* store the sample mask for this loop */
945 LLVMBuildStore(builder, s_mask, s_mask_ptr);
946 lp_build_for_loop_end(&sample_loop_state);
947 }
948
949 mask_val = lp_build_mask_end(&mask);
950 if (!key->multisample)
951 LLVMBuildStore(builder, mask_val, mask_ptr);
952 lp_build_for_loop_end(&loop_state);
953 }
954
955
956 /**
957 * This function will reorder pixels from the fragment shader SoA to memory layout AoS
958 *
959 * Fragment Shader outputs pixels in small 2x2 blocks
960 * e.g. (0, 0), (1, 0), (0, 1), (1, 1) ; (2, 0) ...
961 *
962 * However in memory pixels are stored in rows
963 * e.g. (0, 0), (1, 0), (2, 0), (3, 0) ; (0, 1) ...
964 *
965 * @param type fragment shader type (4x or 8x float)
966 * @param num_fs number of fs_src
967 * @param is_1d whether we're outputting to a 1d resource
968 * @param dst_channels number of output channels
969 * @param fs_src output from fragment shader
970 * @param dst pointer to store result
971 * @param pad_inline is channel padding inline or at end of row
972 * @return the number of dsts
973 */
974 static int
975 generate_fs_twiddle(struct gallivm_state *gallivm,
976 struct lp_type type,
977 unsigned num_fs,
978 unsigned dst_channels,
979 LLVMValueRef fs_src[][4],
980 LLVMValueRef* dst,
981 bool pad_inline)
982 {
983 LLVMValueRef src[16];
984
985 bool swizzle_pad;
986 bool twiddle;
987 bool split;
988
989 unsigned pixels = type.length / 4;
990 unsigned reorder_group;
991 unsigned src_channels;
992 unsigned src_count;
993 unsigned i;
994
995 src_channels = dst_channels < 3 ? dst_channels : 4;
996 src_count = num_fs * src_channels;
997
998 assert(pixels == 2 || pixels == 1);
999 assert(num_fs * src_channels <= ARRAY_SIZE(src));
1000
1001 /*
1002 * Transpose from SoA -> AoS
1003 */
1004 for (i = 0; i < num_fs; ++i) {
1005 lp_build_transpose_aos_n(gallivm, type, &fs_src[i][0], src_channels, &src[i * src_channels]);
1006 }
1007
1008 /*
1009 * Pick transformation options
1010 */
1011 swizzle_pad = false;
1012 twiddle = false;
1013 split = false;
1014 reorder_group = 0;
1015
1016 if (dst_channels == 1) {
1017 twiddle = true;
1018
1019 if (pixels == 2) {
1020 split = true;
1021 }
1022 } else if (dst_channels == 2) {
1023 if (pixels == 1) {
1024 reorder_group = 1;
1025 }
1026 } else if (dst_channels > 2) {
1027 if (pixels == 1) {
1028 reorder_group = 2;
1029 } else {
1030 twiddle = true;
1031 }
1032
1033 if (!pad_inline && dst_channels == 3 && pixels > 1) {
1034 swizzle_pad = true;
1035 }
1036 }
1037
1038 /*
1039 * Split the src in half
1040 */
1041 if (split) {
1042 for (i = num_fs; i > 0; --i) {
1043 src[(i - 1)*2 + 1] = lp_build_extract_range(gallivm, src[i - 1], 4, 4);
1044 src[(i - 1)*2 + 0] = lp_build_extract_range(gallivm, src[i - 1], 0, 4);
1045 }
1046
1047 src_count *= 2;
1048 type.length = 4;
1049 }
1050
1051 /*
1052 * Ensure pixels are in memory order
1053 */
1054 if (reorder_group) {
1055 /* Twiddle pixels by reordering the array, e.g.:
1056 *
1057 * src_count = 8 -> 0 2 1 3 4 6 5 7
1058 * src_count = 16 -> 0 1 4 5 2 3 6 7 8 9 12 13 10 11 14 15
1059 */
1060 const unsigned reorder_sw[] = { 0, 2, 1, 3 };
1061
1062 for (i = 0; i < src_count; ++i) {
1063 unsigned group = i / reorder_group;
1064 unsigned block = (group / 4) * 4 * reorder_group;
1065 unsigned j = block + (reorder_sw[group % 4] * reorder_group) + (i % reorder_group);
1066 dst[i] = src[j];
1067 }
1068 } else if (twiddle) {
1069 /* Twiddle pixels across elements of array */
1070 /*
1071 * XXX: we should avoid this in some cases, but would need to tell
1072 * lp_build_conv to reorder (or deal with it ourselves).
1073 */
1074 lp_bld_quad_twiddle(gallivm, type, src, src_count, dst);
1075 } else {
1076 /* Do nothing */
1077 memcpy(dst, src, sizeof(LLVMValueRef) * src_count);
1078 }
1079
1080 /*
1081 * Moves any padding between pixels to the end
1082 * e.g. RGBXRGBX -> RGBRGBXX
1083 */
1084 if (swizzle_pad) {
1085 unsigned char swizzles[16];
1086 unsigned elems = pixels * dst_channels;
1087
1088 for (i = 0; i < type.length; ++i) {
1089 if (i < elems)
1090 swizzles[i] = i % dst_channels + (i / dst_channels) * 4;
1091 else
1092 swizzles[i] = LP_BLD_SWIZZLE_DONTCARE;
1093 }
1094
1095 for (i = 0; i < src_count; ++i) {
1096 dst[i] = lp_build_swizzle_aos_n(gallivm, dst[i], swizzles, type.length, type.length);
1097 }
1098 }
1099
1100 return src_count;
1101 }
1102
1103
1104 /*
1105 * Untwiddle and transpose, much like the above.
1106 * However, this is after conversion, so we get packed vectors.
1107 * At this time only handle 4x16i8 rgba / 2x16i8 rg / 1x16i8 r data,
1108 * the vectors will look like:
1109 * r0r1r4r5r2r3r6r7r8r9r12... (albeit color channels may
1110 * be swizzled here). Extending to 16bit should be trivial.
1111 * Should also be extended to handle twice wide vectors with AVX2...
1112 */
1113 static void
1114 fs_twiddle_transpose(struct gallivm_state *gallivm,
1115 struct lp_type type,
1116 LLVMValueRef *src,
1117 unsigned src_count,
1118 LLVMValueRef *dst)
1119 {
1120 unsigned i, j;
1121 struct lp_type type64, type16, type32;
1122 LLVMTypeRef type64_t, type8_t, type16_t, type32_t;
1123 LLVMBuilderRef builder = gallivm->builder;
1124 LLVMValueRef tmp[4], shuf[8];
1125 for (j = 0; j < 2; j++) {
1126 shuf[j*4 + 0] = lp_build_const_int32(gallivm, j*4 + 0);
1127 shuf[j*4 + 1] = lp_build_const_int32(gallivm, j*4 + 2);
1128 shuf[j*4 + 2] = lp_build_const_int32(gallivm, j*4 + 1);
1129 shuf[j*4 + 3] = lp_build_const_int32(gallivm, j*4 + 3);
1130 }
1131
1132 assert(src_count == 4 || src_count == 2 || src_count == 1);
1133 assert(type.width == 8);
1134 assert(type.length == 16);
1135
1136 type8_t = lp_build_vec_type(gallivm, type);
1137
1138 type64 = type;
1139 type64.length /= 8;
1140 type64.width *= 8;
1141 type64_t = lp_build_vec_type(gallivm, type64);
1142
1143 type16 = type;
1144 type16.length /= 2;
1145 type16.width *= 2;
1146 type16_t = lp_build_vec_type(gallivm, type16);
1147
1148 type32 = type;
1149 type32.length /= 4;
1150 type32.width *= 4;
1151 type32_t = lp_build_vec_type(gallivm, type32);
1152
1153 lp_build_transpose_aos_n(gallivm, type, src, src_count, tmp);
1154
1155 if (src_count == 1) {
1156 /* transpose was no-op, just untwiddle */
1157 LLVMValueRef shuf_vec;
1158 shuf_vec = LLVMConstVector(shuf, 8);
1159 tmp[0] = LLVMBuildBitCast(builder, src[0], type16_t, "");
1160 tmp[0] = LLVMBuildShuffleVector(builder, tmp[0], tmp[0], shuf_vec, "");
1161 dst[0] = LLVMBuildBitCast(builder, tmp[0], type8_t, "");
1162 } else if (src_count == 2) {
1163 LLVMValueRef shuf_vec;
1164 shuf_vec = LLVMConstVector(shuf, 4);
1165
1166 for (i = 0; i < 2; i++) {
1167 tmp[i] = LLVMBuildBitCast(builder, tmp[i], type32_t, "");
1168 tmp[i] = LLVMBuildShuffleVector(builder, tmp[i], tmp[i], shuf_vec, "");
1169 dst[i] = LLVMBuildBitCast(builder, tmp[i], type8_t, "");
1170 }
1171 } else {
1172 for (j = 0; j < 2; j++) {
1173 LLVMValueRef lo, hi, lo2, hi2;
1174 /*
1175 * Note that if we only really have 3 valid channels (rgb)
1176 * and we don't need alpha we could substitute a undef here
1177 * for the respective channel (causing llvm to drop conversion
1178 * for alpha).
1179 */
1180 /* we now have rgba0rgba1rgba4rgba5 etc, untwiddle */
1181 lo2 = LLVMBuildBitCast(builder, tmp[j*2], type64_t, "");
1182 hi2 = LLVMBuildBitCast(builder, tmp[j*2 + 1], type64_t, "");
1183 lo = lp_build_interleave2(gallivm, type64, lo2, hi2, 0);
1184 hi = lp_build_interleave2(gallivm, type64, lo2, hi2, 1);
1185 dst[j*2] = LLVMBuildBitCast(builder, lo, type8_t, "");
1186 dst[j*2 + 1] = LLVMBuildBitCast(builder, hi, type8_t, "");
1187 }
1188 }
1189 }
1190
1191
1192 /**
1193 * Load an unswizzled block of pixels from memory
1194 */
1195 static void
1196 load_unswizzled_block(struct gallivm_state *gallivm,
1197 LLVMValueRef base_ptr,
1198 LLVMValueRef stride,
1199 unsigned block_width,
1200 unsigned block_height,
1201 LLVMValueRef* dst,
1202 struct lp_type dst_type,
1203 unsigned dst_count,
1204 unsigned dst_alignment)
1205 {
1206 LLVMBuilderRef builder = gallivm->builder;
1207 unsigned row_size = dst_count / block_height;
1208 unsigned i;
1209
1210 /* Ensure block exactly fits into dst */
1211 assert((block_width * block_height) % dst_count == 0);
1212
1213 for (i = 0; i < dst_count; ++i) {
1214 unsigned x = i % row_size;
1215 unsigned y = i / row_size;
1216
1217 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (dst_type.width / 8) * dst_type.length);
1218 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1219
1220 LLVMValueRef gep[2];
1221 LLVMValueRef dst_ptr;
1222
1223 gep[0] = lp_build_const_int32(gallivm, 0);
1224 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1225
1226 dst_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1227 dst_ptr = LLVMBuildBitCast(builder, dst_ptr,
1228 LLVMPointerType(lp_build_vec_type(gallivm, dst_type), 0), "");
1229
1230 dst[i] = LLVMBuildLoad(builder, dst_ptr, "");
1231
1232 LLVMSetAlignment(dst[i], dst_alignment);
1233 }
1234 }
1235
1236
1237 /**
1238 * Store an unswizzled block of pixels to memory
1239 */
1240 static void
1241 store_unswizzled_block(struct gallivm_state *gallivm,
1242 LLVMValueRef base_ptr,
1243 LLVMValueRef stride,
1244 unsigned block_width,
1245 unsigned block_height,
1246 LLVMValueRef* src,
1247 struct lp_type src_type,
1248 unsigned src_count,
1249 unsigned src_alignment)
1250 {
1251 LLVMBuilderRef builder = gallivm->builder;
1252 unsigned row_size = src_count / block_height;
1253 unsigned i;
1254
1255 /* Ensure src exactly fits into block */
1256 assert((block_width * block_height) % src_count == 0);
1257
1258 for (i = 0; i < src_count; ++i) {
1259 unsigned x = i % row_size;
1260 unsigned y = i / row_size;
1261
1262 LLVMValueRef bx = lp_build_const_int32(gallivm, x * (src_type.width / 8) * src_type.length);
1263 LLVMValueRef by = LLVMBuildMul(builder, lp_build_const_int32(gallivm, y), stride, "");
1264
1265 LLVMValueRef gep[2];
1266 LLVMValueRef src_ptr;
1267
1268 gep[0] = lp_build_const_int32(gallivm, 0);
1269 gep[1] = LLVMBuildAdd(builder, bx, by, "");
1270
1271 src_ptr = LLVMBuildGEP(builder, base_ptr, gep, 2, "");
1272 src_ptr = LLVMBuildBitCast(builder, src_ptr,
1273 LLVMPointerType(lp_build_vec_type(gallivm, src_type), 0), "");
1274
1275 src_ptr = LLVMBuildStore(builder, src[i], src_ptr);
1276
1277 LLVMSetAlignment(src_ptr, src_alignment);
1278 }
1279 }
1280
1281
1282 /**
1283 * Checks if a format description is an arithmetic format
1284 *
1285 * A format which has irregular channel sizes such as R3_G3_B2 or R5_G6_B5.
1286 */
1287 static inline boolean
1288 is_arithmetic_format(const struct util_format_description *format_desc)
1289 {
1290 boolean arith = false;
1291 unsigned i;
1292
1293 for (i = 0; i < format_desc->nr_channels; ++i) {
1294 arith |= format_desc->channel[i].size != format_desc->channel[0].size;
1295 arith |= (format_desc->channel[i].size % 8) != 0;
1296 }
1297
1298 return arith;
1299 }
1300
1301
1302 /**
1303 * Checks if this format requires special handling due to required expansion
1304 * to floats for blending, and furthermore has "natural" packed AoS -> unpacked
1305 * SoA conversion.
1306 */
1307 static inline boolean
1308 format_expands_to_float_soa(const struct util_format_description *format_desc)
1309 {
1310 if (format_desc->format == PIPE_FORMAT_R11G11B10_FLOAT ||
1311 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
1312 return true;
1313 }
1314 return false;
1315 }
1316
1317
1318 /**
1319 * Retrieves the type representing the memory layout for a format
1320 *
1321 * e.g. RGBA16F = 4x half-float and R3G3B2 = 1x byte
1322 */
1323 static inline void
1324 lp_mem_type_from_format_desc(const struct util_format_description *format_desc,
1325 struct lp_type* type)
1326 {
1327 unsigned i;
1328 unsigned chan;
1329
1330 if (format_expands_to_float_soa(format_desc)) {
1331 /* just make this a uint with width of block */
1332 type->floating = false;
1333 type->fixed = false;
1334 type->sign = false;
1335 type->norm = false;
1336 type->width = format_desc->block.bits;
1337 type->length = 1;
1338 return;
1339 }
1340
1341 for (i = 0; i < 4; i++)
1342 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1343 break;
1344 chan = i;
1345
1346 memset(type, 0, sizeof(struct lp_type));
1347 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
1348 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
1349 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
1350 type->norm = format_desc->channel[chan].normalized;
1351
1352 if (is_arithmetic_format(format_desc)) {
1353 type->width = 0;
1354 type->length = 1;
1355
1356 for (i = 0; i < format_desc->nr_channels; ++i) {
1357 type->width += format_desc->channel[i].size;
1358 }
1359 } else {
1360 type->width = format_desc->channel[chan].size;
1361 type->length = format_desc->nr_channels;
1362 }
1363 }
1364
1365
1366 /**
1367 * Retrieves the type for a format which is usable in the blending code.
1368 *
1369 * e.g. RGBA16F = 4x float, R3G3B2 = 3x byte
1370 */
1371 static inline void
1372 lp_blend_type_from_format_desc(const struct util_format_description *format_desc,
1373 struct lp_type* type)
1374 {
1375 unsigned i;
1376 unsigned chan;
1377
1378 if (format_expands_to_float_soa(format_desc)) {
1379 /* always use ordinary floats for blending */
1380 type->floating = true;
1381 type->fixed = false;
1382 type->sign = true;
1383 type->norm = false;
1384 type->width = 32;
1385 type->length = 4;
1386 return;
1387 }
1388
1389 for (i = 0; i < 4; i++)
1390 if (format_desc->channel[i].type != UTIL_FORMAT_TYPE_VOID)
1391 break;
1392 chan = i;
1393
1394 memset(type, 0, sizeof(struct lp_type));
1395 type->floating = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FLOAT;
1396 type->fixed = format_desc->channel[chan].type == UTIL_FORMAT_TYPE_FIXED;
1397 type->sign = format_desc->channel[chan].type != UTIL_FORMAT_TYPE_UNSIGNED;
1398 type->norm = format_desc->channel[chan].normalized;
1399 type->width = format_desc->channel[chan].size;
1400 type->length = format_desc->nr_channels;
1401
1402 for (i = 1; i < format_desc->nr_channels; ++i) {
1403 if (format_desc->channel[i].size > type->width)
1404 type->width = format_desc->channel[i].size;
1405 }
1406
1407 if (type->floating) {
1408 type->width = 32;
1409 } else {
1410 if (type->width <= 8) {
1411 type->width = 8;
1412 } else if (type->width <= 16) {
1413 type->width = 16;
1414 } else {
1415 type->width = 32;
1416 }
1417 }
1418
1419 if (is_arithmetic_format(format_desc) && type->length == 3) {
1420 type->length = 4;
1421 }
1422 }
1423
1424
1425 /**
1426 * Scale a normalized value from src_bits to dst_bits.
1427 *
1428 * The exact calculation is
1429 *
1430 * dst = iround(src * dst_mask / src_mask)
1431 *
1432 * or with integer rounding
1433 *
1434 * dst = src * (2*dst_mask + sign(src)*src_mask) / (2*src_mask)
1435 *
1436 * where
1437 *
1438 * src_mask = (1 << src_bits) - 1
1439 * dst_mask = (1 << dst_bits) - 1
1440 *
1441 * but we try to avoid division and multiplication through shifts.
1442 */
1443 static inline LLVMValueRef
1444 scale_bits(struct gallivm_state *gallivm,
1445 int src_bits,
1446 int dst_bits,
1447 LLVMValueRef src,
1448 struct lp_type src_type)
1449 {
1450 LLVMBuilderRef builder = gallivm->builder;
1451 LLVMValueRef result = src;
1452
1453 if (dst_bits < src_bits) {
1454 int delta_bits = src_bits - dst_bits;
1455
1456 if (delta_bits <= dst_bits) {
1457 /*
1458 * Approximate the rescaling with a single shift.
1459 *
1460 * This gives the wrong rounding.
1461 */
1462
1463 result = LLVMBuildLShr(builder,
1464 src,
1465 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1466 "");
1467
1468 } else {
1469 /*
1470 * Try more accurate rescaling.
1471 */
1472
1473 /*
1474 * Drop the least significant bits to make space for the multiplication.
1475 *
1476 * XXX: A better approach would be to use a wider integer type as intermediate. But
1477 * this is enough to convert alpha from 16bits -> 2 when rendering to
1478 * PIPE_FORMAT_R10G10B10A2_UNORM.
1479 */
1480 result = LLVMBuildLShr(builder,
1481 src,
1482 lp_build_const_int_vec(gallivm, src_type, dst_bits),
1483 "");
1484
1485
1486 result = LLVMBuildMul(builder,
1487 result,
1488 lp_build_const_int_vec(gallivm, src_type, (1LL << dst_bits) - 1),
1489 "");
1490
1491 /*
1492 * Add a rounding term before the division.
1493 *
1494 * TODO: Handle signed integers too.
1495 */
1496 if (!src_type.sign) {
1497 result = LLVMBuildAdd(builder,
1498 result,
1499 lp_build_const_int_vec(gallivm, src_type, (1LL << (delta_bits - 1))),
1500 "");
1501 }
1502
1503 /*
1504 * Approximate the division by src_mask with a src_bits shift.
1505 *
1506 * Given the src has already been shifted by dst_bits, all we need
1507 * to do is to shift by the difference.
1508 */
1509
1510 result = LLVMBuildLShr(builder,
1511 result,
1512 lp_build_const_int_vec(gallivm, src_type, delta_bits),
1513 "");
1514 }
1515
1516 } else if (dst_bits > src_bits) {
1517 /* Scale up bits */
1518 int db = dst_bits - src_bits;
1519
1520 /* Shift left by difference in bits */
1521 result = LLVMBuildShl(builder,
1522 src,
1523 lp_build_const_int_vec(gallivm, src_type, db),
1524 "");
1525
1526 if (db <= src_bits) {
1527 /* Enough bits in src to fill the remainder */
1528 LLVMValueRef lower = LLVMBuildLShr(builder,
1529 src,
1530 lp_build_const_int_vec(gallivm, src_type, src_bits - db),
1531 "");
1532
1533 result = LLVMBuildOr(builder, result, lower, "");
1534 } else if (db > src_bits) {
1535 /* Need to repeatedly copy src bits to fill remainder in dst */
1536 unsigned n;
1537
1538 for (n = src_bits; n < dst_bits; n *= 2) {
1539 LLVMValueRef shuv = lp_build_const_int_vec(gallivm, src_type, n);
1540
1541 result = LLVMBuildOr(builder,
1542 result,
1543 LLVMBuildLShr(builder, result, shuv, ""),
1544 "");
1545 }
1546 }
1547 }
1548
1549 return result;
1550 }
1551
1552 /**
1553 * If RT is a smallfloat (needing denorms) format
1554 */
1555 static inline int
1556 have_smallfloat_format(struct lp_type dst_type,
1557 enum pipe_format format)
1558 {
1559 return ((dst_type.floating && dst_type.width != 32) ||
1560 /* due to format handling hacks this format doesn't have floating set
1561 * here (and actually has width set to 32 too) so special case this. */
1562 (format == PIPE_FORMAT_R11G11B10_FLOAT));
1563 }
1564
1565
1566 /**
1567 * Convert from memory format to blending format
1568 *
1569 * e.g. GL_R3G3B2 is 1 byte in memory but 3 bytes for blending
1570 */
1571 static void
1572 convert_to_blend_type(struct gallivm_state *gallivm,
1573 unsigned block_size,
1574 const struct util_format_description *src_fmt,
1575 struct lp_type src_type,
1576 struct lp_type dst_type,
1577 LLVMValueRef* src, // and dst
1578 unsigned num_srcs)
1579 {
1580 LLVMValueRef *dst = src;
1581 LLVMBuilderRef builder = gallivm->builder;
1582 struct lp_type blend_type;
1583 struct lp_type mem_type;
1584 unsigned i, j;
1585 unsigned pixels = block_size / num_srcs;
1586 bool is_arith;
1587
1588 /*
1589 * full custom path for packed floats and srgb formats - none of the later
1590 * functions would do anything useful, and given the lp_type representation they
1591 * can't be fixed. Should really have some SoA blend path for these kind of
1592 * formats rather than hacking them in here.
1593 */
1594 if (format_expands_to_float_soa(src_fmt)) {
1595 LLVMValueRef tmpsrc[4];
1596 /*
1597 * This is pretty suboptimal for this case blending in SoA would be much
1598 * better, since conversion gets us SoA values so need to convert back.
1599 */
1600 assert(src_type.width == 32 || src_type.width == 16);
1601 assert(dst_type.floating);
1602 assert(dst_type.width == 32);
1603 assert(dst_type.length % 4 == 0);
1604 assert(num_srcs % 4 == 0);
1605
1606 if (src_type.width == 16) {
1607 /* expand 4x16bit values to 4x32bit */
1608 struct lp_type type32x4 = src_type;
1609 LLVMTypeRef ltype32x4;
1610 unsigned num_fetch = dst_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
1611 type32x4.width = 32;
1612 ltype32x4 = lp_build_vec_type(gallivm, type32x4);
1613 for (i = 0; i < num_fetch; i++) {
1614 src[i] = LLVMBuildZExt(builder, src[i], ltype32x4, "");
1615 }
1616 src_type.width = 32;
1617 }
1618 for (i = 0; i < 4; i++) {
1619 tmpsrc[i] = src[i];
1620 }
1621 for (i = 0; i < num_srcs / 4; i++) {
1622 LLVMValueRef tmpsoa[4];
1623 LLVMValueRef tmps = tmpsrc[i];
1624 if (dst_type.length == 8) {
1625 LLVMValueRef shuffles[8];
1626 unsigned j;
1627 /* fetch was 4 values but need 8-wide output values */
1628 tmps = lp_build_concat(gallivm, &tmpsrc[i * 2], src_type, 2);
1629 /*
1630 * for 8-wide aos transpose would give us wrong order not matching
1631 * incoming converted fs values and mask. ARGH.
1632 */
1633 for (j = 0; j < 4; j++) {
1634 shuffles[j] = lp_build_const_int32(gallivm, j * 2);
1635 shuffles[j + 4] = lp_build_const_int32(gallivm, j * 2 + 1);
1636 }
1637 tmps = LLVMBuildShuffleVector(builder, tmps, tmps,
1638 LLVMConstVector(shuffles, 8), "");
1639 }
1640 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
1641 lp_build_r11g11b10_to_float(gallivm, tmps, tmpsoa);
1642 }
1643 else {
1644 lp_build_unpack_rgba_soa(gallivm, src_fmt, dst_type, tmps, tmpsoa);
1645 }
1646 lp_build_transpose_aos(gallivm, dst_type, tmpsoa, &src[i * 4]);
1647 }
1648 return;
1649 }
1650
1651 lp_mem_type_from_format_desc(src_fmt, &mem_type);
1652 lp_blend_type_from_format_desc(src_fmt, &blend_type);
1653
1654 /* Is the format arithmetic */
1655 is_arith = blend_type.length * blend_type.width != mem_type.width * mem_type.length;
1656 is_arith &= !(mem_type.width == 16 && mem_type.floating);
1657
1658 /* Pad if necessary */
1659 if (!is_arith && src_type.length < dst_type.length) {
1660 for (i = 0; i < num_srcs; ++i) {
1661 dst[i] = lp_build_pad_vector(gallivm, src[i], dst_type.length);
1662 }
1663
1664 src_type.length = dst_type.length;
1665 }
1666
1667 /* Special case for half-floats */
1668 if (mem_type.width == 16 && mem_type.floating) {
1669 assert(blend_type.width == 32 && blend_type.floating);
1670 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
1671 is_arith = false;
1672 }
1673
1674 if (!is_arith) {
1675 return;
1676 }
1677
1678 src_type.width = blend_type.width * blend_type.length;
1679 blend_type.length *= pixels;
1680 src_type.length *= pixels / (src_type.length / mem_type.length);
1681
1682 for (i = 0; i < num_srcs; ++i) {
1683 LLVMValueRef chans[4];
1684 LLVMValueRef res = NULL;
1685
1686 dst[i] = LLVMBuildZExt(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
1687
1688 for (j = 0; j < src_fmt->nr_channels; ++j) {
1689 unsigned mask = 0;
1690 unsigned sa = src_fmt->channel[j].shift;
1691 #if UTIL_ARCH_LITTLE_ENDIAN
1692 unsigned from_lsb = j;
1693 #else
1694 unsigned from_lsb = src_fmt->nr_channels - j - 1;
1695 #endif
1696
1697 mask = (1 << src_fmt->channel[j].size) - 1;
1698
1699 /* Extract bits from source */
1700 chans[j] = LLVMBuildLShr(builder,
1701 dst[i],
1702 lp_build_const_int_vec(gallivm, src_type, sa),
1703 "");
1704
1705 chans[j] = LLVMBuildAnd(builder,
1706 chans[j],
1707 lp_build_const_int_vec(gallivm, src_type, mask),
1708 "");
1709
1710 /* Scale bits */
1711 if (src_type.norm) {
1712 chans[j] = scale_bits(gallivm, src_fmt->channel[j].size,
1713 blend_type.width, chans[j], src_type);
1714 }
1715
1716 /* Insert bits into correct position */
1717 chans[j] = LLVMBuildShl(builder,
1718 chans[j],
1719 lp_build_const_int_vec(gallivm, src_type, from_lsb * blend_type.width),
1720 "");
1721
1722 if (j == 0) {
1723 res = chans[j];
1724 } else {
1725 res = LLVMBuildOr(builder, res, chans[j], "");
1726 }
1727 }
1728
1729 dst[i] = LLVMBuildBitCast(builder, res, lp_build_vec_type(gallivm, blend_type), "");
1730 }
1731 }
1732
1733
1734 /**
1735 * Convert from blending format to memory format
1736 *
1737 * e.g. GL_R3G3B2 is 3 bytes for blending but 1 byte in memory
1738 */
1739 static void
1740 convert_from_blend_type(struct gallivm_state *gallivm,
1741 unsigned block_size,
1742 const struct util_format_description *src_fmt,
1743 struct lp_type src_type,
1744 struct lp_type dst_type,
1745 LLVMValueRef* src, // and dst
1746 unsigned num_srcs)
1747 {
1748 LLVMValueRef* dst = src;
1749 unsigned i, j, k;
1750 struct lp_type mem_type;
1751 struct lp_type blend_type;
1752 LLVMBuilderRef builder = gallivm->builder;
1753 unsigned pixels = block_size / num_srcs;
1754 bool is_arith;
1755
1756 /*
1757 * full custom path for packed floats and srgb formats - none of the later
1758 * functions would do anything useful, and given the lp_type representation they
1759 * can't be fixed. Should really have some SoA blend path for these kind of
1760 * formats rather than hacking them in here.
1761 */
1762 if (format_expands_to_float_soa(src_fmt)) {
1763 /*
1764 * This is pretty suboptimal for this case blending in SoA would be much
1765 * better - we need to transpose the AoS values back to SoA values for
1766 * conversion/packing.
1767 */
1768 assert(src_type.floating);
1769 assert(src_type.width == 32);
1770 assert(src_type.length % 4 == 0);
1771 assert(dst_type.width == 32 || dst_type.width == 16);
1772
1773 for (i = 0; i < num_srcs / 4; i++) {
1774 LLVMValueRef tmpsoa[4], tmpdst;
1775 lp_build_transpose_aos(gallivm, src_type, &src[i * 4], tmpsoa);
1776 /* really really need SoA here */
1777
1778 if (src_fmt->format == PIPE_FORMAT_R11G11B10_FLOAT) {
1779 tmpdst = lp_build_float_to_r11g11b10(gallivm, tmpsoa);
1780 }
1781 else {
1782 tmpdst = lp_build_float_to_srgb_packed(gallivm, src_fmt,
1783 src_type, tmpsoa);
1784 }
1785
1786 if (src_type.length == 8) {
1787 LLVMValueRef tmpaos, shuffles[8];
1788 unsigned j;
1789 /*
1790 * for 8-wide aos transpose has given us wrong order not matching
1791 * output order. HMPF. Also need to split the output values manually.
1792 */
1793 for (j = 0; j < 4; j++) {
1794 shuffles[j * 2] = lp_build_const_int32(gallivm, j);
1795 shuffles[j * 2 + 1] = lp_build_const_int32(gallivm, j + 4);
1796 }
1797 tmpaos = LLVMBuildShuffleVector(builder, tmpdst, tmpdst,
1798 LLVMConstVector(shuffles, 8), "");
1799 src[i * 2] = lp_build_extract_range(gallivm, tmpaos, 0, 4);
1800 src[i * 2 + 1] = lp_build_extract_range(gallivm, tmpaos, 4, 4);
1801 }
1802 else {
1803 src[i] = tmpdst;
1804 }
1805 }
1806 if (dst_type.width == 16) {
1807 struct lp_type type16x8 = dst_type;
1808 struct lp_type type32x4 = dst_type;
1809 LLVMTypeRef ltype16x4, ltypei64, ltypei128;
1810 unsigned num_fetch = src_type.length == 8 ? num_srcs / 2 : num_srcs / 4;
1811 type16x8.length = 8;
1812 type32x4.width = 32;
1813 ltypei128 = LLVMIntTypeInContext(gallivm->context, 128);
1814 ltypei64 = LLVMIntTypeInContext(gallivm->context, 64);
1815 ltype16x4 = lp_build_vec_type(gallivm, dst_type);
1816 /* We could do vector truncation but it doesn't generate very good code */
1817 for (i = 0; i < num_fetch; i++) {
1818 src[i] = lp_build_pack2(gallivm, type32x4, type16x8,
1819 src[i], lp_build_zero(gallivm, type32x4));
1820 src[i] = LLVMBuildBitCast(builder, src[i], ltypei128, "");
1821 src[i] = LLVMBuildTrunc(builder, src[i], ltypei64, "");
1822 src[i] = LLVMBuildBitCast(builder, src[i], ltype16x4, "");
1823 }
1824 }
1825 return;
1826 }
1827
1828 lp_mem_type_from_format_desc(src_fmt, &mem_type);
1829 lp_blend_type_from_format_desc(src_fmt, &blend_type);
1830
1831 is_arith = (blend_type.length * blend_type.width != mem_type.width * mem_type.length);
1832
1833 /* Special case for half-floats */
1834 if (mem_type.width == 16 && mem_type.floating) {
1835 int length = dst_type.length;
1836 assert(blend_type.width == 32 && blend_type.floating);
1837
1838 dst_type.length = src_type.length;
1839
1840 lp_build_conv_auto(gallivm, src_type, &dst_type, dst, num_srcs, dst);
1841
1842 dst_type.length = length;
1843 is_arith = false;
1844 }
1845
1846 /* Remove any padding */
1847 if (!is_arith && (src_type.length % mem_type.length)) {
1848 src_type.length -= (src_type.length % mem_type.length);
1849
1850 for (i = 0; i < num_srcs; ++i) {
1851 dst[i] = lp_build_extract_range(gallivm, dst[i], 0, src_type.length);
1852 }
1853 }
1854
1855 /* No bit arithmetic to do */
1856 if (!is_arith) {
1857 return;
1858 }
1859
1860 src_type.length = pixels;
1861 src_type.width = blend_type.length * blend_type.width;
1862 dst_type.length = pixels;
1863
1864 for (i = 0; i < num_srcs; ++i) {
1865 LLVMValueRef chans[4];
1866 LLVMValueRef res = NULL;
1867
1868 dst[i] = LLVMBuildBitCast(builder, src[i], lp_build_vec_type(gallivm, src_type), "");
1869
1870 for (j = 0; j < src_fmt->nr_channels; ++j) {
1871 unsigned mask = 0;
1872 unsigned sa = src_fmt->channel[j].shift;
1873 unsigned sz_a = src_fmt->channel[j].size;
1874 #if UTIL_ARCH_LITTLE_ENDIAN
1875 unsigned from_lsb = j;
1876 #else
1877 unsigned from_lsb = src_fmt->nr_channels - j - 1;
1878 #endif
1879
1880 assert(blend_type.width > src_fmt->channel[j].size);
1881
1882 for (k = 0; k < blend_type.width; ++k) {
1883 mask |= 1 << k;
1884 }
1885
1886 /* Extract bits */
1887 chans[j] = LLVMBuildLShr(builder,
1888 dst[i],
1889 lp_build_const_int_vec(gallivm, src_type,
1890 from_lsb * blend_type.width),
1891 "");
1892
1893 chans[j] = LLVMBuildAnd(builder,
1894 chans[j],
1895 lp_build_const_int_vec(gallivm, src_type, mask),
1896 "");
1897
1898 /* Scale down bits */
1899 if (src_type.norm) {
1900 chans[j] = scale_bits(gallivm, blend_type.width,
1901 src_fmt->channel[j].size, chans[j], src_type);
1902 } else if (!src_type.floating && sz_a < blend_type.width) {
1903 LLVMValueRef mask_val = lp_build_const_int_vec(gallivm, src_type, (1UL << sz_a) - 1);
1904 LLVMValueRef mask = LLVMBuildICmp(builder, LLVMIntUGT, chans[j], mask_val, "");
1905 chans[j] = LLVMBuildSelect(builder, mask, mask_val, chans[j], "");
1906 }
1907
1908 /* Insert bits */
1909 chans[j] = LLVMBuildShl(builder,
1910 chans[j],
1911 lp_build_const_int_vec(gallivm, src_type, sa),
1912 "");
1913
1914 sa += src_fmt->channel[j].size;
1915
1916 if (j == 0) {
1917 res = chans[j];
1918 } else {
1919 res = LLVMBuildOr(builder, res, chans[j], "");
1920 }
1921 }
1922
1923 assert (dst_type.width != 24);
1924
1925 dst[i] = LLVMBuildTrunc(builder, res, lp_build_vec_type(gallivm, dst_type), "");
1926 }
1927 }
1928
1929
1930 /**
1931 * Convert alpha to same blend type as src
1932 */
1933 static void
1934 convert_alpha(struct gallivm_state *gallivm,
1935 struct lp_type row_type,
1936 struct lp_type alpha_type,
1937 const unsigned block_size,
1938 const unsigned block_height,
1939 const unsigned src_count,
1940 const unsigned dst_channels,
1941 const bool pad_inline,
1942 LLVMValueRef* src_alpha)
1943 {
1944 LLVMBuilderRef builder = gallivm->builder;
1945 unsigned i, j;
1946 unsigned length = row_type.length;
1947 row_type.length = alpha_type.length;
1948
1949 /* Twiddle the alpha to match pixels */
1950 lp_bld_quad_twiddle(gallivm, alpha_type, src_alpha, block_height, src_alpha);
1951
1952 /*
1953 * TODO this should use single lp_build_conv call for
1954 * src_count == 1 && dst_channels == 1 case (dropping the concat below)
1955 */
1956 for (i = 0; i < block_height; ++i) {
1957 lp_build_conv(gallivm, alpha_type, row_type, &src_alpha[i], 1, &src_alpha[i], 1);
1958 }
1959
1960 alpha_type = row_type;
1961 row_type.length = length;
1962
1963 /* If only one channel we can only need the single alpha value per pixel */
1964 if (src_count == 1 && dst_channels == 1) {
1965
1966 lp_build_concat_n(gallivm, alpha_type, src_alpha, block_height, src_alpha, src_count);
1967 } else {
1968 /* If there are more srcs than rows then we need to split alpha up */
1969 if (src_count > block_height) {
1970 for (i = src_count; i > 0; --i) {
1971 unsigned pixels = block_size / src_count;
1972 unsigned idx = i - 1;
1973
1974 src_alpha[idx] = lp_build_extract_range(gallivm, src_alpha[(idx * pixels) / 4],
1975 (idx * pixels) % 4, pixels);
1976 }
1977 }
1978
1979 /* If there is a src for each pixel broadcast the alpha across whole row */
1980 if (src_count == block_size) {
1981 for (i = 0; i < src_count; ++i) {
1982 src_alpha[i] = lp_build_broadcast(gallivm,
1983 lp_build_vec_type(gallivm, row_type), src_alpha[i]);
1984 }
1985 } else {
1986 unsigned pixels = block_size / src_count;
1987 unsigned channels = pad_inline ? TGSI_NUM_CHANNELS : dst_channels;
1988 unsigned alpha_span = 1;
1989 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
1990
1991 /* Check if we need 2 src_alphas for our shuffles */
1992 if (pixels > alpha_type.length) {
1993 alpha_span = 2;
1994 }
1995
1996 /* Broadcast alpha across all channels, e.g. a1a2 to a1a1a1a1a2a2a2a2 */
1997 for (j = 0; j < row_type.length; ++j) {
1998 if (j < pixels * channels) {
1999 shuffles[j] = lp_build_const_int32(gallivm, j / channels);
2000 } else {
2001 shuffles[j] = LLVMGetUndef(LLVMInt32TypeInContext(gallivm->context));
2002 }
2003 }
2004
2005 for (i = 0; i < src_count; ++i) {
2006 unsigned idx1 = i, idx2 = i;
2007
2008 if (alpha_span > 1){
2009 idx1 *= alpha_span;
2010 idx2 = idx1 + 1;
2011 }
2012
2013 src_alpha[i] = LLVMBuildShuffleVector(builder,
2014 src_alpha[idx1],
2015 src_alpha[idx2],
2016 LLVMConstVector(shuffles, row_type.length),
2017 "");
2018 }
2019 }
2020 }
2021 }
2022
2023
2024 /**
2025 * Generates the blend function for unswizzled colour buffers
2026 * Also generates the read & write from colour buffer
2027 */
2028 static void
2029 generate_unswizzled_blend(struct gallivm_state *gallivm,
2030 unsigned rt,
2031 struct lp_fragment_shader_variant *variant,
2032 enum pipe_format out_format,
2033 unsigned int num_fs,
2034 struct lp_type fs_type,
2035 LLVMValueRef* fs_mask,
2036 LLVMValueRef fs_out_color[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][4],
2037 LLVMValueRef context_ptr,
2038 LLVMValueRef color_ptr,
2039 LLVMValueRef stride,
2040 unsigned partial_mask,
2041 boolean do_branch)
2042 {
2043 const unsigned alpha_channel = 3;
2044 const unsigned block_width = LP_RASTER_BLOCK_SIZE;
2045 const unsigned block_height = LP_RASTER_BLOCK_SIZE;
2046 const unsigned block_size = block_width * block_height;
2047 const unsigned lp_integer_vector_width = 128;
2048
2049 LLVMBuilderRef builder = gallivm->builder;
2050 LLVMValueRef fs_src[4][TGSI_NUM_CHANNELS];
2051 LLVMValueRef fs_src1[4][TGSI_NUM_CHANNELS];
2052 LLVMValueRef src_alpha[4 * 4];
2053 LLVMValueRef src1_alpha[4 * 4] = { NULL };
2054 LLVMValueRef src_mask[4 * 4];
2055 LLVMValueRef src[4 * 4];
2056 LLVMValueRef src1[4 * 4];
2057 LLVMValueRef dst[4 * 4];
2058 LLVMValueRef blend_color;
2059 LLVMValueRef blend_alpha;
2060 LLVMValueRef i32_zero;
2061 LLVMValueRef check_mask;
2062 LLVMValueRef undef_src_val;
2063
2064 struct lp_build_mask_context mask_ctx;
2065 struct lp_type mask_type;
2066 struct lp_type blend_type;
2067 struct lp_type row_type;
2068 struct lp_type dst_type;
2069 struct lp_type ls_type;
2070
2071 unsigned char swizzle[TGSI_NUM_CHANNELS];
2072 unsigned vector_width;
2073 unsigned src_channels = TGSI_NUM_CHANNELS;
2074 unsigned dst_channels;
2075 unsigned dst_count;
2076 unsigned src_count;
2077 unsigned i, j;
2078
2079 const struct util_format_description* out_format_desc = util_format_description(out_format);
2080
2081 unsigned dst_alignment;
2082
2083 bool pad_inline = is_arithmetic_format(out_format_desc);
2084 bool has_alpha = false;
2085 const boolean dual_source_blend = variant->key.blend.rt[0].blend_enable &&
2086 util_blend_state_is_dual(&variant->key.blend, 0);
2087
2088 const boolean is_1d = variant->key.resource_1d;
2089 boolean twiddle_after_convert = FALSE;
2090 unsigned num_fullblock_fs = is_1d ? 2 * num_fs : num_fs;
2091 LLVMValueRef fpstate = 0;
2092
2093 /* Get type from output format */
2094 lp_blend_type_from_format_desc(out_format_desc, &row_type);
2095 lp_mem_type_from_format_desc(out_format_desc, &dst_type);
2096
2097 /*
2098 * Technically this code should go into lp_build_smallfloat_to_float
2099 * and lp_build_float_to_smallfloat but due to the
2100 * http://llvm.org/bugs/show_bug.cgi?id=6393
2101 * llvm reorders the mxcsr intrinsics in a way that breaks the code.
2102 * So the ordering is important here and there shouldn't be any
2103 * llvm ir instrunctions in this function before
2104 * this, otherwise half-float format conversions won't work
2105 * (again due to llvm bug #6393).
2106 */
2107 if (have_smallfloat_format(dst_type, out_format)) {
2108 /* We need to make sure that denorms are ok for half float
2109 conversions */
2110 fpstate = lp_build_fpstate_get(gallivm);
2111 lp_build_fpstate_set_denorms_zero(gallivm, FALSE);
2112 }
2113
2114 mask_type = lp_int32_vec4_type();
2115 mask_type.length = fs_type.length;
2116
2117 for (i = num_fs; i < num_fullblock_fs; i++) {
2118 fs_mask[i] = lp_build_zero(gallivm, mask_type);
2119 }
2120
2121 /* Do not bother executing code when mask is empty.. */
2122 if (do_branch) {
2123 check_mask = LLVMConstNull(lp_build_int_vec_type(gallivm, mask_type));
2124
2125 for (i = 0; i < num_fullblock_fs; ++i) {
2126 check_mask = LLVMBuildOr(builder, check_mask, fs_mask[i], "");
2127 }
2128
2129 lp_build_mask_begin(&mask_ctx, gallivm, mask_type, check_mask);
2130 lp_build_mask_check(&mask_ctx);
2131 }
2132
2133 partial_mask |= !variant->opaque;
2134 i32_zero = lp_build_const_int32(gallivm, 0);
2135
2136 undef_src_val = lp_build_undef(gallivm, fs_type);
2137
2138 row_type.length = fs_type.length;
2139 vector_width = dst_type.floating ? lp_native_vector_width : lp_integer_vector_width;
2140
2141 /* Compute correct swizzle and count channels */
2142 memset(swizzle, LP_BLD_SWIZZLE_DONTCARE, TGSI_NUM_CHANNELS);
2143 dst_channels = 0;
2144
2145 for (i = 0; i < TGSI_NUM_CHANNELS; ++i) {
2146 /* Ensure channel is used */
2147 if (out_format_desc->swizzle[i] >= TGSI_NUM_CHANNELS) {
2148 continue;
2149 }
2150
2151 /* Ensure not already written to (happens in case with GL_ALPHA) */
2152 if (swizzle[out_format_desc->swizzle[i]] < TGSI_NUM_CHANNELS) {
2153 continue;
2154 }
2155
2156 /* Ensure we havn't already found all channels */
2157 if (dst_channels >= out_format_desc->nr_channels) {
2158 continue;
2159 }
2160
2161 swizzle[out_format_desc->swizzle[i]] = i;
2162 ++dst_channels;
2163
2164 if (i == alpha_channel) {
2165 has_alpha = true;
2166 }
2167 }
2168
2169 if (format_expands_to_float_soa(out_format_desc)) {
2170 /*
2171 * the code above can't work for layout_other
2172 * for srgb it would sort of work but we short-circuit swizzles, etc.
2173 * as that is done as part of unpack / pack.
2174 */
2175 dst_channels = 4; /* HACK: this is fake 4 really but need it due to transpose stuff later */
2176 has_alpha = true;
2177 swizzle[0] = 0;
2178 swizzle[1] = 1;
2179 swizzle[2] = 2;
2180 swizzle[3] = 3;
2181 pad_inline = true; /* HACK: prevent rgbxrgbx->rgbrgbxx conversion later */
2182 }
2183
2184 /* If 3 channels then pad to include alpha for 4 element transpose */
2185 if (dst_channels == 3) {
2186 assert (!has_alpha);
2187 for (i = 0; i < TGSI_NUM_CHANNELS; i++) {
2188 if (swizzle[i] > TGSI_NUM_CHANNELS)
2189 swizzle[i] = 3;
2190 }
2191 if (out_format_desc->nr_channels == 4) {
2192 dst_channels = 4;
2193 /*
2194 * We use alpha from the color conversion, not separate one.
2195 * We had to include it for transpose, hence it will get converted
2196 * too (albeit when doing transpose after conversion, that would
2197 * no longer be the case necessarily).
2198 * (It works only with 4 channel dsts, e.g. rgbx formats, because
2199 * otherwise we really have padding, not alpha, included.)
2200 */
2201 has_alpha = true;
2202 }
2203 }
2204
2205 /*
2206 * Load shader output
2207 */
2208 for (i = 0; i < num_fullblock_fs; ++i) {
2209 /* Always load alpha for use in blending */
2210 LLVMValueRef alpha;
2211 if (i < num_fs) {
2212 alpha = LLVMBuildLoad(builder, fs_out_color[rt][alpha_channel][i], "");
2213 }
2214 else {
2215 alpha = undef_src_val;
2216 }
2217
2218 /* Load each channel */
2219 for (j = 0; j < dst_channels; ++j) {
2220 assert(swizzle[j] < 4);
2221 if (i < num_fs) {
2222 fs_src[i][j] = LLVMBuildLoad(builder, fs_out_color[rt][swizzle[j]][i], "");
2223 }
2224 else {
2225 fs_src[i][j] = undef_src_val;
2226 }
2227 }
2228
2229 /* If 3 channels then pad to include alpha for 4 element transpose */
2230 /*
2231 * XXX If we include that here maybe could actually use it instead of
2232 * separate alpha for blending?
2233 * (Difficult though we actually convert pad channels, not alpha.)
2234 */
2235 if (dst_channels == 3 && !has_alpha) {
2236 fs_src[i][3] = alpha;
2237 }
2238
2239 /* We split the row_mask and row_alpha as we want 128bit interleave */
2240 if (fs_type.length == 8) {
2241 src_mask[i*2 + 0] = lp_build_extract_range(gallivm, fs_mask[i],
2242 0, src_channels);
2243 src_mask[i*2 + 1] = lp_build_extract_range(gallivm, fs_mask[i],
2244 src_channels, src_channels);
2245
2246 src_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2247 src_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2248 src_channels, src_channels);
2249 } else {
2250 src_mask[i] = fs_mask[i];
2251 src_alpha[i] = alpha;
2252 }
2253 }
2254 if (dual_source_blend) {
2255 /* same as above except different src/dst, skip masks and comments... */
2256 for (i = 0; i < num_fullblock_fs; ++i) {
2257 LLVMValueRef alpha;
2258 if (i < num_fs) {
2259 alpha = LLVMBuildLoad(builder, fs_out_color[1][alpha_channel][i], "");
2260 }
2261 else {
2262 alpha = undef_src_val;
2263 }
2264
2265 for (j = 0; j < dst_channels; ++j) {
2266 assert(swizzle[j] < 4);
2267 if (i < num_fs) {
2268 fs_src1[i][j] = LLVMBuildLoad(builder, fs_out_color[1][swizzle[j]][i], "");
2269 }
2270 else {
2271 fs_src1[i][j] = undef_src_val;
2272 }
2273 }
2274 if (dst_channels == 3 && !has_alpha) {
2275 fs_src1[i][3] = alpha;
2276 }
2277 if (fs_type.length == 8) {
2278 src1_alpha[i*2 + 0] = lp_build_extract_range(gallivm, alpha, 0, src_channels);
2279 src1_alpha[i*2 + 1] = lp_build_extract_range(gallivm, alpha,
2280 src_channels, src_channels);
2281 } else {
2282 src1_alpha[i] = alpha;
2283 }
2284 }
2285 }
2286
2287 if (util_format_is_pure_integer(out_format)) {
2288 /*
2289 * In this case fs_type was really ints or uints disguised as floats,
2290 * fix that up now.
2291 */
2292 fs_type.floating = 0;
2293 fs_type.sign = dst_type.sign;
2294 for (i = 0; i < num_fullblock_fs; ++i) {
2295 for (j = 0; j < dst_channels; ++j) {
2296 fs_src[i][j] = LLVMBuildBitCast(builder, fs_src[i][j],
2297 lp_build_vec_type(gallivm, fs_type), "");
2298 }
2299 if (dst_channels == 3 && !has_alpha) {
2300 fs_src[i][3] = LLVMBuildBitCast(builder, fs_src[i][3],
2301 lp_build_vec_type(gallivm, fs_type), "");
2302 }
2303 }
2304 }
2305
2306 /*
2307 * We actually should generally do conversion first (for non-1d cases)
2308 * when the blend format is 8 or 16 bits. The reason is obvious,
2309 * there's 2 or 4 times less vectors to deal with for the interleave...
2310 * Albeit for the AVX (not AVX2) case there's no benefit with 16 bit
2311 * vectors (as it can do 32bit unpack with 256bit vectors, but 8/16bit
2312 * unpack only with 128bit vectors).
2313 * Note: for 16bit sizes really need matching pack conversion code
2314 */
2315 if (!is_1d && dst_channels != 3 && dst_type.width == 8) {
2316 twiddle_after_convert = TRUE;
2317 }
2318
2319 /*
2320 * Pixel twiddle from fragment shader order to memory order
2321 */
2322 if (!twiddle_after_convert) {
2323 src_count = generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs,
2324 dst_channels, fs_src, src, pad_inline);
2325 if (dual_source_blend) {
2326 generate_fs_twiddle(gallivm, fs_type, num_fullblock_fs, dst_channels,
2327 fs_src1, src1, pad_inline);
2328 }
2329 } else {
2330 src_count = num_fullblock_fs * dst_channels;
2331 /*
2332 * We reorder things a bit here, so the cases for 4-wide and 8-wide
2333 * (AVX) turn out the same later when untwiddling/transpose (albeit
2334 * for true AVX2 path untwiddle needs to be different).
2335 * For now just order by colors first (so we can use unpack later).
2336 */
2337 for (j = 0; j < num_fullblock_fs; j++) {
2338 for (i = 0; i < dst_channels; i++) {
2339 src[i*num_fullblock_fs + j] = fs_src[j][i];
2340 if (dual_source_blend) {
2341 src1[i*num_fullblock_fs + j] = fs_src1[j][i];
2342 }
2343 }
2344 }
2345 }
2346
2347 src_channels = dst_channels < 3 ? dst_channels : 4;
2348 if (src_count != num_fullblock_fs * src_channels) {
2349 unsigned ds = src_count / (num_fullblock_fs * src_channels);
2350 row_type.length /= ds;
2351 fs_type.length = row_type.length;
2352 }
2353
2354 blend_type = row_type;
2355 mask_type.length = 4;
2356
2357 /* Convert src to row_type */
2358 if (dual_source_blend) {
2359 struct lp_type old_row_type = row_type;
2360 lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2361 src_count = lp_build_conv_auto(gallivm, fs_type, &old_row_type, src1, src_count, src1);
2362 }
2363 else {
2364 src_count = lp_build_conv_auto(gallivm, fs_type, &row_type, src, src_count, src);
2365 }
2366
2367 /* If the rows are not an SSE vector, combine them to become SSE size! */
2368 if ((row_type.width * row_type.length) % 128) {
2369 unsigned bits = row_type.width * row_type.length;
2370 unsigned combined;
2371
2372 assert(src_count >= (vector_width / bits));
2373
2374 dst_count = src_count / (vector_width / bits);
2375
2376 combined = lp_build_concat_n(gallivm, row_type, src, src_count, src, dst_count);
2377 if (dual_source_blend) {
2378 lp_build_concat_n(gallivm, row_type, src1, src_count, src1, dst_count);
2379 }
2380
2381 row_type.length *= combined;
2382 src_count /= combined;
2383
2384 bits = row_type.width * row_type.length;
2385 assert(bits == 128 || bits == 256);
2386 }
2387
2388 if (twiddle_after_convert) {
2389 fs_twiddle_transpose(gallivm, row_type, src, src_count, src);
2390 if (dual_source_blend) {
2391 fs_twiddle_transpose(gallivm, row_type, src1, src_count, src1);
2392 }
2393 }
2394
2395 /*
2396 * Blend Colour conversion
2397 */
2398 blend_color = lp_jit_context_f_blend_color(gallivm, context_ptr);
2399 blend_color = LLVMBuildPointerCast(builder, blend_color,
2400 LLVMPointerType(lp_build_vec_type(gallivm, fs_type), 0), "");
2401 blend_color = LLVMBuildLoad(builder, LLVMBuildGEP(builder, blend_color,
2402 &i32_zero, 1, ""), "");
2403
2404 /* Convert */
2405 lp_build_conv(gallivm, fs_type, blend_type, &blend_color, 1, &blend_color, 1);
2406
2407 if (out_format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB) {
2408 /*
2409 * since blending is done with floats, there was no conversion.
2410 * However, the rules according to fixed point renderbuffers still
2411 * apply, that is we must clamp inputs to 0.0/1.0.
2412 * (This would apply to separate alpha conversion too but we currently
2413 * force has_alpha to be true.)
2414 * TODO: should skip this with "fake" blend, since post-blend conversion
2415 * will clamp anyway.
2416 * TODO: could also skip this if fragment color clamping is enabled. We
2417 * don't support it natively so it gets baked into the shader however, so
2418 * can't really tell here.
2419 */
2420 struct lp_build_context f32_bld;
2421 assert(row_type.floating);
2422 lp_build_context_init(&f32_bld, gallivm, row_type);
2423 for (i = 0; i < src_count; i++) {
2424 src[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src[i]);
2425 }
2426 if (dual_source_blend) {
2427 for (i = 0; i < src_count; i++) {
2428 src1[i] = lp_build_clamp_zero_one_nanzero(&f32_bld, src1[i]);
2429 }
2430 }
2431 /* probably can't be different than row_type but better safe than sorry... */
2432 lp_build_context_init(&f32_bld, gallivm, blend_type);
2433 blend_color = lp_build_clamp(&f32_bld, blend_color, f32_bld.zero, f32_bld.one);
2434 }
2435
2436 /* Extract alpha */
2437 blend_alpha = lp_build_extract_broadcast(gallivm, blend_type, row_type, blend_color, lp_build_const_int32(gallivm, 3));
2438
2439 /* Swizzle to appropriate channels, e.g. from RGBA to BGRA BGRA */
2440 pad_inline &= (dst_channels * (block_size / src_count) * row_type.width) != vector_width;
2441 if (pad_inline) {
2442 /* Use all 4 channels e.g. from RGBA RGBA to RGxx RGxx */
2443 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, TGSI_NUM_CHANNELS, row_type.length);
2444 } else {
2445 /* Only use dst_channels e.g. RGBA RGBA to RG RG xxxx */
2446 blend_color = lp_build_swizzle_aos_n(gallivm, blend_color, swizzle, dst_channels, row_type.length);
2447 }
2448
2449 /*
2450 * Mask conversion
2451 */
2452 lp_bld_quad_twiddle(gallivm, mask_type, &src_mask[0], block_height, &src_mask[0]);
2453
2454 if (src_count < block_height) {
2455 lp_build_concat_n(gallivm, mask_type, src_mask, 4, src_mask, src_count);
2456 } else if (src_count > block_height) {
2457 for (i = src_count; i > 0; --i) {
2458 unsigned pixels = block_size / src_count;
2459 unsigned idx = i - 1;
2460
2461 src_mask[idx] = lp_build_extract_range(gallivm, src_mask[(idx * pixels) / 4],
2462 (idx * pixels) % 4, pixels);
2463 }
2464 }
2465
2466 assert(mask_type.width == 32);
2467
2468 for (i = 0; i < src_count; ++i) {
2469 unsigned pixels = block_size / src_count;
2470 unsigned pixel_width = row_type.width * dst_channels;
2471
2472 if (pixel_width == 24) {
2473 mask_type.width = 8;
2474 mask_type.length = vector_width / mask_type.width;
2475 } else {
2476 mask_type.length = pixels;
2477 mask_type.width = row_type.width * dst_channels;
2478
2479 /*
2480 * If mask_type width is smaller than 32bit, this doesn't quite
2481 * generate the most efficient code (could use some pack).
2482 */
2483 src_mask[i] = LLVMBuildIntCast(builder, src_mask[i],
2484 lp_build_int_vec_type(gallivm, mask_type), "");
2485
2486 mask_type.length *= dst_channels;
2487 mask_type.width /= dst_channels;
2488 }
2489
2490 src_mask[i] = LLVMBuildBitCast(builder, src_mask[i],
2491 lp_build_int_vec_type(gallivm, mask_type), "");
2492 src_mask[i] = lp_build_pad_vector(gallivm, src_mask[i], row_type.length);
2493 }
2494
2495 /*
2496 * Alpha conversion
2497 */
2498 if (!has_alpha) {
2499 struct lp_type alpha_type = fs_type;
2500 alpha_type.length = 4;
2501 convert_alpha(gallivm, row_type, alpha_type,
2502 block_size, block_height,
2503 src_count, dst_channels,
2504 pad_inline, src_alpha);
2505 if (dual_source_blend) {
2506 convert_alpha(gallivm, row_type, alpha_type,
2507 block_size, block_height,
2508 src_count, dst_channels,
2509 pad_inline, src1_alpha);
2510 }
2511 }
2512
2513
2514 /*
2515 * Load dst from memory
2516 */
2517 if (src_count < block_height) {
2518 dst_count = block_height;
2519 } else {
2520 dst_count = src_count;
2521 }
2522
2523 dst_type.length *= block_size / dst_count;
2524
2525 if (format_expands_to_float_soa(out_format_desc)) {
2526 /*
2527 * we need multiple values at once for the conversion, so can as well
2528 * load them vectorized here too instead of concatenating later.
2529 * (Still need concatenation later for 8-wide vectors).
2530 */
2531 dst_count = block_height;
2532 dst_type.length = block_width;
2533 }
2534
2535 /*
2536 * Compute the alignment of the destination pointer in bytes
2537 * We fetch 1-4 pixels, if the format has pot alignment then those fetches
2538 * are always aligned by MIN2(16, fetch_width) except for buffers (not
2539 * 1d tex but can't distinguish here) so need to stick with per-pixel
2540 * alignment in this case.
2541 */
2542 if (is_1d) {
2543 dst_alignment = (out_format_desc->block.bits + 7)/(out_format_desc->block.width * 8);
2544 }
2545 else {
2546 dst_alignment = dst_type.length * dst_type.width / 8;
2547 }
2548 /* Force power-of-two alignment by extracting only the least-significant-bit */
2549 dst_alignment = 1 << (ffs(dst_alignment) - 1);
2550 /*
2551 * Resource base and stride pointers are aligned to 16 bytes, so that's
2552 * the maximum alignment we can guarantee
2553 */
2554 dst_alignment = MIN2(16, dst_alignment);
2555
2556 ls_type = dst_type;
2557
2558 if (dst_count > src_count) {
2559 if ((dst_type.width == 8 || dst_type.width == 16) &&
2560 util_is_power_of_two_or_zero(dst_type.length) &&
2561 dst_type.length * dst_type.width < 128) {
2562 /*
2563 * Never try to load values as 4xi8 which we will then
2564 * concatenate to larger vectors. This gives llvm a real
2565 * headache (the problem is the type legalizer (?) will
2566 * try to load that as 4xi8 zext to 4xi32 to fill the vector,
2567 * then the shuffles to concatenate are more or less impossible
2568 * - llvm is easily capable of generating a sequence of 32
2569 * pextrb/pinsrb instructions for that. Albeit it appears to
2570 * be fixed in llvm 4.0. So, load and concatenate with 32bit
2571 * width to avoid the trouble (16bit seems not as bad, llvm
2572 * probably recognizes the load+shuffle as only one shuffle
2573 * is necessary, but we can do just the same anyway).
2574 */
2575 ls_type.length = dst_type.length * dst_type.width / 32;
2576 ls_type.width = 32;
2577 }
2578 }
2579
2580 if (is_1d) {
2581 load_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2582 dst, ls_type, dst_count / 4, dst_alignment);
2583 for (i = dst_count / 4; i < dst_count; i++) {
2584 dst[i] = lp_build_undef(gallivm, ls_type);
2585 }
2586
2587 }
2588 else {
2589 load_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2590 dst, ls_type, dst_count, dst_alignment);
2591 }
2592
2593
2594 /*
2595 * Convert from dst/output format to src/blending format.
2596 *
2597 * This is necessary as we can only read 1 row from memory at a time,
2598 * so the minimum dst_count will ever be at this point is 4.
2599 *
2600 * With, for example, R8 format you can have all 16 pixels in a 128 bit vector,
2601 * this will take the 4 dsts and combine them into 1 src so we can perform blending
2602 * on all 16 pixels in that single vector at once.
2603 */
2604 if (dst_count > src_count) {
2605 if (ls_type.length != dst_type.length && ls_type.length == 1) {
2606 LLVMTypeRef elem_type = lp_build_elem_type(gallivm, ls_type);
2607 LLVMTypeRef ls_vec_type = LLVMVectorType(elem_type, 1);
2608 for (i = 0; i < dst_count; i++) {
2609 dst[i] = LLVMBuildBitCast(builder, dst[i], ls_vec_type, "");
2610 }
2611 }
2612
2613 lp_build_concat_n(gallivm, ls_type, dst, 4, dst, src_count);
2614
2615 if (ls_type.length != dst_type.length) {
2616 struct lp_type tmp_type = dst_type;
2617 tmp_type.length = dst_type.length * 4 / src_count;
2618 for (i = 0; i < src_count; i++) {
2619 dst[i] = LLVMBuildBitCast(builder, dst[i],
2620 lp_build_vec_type(gallivm, tmp_type), "");
2621 }
2622 }
2623 }
2624
2625 /*
2626 * Blending
2627 */
2628 /* XXX this is broken for RGB8 formats -
2629 * they get expanded from 12 to 16 elements (to include alpha)
2630 * by convert_to_blend_type then reduced to 15 instead of 12
2631 * by convert_from_blend_type (a simple fix though breaks A8...).
2632 * R16G16B16 also crashes differently however something going wrong
2633 * inside llvm handling npot vector sizes seemingly.
2634 * It seems some cleanup could be done here (like skipping conversion/blend
2635 * when not needed).
2636 */
2637 convert_to_blend_type(gallivm, block_size, out_format_desc, dst_type,
2638 row_type, dst, src_count);
2639
2640 /*
2641 * FIXME: Really should get logic ops / masks out of generic blend / row
2642 * format. Logic ops will definitely not work on the blend float format
2643 * used for SRGB here and I think OpenGL expects this to work as expected
2644 * (that is incoming values converted to srgb then logic op applied).
2645 */
2646 for (i = 0; i < src_count; ++i) {
2647 dst[i] = lp_build_blend_aos(gallivm,
2648 &variant->key.blend,
2649 out_format,
2650 row_type,
2651 rt,
2652 src[i],
2653 has_alpha ? NULL : src_alpha[i],
2654 src1[i],
2655 has_alpha ? NULL : src1_alpha[i],
2656 dst[i],
2657 partial_mask ? src_mask[i] : NULL,
2658 blend_color,
2659 has_alpha ? NULL : blend_alpha,
2660 swizzle,
2661 pad_inline ? 4 : dst_channels);
2662 }
2663
2664 convert_from_blend_type(gallivm, block_size, out_format_desc,
2665 row_type, dst_type, dst, src_count);
2666
2667 /* Split the blend rows back to memory rows */
2668 if (dst_count > src_count) {
2669 row_type.length = dst_type.length * (dst_count / src_count);
2670
2671 if (src_count == 1) {
2672 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2673 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2674
2675 row_type.length /= 2;
2676 src_count *= 2;
2677 }
2678
2679 dst[3] = lp_build_extract_range(gallivm, dst[1], row_type.length / 2, row_type.length / 2);
2680 dst[2] = lp_build_extract_range(gallivm, dst[1], 0, row_type.length / 2);
2681 dst[1] = lp_build_extract_range(gallivm, dst[0], row_type.length / 2, row_type.length / 2);
2682 dst[0] = lp_build_extract_range(gallivm, dst[0], 0, row_type.length / 2);
2683
2684 row_type.length /= 2;
2685 src_count *= 2;
2686 }
2687
2688 /*
2689 * Store blend result to memory
2690 */
2691 if (is_1d) {
2692 store_unswizzled_block(gallivm, color_ptr, stride, block_width, 1,
2693 dst, dst_type, dst_count / 4, dst_alignment);
2694 }
2695 else {
2696 store_unswizzled_block(gallivm, color_ptr, stride, block_width, block_height,
2697 dst, dst_type, dst_count, dst_alignment);
2698 }
2699
2700 if (have_smallfloat_format(dst_type, out_format)) {
2701 lp_build_fpstate_set(gallivm, fpstate);
2702 }
2703
2704 if (do_branch) {
2705 lp_build_mask_end(&mask_ctx);
2706 }
2707 }
2708
2709
2710 /**
2711 * Generate the runtime callable function for the whole fragment pipeline.
2712 * Note that the function which we generate operates on a block of 16
2713 * pixels at at time. The block contains 2x2 quads. Each quad contains
2714 * 2x2 pixels.
2715 */
2716 static void
2717 generate_fragment(struct llvmpipe_context *lp,
2718 struct lp_fragment_shader *shader,
2719 struct lp_fragment_shader_variant *variant,
2720 unsigned partial_mask)
2721 {
2722 struct gallivm_state *gallivm = variant->gallivm;
2723 struct lp_fragment_shader_variant_key *key = &variant->key;
2724 struct lp_shader_input inputs[PIPE_MAX_SHADER_INPUTS];
2725 char func_name[64];
2726 struct lp_type fs_type;
2727 struct lp_type blend_type;
2728 LLVMTypeRef fs_elem_type;
2729 LLVMTypeRef blend_vec_type;
2730 LLVMTypeRef arg_types[15];
2731 LLVMTypeRef func_type;
2732 LLVMTypeRef int32_type = LLVMInt32TypeInContext(gallivm->context);
2733 LLVMTypeRef int8_type = LLVMInt8TypeInContext(gallivm->context);
2734 LLVMValueRef context_ptr;
2735 LLVMValueRef x;
2736 LLVMValueRef y;
2737 LLVMValueRef a0_ptr;
2738 LLVMValueRef dadx_ptr;
2739 LLVMValueRef dady_ptr;
2740 LLVMValueRef color_ptr_ptr;
2741 LLVMValueRef stride_ptr;
2742 LLVMValueRef color_sample_stride_ptr;
2743 LLVMValueRef depth_ptr;
2744 LLVMValueRef depth_stride;
2745 LLVMValueRef depth_sample_stride;
2746 LLVMValueRef mask_input;
2747 LLVMValueRef thread_data_ptr;
2748 LLVMBasicBlockRef block;
2749 LLVMBuilderRef builder;
2750 struct lp_build_sampler_soa *sampler;
2751 struct lp_build_image_soa *image;
2752 struct lp_build_interp_soa_context interp;
2753 LLVMValueRef fs_mask[(16 / 4) * LP_MAX_SAMPLES];
2754 LLVMValueRef fs_out_color[LP_MAX_SAMPLES][PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS][16 / 4];
2755 LLVMValueRef function;
2756 LLVMValueRef facing;
2757 unsigned num_fs;
2758 unsigned i;
2759 unsigned chan;
2760 unsigned cbuf;
2761 boolean cbuf0_write_all;
2762 const boolean dual_source_blend = key->blend.rt[0].blend_enable &&
2763 util_blend_state_is_dual(&key->blend, 0);
2764
2765 assert(lp_native_vector_width / 32 >= 4);
2766
2767 /* Adjust color input interpolation according to flatshade state:
2768 */
2769 memcpy(inputs, shader->inputs, shader->info.base.num_inputs * sizeof inputs[0]);
2770 for (i = 0; i < shader->info.base.num_inputs; i++) {
2771 if (inputs[i].interp == LP_INTERP_COLOR) {
2772 if (key->flatshade)
2773 inputs[i].interp = LP_INTERP_CONSTANT;
2774 else
2775 inputs[i].interp = LP_INTERP_PERSPECTIVE;
2776 }
2777 }
2778
2779 /* check if writes to cbuf[0] are to be copied to all cbufs */
2780 cbuf0_write_all =
2781 shader->info.base.properties[TGSI_PROPERTY_FS_COLOR0_WRITES_ALL_CBUFS];
2782
2783 /* TODO: actually pick these based on the fs and color buffer
2784 * characteristics. */
2785
2786 memset(&fs_type, 0, sizeof fs_type);
2787 fs_type.floating = TRUE; /* floating point values */
2788 fs_type.sign = TRUE; /* values are signed */
2789 fs_type.norm = FALSE; /* values are not limited to [0,1] or [-1,1] */
2790 fs_type.width = 32; /* 32-bit float */
2791 fs_type.length = MIN2(lp_native_vector_width / 32, 16); /* n*4 elements per vector */
2792
2793 memset(&blend_type, 0, sizeof blend_type);
2794 blend_type.floating = FALSE; /* values are integers */
2795 blend_type.sign = FALSE; /* values are unsigned */
2796 blend_type.norm = TRUE; /* values are in [0,1] or [-1,1] */
2797 blend_type.width = 8; /* 8-bit ubyte values */
2798 blend_type.length = 16; /* 16 elements per vector */
2799
2800 /*
2801 * Generate the function prototype. Any change here must be reflected in
2802 * lp_jit.h's lp_jit_frag_func function pointer type, and vice-versa.
2803 */
2804
2805 fs_elem_type = lp_build_elem_type(gallivm, fs_type);
2806
2807 blend_vec_type = lp_build_vec_type(gallivm, blend_type);
2808
2809 snprintf(func_name, sizeof(func_name), "fs%u_variant%u_%s",
2810 shader->no, variant->no, partial_mask ? "partial" : "whole");
2811
2812 arg_types[0] = variant->jit_context_ptr_type; /* context */
2813 arg_types[1] = int32_type; /* x */
2814 arg_types[2] = int32_type; /* y */
2815 arg_types[3] = int32_type; /* facing */
2816 arg_types[4] = LLVMPointerType(fs_elem_type, 0); /* a0 */
2817 arg_types[5] = LLVMPointerType(fs_elem_type, 0); /* dadx */
2818 arg_types[6] = LLVMPointerType(fs_elem_type, 0); /* dady */
2819 arg_types[7] = LLVMPointerType(LLVMPointerType(int8_type, 0), 0); /* color */
2820 arg_types[8] = LLVMPointerType(int8_type, 0); /* depth */
2821 arg_types[9] = LLVMInt64TypeInContext(gallivm->context); /* mask_input */
2822 arg_types[10] = variant->jit_thread_data_ptr_type; /* per thread data */
2823 arg_types[11] = LLVMPointerType(int32_type, 0); /* stride */
2824 arg_types[12] = int32_type; /* depth_stride */
2825 arg_types[13] = LLVMPointerType(int32_type, 0); /* color sample strides */
2826 arg_types[14] = int32_type; /* depth sample stride */
2827
2828 func_type = LLVMFunctionType(LLVMVoidTypeInContext(gallivm->context),
2829 arg_types, ARRAY_SIZE(arg_types), 0);
2830
2831 function = LLVMAddFunction(gallivm->module, func_name, func_type);
2832 LLVMSetFunctionCallConv(function, LLVMCCallConv);
2833
2834 variant->function[partial_mask] = function;
2835
2836 /* XXX: need to propagate noalias down into color param now we are
2837 * passing a pointer-to-pointer?
2838 */
2839 for(i = 0; i < ARRAY_SIZE(arg_types); ++i)
2840 if(LLVMGetTypeKind(arg_types[i]) == LLVMPointerTypeKind)
2841 lp_add_function_attr(function, i + 1, LP_FUNC_ATTR_NOALIAS);
2842
2843 context_ptr = LLVMGetParam(function, 0);
2844 x = LLVMGetParam(function, 1);
2845 y = LLVMGetParam(function, 2);
2846 facing = LLVMGetParam(function, 3);
2847 a0_ptr = LLVMGetParam(function, 4);
2848 dadx_ptr = LLVMGetParam(function, 5);
2849 dady_ptr = LLVMGetParam(function, 6);
2850 color_ptr_ptr = LLVMGetParam(function, 7);
2851 depth_ptr = LLVMGetParam(function, 8);
2852 mask_input = LLVMGetParam(function, 9);
2853 thread_data_ptr = LLVMGetParam(function, 10);
2854 stride_ptr = LLVMGetParam(function, 11);
2855 depth_stride = LLVMGetParam(function, 12);
2856 color_sample_stride_ptr = LLVMGetParam(function, 13);
2857 depth_sample_stride = LLVMGetParam(function, 14);
2858
2859 lp_build_name(context_ptr, "context");
2860 lp_build_name(x, "x");
2861 lp_build_name(y, "y");
2862 lp_build_name(a0_ptr, "a0");
2863 lp_build_name(dadx_ptr, "dadx");
2864 lp_build_name(dady_ptr, "dady");
2865 lp_build_name(color_ptr_ptr, "color_ptr_ptr");
2866 lp_build_name(depth_ptr, "depth");
2867 lp_build_name(mask_input, "mask_input");
2868 lp_build_name(thread_data_ptr, "thread_data");
2869 lp_build_name(stride_ptr, "stride_ptr");
2870 lp_build_name(depth_stride, "depth_stride");
2871 lp_build_name(color_sample_stride_ptr, "color_sample_stride_ptr");
2872 lp_build_name(depth_sample_stride, "depth_sample_stride");
2873
2874 /*
2875 * Function body
2876 */
2877
2878 block = LLVMAppendBasicBlockInContext(gallivm->context, function, "entry");
2879 builder = gallivm->builder;
2880 assert(builder);
2881 LLVMPositionBuilderAtEnd(builder, block);
2882
2883 /*
2884 * Must not count ps invocations if there's a null shader.
2885 * (It would be ok to count with null shader if there's d/s tests,
2886 * but only if there's d/s buffers too, which is different
2887 * to implicit rasterization disable which must not depend
2888 * on the d/s buffers.)
2889 * Could use popcount on mask, but pixel accuracy is not required.
2890 * Could disable if there's no stats query, but maybe not worth it.
2891 */
2892 if (shader->info.base.num_instructions > 1) {
2893 LLVMValueRef invocs, val;
2894 invocs = lp_jit_thread_data_invocations(gallivm, thread_data_ptr);
2895 val = LLVMBuildLoad(builder, invocs, "");
2896 val = LLVMBuildAdd(builder, val,
2897 LLVMConstInt(LLVMInt64TypeInContext(gallivm->context), 1, 0),
2898 "invoc_count");
2899 LLVMBuildStore(builder, val, invocs);
2900 }
2901
2902 /* code generated texture sampling */
2903 sampler = lp_llvm_sampler_soa_create(key->samplers);
2904 image = lp_llvm_image_soa_create(lp_fs_variant_key_images(key));
2905
2906 num_fs = 16 / fs_type.length; /* number of loops per 4x4 stamp */
2907 /* for 1d resources only run "upper half" of stamp */
2908 if (key->resource_1d)
2909 num_fs /= 2;
2910
2911 {
2912 LLVMValueRef num_loop = lp_build_const_int32(gallivm, num_fs);
2913 LLVMTypeRef mask_type = lp_build_int_vec_type(gallivm, fs_type);
2914 LLVMValueRef num_loop_samp = lp_build_const_int32(gallivm, num_fs * key->coverage_samples);
2915 LLVMValueRef mask_store = lp_build_array_alloca(gallivm, mask_type,
2916 num_loop_samp, "mask_store");
2917
2918 LLVMTypeRef flt_type = LLVMFloatTypeInContext(gallivm->context);
2919 LLVMValueRef glob_sample_pos = LLVMAddGlobal(gallivm->module, LLVMArrayType(flt_type, key->coverage_samples * 2), "");
2920 LLVMValueRef sample_pos_array;
2921
2922 if (key->multisample && key->coverage_samples == 4) {
2923 LLVMValueRef sample_pos_arr[8];
2924 for (unsigned i = 0; i < 4; i++) {
2925 sample_pos_arr[i * 2] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][0]);
2926 sample_pos_arr[i * 2 + 1] = LLVMConstReal(flt_type, lp_sample_pos_4x[i][1]);
2927 }
2928 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 8);
2929 } else {
2930 LLVMValueRef sample_pos_arr[2];
2931 sample_pos_arr[0] = LLVMConstReal(flt_type, 0.5);
2932 sample_pos_arr[1] = LLVMConstReal(flt_type, 0.5);
2933 sample_pos_array = LLVMConstArray(LLVMFloatTypeInContext(gallivm->context), sample_pos_arr, 2);
2934 }
2935 LLVMSetInitializer(glob_sample_pos, sample_pos_array);
2936
2937 LLVMValueRef color_store[PIPE_MAX_COLOR_BUFS][TGSI_NUM_CHANNELS];
2938 boolean pixel_center_integer =
2939 shader->info.base.properties[TGSI_PROPERTY_FS_COORD_PIXEL_CENTER];
2940
2941 /*
2942 * The shader input interpolation info is not explicitely baked in the
2943 * shader key, but everything it derives from (TGSI, and flatshade) is
2944 * already included in the shader key.
2945 */
2946 lp_build_interp_soa_init(&interp,
2947 gallivm,
2948 shader->info.base.num_inputs,
2949 inputs,
2950 pixel_center_integer,
2951 key->coverage_samples, glob_sample_pos,
2952 num_loop,
2953 key->depth_clamp,
2954 builder, fs_type,
2955 a0_ptr, dadx_ptr, dady_ptr,
2956 x, y);
2957
2958 for (i = 0; i < num_fs; i++) {
2959 if (key->multisample) {
2960 LLVMValueRef smask_val = LLVMBuildLoad(builder, lp_jit_context_sample_mask(gallivm, context_ptr), "");
2961
2962 /*
2963 * For multisampling, extract the per-sample mask from the incoming 64-bit mask,
2964 * store to the per sample mask storage. Or all of them together to generate
2965 * the fragment shader mask. (sample shading TODO).
2966 * Take the incoming state coverage mask into account.
2967 */
2968 for (unsigned s = 0; s < key->coverage_samples; s++) {
2969 LLVMValueRef sindexi = lp_build_const_int32(gallivm, i + (s * num_fs));
2970 LLVMValueRef sample_mask_ptr = LLVMBuildGEP(builder, mask_store,
2971 &sindexi, 1, "sample_mask_ptr");
2972 LLVMValueRef s_mask = generate_quad_mask(gallivm, fs_type,
2973 i*fs_type.length/4, s, mask_input);
2974
2975 LLVMValueRef smask_bit = LLVMBuildAnd(builder, smask_val, lp_build_const_int32(gallivm, (1 << s)), "");
2976 LLVMValueRef cmp = LLVMBuildICmp(builder, LLVMIntNE, smask_bit, lp_build_const_int32(gallivm, 0), "");
2977 smask_bit = LLVMBuildSExt(builder, cmp, int32_type, "");
2978 smask_bit = lp_build_broadcast(gallivm, mask_type, smask_bit);
2979
2980 s_mask = LLVMBuildAnd(builder, s_mask, smask_bit, "");
2981 LLVMBuildStore(builder, s_mask, sample_mask_ptr);
2982 }
2983 } else {
2984 LLVMValueRef mask;
2985 LLVMValueRef indexi = lp_build_const_int32(gallivm, i);
2986 LLVMValueRef mask_ptr = LLVMBuildGEP(builder, mask_store,
2987 &indexi, 1, "mask_ptr");
2988
2989 if (partial_mask) {
2990 mask = generate_quad_mask(gallivm, fs_type,
2991 i*fs_type.length/4, 0, mask_input);
2992 }
2993 else {
2994 mask = lp_build_const_int_vec(gallivm, fs_type, ~0);
2995 }
2996 LLVMBuildStore(builder, mask, mask_ptr);
2997 }
2998 }
2999
3000 generate_fs_loop(gallivm,
3001 shader, key,
3002 builder,
3003 fs_type,
3004 context_ptr,
3005 glob_sample_pos,
3006 num_loop,
3007 &interp,
3008 sampler,
3009 image,
3010 mask_store, /* output */
3011 color_store,
3012 depth_ptr,
3013 depth_stride,
3014 depth_sample_stride,
3015 facing,
3016 thread_data_ptr);
3017
3018 for (i = 0; i < num_fs; i++) {
3019 LLVMValueRef ptr;
3020 for (unsigned s = 0; s < key->coverage_samples; s++) {
3021 int idx = (i + (s * num_fs));
3022 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3023 ptr = LLVMBuildGEP(builder, mask_store, &sindexi, 1, "");
3024
3025 fs_mask[idx] = LLVMBuildLoad(builder, ptr, "smask");
3026 }
3027
3028 for (unsigned s = 0; s < key->min_samples; s++) {
3029 /* This is fucked up need to reorganize things */
3030 int idx = s * num_fs + i;
3031 LLVMValueRef sindexi = lp_build_const_int32(gallivm, idx);
3032 for (cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3033 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3034 ptr = LLVMBuildGEP(builder,
3035 color_store[cbuf * !cbuf0_write_all][chan],
3036 &sindexi, 1, "");
3037 fs_out_color[s][cbuf][chan][i] = ptr;
3038 }
3039 }
3040 if (dual_source_blend) {
3041 /* only support one dual source blend target hence always use output 1 */
3042 for (chan = 0; chan < TGSI_NUM_CHANNELS; ++chan) {
3043 ptr = LLVMBuildGEP(builder,
3044 color_store[1][chan],
3045 &sindexi, 1, "");
3046 fs_out_color[s][1][chan][i] = ptr;
3047 }
3048 }
3049 }
3050 }
3051 }
3052
3053 sampler->destroy(sampler);
3054 image->destroy(image);
3055 /* Loop over color outputs / color buffers to do blending.
3056 */
3057 for(cbuf = 0; cbuf < key->nr_cbufs; cbuf++) {
3058 if (key->cbuf_format[cbuf] != PIPE_FORMAT_NONE) {
3059 LLVMValueRef color_ptr;
3060 LLVMValueRef stride;
3061 LLVMValueRef sample_stride = NULL;
3062 LLVMValueRef index = lp_build_const_int32(gallivm, cbuf);
3063
3064 boolean do_branch = ((key->depth.enabled
3065 || key->stencil[0].enabled
3066 || key->alpha.enabled)
3067 && !shader->info.base.uses_kill);
3068
3069 color_ptr = LLVMBuildLoad(builder,
3070 LLVMBuildGEP(builder, color_ptr_ptr,
3071 &index, 1, ""),
3072 "");
3073
3074 stride = LLVMBuildLoad(builder,
3075 LLVMBuildGEP(builder, stride_ptr, &index, 1, ""),
3076 "");
3077
3078 if (key->multisample)
3079 sample_stride = LLVMBuildLoad(builder,
3080 LLVMBuildGEP(builder, color_sample_stride_ptr,
3081 &index, 1, ""), "");
3082
3083 for (unsigned s = 0; s < key->cbuf_nr_samples[cbuf]; s++) {
3084 unsigned mask_idx = num_fs * (key->multisample ? s : 0);
3085 unsigned out_idx = key->min_samples == 1 ? 0 : s;
3086 LLVMValueRef out_ptr = color_ptr;;
3087
3088 if (key->multisample) {
3089 LLVMValueRef sample_offset = LLVMBuildMul(builder, sample_stride, lp_build_const_int32(gallivm, s), "");
3090 out_ptr = LLVMBuildGEP(builder, out_ptr, &sample_offset, 1, "");
3091 }
3092 out_ptr = LLVMBuildBitCast(builder, out_ptr, LLVMPointerType(blend_vec_type, 0), "");
3093
3094 lp_build_name(out_ptr, "color_ptr%d", cbuf);
3095
3096 generate_unswizzled_blend(gallivm, cbuf, variant,
3097 key->cbuf_format[cbuf],
3098 num_fs, fs_type, &fs_mask[mask_idx], fs_out_color[out_idx],
3099 context_ptr, out_ptr, stride,
3100 partial_mask, do_branch);
3101 }
3102 }
3103 }
3104
3105 LLVMBuildRetVoid(builder);
3106
3107 gallivm_verify_function(gallivm, function);
3108 }
3109
3110
3111 static void
3112 dump_fs_variant_key(struct lp_fragment_shader_variant_key *key)
3113 {
3114 unsigned i;
3115
3116 debug_printf("fs variant %p:\n", (void *) key);
3117
3118 if (key->flatshade) {
3119 debug_printf("flatshade = 1\n");
3120 }
3121 if (key->multisample) {
3122 debug_printf("multisample = 1\n");
3123 debug_printf("coverage samples = %d\n", key->coverage_samples);
3124 debug_printf("min samples = %d\n", key->min_samples);
3125 }
3126 for (i = 0; i < key->nr_cbufs; ++i) {
3127 debug_printf("cbuf_format[%u] = %s\n", i, util_format_name(key->cbuf_format[i]));
3128 debug_printf("cbuf nr_samples[%u] = %d\n", i, key->cbuf_nr_samples[i]);
3129 }
3130 if (key->depth.enabled || key->stencil[0].enabled) {
3131 debug_printf("depth.format = %s\n", util_format_name(key->zsbuf_format));
3132 debug_printf("depth nr_samples = %d\n", key->zsbuf_nr_samples);
3133 }
3134 if (key->depth.enabled) {
3135 debug_printf("depth.func = %s\n", util_str_func(key->depth.func, TRUE));
3136 debug_printf("depth.writemask = %u\n", key->depth.writemask);
3137 }
3138
3139 for (i = 0; i < 2; ++i) {
3140 if (key->stencil[i].enabled) {
3141 debug_printf("stencil[%u].func = %s\n", i, util_str_func(key->stencil[i].func, TRUE));
3142 debug_printf("stencil[%u].fail_op = %s\n", i, util_str_stencil_op(key->stencil[i].fail_op, TRUE));
3143 debug_printf("stencil[%u].zpass_op = %s\n", i, util_str_stencil_op(key->stencil[i].zpass_op, TRUE));
3144 debug_printf("stencil[%u].zfail_op = %s\n", i, util_str_stencil_op(key->stencil[i].zfail_op, TRUE));
3145 debug_printf("stencil[%u].valuemask = 0x%x\n", i, key->stencil[i].valuemask);
3146 debug_printf("stencil[%u].writemask = 0x%x\n", i, key->stencil[i].writemask);
3147 }
3148 }
3149
3150 if (key->alpha.enabled) {
3151 debug_printf("alpha.func = %s\n", util_str_func(key->alpha.func, TRUE));
3152 }
3153
3154 if (key->occlusion_count) {
3155 debug_printf("occlusion_count = 1\n");
3156 }
3157
3158 if (key->blend.logicop_enable) {
3159 debug_printf("blend.logicop_func = %s\n", util_str_logicop(key->blend.logicop_func, TRUE));
3160 }
3161 else if (key->blend.rt[0].blend_enable) {
3162 debug_printf("blend.rgb_func = %s\n", util_str_blend_func (key->blend.rt[0].rgb_func, TRUE));
3163 debug_printf("blend.rgb_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_src_factor, TRUE));
3164 debug_printf("blend.rgb_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].rgb_dst_factor, TRUE));
3165 debug_printf("blend.alpha_func = %s\n", util_str_blend_func (key->blend.rt[0].alpha_func, TRUE));
3166 debug_printf("blend.alpha_src_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_src_factor, TRUE));
3167 debug_printf("blend.alpha_dst_factor = %s\n", util_str_blend_factor(key->blend.rt[0].alpha_dst_factor, TRUE));
3168 }
3169 debug_printf("blend.colormask = 0x%x\n", key->blend.rt[0].colormask);
3170 if (key->blend.alpha_to_coverage) {
3171 debug_printf("blend.alpha_to_coverage is enabled\n");
3172 }
3173 for (i = 0; i < key->nr_samplers; ++i) {
3174 const struct lp_static_sampler_state *sampler = &key->samplers[i].sampler_state;
3175 debug_printf("sampler[%u] = \n", i);
3176 debug_printf(" .wrap = %s %s %s\n",
3177 util_str_tex_wrap(sampler->wrap_s, TRUE),
3178 util_str_tex_wrap(sampler->wrap_t, TRUE),
3179 util_str_tex_wrap(sampler->wrap_r, TRUE));
3180 debug_printf(" .min_img_filter = %s\n",
3181 util_str_tex_filter(sampler->min_img_filter, TRUE));
3182 debug_printf(" .min_mip_filter = %s\n",
3183 util_str_tex_mipfilter(sampler->min_mip_filter, TRUE));
3184 debug_printf(" .mag_img_filter = %s\n",
3185 util_str_tex_filter(sampler->mag_img_filter, TRUE));
3186 if (sampler->compare_mode != PIPE_TEX_COMPARE_NONE)
3187 debug_printf(" .compare_func = %s\n", util_str_func(sampler->compare_func, TRUE));
3188 debug_printf(" .normalized_coords = %u\n", sampler->normalized_coords);
3189 debug_printf(" .min_max_lod_equal = %u\n", sampler->min_max_lod_equal);
3190 debug_printf(" .lod_bias_non_zero = %u\n", sampler->lod_bias_non_zero);
3191 debug_printf(" .apply_min_lod = %u\n", sampler->apply_min_lod);
3192 debug_printf(" .apply_max_lod = %u\n", sampler->apply_max_lod);
3193 }
3194 for (i = 0; i < key->nr_sampler_views; ++i) {
3195 const struct lp_static_texture_state *texture = &key->samplers[i].texture_state;
3196 debug_printf("texture[%u] = \n", i);
3197 debug_printf(" .format = %s\n",
3198 util_format_name(texture->format));
3199 debug_printf(" .target = %s\n",
3200 util_str_tex_target(texture->target, TRUE));
3201 debug_printf(" .level_zero_only = %u\n",
3202 texture->level_zero_only);
3203 debug_printf(" .pot = %u %u %u\n",
3204 texture->pot_width,
3205 texture->pot_height,
3206 texture->pot_depth);
3207 }
3208 struct lp_image_static_state *images = lp_fs_variant_key_images(key);
3209 for (i = 0; i < key->nr_images; ++i) {
3210 const struct lp_static_texture_state *image = &images[i].image_state;
3211 debug_printf("image[%u] = \n", i);
3212 debug_printf(" .format = %s\n",
3213 util_format_name(image->format));
3214 debug_printf(" .target = %s\n",
3215 util_str_tex_target(image->target, TRUE));
3216 debug_printf(" .level_zero_only = %u\n",
3217 image->level_zero_only);
3218 debug_printf(" .pot = %u %u %u\n",
3219 image->pot_width,
3220 image->pot_height,
3221 image->pot_depth);
3222 }
3223 }
3224
3225
3226 void
3227 lp_debug_fs_variant(struct lp_fragment_shader_variant *variant)
3228 {
3229 debug_printf("llvmpipe: Fragment shader #%u variant #%u:\n",
3230 variant->shader->no, variant->no);
3231 if (variant->shader->base.type == PIPE_SHADER_IR_TGSI)
3232 tgsi_dump(variant->shader->base.tokens, 0);
3233 else
3234 nir_print_shader(variant->shader->base.ir.nir, stderr);
3235 dump_fs_variant_key(&variant->key);
3236 debug_printf("variant->opaque = %u\n", variant->opaque);
3237 debug_printf("\n");
3238 }
3239
3240
3241 /**
3242 * Generate a new fragment shader variant from the shader code and
3243 * other state indicated by the key.
3244 */
3245 static struct lp_fragment_shader_variant *
3246 generate_variant(struct llvmpipe_context *lp,
3247 struct lp_fragment_shader *shader,
3248 const struct lp_fragment_shader_variant_key *key)
3249 {
3250 struct lp_fragment_shader_variant *variant;
3251 const struct util_format_description *cbuf0_format_desc = NULL;
3252 boolean fullcolormask;
3253 char module_name[64];
3254
3255 variant = MALLOC(sizeof *variant + shader->variant_key_size - sizeof variant->key);
3256 if (!variant)
3257 return NULL;
3258
3259 memset(variant, 0, sizeof(*variant));
3260 snprintf(module_name, sizeof(module_name), "fs%u_variant%u",
3261 shader->no, shader->variants_created);
3262
3263 variant->gallivm = gallivm_create(module_name, lp->context);
3264 if (!variant->gallivm) {
3265 FREE(variant);
3266 return NULL;
3267 }
3268
3269 variant->shader = shader;
3270 variant->list_item_global.base = variant;
3271 variant->list_item_local.base = variant;
3272 variant->no = shader->variants_created++;
3273
3274 memcpy(&variant->key, key, shader->variant_key_size);
3275
3276 /*
3277 * Determine whether we are touching all channels in the color buffer.
3278 */
3279 fullcolormask = FALSE;
3280 if (key->nr_cbufs == 1) {
3281 cbuf0_format_desc = util_format_description(key->cbuf_format[0]);
3282 fullcolormask = util_format_colormask_full(cbuf0_format_desc, key->blend.rt[0].colormask);
3283 }
3284
3285 variant->opaque =
3286 !key->blend.logicop_enable &&
3287 !key->blend.rt[0].blend_enable &&
3288 fullcolormask &&
3289 !key->stencil[0].enabled &&
3290 !key->alpha.enabled &&
3291 !key->multisample &&
3292 !key->blend.alpha_to_coverage &&
3293 !key->depth.enabled &&
3294 !shader->info.base.uses_kill &&
3295 !shader->info.base.writes_samplemask
3296 ? TRUE : FALSE;
3297
3298 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3299 lp_debug_fs_variant(variant);
3300 }
3301
3302 lp_jit_init_types(variant);
3303
3304 if (variant->jit_function[RAST_EDGE_TEST] == NULL)
3305 generate_fragment(lp, shader, variant, RAST_EDGE_TEST);
3306
3307 if (variant->jit_function[RAST_WHOLE] == NULL) {
3308 if (variant->opaque) {
3309 /* Specialized shader, which doesn't need to read the color buffer. */
3310 generate_fragment(lp, shader, variant, RAST_WHOLE);
3311 }
3312 }
3313
3314 /*
3315 * Compile everything
3316 */
3317
3318 gallivm_compile_module(variant->gallivm);
3319
3320 variant->nr_instrs += lp_build_count_ir_module(variant->gallivm->module);
3321
3322 if (variant->function[RAST_EDGE_TEST]) {
3323 variant->jit_function[RAST_EDGE_TEST] = (lp_jit_frag_func)
3324 gallivm_jit_function(variant->gallivm,
3325 variant->function[RAST_EDGE_TEST]);
3326 }
3327
3328 if (variant->function[RAST_WHOLE]) {
3329 variant->jit_function[RAST_WHOLE] = (lp_jit_frag_func)
3330 gallivm_jit_function(variant->gallivm,
3331 variant->function[RAST_WHOLE]);
3332 } else if (!variant->jit_function[RAST_WHOLE]) {
3333 variant->jit_function[RAST_WHOLE] = variant->jit_function[RAST_EDGE_TEST];
3334 }
3335
3336 gallivm_free_ir(variant->gallivm);
3337
3338 return variant;
3339 }
3340
3341
3342 static void *
3343 llvmpipe_create_fs_state(struct pipe_context *pipe,
3344 const struct pipe_shader_state *templ)
3345 {
3346 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3347 struct lp_fragment_shader *shader;
3348 int nr_samplers;
3349 int nr_sampler_views;
3350 int nr_images;
3351 int i;
3352
3353 shader = CALLOC_STRUCT(lp_fragment_shader);
3354 if (!shader)
3355 return NULL;
3356
3357 shader->no = fs_no++;
3358 make_empty_list(&shader->variants);
3359
3360 shader->base.type = templ->type;
3361 if (templ->type == PIPE_SHADER_IR_TGSI) {
3362 /* get/save the summary info for this shader */
3363 lp_build_tgsi_info(templ->tokens, &shader->info);
3364
3365 /* we need to keep a local copy of the tokens */
3366 shader->base.tokens = tgsi_dup_tokens(templ->tokens);
3367 } else {
3368 shader->base.ir.nir = templ->ir.nir;
3369 nir_tgsi_scan_shader(templ->ir.nir, &shader->info.base, true);
3370 }
3371
3372 shader->draw_data = draw_create_fragment_shader(llvmpipe->draw, templ);
3373 if (shader->draw_data == NULL) {
3374 FREE((void *) shader->base.tokens);
3375 FREE(shader);
3376 return NULL;
3377 }
3378
3379 nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
3380 nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
3381 nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
3382 shader->variant_key_size = lp_fs_variant_key_size(MAX2(nr_samplers, nr_sampler_views), nr_images);
3383
3384 for (i = 0; i < shader->info.base.num_inputs; i++) {
3385 shader->inputs[i].usage_mask = shader->info.base.input_usage_mask[i];
3386 shader->inputs[i].cyl_wrap = shader->info.base.input_cylindrical_wrap[i];
3387 shader->inputs[i].location = shader->info.base.input_interpolate_loc[i];
3388
3389 switch (shader->info.base.input_interpolate[i]) {
3390 case TGSI_INTERPOLATE_CONSTANT:
3391 shader->inputs[i].interp = LP_INTERP_CONSTANT;
3392 break;
3393 case TGSI_INTERPOLATE_LINEAR:
3394 shader->inputs[i].interp = LP_INTERP_LINEAR;
3395 break;
3396 case TGSI_INTERPOLATE_PERSPECTIVE:
3397 shader->inputs[i].interp = LP_INTERP_PERSPECTIVE;
3398 break;
3399 case TGSI_INTERPOLATE_COLOR:
3400 shader->inputs[i].interp = LP_INTERP_COLOR;
3401 break;
3402 default:
3403 assert(0);
3404 break;
3405 }
3406
3407 switch (shader->info.base.input_semantic_name[i]) {
3408 case TGSI_SEMANTIC_FACE:
3409 shader->inputs[i].interp = LP_INTERP_FACING;
3410 break;
3411 case TGSI_SEMANTIC_POSITION:
3412 /* Position was already emitted above
3413 */
3414 shader->inputs[i].interp = LP_INTERP_POSITION;
3415 shader->inputs[i].src_index = 0;
3416 continue;
3417 }
3418
3419 /* XXX this is a completely pointless index map... */
3420 shader->inputs[i].src_index = i+1;
3421 }
3422
3423 if (LP_DEBUG & DEBUG_TGSI) {
3424 unsigned attrib;
3425 debug_printf("llvmpipe: Create fragment shader #%u %p:\n",
3426 shader->no, (void *) shader);
3427 tgsi_dump(templ->tokens, 0);
3428 debug_printf("usage masks:\n");
3429 for (attrib = 0; attrib < shader->info.base.num_inputs; ++attrib) {
3430 unsigned usage_mask = shader->info.base.input_usage_mask[attrib];
3431 debug_printf(" IN[%u].%s%s%s%s\n",
3432 attrib,
3433 usage_mask & TGSI_WRITEMASK_X ? "x" : "",
3434 usage_mask & TGSI_WRITEMASK_Y ? "y" : "",
3435 usage_mask & TGSI_WRITEMASK_Z ? "z" : "",
3436 usage_mask & TGSI_WRITEMASK_W ? "w" : "");
3437 }
3438 debug_printf("\n");
3439 }
3440
3441 return shader;
3442 }
3443
3444
3445 static void
3446 llvmpipe_bind_fs_state(struct pipe_context *pipe, void *fs)
3447 {
3448 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3449 struct lp_fragment_shader *lp_fs = (struct lp_fragment_shader *)fs;
3450 if (llvmpipe->fs == lp_fs)
3451 return;
3452
3453 draw_bind_fragment_shader(llvmpipe->draw,
3454 (lp_fs ? lp_fs->draw_data : NULL));
3455
3456 llvmpipe->fs = lp_fs;
3457
3458 llvmpipe->dirty |= LP_NEW_FS;
3459 }
3460
3461
3462 /**
3463 * Remove shader variant from two lists: the shader's variant list
3464 * and the context's variant list.
3465 */
3466 static void
3467 llvmpipe_remove_shader_variant(struct llvmpipe_context *lp,
3468 struct lp_fragment_shader_variant *variant)
3469 {
3470 if ((LP_DEBUG & DEBUG_FS) || (gallivm_debug & GALLIVM_DEBUG_IR)) {
3471 debug_printf("llvmpipe: del fs #%u var %u v created %u v cached %u "
3472 "v total cached %u inst %u total inst %u\n",
3473 variant->shader->no, variant->no,
3474 variant->shader->variants_created,
3475 variant->shader->variants_cached,
3476 lp->nr_fs_variants, variant->nr_instrs, lp->nr_fs_instrs);
3477 }
3478
3479 gallivm_destroy(variant->gallivm);
3480
3481 /* remove from shader's list */
3482 remove_from_list(&variant->list_item_local);
3483 variant->shader->variants_cached--;
3484
3485 /* remove from context's list */
3486 remove_from_list(&variant->list_item_global);
3487 lp->nr_fs_variants--;
3488 lp->nr_fs_instrs -= variant->nr_instrs;
3489
3490 FREE(variant);
3491 }
3492
3493
3494 static void
3495 llvmpipe_delete_fs_state(struct pipe_context *pipe, void *fs)
3496 {
3497 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3498 struct lp_fragment_shader *shader = fs;
3499 struct lp_fs_variant_list_item *li;
3500
3501 assert(fs != llvmpipe->fs);
3502
3503 /*
3504 * XXX: we need to flush the context until we have some sort of reference
3505 * counting in fragment shaders as they may still be binned
3506 * Flushing alone might not sufficient we need to wait on it too.
3507 */
3508 llvmpipe_finish(pipe, __FUNCTION__);
3509
3510 /* Delete all the variants */
3511 li = first_elem(&shader->variants);
3512 while(!at_end(&shader->variants, li)) {
3513 struct lp_fs_variant_list_item *next = next_elem(li);
3514 llvmpipe_remove_shader_variant(llvmpipe, li->base);
3515 li = next;
3516 }
3517
3518 /* Delete draw module's data */
3519 draw_delete_fragment_shader(llvmpipe->draw, shader->draw_data);
3520
3521 if (shader->base.ir.nir)
3522 ralloc_free(shader->base.ir.nir);
3523 assert(shader->variants_cached == 0);
3524 FREE((void *) shader->base.tokens);
3525 FREE(shader);
3526 }
3527
3528
3529
3530 static void
3531 llvmpipe_set_constant_buffer(struct pipe_context *pipe,
3532 enum pipe_shader_type shader, uint index,
3533 const struct pipe_constant_buffer *cb)
3534 {
3535 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3536 struct pipe_resource *constants = cb ? cb->buffer : NULL;
3537
3538 assert(shader < PIPE_SHADER_TYPES);
3539 assert(index < ARRAY_SIZE(llvmpipe->constants[shader]));
3540
3541 /* note: reference counting */
3542 util_copy_constant_buffer(&llvmpipe->constants[shader][index], cb);
3543
3544 if (constants) {
3545 if (!(constants->bind & PIPE_BIND_CONSTANT_BUFFER)) {
3546 debug_printf("Illegal set constant without bind flag\n");
3547 constants->bind |= PIPE_BIND_CONSTANT_BUFFER;
3548 }
3549 }
3550
3551 if (shader == PIPE_SHADER_VERTEX ||
3552 shader == PIPE_SHADER_GEOMETRY ||
3553 shader == PIPE_SHADER_TESS_CTRL ||
3554 shader == PIPE_SHADER_TESS_EVAL) {
3555 /* Pass the constants to the 'draw' module */
3556 const unsigned size = cb ? cb->buffer_size : 0;
3557 const ubyte *data;
3558
3559 if (constants) {
3560 data = (ubyte *) llvmpipe_resource_data(constants);
3561 }
3562 else if (cb && cb->user_buffer) {
3563 data = (ubyte *) cb->user_buffer;
3564 }
3565 else {
3566 data = NULL;
3567 }
3568
3569 if (data)
3570 data += cb->buffer_offset;
3571
3572 draw_set_mapped_constant_buffer(llvmpipe->draw, shader,
3573 index, data, size);
3574 }
3575 else if (shader == PIPE_SHADER_COMPUTE)
3576 llvmpipe->cs_dirty |= LP_CSNEW_CONSTANTS;
3577 else
3578 llvmpipe->dirty |= LP_NEW_FS_CONSTANTS;
3579
3580 if (cb && cb->user_buffer) {
3581 pipe_resource_reference(&constants, NULL);
3582 }
3583 }
3584
3585 static void
3586 llvmpipe_set_shader_buffers(struct pipe_context *pipe,
3587 enum pipe_shader_type shader, unsigned start_slot,
3588 unsigned count, const struct pipe_shader_buffer *buffers,
3589 unsigned writable_bitmask)
3590 {
3591 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3592 unsigned i, idx;
3593 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
3594 const struct pipe_shader_buffer *buffer = buffers ? &buffers[idx] : NULL;
3595
3596 util_copy_shader_buffer(&llvmpipe->ssbos[shader][i], buffer);
3597
3598 if (shader == PIPE_SHADER_VERTEX ||
3599 shader == PIPE_SHADER_GEOMETRY ||
3600 shader == PIPE_SHADER_TESS_CTRL ||
3601 shader == PIPE_SHADER_TESS_EVAL) {
3602 const unsigned size = buffer ? buffer->buffer_size : 0;
3603 const ubyte *data = NULL;
3604 if (buffer && buffer->buffer)
3605 data = (ubyte *) llvmpipe_resource_data(buffer->buffer);
3606 if (data)
3607 data += buffer->buffer_offset;
3608 draw_set_mapped_shader_buffer(llvmpipe->draw, shader,
3609 i, data, size);
3610 } else if (shader == PIPE_SHADER_COMPUTE) {
3611 llvmpipe->cs_dirty |= LP_CSNEW_SSBOS;
3612 } else if (shader == PIPE_SHADER_FRAGMENT) {
3613 llvmpipe->dirty |= LP_NEW_FS_SSBOS;
3614 }
3615 }
3616 }
3617
3618 static void
3619 llvmpipe_set_shader_images(struct pipe_context *pipe,
3620 enum pipe_shader_type shader, unsigned start_slot,
3621 unsigned count, const struct pipe_image_view *images)
3622 {
3623 struct llvmpipe_context *llvmpipe = llvmpipe_context(pipe);
3624 unsigned i, idx;
3625
3626 draw_flush(llvmpipe->draw);
3627 for (i = start_slot, idx = 0; i < start_slot + count; i++, idx++) {
3628 const struct pipe_image_view *image = images ? &images[idx] : NULL;
3629
3630 util_copy_image_view(&llvmpipe->images[shader][i], image);
3631 }
3632
3633 llvmpipe->num_images[shader] = start_slot + count;
3634 if (shader == PIPE_SHADER_VERTEX ||
3635 shader == PIPE_SHADER_GEOMETRY ||
3636 shader == PIPE_SHADER_TESS_CTRL ||
3637 shader == PIPE_SHADER_TESS_EVAL) {
3638 draw_set_images(llvmpipe->draw,
3639 shader,
3640 llvmpipe->images[shader],
3641 start_slot + count);
3642 } else if (shader == PIPE_SHADER_COMPUTE)
3643 llvmpipe->cs_dirty |= LP_CSNEW_IMAGES;
3644 else
3645 llvmpipe->dirty |= LP_NEW_FS_IMAGES;
3646 }
3647
3648 /**
3649 * Return the blend factor equivalent to a destination alpha of one.
3650 */
3651 static inline unsigned
3652 force_dst_alpha_one(unsigned factor, boolean clamped_zero)
3653 {
3654 switch(factor) {
3655 case PIPE_BLENDFACTOR_DST_ALPHA:
3656 return PIPE_BLENDFACTOR_ONE;
3657 case PIPE_BLENDFACTOR_INV_DST_ALPHA:
3658 return PIPE_BLENDFACTOR_ZERO;
3659 case PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE:
3660 if (clamped_zero)
3661 return PIPE_BLENDFACTOR_ZERO;
3662 else
3663 return PIPE_BLENDFACTOR_SRC_ALPHA_SATURATE;
3664 }
3665
3666 return factor;
3667 }
3668
3669
3670 /**
3671 * We need to generate several variants of the fragment pipeline to match
3672 * all the combinations of the contributing state atoms.
3673 *
3674 * TODO: there is actually no reason to tie this to context state -- the
3675 * generated code could be cached globally in the screen.
3676 */
3677 static struct lp_fragment_shader_variant_key *
3678 make_variant_key(struct llvmpipe_context *lp,
3679 struct lp_fragment_shader *shader,
3680 char *store)
3681 {
3682 unsigned i;
3683 struct lp_fragment_shader_variant_key *key;
3684
3685 key = (struct lp_fragment_shader_variant_key *)store;
3686
3687 memset(key, 0, offsetof(struct lp_fragment_shader_variant_key, samplers[1]));
3688
3689 if (lp->framebuffer.zsbuf) {
3690 enum pipe_format zsbuf_format = lp->framebuffer.zsbuf->format;
3691 const struct util_format_description *zsbuf_desc =
3692 util_format_description(zsbuf_format);
3693
3694 if (lp->depth_stencil->depth.enabled &&
3695 util_format_has_depth(zsbuf_desc)) {
3696 key->zsbuf_format = zsbuf_format;
3697 memcpy(&key->depth, &lp->depth_stencil->depth, sizeof key->depth);
3698 }
3699 if (lp->depth_stencil->stencil[0].enabled &&
3700 util_format_has_stencil(zsbuf_desc)) {
3701 key->zsbuf_format = zsbuf_format;
3702 memcpy(&key->stencil, &lp->depth_stencil->stencil, sizeof key->stencil);
3703 }
3704 if (llvmpipe_resource_is_1d(lp->framebuffer.zsbuf->texture)) {
3705 key->resource_1d = TRUE;
3706 }
3707 key->zsbuf_nr_samples = util_res_sample_count(lp->framebuffer.zsbuf->texture);
3708 }
3709
3710 /*
3711 * Propagate the depth clamp setting from the rasterizer state.
3712 * depth_clip == 0 implies depth clamping is enabled.
3713 *
3714 * When clip_halfz is enabled, then always clamp the depth values.
3715 *
3716 * XXX: This is incorrect for GL, but correct for d3d10 (depth
3717 * clamp is always active in d3d10, regardless if depth clip is
3718 * enabled or not).
3719 * (GL has an always-on [0,1] clamp on fs depth output instead
3720 * to ensure the depth values stay in range. Doesn't look like
3721 * we do that, though...)
3722 */
3723 if (lp->rasterizer->clip_halfz) {
3724 key->depth_clamp = 1;
3725 } else {
3726 key->depth_clamp = (lp->rasterizer->depth_clip_near == 0) ? 1 : 0;
3727 }
3728
3729 /* alpha test only applies if render buffer 0 is non-integer (or does not exist) */
3730 if (!lp->framebuffer.nr_cbufs ||
3731 !lp->framebuffer.cbufs[0] ||
3732 !util_format_is_pure_integer(lp->framebuffer.cbufs[0]->format)) {
3733 key->alpha.enabled = lp->depth_stencil->alpha.enabled;
3734 }
3735 if(key->alpha.enabled)
3736 key->alpha.func = lp->depth_stencil->alpha.func;
3737 /* alpha.ref_value is passed in jit_context */
3738
3739 key->flatshade = lp->rasterizer->flatshade;
3740 key->multisample = lp->rasterizer->multisample;
3741 if (lp->active_occlusion_queries && !lp->queries_disabled) {
3742 key->occlusion_count = TRUE;
3743 }
3744
3745 if (lp->framebuffer.nr_cbufs) {
3746 memcpy(&key->blend, lp->blend, sizeof key->blend);
3747 }
3748
3749 key->coverage_samples = 1;
3750 key->min_samples = 1;
3751 if (key->multisample) {
3752 key->coverage_samples = util_framebuffer_get_num_samples(&lp->framebuffer);
3753 key->min_samples = lp->min_samples == 1 ? 1 : key->coverage_samples;
3754 }
3755 key->nr_cbufs = lp->framebuffer.nr_cbufs;
3756
3757 if (!key->blend.independent_blend_enable) {
3758 /* we always need independent blend otherwise the fixups below won't work */
3759 for (i = 1; i < key->nr_cbufs; i++) {
3760 memcpy(&key->blend.rt[i], &key->blend.rt[0], sizeof(key->blend.rt[0]));
3761 }
3762 key->blend.independent_blend_enable = 1;
3763 }
3764
3765 for (i = 0; i < lp->framebuffer.nr_cbufs; i++) {
3766 struct pipe_rt_blend_state *blend_rt = &key->blend.rt[i];
3767
3768 if (lp->framebuffer.cbufs[i]) {
3769 enum pipe_format format = lp->framebuffer.cbufs[i]->format;
3770 const struct util_format_description *format_desc;
3771
3772 key->cbuf_format[i] = format;
3773 key->cbuf_nr_samples[i] = util_res_sample_count(lp->framebuffer.cbufs[i]->texture);
3774
3775 /*
3776 * Figure out if this is a 1d resource. Note that OpenGL allows crazy
3777 * mixing of 2d textures with height 1 and 1d textures, so make sure
3778 * we pick 1d if any cbuf or zsbuf is 1d.
3779 */
3780 if (llvmpipe_resource_is_1d(lp->framebuffer.cbufs[i]->texture)) {
3781 key->resource_1d = TRUE;
3782 }
3783
3784 format_desc = util_format_description(format);
3785 assert(format_desc->colorspace == UTIL_FORMAT_COLORSPACE_RGB ||
3786 format_desc->colorspace == UTIL_FORMAT_COLORSPACE_SRGB);
3787
3788 /*
3789 * Mask out color channels not present in the color buffer.
3790 */
3791 blend_rt->colormask &= util_format_colormask(format_desc);
3792
3793 /*
3794 * Disable blend for integer formats.
3795 */
3796 if (util_format_is_pure_integer(format)) {
3797 blend_rt->blend_enable = 0;
3798 }
3799
3800 /*
3801 * Our swizzled render tiles always have an alpha channel, but the
3802 * linear render target format often does not, so force here the dst
3803 * alpha to be one.
3804 *
3805 * This is not a mere optimization. Wrong results will be produced if
3806 * the dst alpha is used, the dst format does not have alpha, and the
3807 * previous rendering was not flushed from the swizzled to linear
3808 * buffer. For example, NonPowTwo DCT.
3809 *
3810 * TODO: This should be generalized to all channels for better
3811 * performance, but only alpha causes correctness issues.
3812 *
3813 * Also, force rgb/alpha func/factors match, to make AoS blending
3814 * easier.
3815 */
3816 if (format_desc->swizzle[3] > PIPE_SWIZZLE_W ||
3817 format_desc->swizzle[3] == format_desc->swizzle[0]) {
3818 /* Doesn't cover mixed snorm/unorm but can't render to them anyway */
3819 boolean clamped_zero = !util_format_is_float(format) &&
3820 !util_format_is_snorm(format);
3821 blend_rt->rgb_src_factor =
3822 force_dst_alpha_one(blend_rt->rgb_src_factor, clamped_zero);
3823 blend_rt->rgb_dst_factor =
3824 force_dst_alpha_one(blend_rt->rgb_dst_factor, clamped_zero);
3825 blend_rt->alpha_func = blend_rt->rgb_func;
3826 blend_rt->alpha_src_factor = blend_rt->rgb_src_factor;
3827 blend_rt->alpha_dst_factor = blend_rt->rgb_dst_factor;
3828 }
3829 }
3830 else {
3831 /* no color buffer for this fragment output */
3832 key->cbuf_format[i] = PIPE_FORMAT_NONE;
3833 key->cbuf_nr_samples[i] = 0;
3834 blend_rt->colormask = 0x0;
3835 blend_rt->blend_enable = 0;
3836 }
3837 }
3838
3839 /* This value will be the same for all the variants of a given shader:
3840 */
3841 key->nr_samplers = shader->info.base.file_max[TGSI_FILE_SAMPLER] + 1;
3842
3843 struct lp_sampler_static_state *fs_sampler;
3844
3845 fs_sampler = key->samplers;
3846
3847 memset(fs_sampler, 0, MAX2(key->nr_samplers, key->nr_sampler_views) * sizeof *fs_sampler);
3848
3849 for(i = 0; i < key->nr_samplers; ++i) {
3850 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
3851 lp_sampler_static_sampler_state(&fs_sampler[i].sampler_state,
3852 lp->samplers[PIPE_SHADER_FRAGMENT][i]);
3853 }
3854 }
3855
3856 /*
3857 * XXX If TGSI_FILE_SAMPLER_VIEW exists assume all texture opcodes
3858 * are dx10-style? Can't really have mixed opcodes, at least not
3859 * if we want to skip the holes here (without rescanning tgsi).
3860 */
3861 if (shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] != -1) {
3862 key->nr_sampler_views = shader->info.base.file_max[TGSI_FILE_SAMPLER_VIEW] + 1;
3863 for(i = 0; i < key->nr_sampler_views; ++i) {
3864 /*
3865 * Note sview may exceed what's representable by file_mask.
3866 * This will still work, the only downside is that not actually
3867 * used views may be included in the shader key.
3868 */
3869 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER_VIEW] & (1u << (i & 31))) {
3870 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
3871 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
3872 }
3873 }
3874 }
3875 else {
3876 key->nr_sampler_views = key->nr_samplers;
3877 for(i = 0; i < key->nr_sampler_views; ++i) {
3878 if(shader->info.base.file_mask[TGSI_FILE_SAMPLER] & (1 << i)) {
3879 lp_sampler_static_texture_state(&fs_sampler[i].texture_state,
3880 lp->sampler_views[PIPE_SHADER_FRAGMENT][i]);
3881 }
3882 }
3883 }
3884
3885 struct lp_image_static_state *lp_image;
3886 lp_image = lp_fs_variant_key_images(key);
3887 key->nr_images = shader->info.base.file_max[TGSI_FILE_IMAGE] + 1;
3888 for (i = 0; i < key->nr_images; ++i) {
3889 if (shader->info.base.file_mask[TGSI_FILE_IMAGE] & (1 << i)) {
3890 lp_sampler_static_texture_state_image(&lp_image[i].image_state,
3891 &lp->images[PIPE_SHADER_FRAGMENT][i]);
3892 }
3893 }
3894 return key;
3895 }
3896
3897
3898
3899 /**
3900 * Update fragment shader state. This is called just prior to drawing
3901 * something when some fragment-related state has changed.
3902 */
3903 void
3904 llvmpipe_update_fs(struct llvmpipe_context *lp)
3905 {
3906 struct lp_fragment_shader *shader = lp->fs;
3907 struct lp_fragment_shader_variant_key *key;
3908 struct lp_fragment_shader_variant *variant = NULL;
3909 struct lp_fs_variant_list_item *li;
3910 char store[LP_FS_MAX_VARIANT_KEY_SIZE];
3911
3912 key = make_variant_key(lp, shader, store);
3913
3914 /* Search the variants for one which matches the key */
3915 li = first_elem(&shader->variants);
3916 while(!at_end(&shader->variants, li)) {
3917 if(memcmp(&li->base->key, key, shader->variant_key_size) == 0) {
3918 variant = li->base;
3919 break;
3920 }
3921 li = next_elem(li);
3922 }
3923
3924 if (variant) {
3925 /* Move this variant to the head of the list to implement LRU
3926 * deletion of shader's when we have too many.
3927 */
3928 move_to_head(&lp->fs_variants_list, &variant->list_item_global);
3929 }
3930 else {
3931 /* variant not found, create it now */
3932 int64_t t0, t1, dt;
3933 unsigned i;
3934 unsigned variants_to_cull;
3935
3936 if (LP_DEBUG & DEBUG_FS) {
3937 debug_printf("%u variants,\t%u instrs,\t%u instrs/variant\n",
3938 lp->nr_fs_variants,
3939 lp->nr_fs_instrs,
3940 lp->nr_fs_variants ? lp->nr_fs_instrs / lp->nr_fs_variants : 0);
3941 }
3942
3943 /* First, check if we've exceeded the max number of shader variants.
3944 * If so, free 6.25% of them (the least recently used ones).
3945 */
3946 variants_to_cull = lp->nr_fs_variants >= LP_MAX_SHADER_VARIANTS ? LP_MAX_SHADER_VARIANTS / 16 : 0;
3947
3948 if (variants_to_cull ||
3949 lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS) {
3950 struct pipe_context *pipe = &lp->pipe;
3951
3952 if (gallivm_debug & GALLIVM_DEBUG_PERF) {
3953 debug_printf("Evicting FS: %u fs variants,\t%u total variants,"
3954 "\t%u instrs,\t%u instrs/variant\n",
3955 shader->variants_cached,
3956 lp->nr_fs_variants, lp->nr_fs_instrs,
3957 lp->nr_fs_instrs / lp->nr_fs_variants);
3958 }
3959
3960 /*
3961 * XXX: we need to flush the context until we have some sort of
3962 * reference counting in fragment shaders as they may still be binned
3963 * Flushing alone might not be sufficient we need to wait on it too.
3964 */
3965 llvmpipe_finish(pipe, __FUNCTION__);
3966
3967 /*
3968 * We need to re-check lp->nr_fs_variants because an arbitrarliy large
3969 * number of shader variants (potentially all of them) could be
3970 * pending for destruction on flush.
3971 */
3972
3973 for (i = 0; i < variants_to_cull || lp->nr_fs_instrs >= LP_MAX_SHADER_INSTRUCTIONS; i++) {
3974 struct lp_fs_variant_list_item *item;
3975 if (is_empty_list(&lp->fs_variants_list)) {
3976 break;
3977 }
3978 item = last_elem(&lp->fs_variants_list);
3979 assert(item);
3980 assert(item->base);
3981 llvmpipe_remove_shader_variant(lp, item->base);
3982 }
3983 }
3984
3985 /*
3986 * Generate the new variant.
3987 */
3988 t0 = os_time_get();
3989 variant = generate_variant(lp, shader, key);
3990 t1 = os_time_get();
3991 dt = t1 - t0;
3992 LP_COUNT_ADD(llvm_compile_time, dt);
3993 LP_COUNT_ADD(nr_llvm_compiles, 2); /* emit vs. omit in/out test */
3994
3995 /* Put the new variant into the list */
3996 if (variant) {
3997 insert_at_head(&shader->variants, &variant->list_item_local);
3998 insert_at_head(&lp->fs_variants_list, &variant->list_item_global);
3999 lp->nr_fs_variants++;
4000 lp->nr_fs_instrs += variant->nr_instrs;
4001 shader->variants_cached++;
4002 }
4003 }
4004
4005 /* Bind this variant */
4006 lp_setup_set_fs_variant(lp->setup, variant);
4007 }
4008
4009
4010
4011
4012
4013 void
4014 llvmpipe_init_fs_funcs(struct llvmpipe_context *llvmpipe)
4015 {
4016 llvmpipe->pipe.create_fs_state = llvmpipe_create_fs_state;
4017 llvmpipe->pipe.bind_fs_state = llvmpipe_bind_fs_state;
4018 llvmpipe->pipe.delete_fs_state = llvmpipe_delete_fs_state;
4019
4020 llvmpipe->pipe.set_constant_buffer = llvmpipe_set_constant_buffer;
4021
4022 llvmpipe->pipe.set_shader_buffers = llvmpipe_set_shader_buffers;
4023 llvmpipe->pipe.set_shader_images = llvmpipe_set_shader_images;
4024 }
4025
4026