projects / mesa.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
history | raw | HEAD
gallivm: optimize gather a bit, by using supplied destination type
[mesa.git] / src / gallium / auxiliary / gallivm / lp_bld_sample_aos.c
1 /**************************************************************************
2 *
3 * Copyright 2010 VMware, Inc.
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28 /**
29 * @file
30 * Texture sampling -- AoS.
31 *
32 * @author Jose Fonseca <jfonseca@vmware.com>
33 * @author Brian Paul <brianp@vmware.com>
34 */
35
36 #include "pipe/p_defines.h"
37 #include "pipe/p_state.h"
38 #include "util/u_debug.h"
39 #include "util/u_dump.h"
40 #include "util/u_memory.h"
41 #include "util/u_math.h"
42 #include "util/u_format.h"
43 #include "util/u_cpu_detect.h"
44 #include "lp_bld_debug.h"
45 #include "lp_bld_type.h"
46 #include "lp_bld_const.h"
47 #include "lp_bld_conv.h"
48 #include "lp_bld_arit.h"
49 #include "lp_bld_bitarit.h"
50 #include "lp_bld_logic.h"
51 #include "lp_bld_swizzle.h"
52 #include "lp_bld_pack.h"
53 #include "lp_bld_flow.h"
54 #include "lp_bld_gather.h"
55 #include "lp_bld_format.h"
56 #include "lp_bld_init.h"
57 #include "lp_bld_sample.h"
58 #include "lp_bld_sample_aos.h"
59 #include "lp_bld_quad.h"
60
61
62 /**
63 * Build LLVM code for texture coord wrapping, for nearest filtering,
64 * for scaled integer texcoords.
65 * \param block_length is the length of the pixel block along the
66 * coordinate axis
67 * \param coord the incoming texcoord (s,t or r) scaled to the texture size
68 * \param coord_f the incoming texcoord (s,t or r) as float vec
69 * \param length the texture size along one dimension
70 * \param stride pixel stride along the coordinate axis (in bytes)
71 * \param offset the texel offset along the coord axis
72 * \param is_pot if TRUE, length is a power of two
73 * \param wrap_mode one of PIPE_TEX_WRAP_x
74 * \param out_offset byte offset for the wrapped coordinate
75 * \param out_i resulting sub-block pixel coordinate for coord0
76 */
77 static void
78 lp_build_sample_wrap_nearest_int(struct lp_build_sample_context *bld,
79 unsigned block_length,
80 LLVMValueRef coord,
81 LLVMValueRef coord_f,
82 LLVMValueRef length,
83 LLVMValueRef stride,
84 LLVMValueRef offset,
85 boolean is_pot,
86 unsigned wrap_mode,
87 LLVMValueRef *out_offset,
88 LLVMValueRef *out_i)
89 {
90 struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
91 LLVMBuilderRef builder = bld->gallivm->builder;
92 LLVMValueRef length_minus_one;
93
94 length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
95
96 switch(wrap_mode) {
97 case PIPE_TEX_WRAP_REPEAT:
98 if(is_pot)
99 coord = LLVMBuildAnd(builder, coord, length_minus_one, "");
100 else {
101 struct lp_build_context *coord_bld = &bld->coord_bld;
102 LLVMValueRef length_f = lp_build_int_to_float(coord_bld, length);
103 if (offset) {
104 offset = lp_build_int_to_float(coord_bld, offset);
105 offset = lp_build_div(coord_bld, offset, length_f);
106 coord_f = lp_build_add(coord_bld, coord_f, offset);
107 }
108 coord = lp_build_fract_safe(coord_bld, coord_f);
109 coord = lp_build_mul(coord_bld, coord, length_f);
110 coord = lp_build_itrunc(coord_bld, coord);
111 }
112 break;
113
114 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
115 coord = lp_build_max(int_coord_bld, coord, int_coord_bld->zero);
116 coord = lp_build_min(int_coord_bld, coord, length_minus_one);
117 break;
118
119 case PIPE_TEX_WRAP_CLAMP:
120 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
121 case PIPE_TEX_WRAP_MIRROR_REPEAT:
122 case PIPE_TEX_WRAP_MIRROR_CLAMP:
123 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
124 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
125 default:
126 assert(0);
127 }
128
129 lp_build_sample_partial_offset(int_coord_bld, block_length, coord, stride,
130 out_offset, out_i);
131 }
132
133
134 /**
135 * Build LLVM code for texture coord wrapping, for nearest filtering,
136 * for float texcoords.
137 * \param coord the incoming texcoord (s,t or r)
138 * \param length the texture size along one dimension
139 * \param offset the texel offset along the coord axis
140 * \param is_pot if TRUE, length is a power of two
141 * \param wrap_mode one of PIPE_TEX_WRAP_x
142 * \param icoord the texcoord after wrapping, as int
143 */
144 static void
145 lp_build_sample_wrap_nearest_float(struct lp_build_sample_context *bld,
146 LLVMValueRef coord,
147 LLVMValueRef length,
148 LLVMValueRef offset,
149 boolean is_pot,
150 unsigned wrap_mode,
151 LLVMValueRef *icoord)
152 {
153 struct lp_build_context *coord_bld = &bld->coord_bld;
154 LLVMValueRef length_minus_one;
155
156 switch(wrap_mode) {
157 case PIPE_TEX_WRAP_REPEAT:
158 if (offset) {
159 /* this is definitely not ideal for POT case */
160 offset = lp_build_int_to_float(coord_bld, offset);
161 offset = lp_build_div(coord_bld, offset, length);
162 coord = lp_build_add(coord_bld, coord, offset);
163 }
164 /* take fraction, unnormalize */
165 coord = lp_build_fract_safe(coord_bld, coord);
166 coord = lp_build_mul(coord_bld, coord, length);
167 *icoord = lp_build_itrunc(coord_bld, coord);
168 break;
169 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
170 length_minus_one = lp_build_sub(coord_bld, length, coord_bld->one);
171 if (bld->static_sampler_state->normalized_coords) {
172 /* scale coord to length */
173 coord = lp_build_mul(coord_bld, coord, length);
174 }
175 if (offset) {
176 offset = lp_build_int_to_float(coord_bld, offset);
177 coord = lp_build_add(coord_bld, coord, offset);
178 }
179 coord = lp_build_clamp(coord_bld, coord, coord_bld->zero,
180 length_minus_one);
181 *icoord = lp_build_itrunc(coord_bld, coord);
182 break;
183
184 case PIPE_TEX_WRAP_CLAMP:
185 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
186 case PIPE_TEX_WRAP_MIRROR_REPEAT:
187 case PIPE_TEX_WRAP_MIRROR_CLAMP:
188 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
189 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
190 default:
191 assert(0);
192 }
193 }
194
195
196 /**
197 * Helper to compute the first coord and the weight for
198 * linear wrap repeat npot textures
199 */
200 static void
201 lp_build_coord_repeat_npot_linear_int(struct lp_build_sample_context *bld,
202 LLVMValueRef coord_f,
203 LLVMValueRef length_i,
204 LLVMValueRef length_f,
205 LLVMValueRef *coord0_i,
206 LLVMValueRef *weight_i)
207 {
208 struct lp_build_context *coord_bld = &bld->coord_bld;
209 struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
210 struct lp_build_context abs_coord_bld;
211 struct lp_type abs_type;
212 LLVMValueRef length_minus_one = lp_build_sub(int_coord_bld, length_i,
213 int_coord_bld->one);
214 LLVMValueRef mask, i32_c8, i32_c128, i32_c255;
215
216 /* wrap with normalized floats is just fract */
217 coord_f = lp_build_fract(coord_bld, coord_f);
218 /* mul by size */
219 coord_f = lp_build_mul(coord_bld, coord_f, length_f);
220 /* convert to int, compute lerp weight */
221 coord_f = lp_build_mul_imm(&bld->coord_bld, coord_f, 256);
222
223 /* At this point we don't have any negative numbers so use non-signed
224 * build context which might help on some archs.
225 */
226 abs_type = coord_bld->type;
227 abs_type.sign = 0;
228 lp_build_context_init(&abs_coord_bld, bld->gallivm, abs_type);
229 *coord0_i = lp_build_iround(&abs_coord_bld, coord_f);
230
231 /* subtract 0.5 (add -128) */
232 i32_c128 = lp_build_const_int_vec(bld->gallivm, bld->int_coord_type, -128);
233 *coord0_i = LLVMBuildAdd(bld->gallivm->builder, *coord0_i, i32_c128, "");
234
235 /* compute fractional part (AND with 0xff) */
236 i32_c255 = lp_build_const_int_vec(bld->gallivm, bld->int_coord_type, 255);
237 *weight_i = LLVMBuildAnd(bld->gallivm->builder, *coord0_i, i32_c255, "");
238
239 /* compute floor (shift right 8) */
240 i32_c8 = lp_build_const_int_vec(bld->gallivm, bld->int_coord_type, 8);
241 *coord0_i = LLVMBuildAShr(bld->gallivm->builder, *coord0_i, i32_c8, "");
242 /*
243 * we avoided the 0.5/length division before the repeat wrap,
244 * now need to fix up edge cases with selects
245 */
246 mask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
247 PIPE_FUNC_LESS, *coord0_i, int_coord_bld->zero);
248 *coord0_i = lp_build_select(int_coord_bld, mask, length_minus_one, *coord0_i);
249 /*
250 * We should never get values too large - except if coord was nan or inf,
251 * in which case things go terribly wrong...
252 * Alternatively, could use fract_safe above...
253 */
254 *coord0_i = lp_build_min(int_coord_bld, *coord0_i, length_minus_one);
255 }
256
257
258 /**
259 * Build LLVM code for texture coord wrapping, for linear filtering,
260 * for scaled integer texcoords.
261 * \param block_length is the length of the pixel block along the
262 * coordinate axis
263 * \param coord0 the incoming texcoord (s,t or r) scaled to the texture size
264 * \param coord_f the incoming texcoord (s,t or r) as float vec
265 * \param length the texture size along one dimension
266 * \param stride pixel stride along the coordinate axis (in bytes)
267 * \param offset the texel offset along the coord axis
268 * \param is_pot if TRUE, length is a power of two
269 * \param wrap_mode one of PIPE_TEX_WRAP_x
270 * \param offset0 resulting relative offset for coord0
271 * \param offset1 resulting relative offset for coord0 + 1
272 * \param i0 resulting sub-block pixel coordinate for coord0
273 * \param i1 resulting sub-block pixel coordinate for coord0 + 1
274 */
275 static void
276 lp_build_sample_wrap_linear_int(struct lp_build_sample_context *bld,
277 unsigned block_length,
278 LLVMValueRef coord0,
279 LLVMValueRef *weight_i,
280 LLVMValueRef coord_f,
281 LLVMValueRef length,
282 LLVMValueRef stride,
283 LLVMValueRef offset,
284 boolean is_pot,
285 unsigned wrap_mode,
286 LLVMValueRef *offset0,
287 LLVMValueRef *offset1,
288 LLVMValueRef *i0,
289 LLVMValueRef *i1)
290 {
291 struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
292 LLVMBuilderRef builder = bld->gallivm->builder;
293 LLVMValueRef length_minus_one;
294 LLVMValueRef lmask, umask, mask;
295
296 /*
297 * If the pixel block covers more than one pixel then there is no easy
298 * way to calculate offset1 relative to offset0. Instead, compute them
299 * independently. Otherwise, try to compute offset0 and offset1 with
300 * a single stride multiplication.
301 */
302
303 length_minus_one = lp_build_sub(int_coord_bld, length, int_coord_bld->one);
304
305 if (block_length != 1) {
306 LLVMValueRef coord1;
307 switch(wrap_mode) {
308 case PIPE_TEX_WRAP_REPEAT:
309 if (is_pot) {
310 coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
311 coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, "");
312 coord1 = LLVMBuildAnd(builder, coord1, length_minus_one, "");
313 }
314 else {
315 LLVMValueRef mask;
316 LLVMValueRef length_f = lp_build_int_to_float(&bld->coord_bld, length);
317 if (offset) {
318 offset = lp_build_int_to_float(&bld->coord_bld, offset);
319 offset = lp_build_div(&bld->coord_bld, offset, length_f);
320 coord_f = lp_build_add(&bld->coord_bld, coord_f, offset);
321 }
322 lp_build_coord_repeat_npot_linear_int(bld, coord_f,
323 length, length_f,
324 &coord0, weight_i);
325 mask = lp_build_compare(bld->gallivm, int_coord_bld->type,
326 PIPE_FUNC_NOTEQUAL, coord0, length_minus_one);
327 coord1 = LLVMBuildAnd(builder,
328 lp_build_add(int_coord_bld, coord0,
329 int_coord_bld->one),
330 mask, "");
331 }
332 break;
333
334 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
335 coord1 = lp_build_add(int_coord_bld, coord0, int_coord_bld->one);
336 coord0 = lp_build_clamp(int_coord_bld, coord0, int_coord_bld->zero,
337 length_minus_one);
338 coord1 = lp_build_clamp(int_coord_bld, coord1, int_coord_bld->zero,
339 length_minus_one);
340 break;
341
342 case PIPE_TEX_WRAP_CLAMP:
343 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
344 case PIPE_TEX_WRAP_MIRROR_REPEAT:
345 case PIPE_TEX_WRAP_MIRROR_CLAMP:
346 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
347 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
348 default:
349 assert(0);
350 coord0 = int_coord_bld->zero;
351 coord1 = int_coord_bld->zero;
352 break;
353 }
354 lp_build_sample_partial_offset(int_coord_bld, block_length, coord0, stride,
355 offset0, i0);
356 lp_build_sample_partial_offset(int_coord_bld, block_length, coord1, stride,
357 offset1, i1);
358 return;
359 }
360
361 *i0 = int_coord_bld->zero;
362 *i1 = int_coord_bld->zero;
363
364 switch(wrap_mode) {
365 case PIPE_TEX_WRAP_REPEAT:
366 if (is_pot) {
367 coord0 = LLVMBuildAnd(builder, coord0, length_minus_one, "");
368 }
369 else {
370 LLVMValueRef length_f = lp_build_int_to_float(&bld->coord_bld, length);
371 if (offset) {
372 offset = lp_build_int_to_float(&bld->coord_bld, offset);
373 offset = lp_build_div(&bld->coord_bld, offset, length_f);
374 coord_f = lp_build_add(&bld->coord_bld, coord_f, offset);
375 }
376 lp_build_coord_repeat_npot_linear_int(bld, coord_f,
377 length, length_f,
378 &coord0, weight_i);
379 }
380
381 mask = lp_build_compare(bld->gallivm, int_coord_bld->type,
382 PIPE_FUNC_NOTEQUAL, coord0, length_minus_one);
383
384 *offset0 = lp_build_mul(int_coord_bld, coord0, stride);
385 *offset1 = LLVMBuildAnd(builder,
386 lp_build_add(int_coord_bld, *offset0, stride),
387 mask, "");
388 break;
389
390 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
391 /* XXX this might be slower than the separate path
392 * on some newer cpus. With sse41 this is 8 instructions vs. 7
393 * - at least on SNB this is almost certainly slower since
394 * min/max are cheaper than selects, and the muls aren't bad.
395 */
396 lmask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
397 PIPE_FUNC_GEQUAL, coord0, int_coord_bld->zero);
398 umask = lp_build_compare(int_coord_bld->gallivm, int_coord_bld->type,
399 PIPE_FUNC_LESS, coord0, length_minus_one);
400
401 coord0 = lp_build_select(int_coord_bld, lmask, coord0, int_coord_bld->zero);
402 coord0 = lp_build_select(int_coord_bld, umask, coord0, length_minus_one);
403
404 mask = LLVMBuildAnd(builder, lmask, umask, "");
405
406 *offset0 = lp_build_mul(int_coord_bld, coord0, stride);
407 *offset1 = lp_build_add(int_coord_bld,
408 *offset0,
409 LLVMBuildAnd(builder, stride, mask, ""));
410 break;
411
412 case PIPE_TEX_WRAP_CLAMP:
413 case PIPE_TEX_WRAP_CLAMP_TO_BORDER:
414 case PIPE_TEX_WRAP_MIRROR_REPEAT:
415 case PIPE_TEX_WRAP_MIRROR_CLAMP:
416 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE:
417 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER:
418 default:
419 assert(0);
420 *offset0 = int_coord_bld->zero;
421 *offset1 = int_coord_bld->zero;
422 break;
423 }
424 }
425
426
427 /**
428 * Build LLVM code for texture coord wrapping, for linear filtering,
429 * for float texcoords.
430 * \param block_length is the length of the pixel block along the
431 * coordinate axis
432 * \param coord the incoming texcoord (s,t or r)
433 * \param length the texture size along one dimension
434 * \param offset the texel offset along the coord axis
435 * \param is_pot if TRUE, length is a power of two
436 * \param wrap_mode one of PIPE_TEX_WRAP_x
437 * \param coord0 the first texcoord after wrapping, as int
438 * \param coord1 the second texcoord after wrapping, as int
439 * \param weight the filter weight as int (0-255)
440 * \param force_nearest if this coord actually uses nearest filtering
441 */
442 static void
443 lp_build_sample_wrap_linear_float(struct lp_build_sample_context *bld,
444 unsigned block_length,
445 LLVMValueRef coord,
446 LLVMValueRef length,
447 LLVMValueRef offset,
448 boolean is_pot,
449 unsigned wrap_mode,
450 LLVMValueRef *coord0,
451 LLVMValueRef *coord1,
452 LLVMValueRef *weight,
453 unsigned force_nearest)
454 {
455 struct lp_build_context *int_coord_bld = &bld->int_coord_bld;
456 struct lp_build_context *coord_bld = &bld->coord_bld;
457 LLVMBuilderRef builder = bld->gallivm->builder;
458 LLVMValueRef half = lp_build_const_vec(bld->gallivm, coord_bld->type, 0.5);
459 LLVMValueRef length_minus_one = lp_build_sub(coord_bld, length, coord_bld->one);
460
461 switch(wrap_mode) {
462 case PIPE_TEX_WRAP_REPEAT:
463 if (is_pot) {
464 /* mul by size and subtract 0.5 */
465 coord = lp_build_mul(coord_bld, coord, length);
466 if (offset) {
467 offset = lp_build_int_to_float(coord_bld, offset);
468 coord = lp_build_add(coord_bld, coord, offset);
469 }
470 if (!force_nearest)
471 coord = lp_build_sub(coord_bld, coord, half);
472 *coord1 = lp_build_add(coord_bld, coord, coord_bld->one);
473 /* convert to int, compute lerp weight */
474 lp_build_ifloor_fract(coord_bld, coord, coord0, weight);
475 *coord1 = lp_build_ifloor(coord_bld, *coord1);
476 /* repeat wrap */
477 length_minus_one = lp_build_itrunc(coord_bld, length_minus_one);
478 *coord0 = LLVMBuildAnd(builder, *coord0, length_minus_one, "");
479 *coord1 = LLVMBuildAnd(builder, *coord1, length_minus_one, "");
480 }
481 else {
482 LLVMValueRef mask;
483 if (offset) {
484 offset = lp_build_int_to_float(coord_bld, offset);
485 offset = lp_build_div(coord_bld, offset, length);
486 coord = lp_build_add(coord_bld, coord, offset);
487 }
488 /* wrap with normalized floats is just fract */
489 coord = lp_build_fract(coord_bld, coord);
490 /* unnormalize */
491 coord = lp_build_mul(coord_bld, coord, length);
492 /*
493 * we avoided the 0.5/length division, have to fix up wrong
494 * edge cases with selects
495 */
496 *coord1 = lp_build_add(coord_bld, coord, half);
497 coord = lp_build_sub(coord_bld, coord, half);
498 *weight = lp_build_fract(coord_bld, coord);
499 /*
500 * It is important for this comparison to be unordered
501 * (or need fract_safe above).
502 */
503 mask = lp_build_compare(coord_bld->gallivm, coord_bld->type,
504 PIPE_FUNC_LESS, coord, coord_bld->zero);
505 *coord0 = lp_build_select(coord_bld, mask, length_minus_one, coord);
506 *coord0 = lp_build_itrunc(coord_bld, *coord0);
507 mask = lp_build_compare(coord_bld->gallivm, coord_bld->type,
508 PIPE_FUNC_LESS, *coord1, length);
509 *coord1 = lp_build_select(coord_bld, mask, *coord1, coord_bld->zero);
510 *coord1 = lp_build_itrunc(coord_bld, *coord1);
511 }
512 break;
513 case PIPE_TEX_WRAP_CLAMP_TO_EDGE:
514 if (bld->static_sampler_state->normalized_coords) {
515 /* mul by tex size */
516 coord = lp_build_mul(coord_bld, coord, length);
517 }
518 if (offset) {
519 offset = lp_build_int_to_float(coord_bld, offset);
520 coord = lp_build_add(coord_bld, coord, offset);
521 }
522 /* subtract 0.5 */
523 if (!force_nearest) {
524 coord = lp_build_sub(coord_bld, coord, half);
525 }
526 /* clamp to [0, length - 1] */
527 coord = lp_build_min_ext(coord_bld, coord, length_minus_one,
528 GALLIVM_NAN_RETURN_OTHER_SECOND_NONNAN);
529 coord = lp_build_max(coord_bld, coord, coord_bld->zero);
530 *coord1 = lp_build_add(coord_bld, coord, coord_bld->one);
531 /* convert to int, compute lerp weight */
532 lp_build_ifloor_fract(coord_bld, coord, coord0, weight);
533 /* coord1 = min(coord1, length-1) */
534 *coord1 = lp_build_min(coord_bld, *coord1, length_minus_one);
535 *coord1 = lp_build_itrunc(coord_bld, *coord1);
536 break;
537 default:
538 assert(0);
539 *coord0 = int_coord_bld->zero;
540 *coord1 = int_coord_bld->zero;
541 *weight = coord_bld->zero;
542 break;
543 }
544 *weight = lp_build_mul_imm(coord_bld, *weight, 256);
545 *weight = lp_build_itrunc(coord_bld, *weight);
546 return;
547 }
548
549
550 /**
551 * Fetch texels for image with nearest sampling.
552 * Return filtered color as two vectors of 16-bit fixed point values.
553 */
554 static void
555 lp_build_sample_fetch_image_nearest(struct lp_build_sample_context *bld,
556 LLVMValueRef data_ptr,
557 LLVMValueRef offset,
558 LLVMValueRef x_subcoord,
559 LLVMValueRef y_subcoord,
560 LLVMValueRef *colors)
561 {
562 /*
563 * Fetch the pixels as 4 x 32bit (rgba order might differ):
564 *
565 * rgba0 rgba1 rgba2 rgba3
566 *
567 * bit cast them into 16 x u8
568 *
569 * r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3
570 *
571 * unpack them into two 8 x i16:
572 *
573 * r0 g0 b0 a0 r1 g1 b1 a1
574 * r2 g2 b2 a2 r3 g3 b3 a3
575 *
576 * The higher 8 bits of the resulting elements will be zero.
577 */
578 LLVMBuilderRef builder = bld->gallivm->builder;
579 LLVMValueRef rgba8;
580 struct lp_build_context u8n;
581 LLVMTypeRef u8n_vec_type;
582 struct lp_type fetch_type;
583
584 lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8, bld->vector_width));
585 u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type);
586
587 fetch_type = lp_type_uint(bld->texel_type.width);
588 if (util_format_is_rgba8_variant(bld->format_desc)) {
589 /*
590 * Given the format is a rgba8, just read the pixels as is,
591 * without any swizzling. Swizzling will be done later.
592 */
593 rgba8 = lp_build_gather(bld->gallivm,
594 bld->texel_type.length,
595 bld->format_desc->block.bits,
596 fetch_type,
597 TRUE,
598 data_ptr, offset, TRUE);
599
600 rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, "");
601 }
602 else {
603 rgba8 = lp_build_fetch_rgba_aos(bld->gallivm,
604 bld->format_desc,
605 u8n.type,
606 TRUE,
607 data_ptr, offset,
608 x_subcoord,
609 y_subcoord,
610 bld->cache);
611 }
612
613 *colors = rgba8;
614 }
615
616
617 /**
618 * Sample a single texture image with nearest sampling.
619 * If sampling a cube texture, r = cube face in [0,5].
620 * Return filtered color as two vectors of 16-bit fixed point values.
621 */
622 static void
623 lp_build_sample_image_nearest(struct lp_build_sample_context *bld,
624 LLVMValueRef int_size,
625 LLVMValueRef row_stride_vec,
626 LLVMValueRef img_stride_vec,
627 LLVMValueRef data_ptr,
628 LLVMValueRef mipoffsets,
629 LLVMValueRef s,
630 LLVMValueRef t,
631 LLVMValueRef r,
632 const LLVMValueRef *offsets,
633 LLVMValueRef *colors)
634 {
635 const unsigned dims = bld->dims;
636 struct lp_build_context i32;
637 LLVMValueRef width_vec, height_vec, depth_vec;
638 LLVMValueRef s_ipart, t_ipart = NULL, r_ipart = NULL;
639 LLVMValueRef s_float, t_float = NULL, r_float = NULL;
640 LLVMValueRef x_stride;
641 LLVMValueRef x_offset, offset;
642 LLVMValueRef x_subcoord, y_subcoord, z_subcoord;
643
644 lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32, bld->vector_width));
645
646 lp_build_extract_image_sizes(bld,
647 &bld->int_size_bld,
648 bld->int_coord_type,
649 int_size,
650 &width_vec,
651 &height_vec,
652 &depth_vec);
653
654 s_float = s; t_float = t; r_float = r;
655
656 if (bld->static_sampler_state->normalized_coords) {
657 LLVMValueRef flt_size;
658
659 flt_size = lp_build_int_to_float(&bld->float_size_bld, int_size);
660
661 lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r);
662 }
663
664 /* convert float to int */
665 /* For correct rounding, need floor, not truncation here.
666 * Note that in some cases (clamp to edge, no texel offsets) we
667 * could use a non-signed build context which would help archs
668 * greatly which don't have arch rounding.
669 */
670 s_ipart = lp_build_ifloor(&bld->coord_bld, s);
671 if (dims >= 2)
672 t_ipart = lp_build_ifloor(&bld->coord_bld, t);
673 if (dims >= 3)
674 r_ipart = lp_build_ifloor(&bld->coord_bld, r);
675
676 /* add texel offsets */
677 if (offsets[0]) {
678 s_ipart = lp_build_add(&i32, s_ipart, offsets[0]);
679 if (dims >= 2) {
680 t_ipart = lp_build_add(&i32, t_ipart, offsets[1]);
681 if (dims >= 3) {
682 r_ipart = lp_build_add(&i32, r_ipart, offsets[2]);
683 }
684 }
685 }
686
687 /* get pixel, row, image strides */
688 x_stride = lp_build_const_vec(bld->gallivm,
689 bld->int_coord_bld.type,
690 bld->format_desc->block.bits/8);
691
692 /* Do texcoord wrapping, compute texel offset */
693 lp_build_sample_wrap_nearest_int(bld,
694 bld->format_desc->block.width,
695 s_ipart, s_float,
696 width_vec, x_stride, offsets[0],
697 bld->static_texture_state->pot_width,
698 bld->static_sampler_state->wrap_s,
699 &x_offset, &x_subcoord);
700 offset = x_offset;
701 if (dims >= 2) {
702 LLVMValueRef y_offset;
703 lp_build_sample_wrap_nearest_int(bld,
704 bld->format_desc->block.height,
705 t_ipart, t_float,
706 height_vec, row_stride_vec, offsets[1],
707 bld->static_texture_state->pot_height,
708 bld->static_sampler_state->wrap_t,
709 &y_offset, &y_subcoord);
710 offset = lp_build_add(&bld->int_coord_bld, offset, y_offset);
711 if (dims >= 3) {
712 LLVMValueRef z_offset;
713 lp_build_sample_wrap_nearest_int(bld,
714 1, /* block length (depth) */
715 r_ipart, r_float,
716 depth_vec, img_stride_vec, offsets[2],
717 bld->static_texture_state->pot_depth,
718 bld->static_sampler_state->wrap_r,
719 &z_offset, &z_subcoord);
720 offset = lp_build_add(&bld->int_coord_bld, offset, z_offset);
721 }
722 }
723 if (has_layer_coord(bld->static_texture_state->target)) {
724 LLVMValueRef z_offset;
725 /* The r coord is the cube face in [0,5] or array layer */
726 z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec);
727 offset = lp_build_add(&bld->int_coord_bld, offset, z_offset);
728 }
729 if (mipoffsets) {
730 offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets);
731 }
732
733 lp_build_sample_fetch_image_nearest(bld, data_ptr, offset,
734 x_subcoord, y_subcoord,
735 colors);
736 }
737
738
739 /**
740 * Sample a single texture image with nearest sampling.
741 * If sampling a cube texture, r = cube face in [0,5].
742 * Return filtered color as two vectors of 16-bit fixed point values.
743 * Does address calcs (except offsets) with floats.
744 * Useful for AVX which has support for 8x32 floats but not 8x32 ints.
745 */
746 static void
747 lp_build_sample_image_nearest_afloat(struct lp_build_sample_context *bld,
748 LLVMValueRef int_size,
749 LLVMValueRef row_stride_vec,
750 LLVMValueRef img_stride_vec,
751 LLVMValueRef data_ptr,
752 LLVMValueRef mipoffsets,
753 LLVMValueRef s,
754 LLVMValueRef t,
755 LLVMValueRef r,
756 const LLVMValueRef *offsets,
757 LLVMValueRef *colors)
758 {
759 const unsigned dims = bld->dims;
760 LLVMValueRef width_vec, height_vec, depth_vec;
761 LLVMValueRef offset;
762 LLVMValueRef x_subcoord, y_subcoord;
763 LLVMValueRef x_icoord = NULL, y_icoord = NULL, z_icoord = NULL;
764 LLVMValueRef flt_size;
765
766 flt_size = lp_build_int_to_float(&bld->float_size_bld, int_size);
767
768 lp_build_extract_image_sizes(bld,
769 &bld->float_size_bld,
770 bld->coord_type,
771 flt_size,
772 &width_vec,
773 &height_vec,
774 &depth_vec);
775
776 /* Do texcoord wrapping */
777 lp_build_sample_wrap_nearest_float(bld,
778 s, width_vec, offsets[0],
779 bld->static_texture_state->pot_width,
780 bld->static_sampler_state->wrap_s,
781 &x_icoord);
782
783 if (dims >= 2) {
784 lp_build_sample_wrap_nearest_float(bld,
785 t, height_vec, offsets[1],
786 bld->static_texture_state->pot_height,
787 bld->static_sampler_state->wrap_t,
788 &y_icoord);
789
790 if (dims >= 3) {
791 lp_build_sample_wrap_nearest_float(bld,
792 r, depth_vec, offsets[2],
793 bld->static_texture_state->pot_depth,
794 bld->static_sampler_state->wrap_r,
795 &z_icoord);
796 }
797 }
798 if (has_layer_coord(bld->static_texture_state->target)) {
799 z_icoord = r;
800 }
801
802 /*
803 * From here on we deal with ints, and we should split up the 256bit
804 * vectors manually for better generated code.
805 */
806
807 /*
808 * compute texel offsets -
809 * cannot do offset calc with floats, difficult for block-based formats,
810 * and not enough precision anyway.
811 */
812 lp_build_sample_offset(&bld->int_coord_bld,
813 bld->format_desc,
814 x_icoord, y_icoord,
815 z_icoord,
816 row_stride_vec, img_stride_vec,
817 &offset,
818 &x_subcoord, &y_subcoord);
819 if (mipoffsets) {
820 offset = lp_build_add(&bld->int_coord_bld, offset, mipoffsets);
821 }
822
823 lp_build_sample_fetch_image_nearest(bld, data_ptr, offset,
824 x_subcoord, y_subcoord,
825 colors);
826 }
827
828
829 /**
830 * Fetch texels for image with linear sampling.
831 * Return filtered color as two vectors of 16-bit fixed point values.
832 */
833 static void
834 lp_build_sample_fetch_image_linear(struct lp_build_sample_context *bld,
835 LLVMValueRef data_ptr,
836 LLVMValueRef offset[2][2][2],
837 LLVMValueRef x_subcoord[2],
838 LLVMValueRef y_subcoord[2],
839 LLVMValueRef s_fpart,
840 LLVMValueRef t_fpart,
841 LLVMValueRef r_fpart,
842 LLVMValueRef *colors)
843 {
844 const unsigned dims = bld->dims;
845 LLVMBuilderRef builder = bld->gallivm->builder;
846 struct lp_build_context u8n;
847 LLVMTypeRef u8n_vec_type;
848 LLVMTypeRef elem_type = LLVMInt32TypeInContext(bld->gallivm->context);
849 LLVMValueRef shuffles[LP_MAX_VECTOR_LENGTH];
850 LLVMValueRef shuffle;
851 LLVMValueRef neighbors[2][2][2]; /* [z][y][x] */
852 LLVMValueRef packed;
853 unsigned i, j, k;
854 unsigned numj, numk;
855
856 lp_build_context_init(&u8n, bld->gallivm, lp_type_unorm(8, bld->vector_width));
857 u8n_vec_type = lp_build_vec_type(bld->gallivm, u8n.type);
858
859 /*
860 * Transform 4 x i32 in
861 *
862 * s_fpart = {s0, s1, s2, s3}
863 *
864 * where each value is between 0 and 0xff,
865 *
866 * into one 16 x i20
867 *
868 * s_fpart = {s0, s0, s0, s0, s1, s1, s1, s1, s2, s2, s2, s2, s3, s3, s3, s3}
869 *
870 * and likewise for t_fpart. There is no risk of loosing precision here
871 * since the fractional parts only use the lower 8bits.
872 */
873 s_fpart = LLVMBuildBitCast(builder, s_fpart, u8n_vec_type, "");
874 if (dims >= 2)
875 t_fpart = LLVMBuildBitCast(builder, t_fpart, u8n_vec_type, "");
876 if (dims >= 3)
877 r_fpart = LLVMBuildBitCast(builder, r_fpart, u8n_vec_type, "");
878
879 for (j = 0; j < u8n.type.length; j += 4) {
880 #ifdef PIPE_ARCH_LITTLE_ENDIAN
881 unsigned subindex = 0;
882 #else
883 unsigned subindex = 3;
884 #endif
885 LLVMValueRef index;
886
887 index = LLVMConstInt(elem_type, j + subindex, 0);
888 for (i = 0; i < 4; ++i)
889 shuffles[j + i] = index;
890 }
891
892 shuffle = LLVMConstVector(shuffles, u8n.type.length);
893
894 s_fpart = LLVMBuildShuffleVector(builder, s_fpart, u8n.undef,
895 shuffle, "");
896 if (dims >= 2) {
897 t_fpart = LLVMBuildShuffleVector(builder, t_fpart, u8n.undef,
898 shuffle, "");
899 }
900 if (dims >= 3) {
901 r_fpart = LLVMBuildShuffleVector(builder, r_fpart, u8n.undef,
902 shuffle, "");
903 }
904
905 /*
906 * Fetch the pixels as 4 x 32bit (rgba order might differ):
907 *
908 * rgba0 rgba1 rgba2 rgba3
909 *
910 * bit cast them into 16 x u8
911 *
912 * r0 g0 b0 a0 r1 g1 b1 a1 r2 g2 b2 a2 r3 g3 b3 a3
913 *
914 * unpack them into two 8 x i16:
915 *
916 * r0 g0 b0 a0 r1 g1 b1 a1
917 * r2 g2 b2 a2 r3 g3 b3 a3
918 *
919 * The higher 8 bits of the resulting elements will be zero.
920 */
921 numj = 1 + (dims >= 2);
922 numk = 1 + (dims >= 3);
923
924 for (k = 0; k < numk; k++) {
925 for (j = 0; j < numj; j++) {
926 for (i = 0; i < 2; i++) {
927 LLVMValueRef rgba8;
928
929 if (util_format_is_rgba8_variant(bld->format_desc)) {
930 struct lp_type fetch_type;
931 /*
932 * Given the format is a rgba8, just read the pixels as is,
933 * without any swizzling. Swizzling will be done later.
934 */
935 fetch_type = lp_type_uint(bld->texel_type.width);
936 rgba8 = lp_build_gather(bld->gallivm,
937 bld->texel_type.length,
938 bld->format_desc->block.bits,
939 fetch_type,
940 TRUE,
941 data_ptr, offset[k][j][i], TRUE);
942
943 rgba8 = LLVMBuildBitCast(builder, rgba8, u8n_vec_type, "");
944 }
945 else {
946 rgba8 = lp_build_fetch_rgba_aos(bld->gallivm,
947 bld->format_desc,
948 u8n.type,
949 TRUE,
950 data_ptr, offset[k][j][i],
951 x_subcoord[i],
952 y_subcoord[j],
953 bld->cache);
954 }
955
956 neighbors[k][j][i] = rgba8;
957 }
958 }
959 }
960
961 /*
962 * Linear interpolation with 8.8 fixed point.
963 */
964 if (bld->static_sampler_state->force_nearest_s) {
965 /* special case 1-D lerp */
966 packed = lp_build_lerp(&u8n,
967 t_fpart,
968 neighbors[0][0][0],
969 neighbors[0][0][1],
970 LP_BLD_LERP_PRESCALED_WEIGHTS);
971 }
972 else if (bld->static_sampler_state->force_nearest_t) {
973 /* special case 1-D lerp */
974 packed = lp_build_lerp(&u8n,
975 s_fpart,
976 neighbors[0][0][0],
977 neighbors[0][0][1],
978 LP_BLD_LERP_PRESCALED_WEIGHTS);
979 }
980 else {
981 /* general 1/2/3-D lerping */
982 if (dims == 1) {
983 packed = lp_build_lerp(&u8n,
984 s_fpart,
985 neighbors[0][0][0],
986 neighbors[0][0][1],
987 LP_BLD_LERP_PRESCALED_WEIGHTS);
988 } else if (dims == 2) {
989 /* 2-D lerp */
990 packed = lp_build_lerp_2d(&u8n,
991 s_fpart, t_fpart,
992 neighbors[0][0][0],
993 neighbors[0][0][1],
994 neighbors[0][1][0],
995 neighbors[0][1][1],
996 LP_BLD_LERP_PRESCALED_WEIGHTS);
997 } else {
998 /* 3-D lerp */
999 assert(dims == 3);
1000 packed = lp_build_lerp_3d(&u8n,
1001 s_fpart, t_fpart, r_fpart,
1002 neighbors[0][0][0],
1003 neighbors[0][0][1],
1004 neighbors[0][1][0],
1005 neighbors[0][1][1],
1006 neighbors[1][0][0],
1007 neighbors[1][0][1],
1008 neighbors[1][1][0],
1009 neighbors[1][1][1],
1010 LP_BLD_LERP_PRESCALED_WEIGHTS);
1011 }
1012 }
1013
1014 *colors = packed;
1015 }
1016
1017 /**
1018 * Sample a single texture image with (bi-)(tri-)linear sampling.
1019 * Return filtered color as two vectors of 16-bit fixed point values.
1020 */
1021 static void
1022 lp_build_sample_image_linear(struct lp_build_sample_context *bld,
1023 LLVMValueRef int_size,
1024 LLVMValueRef row_stride_vec,
1025 LLVMValueRef img_stride_vec,
1026 LLVMValueRef data_ptr,
1027 LLVMValueRef mipoffsets,
1028 LLVMValueRef s,
1029 LLVMValueRef t,
1030 LLVMValueRef r,
1031 const LLVMValueRef *offsets,
1032 LLVMValueRef *colors)
1033 {
1034 const unsigned dims = bld->dims;
1035 LLVMBuilderRef builder = bld->gallivm->builder;
1036 struct lp_build_context i32;
1037 LLVMValueRef i32_c8, i32_c128, i32_c255;
1038 LLVMValueRef width_vec, height_vec, depth_vec;
1039 LLVMValueRef s_ipart, s_fpart, s_float;
1040 LLVMValueRef t_ipart = NULL, t_fpart = NULL, t_float = NULL;
1041 LLVMValueRef r_ipart = NULL, r_fpart = NULL, r_float = NULL;
1042 LLVMValueRef x_stride, y_stride, z_stride;
1043 LLVMValueRef x_offset0, x_offset1;
1044 LLVMValueRef y_offset0, y_offset1;
1045 LLVMValueRef z_offset0, z_offset1;
1046 LLVMValueRef offset[2][2][2]; /* [z][y][x] */
1047 LLVMValueRef x_subcoord[2], y_subcoord[2], z_subcoord[2];
1048 unsigned x, y, z;
1049
1050 lp_build_context_init(&i32, bld->gallivm, lp_type_int_vec(32, bld->vector_width));
1051
1052 lp_build_extract_image_sizes(bld,
1053 &bld->int_size_bld,
1054 bld->int_coord_type,
1055 int_size,
1056 &width_vec,
1057 &height_vec,
1058 &depth_vec);
1059
1060 s_float = s; t_float = t; r_float = r;
1061
1062 if (bld->static_sampler_state->normalized_coords) {
1063 LLVMValueRef scaled_size;
1064 LLVMValueRef flt_size;
1065
1066 /* scale size by 256 (8 fractional bits) */
1067 scaled_size = lp_build_shl_imm(&bld->int_size_bld, int_size, 8);
1068
1069 flt_size = lp_build_int_to_float(&bld->float_size_bld, scaled_size);
1070
1071 lp_build_unnormalized_coords(bld, flt_size, &s, &t, &r);
1072 }
1073 else {
1074 /* scale coords by 256 (8 fractional bits) */
1075 s = lp_build_mul_imm(&bld->coord_bld, s, 256);
1076 if (dims >= 2)
1077 t = lp_build_mul_imm(&bld->coord_bld, t, 256);
1078 if (dims >= 3)
1079 r = lp_build_mul_imm(&bld->coord_bld, r, 256);
1080 }
1081
1082 /* convert float to int */
1083 /* For correct rounding, need round to nearest, not truncation here.
1084 * Note that in some cases (clamp to edge, no texel offsets) we
1085 * could use a non-signed build context which would help archs which
1086 * don't have fptosi intrinsic with nearest rounding implemented.
1087 */
1088 s = lp_build_iround(&bld->coord_bld, s);
1089 if (dims >= 2)
1090 t = lp_build_iround(&bld->coord_bld, t);
1091 if (dims >= 3)
1092 r = lp_build_iround(&bld->coord_bld, r);
1093
1094 /* subtract 0.5 (add -128) */
1095 i32_c128 = lp_build_const_int_vec(bld->gallivm, i32.type, -128);
1096 if (!bld->static_sampler_state->force_nearest_s) {
1097 s = LLVMBuildAdd(builder, s, i32_c128, "");
1098 }
1099 if (dims >= 2 && !bld->static_sampler_state->force_nearest_t) {
1100 t = LLVMBuildAdd(builder, t, i32_c128, "");
1101 }
1102 if (dims >= 3) {
1103 r = LLVMBuildAdd(builder, r, i32_c128, "");
1104 }
1105
1106 /* compute floor (shift right 8) */
1107 i32_c8 = lp_build_const_int_vec(bld->gallivm, i32.type, 8);
1108 s_ipart = LLVMBuildAShr(builder, s, i32_c8, "");
1109 if (dims >= 2)
1110 t_ipart = LLVMBuildAShr(builder, t, i32_c8, "");
1111 if (dims >= 3)
1112 r_ipart = LLVMBuildAShr(builder, r, i32_c8, "");
1113
1114 /* add texel offsets */
1115 if (offsets[0]) {
1116 s_ipart = lp_build_add(&i32, s_ipart, offsets[0]);
1117 if (dims >= 2) {
1118 t_ipart = lp_build_add(&i32, t_ipart, offsets[1]);
1119 if (dims >= 3) {
1120 r_ipart = lp_build_add(&i32, r_ipart, offsets[2]);
1121 }
1122 }
1123 }
1124
1125 /* compute fractional part (AND with 0xff) */
1126 i32_c255 = lp_build_const_int_vec(bld->gallivm, i32.type, 255);
1127 s_fpart = LLVMBuildAnd(builder, s, i32_c255, "");
1128 if (dims >= 2)
1129 t_fpart = LLVMBuildAnd(builder, t, i32_c255, "");
1130 if (dims >= 3)
1131 r_fpart = LLVMBuildAnd(builder, r, i32_c255, "");
1132
1133 /* get pixel, row and image strides */
1134 x_stride = lp_build_const_vec(bld->gallivm, bld->int_coord_bld.type,
1135 bld->format_desc->block.bits/8);
1136 y_stride = row_stride_vec;
1137 z_stride = img_stride_vec;
1138
1139 /* do texcoord wrapping and compute texel offsets */
1140 lp_build_sample_wrap_linear_int(bld,
1141 bld->format_desc->block.width,
1142 s_ipart, &s_fpart, s_float,
1143 width_vec, x_stride, offsets[0],
1144 bld->static_texture_state->pot_width,
1145 bld->static_sampler_state->wrap_s,
1146 &x_offset0, &x_offset1,
1147 &x_subcoord[0], &x_subcoord[1]);
1148
1149 /* add potential cube/array/mip offsets now as they are constant per pixel */
1150 if (has_layer_coord(bld->static_texture_state->target)) {
1151 LLVMValueRef z_offset;
1152 z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec);
1153 /* The r coord is the cube face in [0,5] or array layer */
1154 x_offset0 = lp_build_add(&bld->int_coord_bld, x_offset0, z_offset);
1155 x_offset1 = lp_build_add(&bld->int_coord_bld, x_offset1, z_offset);
1156 }
1157 if (mipoffsets) {
1158 x_offset0 = lp_build_add(&bld->int_coord_bld, x_offset0, mipoffsets);
1159 x_offset1 = lp_build_add(&bld->int_coord_bld, x_offset1, mipoffsets);
1160 }
1161
1162 for (z = 0; z < 2; z++) {
1163 for (y = 0; y < 2; y++) {
1164 offset[z][y][0] = x_offset0;
1165 offset[z][y][1] = x_offset1;
1166 }
1167 }
1168
1169 if (dims >= 2) {
1170 lp_build_sample_wrap_linear_int(bld,
1171 bld->format_desc->block.height,
1172 t_ipart, &t_fpart, t_float,
1173 height_vec, y_stride, offsets[1],
1174 bld->static_texture_state->pot_height,
1175 bld->static_sampler_state->wrap_t,
1176 &y_offset0, &y_offset1,
1177 &y_subcoord[0], &y_subcoord[1]);
1178
1179 for (z = 0; z < 2; z++) {
1180 for (x = 0; x < 2; x++) {
1181 offset[z][0][x] = lp_build_add(&bld->int_coord_bld,
1182 offset[z][0][x], y_offset0);
1183 offset[z][1][x] = lp_build_add(&bld->int_coord_bld,
1184 offset[z][1][x], y_offset1);
1185 }
1186 }
1187 }
1188
1189 if (dims >= 3) {
1190 lp_build_sample_wrap_linear_int(bld,
1191 1, /* block length (depth) */
1192 r_ipart, &r_fpart, r_float,
1193 depth_vec, z_stride, offsets[2],
1194 bld->static_texture_state->pot_depth,
1195 bld->static_sampler_state->wrap_r,
1196 &z_offset0, &z_offset1,
1197 &z_subcoord[0], &z_subcoord[1]);
1198 for (y = 0; y < 2; y++) {
1199 for (x = 0; x < 2; x++) {
1200 offset[0][y][x] = lp_build_add(&bld->int_coord_bld,
1201 offset[0][y][x], z_offset0);
1202 offset[1][y][x] = lp_build_add(&bld->int_coord_bld,
1203 offset[1][y][x], z_offset1);
1204 }
1205 }
1206 }
1207
1208 lp_build_sample_fetch_image_linear(bld, data_ptr, offset,
1209 x_subcoord, y_subcoord,
1210 s_fpart, t_fpart, r_fpart,
1211 colors);
1212 }
1213
1214
1215 /**
1216 * Sample a single texture image with (bi-)(tri-)linear sampling.
1217 * Return filtered color as two vectors of 16-bit fixed point values.
1218 * Does address calcs (except offsets) with floats.
1219 * Useful for AVX which has support for 8x32 floats but not 8x32 ints.
1220 */
1221 static void
1222 lp_build_sample_image_linear_afloat(struct lp_build_sample_context *bld,
1223 LLVMValueRef int_size,
1224 LLVMValueRef row_stride_vec,
1225 LLVMValueRef img_stride_vec,
1226 LLVMValueRef data_ptr,
1227 LLVMValueRef mipoffsets,
1228 LLVMValueRef s,
1229 LLVMValueRef t,
1230 LLVMValueRef r,
1231 const LLVMValueRef *offsets,
1232 LLVMValueRef *colors)
1233 {
1234 const unsigned dims = bld->dims;
1235 LLVMValueRef width_vec, height_vec, depth_vec;
1236 LLVMValueRef s_fpart;
1237 LLVMValueRef t_fpart = NULL;
1238 LLVMValueRef r_fpart = NULL;
1239 LLVMValueRef x_stride, y_stride, z_stride;
1240 LLVMValueRef x_offset0, x_offset1;
1241 LLVMValueRef y_offset0, y_offset1;
1242 LLVMValueRef z_offset0, z_offset1;
1243 LLVMValueRef offset[2][2][2]; /* [z][y][x] */
1244 LLVMValueRef x_subcoord[2], y_subcoord[2];
1245 LLVMValueRef flt_size;
1246 LLVMValueRef x_icoord0, x_icoord1;
1247 LLVMValueRef y_icoord0, y_icoord1;
1248 LLVMValueRef z_icoord0, z_icoord1;
1249 unsigned x, y, z;
1250
1251 flt_size = lp_build_int_to_float(&bld->float_size_bld, int_size);
1252
1253 lp_build_extract_image_sizes(bld,
1254 &bld->float_size_bld,
1255 bld->coord_type,
1256 flt_size,
1257 &width_vec,
1258 &height_vec,
1259 &depth_vec);
1260
1261 /* do texcoord wrapping and compute texel offsets */
1262 lp_build_sample_wrap_linear_float(bld,
1263 bld->format_desc->block.width,
1264 s, width_vec, offsets[0],
1265 bld->static_texture_state->pot_width,
1266 bld->static_sampler_state->wrap_s,
1267 &x_icoord0, &x_icoord1,
1268 &s_fpart,
1269 bld->static_sampler_state->force_nearest_s);
1270
1271 if (dims >= 2) {
1272 lp_build_sample_wrap_linear_float(bld,
1273 bld->format_desc->block.height,
1274 t, height_vec, offsets[1],
1275 bld->static_texture_state->pot_height,
1276 bld->static_sampler_state->wrap_t,
1277 &y_icoord0, &y_icoord1,
1278 &t_fpart,
1279 bld->static_sampler_state->force_nearest_t);
1280
1281 if (dims >= 3) {
1282 lp_build_sample_wrap_linear_float(bld,
1283 1, /* block length (depth) */
1284 r, depth_vec, offsets[2],
1285 bld->static_texture_state->pot_depth,
1286 bld->static_sampler_state->wrap_r,
1287 &z_icoord0, &z_icoord1,
1288 &r_fpart, 0);
1289 }
1290 }
1291
1292 /*
1293 * From here on we deal with ints, and we should split up the 256bit
1294 * vectors manually for better generated code.
1295 */
1296
1297 /* get pixel, row and image strides */
1298 x_stride = lp_build_const_vec(bld->gallivm,
1299 bld->int_coord_bld.type,
1300 bld->format_desc->block.bits/8);
1301 y_stride = row_stride_vec;
1302 z_stride = img_stride_vec;
1303
1304 /*
1305 * compute texel offset -
1306 * cannot do offset calc with floats, difficult for block-based formats,
1307 * and not enough precision anyway.
1308 */
1309 lp_build_sample_partial_offset(&bld->int_coord_bld,
1310 bld->format_desc->block.width,
1311 x_icoord0, x_stride,
1312 &x_offset0, &x_subcoord[0]);
1313 lp_build_sample_partial_offset(&bld->int_coord_bld,
1314 bld->format_desc->block.width,
1315 x_icoord1, x_stride,
1316 &x_offset1, &x_subcoord[1]);
1317
1318 /* add potential cube/array/mip offsets now as they are constant per pixel */
1319 if (has_layer_coord(bld->static_texture_state->target)) {
1320 LLVMValueRef z_offset;
1321 z_offset = lp_build_mul(&bld->int_coord_bld, r, img_stride_vec);
1322 /* The r coord is the cube face in [0,5] or array layer */
1323 x_offset0 = lp_build_add(&bld->int_coord_bld, x_offset0, z_offset);
1324 x_offset1 = lp_build_add(&bld->int_coord_bld, x_offset1, z_offset);
1325 }
1326 if (mipoffsets) {
1327 x_offset0 = lp_build_add(&bld->int_coord_bld, x_offset0, mipoffsets);
1328 x_offset1 = lp_build_add(&bld->int_coord_bld, x_offset1, mipoffsets);
1329 }
1330
1331 for (z = 0; z < 2; z++) {
1332 for (y = 0; y < 2; y++) {
1333 offset[z][y][0] = x_offset0;
1334 offset[z][y][1] = x_offset1;
1335 }
1336 }
1337
1338 if (dims >= 2) {
1339 lp_build_sample_partial_offset(&bld->int_coord_bld,
1340 bld->format_desc->block.height,
1341 y_icoord0, y_stride,
1342 &y_offset0, &y_subcoord[0]);
1343 lp_build_sample_partial_offset(&bld->int_coord_bld,
1344 bld->format_desc->block.height,
1345 y_icoord1, y_stride,
1346 &y_offset1, &y_subcoord[1]);
1347 for (z = 0; z < 2; z++) {
1348 for (x = 0; x < 2; x++) {
1349 offset[z][0][x] = lp_build_add(&bld->int_coord_bld,
1350 offset[z][0][x], y_offset0);
1351 offset[z][1][x] = lp_build_add(&bld->int_coord_bld,
1352 offset[z][1][x], y_offset1);
1353 }
1354 }
1355 }
1356
1357 if (dims >= 3) {
1358 LLVMValueRef z_subcoord[2];
1359 lp_build_sample_partial_offset(&bld->int_coord_bld,
1360 1,
1361 z_icoord0, z_stride,
1362 &z_offset0, &z_subcoord[0]);
1363 lp_build_sample_partial_offset(&bld->int_coord_bld,
1364 1,
1365 z_icoord1, z_stride,
1366 &z_offset1, &z_subcoord[1]);
1367 for (y = 0; y < 2; y++) {
1368 for (x = 0; x < 2; x++) {
1369 offset[0][y][x] = lp_build_add(&bld->int_coord_bld,
1370 offset[0][y][x], z_offset0);
1371 offset[1][y][x] = lp_build_add(&bld->int_coord_bld,
1372 offset[1][y][x], z_offset1);
1373 }
1374 }
1375 }
1376
1377 lp_build_sample_fetch_image_linear(bld, data_ptr, offset,
1378 x_subcoord, y_subcoord,
1379 s_fpart, t_fpart, r_fpart,
1380 colors);
1381 }
1382
1383
1384 /**
1385 * Sample the texture/mipmap using given image filter and mip filter.
1386 * data0_ptr and data1_ptr point to the two mipmap levels to sample
1387 * from. width0/1_vec, height0/1_vec, depth0/1_vec indicate their sizes.
1388 * If we're using nearest miplevel sampling the '1' values will be null/unused.
1389 */
1390 static void
1391 lp_build_sample_mipmap(struct lp_build_sample_context *bld,
1392 unsigned img_filter,
1393 unsigned mip_filter,
1394 LLVMValueRef s,
1395 LLVMValueRef t,
1396 LLVMValueRef r,
1397 const LLVMValueRef *offsets,
1398 LLVMValueRef ilevel0,
1399 LLVMValueRef ilevel1,
1400 LLVMValueRef lod_fpart,
1401 LLVMValueRef colors_var)
1402 {
1403 LLVMBuilderRef builder = bld->gallivm->builder;
1404 LLVMValueRef size0;
1405 LLVMValueRef size1;
1406 LLVMValueRef row_stride0_vec = NULL;
1407 LLVMValueRef row_stride1_vec = NULL;
1408 LLVMValueRef img_stride0_vec = NULL;
1409 LLVMValueRef img_stride1_vec = NULL;
1410 LLVMValueRef data_ptr0;
1411 LLVMValueRef data_ptr1;
1412 LLVMValueRef mipoff0 = NULL;
1413 LLVMValueRef mipoff1 = NULL;
1414 LLVMValueRef colors0;
1415 LLVMValueRef colors1;
1416 boolean use_floats = util_cpu_caps.has_avx &&
1417 !util_cpu_caps.has_avx2 &&
1418 bld->coord_type.length > 4;
1419
1420 /* sample the first mipmap level */
1421 lp_build_mipmap_level_sizes(bld, ilevel0,
1422 &size0,
1423 &row_stride0_vec, &img_stride0_vec);
1424 if (bld->num_mips == 1) {
1425 data_ptr0 = lp_build_get_mipmap_level(bld, ilevel0);
1426 }
1427 else {
1428 /* This path should work for num_lods 1 too but slightly less efficient */
1429 data_ptr0 = bld->base_ptr;
1430 mipoff0 = lp_build_get_mip_offsets(bld, ilevel0);
1431 }
1432
1433 if (use_floats) {
1434 if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1435 lp_build_sample_image_nearest_afloat(bld,
1436 size0,
1437 row_stride0_vec, img_stride0_vec,
1438 data_ptr0, mipoff0, s, t, r, offsets,
1439 &colors0);
1440 }
1441 else {
1442 assert(img_filter == PIPE_TEX_FILTER_LINEAR);
1443 lp_build_sample_image_linear_afloat(bld,
1444 size0,
1445 row_stride0_vec, img_stride0_vec,
1446 data_ptr0, mipoff0, s, t, r, offsets,
1447 &colors0);
1448 }
1449 }
1450 else {
1451 if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1452 lp_build_sample_image_nearest(bld,
1453 size0,
1454 row_stride0_vec, img_stride0_vec,
1455 data_ptr0, mipoff0, s, t, r, offsets,
1456 &colors0);
1457 }
1458 else {
1459 assert(img_filter == PIPE_TEX_FILTER_LINEAR);
1460 lp_build_sample_image_linear(bld,
1461 size0,
1462 row_stride0_vec, img_stride0_vec,
1463 data_ptr0, mipoff0, s, t, r, offsets,
1464 &colors0);
1465 }
1466 }
1467
1468 /* Store the first level's colors in the output variables */
1469 LLVMBuildStore(builder, colors0, colors_var);
1470
1471 if (mip_filter == PIPE_TEX_MIPFILTER_LINEAR) {
1472 LLVMValueRef h16vec_scale = lp_build_const_vec(bld->gallivm,
1473 bld->lodf_bld.type, 256.0);
1474 LLVMTypeRef i32vec_type = bld->lodi_bld.vec_type;
1475 struct lp_build_if_state if_ctx;
1476 LLVMValueRef need_lerp;
1477 unsigned num_quads = bld->coord_bld.type.length / 4;
1478 unsigned i;
1479
1480 lod_fpart = LLVMBuildFMul(builder, lod_fpart, h16vec_scale, "");
1481 lod_fpart = LLVMBuildFPToSI(builder, lod_fpart, i32vec_type, "lod_fpart.fixed16");
1482
1483 /* need_lerp = lod_fpart > 0 */
1484 if (bld->num_lods == 1) {
1485 need_lerp = LLVMBuildICmp(builder, LLVMIntSGT,
1486 lod_fpart, bld->lodi_bld.zero,
1487 "need_lerp");
1488 }
1489 else {
1490 /*
1491 * We'll do mip filtering if any of the quads need it.
1492 * It might be better to split the vectors here and only fetch/filter
1493 * quads which need it.
1494 */
1495 /*
1496 * We need to clamp lod_fpart here since we can get negative
1497 * values which would screw up filtering if not all
1498 * lod_fpart values have same sign.
1499 * We can however then skip the greater than comparison.
1500 */
1501 lod_fpart = lp_build_max(&bld->lodi_bld, lod_fpart,
1502 bld->lodi_bld.zero);
1503 need_lerp = lp_build_any_true_range(&bld->lodi_bld, bld->num_lods, lod_fpart);
1504 }
1505
1506 lp_build_if(&if_ctx, bld->gallivm, need_lerp);
1507 {
1508 struct lp_build_context u8n_bld;
1509
1510 lp_build_context_init(&u8n_bld, bld->gallivm, lp_type_unorm(8, bld->vector_width));
1511
1512 /* sample the second mipmap level */
1513 lp_build_mipmap_level_sizes(bld, ilevel1,
1514 &size1,
1515 &row_stride1_vec, &img_stride1_vec);
1516 if (bld->num_mips == 1) {
1517 data_ptr1 = lp_build_get_mipmap_level(bld, ilevel1);
1518 }
1519 else {
1520 data_ptr1 = bld->base_ptr;
1521 mipoff1 = lp_build_get_mip_offsets(bld, ilevel1);
1522 }
1523
1524 if (use_floats) {
1525 if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1526 lp_build_sample_image_nearest_afloat(bld,
1527 size1,
1528 row_stride1_vec, img_stride1_vec,
1529 data_ptr1, mipoff1, s, t, r, offsets,
1530 &colors1);
1531 }
1532 else {
1533 lp_build_sample_image_linear_afloat(bld,
1534 size1,
1535 row_stride1_vec, img_stride1_vec,
1536 data_ptr1, mipoff1, s, t, r, offsets,
1537 &colors1);
1538 }
1539 }
1540 else {
1541 if (img_filter == PIPE_TEX_FILTER_NEAREST) {
1542 lp_build_sample_image_nearest(bld,
1543 size1,
1544 row_stride1_vec, img_stride1_vec,
1545 data_ptr1, mipoff1, s, t, r, offsets,
1546 &colors1);
1547 }
1548 else {
1549 lp_build_sample_image_linear(bld,
1550 size1,
1551 row_stride1_vec, img_stride1_vec,
1552 data_ptr1, mipoff1, s, t, r, offsets,
1553 &colors1);
1554 }
1555 }
1556
1557 /* interpolate samples from the two mipmap levels */
1558
1559 if (num_quads == 1 && bld->num_lods == 1) {
1560 lod_fpart = LLVMBuildTrunc(builder, lod_fpart, u8n_bld.elem_type, "");
1561 lod_fpart = lp_build_broadcast_scalar(&u8n_bld, lod_fpart);
1562 }
1563 else {
1564 unsigned num_chans_per_lod = 4 * bld->coord_type.length / bld->num_lods;
1565 LLVMTypeRef tmp_vec_type = LLVMVectorType(u8n_bld.elem_type, bld->lodi_bld.type.length);
1566 LLVMValueRef shuffle[LP_MAX_VECTOR_LENGTH];
1567
1568 /* Take the LSB of lod_fpart */
1569 lod_fpart = LLVMBuildTrunc(builder, lod_fpart, tmp_vec_type, "");
1570
1571 /* Broadcast each lod weight into their respective channels */
1572 for (i = 0; i < u8n_bld.type.length; ++i) {
1573 shuffle[i] = lp_build_const_int32(bld->gallivm, i / num_chans_per_lod);
1574 }
1575 lod_fpart = LLVMBuildShuffleVector(builder, lod_fpart, LLVMGetUndef(tmp_vec_type),
1576 LLVMConstVector(shuffle, u8n_bld.type.length), "");
1577 }
1578
1579 colors0 = lp_build_lerp(&u8n_bld, lod_fpart,
1580 colors0, colors1,
1581 LP_BLD_LERP_PRESCALED_WEIGHTS);
1582
1583 LLVMBuildStore(builder, colors0, colors_var);
1584 }
1585 lp_build_endif(&if_ctx);
1586 }
1587 }
1588
1589
1590
1591 /**
1592 * Texture sampling in AoS format. Used when sampling common 32-bit/texel
1593 * formats. 1D/2D/3D/cube texture supported. All mipmap sampling modes
1594 * but only limited texture coord wrap modes.
1595 */
1596 void
1597 lp_build_sample_aos(struct lp_build_sample_context *bld,
1598 unsigned sampler_unit,
1599 LLVMValueRef s,
1600 LLVMValueRef t,
1601 LLVMValueRef r,
1602 const LLVMValueRef *offsets,
1603 LLVMValueRef lod_positive,
1604 LLVMValueRef lod_fpart,
1605 LLVMValueRef ilevel0,
1606 LLVMValueRef ilevel1,
1607 LLVMValueRef texel_out[4])
1608 {
1609 LLVMBuilderRef builder = bld->gallivm->builder;
1610 const unsigned mip_filter = bld->static_sampler_state->min_mip_filter;
1611 const unsigned min_filter = bld->static_sampler_state->min_img_filter;
1612 const unsigned mag_filter = bld->static_sampler_state->mag_img_filter;
1613 const unsigned dims = bld->dims;
1614 LLVMValueRef packed_var, packed;
1615 LLVMValueRef unswizzled[4];
1616 struct lp_build_context u8n_bld;
1617
1618 /* we only support the common/simple wrap modes at this time */
1619 assert(lp_is_simple_wrap_mode(bld->static_sampler_state->wrap_s));
1620 if (dims >= 2)
1621 assert(lp_is_simple_wrap_mode(bld->static_sampler_state->wrap_t));
1622 if (dims >= 3)
1623 assert(lp_is_simple_wrap_mode(bld->static_sampler_state->wrap_r));
1624
1625
1626 /* make 8-bit unorm builder context */
1627 lp_build_context_init(&u8n_bld, bld->gallivm, lp_type_unorm(8, bld->vector_width));
1628
1629 /*
1630 * Get/interpolate texture colors.
1631 */
1632
1633 packed_var = lp_build_alloca(bld->gallivm, u8n_bld.vec_type, "packed_var");
1634
1635 if (min_filter == mag_filter) {
1636 /* no need to distinguish between minification and magnification */
1637 lp_build_sample_mipmap(bld,
1638 min_filter, mip_filter,
1639 s, t, r, offsets,
1640 ilevel0, ilevel1, lod_fpart,
1641 packed_var);
1642 }
1643 else {
1644 /* Emit conditional to choose min image filter or mag image filter
1645 * depending on the lod being > 0 or <= 0, respectively.
1646 */
1647 struct lp_build_if_state if_ctx;
1648
1649 /*
1650 * FIXME this should take all lods into account, if some are min
1651 * some max probably could hack up the weights in the linear
1652 * path with selects to work for nearest.
1653 */
1654 if (bld->num_lods > 1)
1655 lod_positive = LLVMBuildExtractElement(builder, lod_positive,
1656 lp_build_const_int32(bld->gallivm, 0), "");
1657
1658 lod_positive = LLVMBuildTrunc(builder, lod_positive,
1659 LLVMInt1TypeInContext(bld->gallivm->context), "");
1660
1661 lp_build_if(&if_ctx, bld->gallivm, lod_positive);
1662 {
1663 /* Use the minification filter */
1664 lp_build_sample_mipmap(bld,
1665 min_filter, mip_filter,
1666 s, t, r, offsets,
1667 ilevel0, ilevel1, lod_fpart,
1668 packed_var);
1669 }
1670 lp_build_else(&if_ctx);
1671 {
1672 /* Use the magnification filter */
1673 lp_build_sample_mipmap(bld,
1674 mag_filter, PIPE_TEX_MIPFILTER_NONE,
1675 s, t, r, offsets,
1676 ilevel0, NULL, NULL,
1677 packed_var);
1678 }
1679 lp_build_endif(&if_ctx);
1680 }
1681
1682 packed = LLVMBuildLoad(builder, packed_var, "");
1683
1684 /*
1685 * Convert to SoA and swizzle.
1686 */
1687 lp_build_rgba8_to_fi32_soa(bld->gallivm,
1688 bld->texel_type,
1689 packed, unswizzled);
1690
1691 if (util_format_is_rgba8_variant(bld->format_desc)) {
1692 lp_build_format_swizzle_soa(bld->format_desc,
1693 &bld->texel_bld,
1694 unswizzled, texel_out);
1695 }
1696 else {
1697 texel_out[0] = unswizzled[0];
1698 texel_out[1] = unswizzled[1];
1699 texel_out[2] = unswizzled[2];
1700 texel_out[3] = unswizzled[3];
1701 }
1702 }