1 /**************************************************************************
3 * Copyright 2007-2009 VMware, Inc.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21 * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
22 * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23 * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24 * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 **************************************************************************/
29 * Rasterization for binned triangles within a tile
33 #include "util/u_math.h"
35 #include "lp_debug_intrin.h"
37 #include "lp_rast_priv.h"
38 #include "lp_tile_soa.h"
44 * Shade all pixels in a 4x4 block.
47 block_full_4(struct lp_rasterizer_task
*task
,
48 const struct lp_rast_triangle
*tri
,
51 lp_rast_shade_quads_all(task
, &tri
->inputs
, x
, y
);
56 * Shade all pixels in a 16x16 block.
59 block_full_16(struct lp_rasterizer_task
*task
,
60 const struct lp_rast_triangle
*tri
,
66 for (iy
= 0; iy
< 16; iy
+= 4)
67 for (ix
= 0; ix
< 16; ix
+= 4)
68 block_full_4(task
, tri
, x
+ ix
, y
+ iy
);
71 #if !defined(PIPE_ARCH_SSE)
73 static INLINE
unsigned
74 build_mask_linear(int c
, int dcdx
, int dcdy
)
83 mask
|= ((c0
+ 0 * dcdx
) >> 31) & (1 << 0);
84 mask
|= ((c0
+ 1 * dcdx
) >> 31) & (1 << 1);
85 mask
|= ((c0
+ 2 * dcdx
) >> 31) & (1 << 2);
86 mask
|= ((c0
+ 3 * dcdx
) >> 31) & (1 << 3);
87 mask
|= ((c1
+ 0 * dcdx
) >> 31) & (1 << 4);
88 mask
|= ((c1
+ 1 * dcdx
) >> 31) & (1 << 5);
89 mask
|= ((c1
+ 2 * dcdx
) >> 31) & (1 << 6);
90 mask
|= ((c1
+ 3 * dcdx
) >> 31) & (1 << 7);
91 mask
|= ((c2
+ 0 * dcdx
) >> 31) & (1 << 8);
92 mask
|= ((c2
+ 1 * dcdx
) >> 31) & (1 << 9);
93 mask
|= ((c2
+ 2 * dcdx
) >> 31) & (1 << 10);
94 mask
|= ((c2
+ 3 * dcdx
) >> 31) & (1 << 11);
95 mask
|= ((c3
+ 0 * dcdx
) >> 31) & (1 << 12);
96 mask
|= ((c3
+ 1 * dcdx
) >> 31) & (1 << 13);
97 mask
|= ((c3
+ 2 * dcdx
) >> 31) & (1 << 14);
98 mask
|= ((c3
+ 3 * dcdx
) >> 31) & (1 << 15);
112 *outmask
|= build_mask_linear(c
, dcdx
, dcdy
);
113 *partmask
|= build_mask_linear(c
+ cdiff
, dcdx
, dcdy
);
117 lp_rast_triangle_3_16(struct lp_rasterizer_task
*task
,
118 const union lp_rast_cmd_arg arg
)
120 union lp_rast_cmd_arg arg2
;
121 arg2
.triangle
.tri
= arg
.triangle
.tri
;
122 arg2
.triangle
.plane_mask
= (1<<3)-1;
123 lp_rast_triangle_3(task
, arg2
);
127 lp_rast_triangle_4_16(struct lp_rasterizer_task
*task
,
128 const union lp_rast_cmd_arg arg
)
130 union lp_rast_cmd_arg arg2
;
131 arg2
.triangle
.tri
= arg
.triangle
.tri
;
132 arg2
.triangle
.plane_mask
= (1<<4)-1;
133 lp_rast_triangle_3(task
, arg2
);
137 lp_rast_triangle_3_4(struct lp_rasterizer_task
*task
,
138 const union lp_rast_cmd_arg arg
)
140 lp_rast_triangle_3_16(task
, arg
);
144 #include <emmintrin.h>
145 #include "util/u_sse.h"
156 __m128i cstep0
= _mm_setr_epi32(c
, c
+dcdx
, c
+dcdx
*2, c
+dcdx
*3);
157 __m128i xdcdy
= _mm_set1_epi32(dcdy
);
159 /* Get values across the quad
161 __m128i cstep1
= _mm_add_epi32(cstep0
, xdcdy
);
162 __m128i cstep2
= _mm_add_epi32(cstep1
, xdcdy
);
163 __m128i cstep3
= _mm_add_epi32(cstep2
, xdcdy
);
166 __m128i cstep01
, cstep23
, result
;
168 cstep01
= _mm_packs_epi32(cstep0
, cstep1
);
169 cstep23
= _mm_packs_epi32(cstep2
, cstep3
);
170 result
= _mm_packs_epi16(cstep01
, cstep23
);
172 *outmask
|= _mm_movemask_epi8(result
);
177 __m128i cio4
= _mm_set1_epi32(cdiff
);
178 __m128i cstep01
, cstep23
, result
;
180 cstep0
= _mm_add_epi32(cstep0
, cio4
);
181 cstep1
= _mm_add_epi32(cstep1
, cio4
);
182 cstep2
= _mm_add_epi32(cstep2
, cio4
);
183 cstep3
= _mm_add_epi32(cstep3
, cio4
);
185 cstep01
= _mm_packs_epi32(cstep0
, cstep1
);
186 cstep23
= _mm_packs_epi32(cstep2
, cstep3
);
187 result
= _mm_packs_epi16(cstep01
, cstep23
);
189 *partmask
|= _mm_movemask_epi8(result
);
194 static INLINE
unsigned
195 build_mask_linear(int c
, int dcdx
, int dcdy
)
197 __m128i cstep0
= _mm_setr_epi32(c
, c
+dcdx
, c
+dcdx
*2, c
+dcdx
*3);
198 __m128i xdcdy
= _mm_set1_epi32(dcdy
);
200 /* Get values across the quad
202 __m128i cstep1
= _mm_add_epi32(cstep0
, xdcdy
);
203 __m128i cstep2
= _mm_add_epi32(cstep1
, xdcdy
);
204 __m128i cstep3
= _mm_add_epi32(cstep2
, xdcdy
);
206 /* pack pairs of results into epi16
208 __m128i cstep01
= _mm_packs_epi32(cstep0
, cstep1
);
209 __m128i cstep23
= _mm_packs_epi32(cstep2
, cstep3
);
211 /* pack into epi8, preserving sign bits
213 __m128i result
= _mm_packs_epi16(cstep01
, cstep23
);
215 /* extract sign bits to create mask
217 return _mm_movemask_epi8(result
);
220 static INLINE
unsigned
221 sign_bits4(const __m128i
*cstep
, int cdiff
)
224 /* Adjust the step values
226 __m128i cio4
= _mm_set1_epi32(cdiff
);
227 __m128i cstep0
= _mm_add_epi32(cstep
[0], cio4
);
228 __m128i cstep1
= _mm_add_epi32(cstep
[1], cio4
);
229 __m128i cstep2
= _mm_add_epi32(cstep
[2], cio4
);
230 __m128i cstep3
= _mm_add_epi32(cstep
[3], cio4
);
234 __m128i cstep01
= _mm_packs_epi32(cstep0
, cstep1
);
235 __m128i cstep23
= _mm_packs_epi32(cstep2
, cstep3
);
236 __m128i result
= _mm_packs_epi16(cstep01
, cstep23
);
238 /* Extract the sign bits
240 return _mm_movemask_epi8(result
);
250 #include "lp_rast_tri_tmp.h"
254 #include "lp_rast_tri_tmp.h"
258 /*#define TRI_4 lp_rast_triangle_3_4*/
259 /*#define TRI_16 lp_rast_triangle_3_16*/
260 #include "lp_rast_tri_tmp.h"
264 #define TRI_16 lp_rast_triangle_4_16
265 #include "lp_rast_tri_tmp.h"
269 #include "lp_rast_tri_tmp.h"
273 #include "lp_rast_tri_tmp.h"
277 #include "lp_rast_tri_tmp.h"
281 #include "lp_rast_tri_tmp.h"
285 transpose4_epi32(const __m128i
* restrict a
,
286 const __m128i
* restrict b
,
287 const __m128i
* restrict c
,
288 const __m128i
* restrict d
,
289 __m128i
* restrict o
,
290 __m128i
* restrict p
,
291 __m128i
* restrict q
,
292 __m128i
* restrict r
)
294 __m128i t0
= _mm_unpacklo_epi32(*a
, *b
);
295 __m128i t1
= _mm_unpacklo_epi32(*c
, *d
);
296 __m128i t2
= _mm_unpackhi_epi32(*a
, *b
);
297 __m128i t3
= _mm_unpackhi_epi32(*c
, *d
);
299 *o
= _mm_unpacklo_epi64(t0
, t1
);
300 *p
= _mm_unpackhi_epi64(t0
, t1
);
301 *q
= _mm_unpacklo_epi64(t2
, t3
);
302 *r
= _mm_unpackhi_epi64(t2
, t3
);
306 #define SCALAR_EPI32(m, i) _mm_shuffle_epi32((m), _MM_SHUFFLE(i,i,i,i))
312 /* Provide an SSE2 implementation of _mm_mullo_epi32() in terms of
315 * I suspect this works fine for us because one of our operands is
316 * always positive, but not sure that this can be used for general
317 * signed integer multiplication.
319 * This seems close enough to the speed of SSE4 and the real
320 * _mm_mullo_epi32() intrinsic as to not justify adding an sse4
321 * dependency at this point.
323 static INLINE __m128i
mm_mullo_epi32(const __m128i a
, const __m128i b
)
325 __m128i a4
= _mm_srli_si128(a
, 4); /* shift by one dword */
326 __m128i b4
= _mm_srli_si128(b
, 4); /* shift by one dword */
327 __m128i ba
= _mm_mul_epu32(b
, a
); /* multply dwords 0, 2 */
328 __m128i b4a4
= _mm_mul_epu32(b4
, a4
); /* multiply dwords 1, 3 */
330 /* Interleave the results, either with shuffles or (slightly
331 * faster) direct bit operations:
334 __m128i ba8
= _mm_shuffle_epi32(ba
, 8);
335 __m128i b4a48
= _mm_shuffle_epi32(b4a4
, 8);
336 __m128i result
= _mm_unpacklo_epi32(ba8
, b4a48
);
338 __m128i mask
= _mm_setr_epi32(~0,0,~0,0);
339 __m128i ba_mask
= _mm_and_si128(ba
, mask
);
340 __m128i b4a4_mask
= _mm_and_si128(b4a4
, mask
);
341 __m128i b4a4_mask_shift
= _mm_slli_si128(b4a4_mask
, 4);
342 __m128i result
= _mm_or_si128(ba_mask
, b4a4_mask_shift
);
352 lp_rast_triangle_3_16(struct lp_rasterizer_task
*task
,
353 const union lp_rast_cmd_arg arg
)
355 const struct lp_rast_triangle
*tri
= arg
.triangle
.tri
;
356 const struct lp_rast_plane
*plane
= tri
->plane
;
357 int x
= (arg
.triangle
.plane_mask
& 0xff) + task
->x
;
358 int y
= (arg
.triangle
.plane_mask
>> 8) + task
->y
;
361 struct { unsigned mask
:16; unsigned i
:8; unsigned j
:8; } out
[16];
364 __m128i p0
= _mm_loadu_si128((__m128i
*)&plane
[0]); /* c, dcdx, dcdy, eo */
365 __m128i p1
= _mm_loadu_si128((__m128i
*)&plane
[1]); /* c, dcdx, dcdy, eo */
366 __m128i p2
= _mm_loadu_si128((__m128i
*)&plane
[2]); /* c, dcdx, dcdy, eo */
367 __m128i zero
= _mm_setzero_si128();
377 __m128i span_0
; /* 0,dcdx,2dcdx,3dcdx for plane 0 */
378 __m128i span_1
; /* 0,dcdx,2dcdx,3dcdx for plane 1 */
379 __m128i span_2
; /* 0,dcdx,2dcdx,3dcdx for plane 2 */
382 transpose4_epi32(&p0
, &p1
, &p2
, &zero
,
383 &c
, &dcdx
, &dcdy
, &rej4
);
387 dcdx
= _mm_sub_epi32(zero
, dcdx
);
389 c
= _mm_add_epi32(c
, mm_mullo_epi32(dcdx
, _mm_set1_epi32(x
)));
390 c
= _mm_add_epi32(c
, mm_mullo_epi32(dcdy
, _mm_set1_epi32(y
)));
391 rej4
= _mm_slli_epi32(rej4
, 2);
393 dcdx2
= _mm_add_epi32(dcdx
, dcdx
);
394 dcdx3
= _mm_add_epi32(dcdx2
, dcdx
);
396 transpose4_epi32(&zero
, &dcdx
, &dcdx2
, &dcdx3
,
397 &span_0
, &span_1
, &span_2
, &unused
);
399 for (i
= 0; i
< 4; i
++) {
402 for (j
= 0; j
< 4; j
++) {
403 __m128i c4rej
= _mm_add_epi32(cx
, rej4
);
404 __m128i rej_masks
= _mm_srai_epi32(c4rej
, 31);
406 /* if (is_zero(rej_masks)) */
407 if (_mm_movemask_epi8(rej_masks
) == 0) {
408 __m128i c0_0
= _mm_add_epi32(SCALAR_EPI32(cx
, 0), span_0
);
409 __m128i c1_0
= _mm_add_epi32(SCALAR_EPI32(cx
, 1), span_1
);
410 __m128i c2_0
= _mm_add_epi32(SCALAR_EPI32(cx
, 2), span_2
);
412 __m128i c_0
= _mm_or_si128(_mm_or_si128(c0_0
, c1_0
), c2_0
);
414 __m128i c0_1
= _mm_add_epi32(c0_0
, SCALAR_EPI32(dcdy
, 0));
415 __m128i c1_1
= _mm_add_epi32(c1_0
, SCALAR_EPI32(dcdy
, 1));
416 __m128i c2_1
= _mm_add_epi32(c2_0
, SCALAR_EPI32(dcdy
, 2));
418 __m128i c_1
= _mm_or_si128(_mm_or_si128(c0_1
, c1_1
), c2_1
);
419 __m128i c_01
= _mm_packs_epi32(c_0
, c_1
);
421 __m128i c0_2
= _mm_add_epi32(c0_1
, SCALAR_EPI32(dcdy
, 0));
422 __m128i c1_2
= _mm_add_epi32(c1_1
, SCALAR_EPI32(dcdy
, 1));
423 __m128i c2_2
= _mm_add_epi32(c2_1
, SCALAR_EPI32(dcdy
, 2));
425 __m128i c_2
= _mm_or_si128(_mm_or_si128(c0_2
, c1_2
), c2_2
);
427 __m128i c0_3
= _mm_add_epi32(c0_2
, SCALAR_EPI32(dcdy
, 0));
428 __m128i c1_3
= _mm_add_epi32(c1_2
, SCALAR_EPI32(dcdy
, 1));
429 __m128i c2_3
= _mm_add_epi32(c2_2
, SCALAR_EPI32(dcdy
, 2));
431 __m128i c_3
= _mm_or_si128(_mm_or_si128(c0_3
, c1_3
), c2_3
);
432 __m128i c_23
= _mm_packs_epi32(c_2
, c_3
);
433 __m128i c_0123
= _mm_packs_epi16(c_01
, c_23
);
435 unsigned mask
= _mm_movemask_epi8(c_0123
);
443 cx
= _mm_add_epi32(cx
, _mm_slli_epi32(dcdx
, 2));
446 c
= _mm_add_epi32(c
, _mm_slli_epi32(dcdy
, 2));
449 for (i
= 0; i
< nr
; i
++)
450 lp_rast_shade_quads_mask(task
,
454 0xffff & ~out
[i
].mask
);
462 lp_rast_triangle_3_4(struct lp_rasterizer_task
*task
,
463 const union lp_rast_cmd_arg arg
)
465 const struct lp_rast_triangle
*tri
= arg
.triangle
.tri
;
466 const struct lp_rast_plane
*plane
= tri
->plane
;
467 int x
= (arg
.triangle
.plane_mask
& 0xff) + task
->x
;
468 int y
= (arg
.triangle
.plane_mask
>> 8) + task
->y
;
470 __m128i p0
= _mm_loadu_si128((__m128i
*)&plane
[0]); /* c, dcdx, dcdy, eo */
471 __m128i p1
= _mm_loadu_si128((__m128i
*)&plane
[1]); /* c, dcdx, dcdy, eo */
472 __m128i p2
= _mm_loadu_si128((__m128i
*)&plane
[2]); /* c, dcdx, dcdy, eo */
473 __m128i zero
= _mm_setzero_si128();
482 __m128i span_0
; /* 0,dcdx,2dcdx,3dcdx for plane 0 */
483 __m128i span_1
; /* 0,dcdx,2dcdx,3dcdx for plane 1 */
484 __m128i span_2
; /* 0,dcdx,2dcdx,3dcdx for plane 2 */
487 transpose4_epi32(&p0
, &p1
, &p2
, &zero
,
488 &c
, &dcdx
, &dcdy
, &unused
);
492 dcdx
= _mm_sub_epi32(zero
, dcdx
);
494 c
= _mm_add_epi32(c
, mm_mullo_epi32(dcdx
, _mm_set1_epi32(x
)));
495 c
= _mm_add_epi32(c
, mm_mullo_epi32(dcdy
, _mm_set1_epi32(y
)));
497 dcdx2
= _mm_add_epi32(dcdx
, dcdx
);
498 dcdx3
= _mm_add_epi32(dcdx2
, dcdx
);
500 transpose4_epi32(&zero
, &dcdx
, &dcdx2
, &dcdx3
,
501 &span_0
, &span_1
, &span_2
, &unused
);
505 __m128i c0_0
= _mm_add_epi32(SCALAR_EPI32(c
, 0), span_0
);
506 __m128i c1_0
= _mm_add_epi32(SCALAR_EPI32(c
, 1), span_1
);
507 __m128i c2_0
= _mm_add_epi32(SCALAR_EPI32(c
, 2), span_2
);
509 __m128i c_0
= _mm_or_si128(_mm_or_si128(c0_0
, c1_0
), c2_0
);
511 __m128i c0_1
= _mm_add_epi32(c0_0
, SCALAR_EPI32(dcdy
, 0));
512 __m128i c1_1
= _mm_add_epi32(c1_0
, SCALAR_EPI32(dcdy
, 1));
513 __m128i c2_1
= _mm_add_epi32(c2_0
, SCALAR_EPI32(dcdy
, 2));
515 __m128i c_1
= _mm_or_si128(_mm_or_si128(c0_1
, c1_1
), c2_1
);
516 __m128i c_01
= _mm_packs_epi32(c_0
, c_1
);
518 __m128i c0_2
= _mm_add_epi32(c0_1
, SCALAR_EPI32(dcdy
, 0));
519 __m128i c1_2
= _mm_add_epi32(c1_1
, SCALAR_EPI32(dcdy
, 1));
520 __m128i c2_2
= _mm_add_epi32(c2_1
, SCALAR_EPI32(dcdy
, 2));
522 __m128i c_2
= _mm_or_si128(_mm_or_si128(c0_2
, c1_2
), c2_2
);
524 __m128i c0_3
= _mm_add_epi32(c0_2
, SCALAR_EPI32(dcdy
, 0));
525 __m128i c1_3
= _mm_add_epi32(c1_2
, SCALAR_EPI32(dcdy
, 1));
526 __m128i c2_3
= _mm_add_epi32(c2_2
, SCALAR_EPI32(dcdy
, 2));
528 __m128i c_3
= _mm_or_si128(_mm_or_si128(c0_3
, c1_3
), c2_3
);
529 __m128i c_23
= _mm_packs_epi32(c_2
, c_3
);
530 __m128i c_0123
= _mm_packs_epi16(c_01
, c_23
);
532 unsigned mask
= _mm_movemask_epi8(c_0123
);
535 lp_rast_shade_quads_mask(task
,