2 * Mesa 3-D graphics library
5 * Copyright (C) 1999-2004 Brian Paul All Rights Reserved.
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the "Software"),
9 * to deal in the Software without restriction, including without limitation
10 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 * and/or sell copies of the Software, and to permit persons to whom the
12 * Software is furnished to do so, subject to the following conditions:
14 * The above copyright notice and this permission notice shall be included
15 * in all copies or substantial portions of the Software.
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18 * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20 * BRIAN PAUL BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN
21 * AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
22 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26 * Translate tgsi vertex programs to x86/x87/SSE/SSE2 machine code
27 * using the rtasm runtime assembler. Based on the old
28 * t_vb_arb_program_sse.c
32 #include "util/u_memory.h"
33 #include "util/u_math.h"
34 #include "pipe/p_shader_tokens.h"
35 #include "pipe/p_debug.h"
36 #include "tgsi/tgsi_parse.h"
37 #include "tgsi/tgsi_util.h"
38 #include "tgsi/tgsi_exec.h"
39 #include "tgsi/tgsi_dump.h"
42 #include "draw_vs_aos.h"
44 #include "rtasm/rtasm_x86sse.h"
50 static const char *files
[] =
63 static INLINE boolean
eq( struct x86_reg a
,
66 return (a
.file
== b
.file
&&
72 struct x86_reg
aos_get_x86( struct aos_compilation
*cp
,
73 unsigned which_reg
, /* quick hack */
83 if (cp
->x86_reg
[which_reg
] != value
) {
88 assert(which_reg
== 0);
89 offset
= Offset(struct aos_machine
, immediates
);
92 assert(which_reg
== 1);
93 offset
= Offset(struct aos_machine
, constants
);
96 assert(which_reg
== 0);
97 offset
= Offset(struct aos_machine
, buffer
);
105 x86_mov(cp
->func
, reg
,
106 x86_make_disp(cp
->machine_EDX
, offset
));
108 cp
->x86_reg
[which_reg
] = value
;
115 static struct x86_reg
get_reg_ptr(struct aos_compilation
*cp
,
119 struct x86_reg ptr
= cp
->machine_EDX
;
122 case TGSI_FILE_INPUT
:
123 assert(idx
< MAX_INPUTS
);
124 return x86_make_disp(ptr
, Offset(struct aos_machine
, input
[idx
]));
126 case TGSI_FILE_OUTPUT
:
127 return x86_make_disp(ptr
, Offset(struct aos_machine
, output
[idx
]));
129 case TGSI_FILE_TEMPORARY
:
130 assert(idx
< MAX_TEMPS
);
131 return x86_make_disp(ptr
, Offset(struct aos_machine
, temp
[idx
]));
133 case AOS_FILE_INTERNAL
:
134 assert(idx
< MAX_INTERNALS
);
135 return x86_make_disp(ptr
, Offset(struct aos_machine
, internal
[idx
]));
137 case TGSI_FILE_IMMEDIATE
:
138 assert(idx
< MAX_IMMEDIATES
); /* just a sanity check */
139 return x86_make_disp(aos_get_x86(cp
, 0, X86_IMMEDIATES
), idx
* 4 * sizeof(float));
141 case TGSI_FILE_CONSTANT
:
142 assert(idx
< MAX_CONSTANTS
); /* just a sanity check */
143 return x86_make_disp(aos_get_x86(cp
, 1, X86_CONSTANTS
), idx
* 4 * sizeof(float));
146 ERROR(cp
, "unknown reg file");
147 return x86_make_reg(0,0);
153 #define X87_CW_EXCEPTION_INV_OP (1<<0)
154 #define X87_CW_EXCEPTION_DENORM_OP (1<<1)
155 #define X87_CW_EXCEPTION_ZERO_DIVIDE (1<<2)
156 #define X87_CW_EXCEPTION_OVERFLOW (1<<3)
157 #define X87_CW_EXCEPTION_UNDERFLOW (1<<4)
158 #define X87_CW_EXCEPTION_PRECISION (1<<5)
159 #define X87_CW_PRECISION_SINGLE (0<<8)
160 #define X87_CW_PRECISION_RESERVED (1<<8)
161 #define X87_CW_PRECISION_DOUBLE (2<<8)
162 #define X87_CW_PRECISION_DOUBLE_EXT (3<<8)
163 #define X87_CW_PRECISION_MASK (3<<8)
164 #define X87_CW_ROUND_NEAREST (0<<10)
165 #define X87_CW_ROUND_DOWN (1<<10)
166 #define X87_CW_ROUND_UP (2<<10)
167 #define X87_CW_ROUND_ZERO (3<<10)
168 #define X87_CW_ROUND_MASK (3<<10)
169 #define X87_CW_INFINITY (1<<12)
174 static void spill( struct aos_compilation
*cp
, unsigned idx
)
176 if (!cp
->xmm
[idx
].dirty
||
177 (cp
->xmm
[idx
].file
!= TGSI_FILE_INPUT
&& /* inputs are fetched into xmm & set dirty */
178 cp
->xmm
[idx
].file
!= TGSI_FILE_OUTPUT
&&
179 cp
->xmm
[idx
].file
!= TGSI_FILE_TEMPORARY
)) {
180 ERROR(cp
, "invalid spill");
184 struct x86_reg oldval
= get_reg_ptr(cp
,
188 if (0) debug_printf("\nspill %s[%d]",
189 files
[cp
->xmm
[idx
].file
],
192 assert(cp
->xmm
[idx
].dirty
);
193 sse_movaps(cp
->func
, oldval
, x86_make_reg(file_XMM
, idx
));
194 cp
->xmm
[idx
].dirty
= 0;
199 void aos_spill_all( struct aos_compilation
*cp
)
203 for (i
= 0; i
< 8; i
++) {
204 if (cp
->xmm
[i
].dirty
)
206 aos_release_xmm_reg(cp
, i
);
211 static struct x86_reg
get_xmm_writable( struct aos_compilation
*cp
,
214 if (reg
.file
!= file_XMM
||
215 cp
->xmm
[reg
.idx
].file
!= TGSI_FILE_NULL
)
217 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
218 sse_movaps(cp
->func
, tmp
, reg
);
222 cp
->xmm
[reg
.idx
].last_used
= cp
->insn_counter
;
226 static struct x86_reg
get_xmm( struct aos_compilation
*cp
,
229 if (reg
.file
!= file_XMM
)
231 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
232 sse_movaps(cp
->func
, tmp
, reg
);
236 cp
->xmm
[reg
.idx
].last_used
= cp
->insn_counter
;
241 /* Allocate an empty xmm register, either as a temporary or later to
242 * "adopt" as a shader reg.
244 struct x86_reg
aos_get_xmm_reg( struct aos_compilation
*cp
)
248 boolean found
= FALSE
;
250 for (i
= 0; i
< 8; i
++)
251 if (cp
->xmm
[i
].last_used
!= cp
->insn_counter
&&
252 cp
->xmm
[i
].file
== TGSI_FILE_NULL
) {
258 for (i
= 0; i
< 8; i
++)
259 if (cp
->xmm
[i
].last_used
< cp
->xmm
[oldest
].last_used
)
263 /* Need to write out the old value?
265 if (cp
->xmm
[oldest
].dirty
)
268 assert(cp
->xmm
[oldest
].last_used
!= cp
->insn_counter
);
270 cp
->xmm
[oldest
].file
= TGSI_FILE_NULL
;
271 cp
->xmm
[oldest
].idx
= 0;
272 cp
->xmm
[oldest
].dirty
= 0;
273 cp
->xmm
[oldest
].last_used
= cp
->insn_counter
;
274 return x86_make_reg(file_XMM
, oldest
);
277 void aos_release_xmm_reg( struct aos_compilation
*cp
,
280 cp
->xmm
[idx
].file
= TGSI_FILE_NULL
;
281 cp
->xmm
[idx
].idx
= 0;
282 cp
->xmm
[idx
].dirty
= 0;
283 cp
->xmm
[idx
].last_used
= 0;
289 /* Mark an xmm reg as holding the current copy of a shader reg.
291 void aos_adopt_xmm_reg( struct aos_compilation
*cp
,
299 if (reg
.file
!= file_XMM
) {
305 /* If any xmm reg thinks it holds this shader reg, break the
308 for (i
= 0; i
< 8; i
++) {
309 if (cp
->xmm
[i
].file
== file
&&
310 cp
->xmm
[i
].idx
== idx
)
312 /* If an xmm reg is already holding this shader reg, take into account its
315 dirty
|= cp
->xmm
[i
].dirty
;
316 aos_release_xmm_reg(cp
, i
);
320 cp
->xmm
[reg
.idx
].file
= file
;
321 cp
->xmm
[reg
.idx
].idx
= idx
;
322 cp
->xmm
[reg
.idx
].dirty
= dirty
;
323 cp
->xmm
[reg
.idx
].last_used
= cp
->insn_counter
;
327 /* Return a pointer to the in-memory copy of the reg, making sure it is uptodate.
329 static struct x86_reg
aos_get_shader_reg_ptr( struct aos_compilation
*cp
,
335 /* Ensure the in-memory copy of this reg is up-to-date
337 for (i
= 0; i
< 8; i
++) {
338 if (cp
->xmm
[i
].file
== file
&&
339 cp
->xmm
[i
].idx
== idx
&&
345 return get_reg_ptr( cp
, file
, idx
);
349 /* As above, but return a pointer. Note - this pointer may alias
350 * those returned by get_arg_ptr().
352 static struct x86_reg
get_dst_ptr( struct aos_compilation
*cp
,
353 const struct tgsi_full_dst_register
*dst
)
355 unsigned file
= dst
->DstRegister
.File
;
356 unsigned idx
= dst
->DstRegister
.Index
;
360 /* Ensure in-memory copy of this reg is up-to-date and invalidate
363 for (i
= 0; i
< 8; i
++) {
364 if (cp
->xmm
[i
].file
== file
&&
365 cp
->xmm
[i
].idx
== idx
)
367 if (cp
->xmm
[i
].dirty
)
370 aos_release_xmm_reg(cp
, i
);
374 return get_reg_ptr( cp
, file
, idx
);
381 /* Return an XMM reg if the argument is resident, otherwise return a
382 * base+offset pointer to the saved value.
384 struct x86_reg
aos_get_shader_reg( struct aos_compilation
*cp
,
390 for (i
= 0; i
< 8; i
++) {
391 if (cp
->xmm
[i
].file
== file
&&
392 cp
->xmm
[i
].idx
== idx
)
394 cp
->xmm
[i
].last_used
= cp
->insn_counter
;
395 return x86_make_reg(file_XMM
, i
);
399 /* If not found in the XMM register file, return an indirect
400 * reference to the in-memory copy:
402 return get_reg_ptr( cp
, file
, idx
);
407 static struct x86_reg
aos_get_shader_reg_xmm( struct aos_compilation
*cp
,
411 struct x86_reg reg
= get_xmm( cp
,
412 aos_get_shader_reg( cp
, file
, idx
) );
414 aos_adopt_xmm_reg( cp
,
425 struct x86_reg
aos_get_internal_xmm( struct aos_compilation
*cp
,
428 return aos_get_shader_reg_xmm( cp
, AOS_FILE_INTERNAL
, imm
);
432 struct x86_reg
aos_get_internal( struct aos_compilation
*cp
,
435 return aos_get_shader_reg( cp
, AOS_FILE_INTERNAL
, imm
);
442 /* Emulate pshufd insn in regular SSE, if necessary:
444 static void emit_pshufd( struct aos_compilation
*cp
,
450 sse2_pshufd(cp
->func
, dst
, arg0
, shuf
);
454 sse_movaps(cp
->func
, dst
, arg0
);
456 sse_shufps(cp
->func
, dst
, dst
, shuf
);
460 /* load masks (pack into negs??)
461 * pshufd - shuffle according to writemask
466 static boolean
mask_write( struct aos_compilation
*cp
,
468 struct x86_reg result
,
471 struct x86_reg imm_swz
= aos_get_internal_xmm(cp
, IMM_SWZ
);
472 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
474 emit_pshufd(cp
, tmp
, imm_swz
,
475 SHUF((mask
& 1) ? 2 : 3,
478 (mask
& 8) ? 2 : 3));
480 sse_andps(cp
->func
, dst
, tmp
);
481 sse_andnps(cp
->func
, tmp
, result
);
482 sse_orps(cp
->func
, dst
, tmp
);
484 aos_release_xmm_reg(cp
, tmp
.idx
);
491 /* Helper for writemask:
493 static boolean
emit_shuf_copy2( struct aos_compilation
*cp
,
499 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
501 emit_pshufd(cp
, dst
, arg1
, shuf
);
502 emit_pshufd(cp
, tmp
, arg0
, shuf
);
503 sse_shufps(cp
->func
, dst
, tmp
, SHUF(X
, Y
, Z
, W
));
504 emit_pshufd(cp
, dst
, dst
, shuf
);
506 aos_release_xmm_reg(cp
, tmp
.idx
);
512 #define SSE_SWIZZLE_NOOP ((0<<0) | (1<<2) | (2<<4) | (3<<6))
515 /* Locate a source register and perform any required (simple) swizzle.
517 * Just fail on complex swizzles at this point.
519 static struct x86_reg
fetch_src( struct aos_compilation
*cp
,
520 const struct tgsi_full_src_register
*src
)
522 struct x86_reg arg0
= aos_get_shader_reg(cp
,
523 src
->SrcRegister
.File
,
524 src
->SrcRegister
.Index
);
530 for (i
= 0; i
< 4; i
++) {
531 unsigned swizzle
= tgsi_util_get_full_src_register_extswizzle( src
, i
);
532 unsigned neg
= tgsi_util_get_full_src_register_sign_mode( src
, i
);
535 case TGSI_EXTSWIZZLE_ZERO
:
536 case TGSI_EXTSWIZZLE_ONE
:
537 ERROR(cp
, "not supporting full swizzles yet in tgsi_aos_sse2");
541 swz
|= (swizzle
& 0x3) << (i
* 2);
546 case TGSI_UTIL_SIGN_TOGGLE
:
550 case TGSI_UTIL_SIGN_KEEP
:
553 case TGSI_UTIL_SIGN_CLEAR
:
558 ERROR(cp
, "unsupported sign-mode");
563 if (swz
!= SSE_SWIZZLE_NOOP
|| negs
!= 0 || abs
!= 0) {
564 struct x86_reg dst
= aos_get_xmm_reg(cp
);
566 if (swz
!= SSE_SWIZZLE_NOOP
)
567 emit_pshufd(cp
, dst
, arg0
, swz
);
569 sse_movaps(cp
->func
, dst
, arg0
);
571 if (negs
&& negs
!= 0xf) {
572 struct x86_reg imm_swz
= aos_get_internal_xmm(cp
, IMM_SWZ
);
573 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
576 * Use neg as arg to pshufd
579 emit_pshufd(cp
, tmp
, imm_swz
,
580 SHUF((negs
& 1) ? 1 : 0,
583 (negs
& 8) ? 1 : 0));
584 sse_mulps(cp
->func
, dst
, tmp
);
586 aos_release_xmm_reg(cp
, tmp
.idx
);
589 struct x86_reg imm_negs
= aos_get_internal_xmm(cp
, IMM_NEGS
);
590 sse_mulps(cp
->func
, dst
, imm_negs
);
594 if (abs
&& abs
!= 0xf) {
595 ERROR(cp
, "unsupported partial abs");
598 struct x86_reg neg
= aos_get_internal(cp
, IMM_NEGS
);
599 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
601 sse_movaps(cp
->func
, tmp
, dst
);
602 sse_mulps(cp
->func
, tmp
, neg
);
603 sse_maxps(cp
->func
, dst
, tmp
);
605 aos_release_xmm_reg(cp
, tmp
.idx
);
614 static void x87_fld_src( struct aos_compilation
*cp
,
615 const struct tgsi_full_src_register
*src
,
618 struct x86_reg arg0
= aos_get_shader_reg_ptr(cp
,
619 src
->SrcRegister
.File
,
620 src
->SrcRegister
.Index
);
622 unsigned swizzle
= tgsi_util_get_full_src_register_extswizzle( src
, channel
);
623 unsigned neg
= tgsi_util_get_full_src_register_sign_mode( src
, channel
);
626 case TGSI_EXTSWIZZLE_ZERO
:
627 x87_fldz( cp
->func
);
630 case TGSI_EXTSWIZZLE_ONE
:
631 x87_fld1( cp
->func
);
635 x87_fld( cp
->func
, x86_make_disp(arg0
, (swizzle
& 3) * sizeof(float)) );
641 case TGSI_UTIL_SIGN_TOGGLE
:
644 x87_fchs( cp
->func
);
647 case TGSI_UTIL_SIGN_KEEP
:
650 case TGSI_UTIL_SIGN_CLEAR
:
651 x87_fabs( cp
->func
);
654 case TGSI_UTIL_SIGN_SET
:
655 x87_fabs( cp
->func
);
656 x87_fchs( cp
->func
);
660 ERROR(cp
, "unsupported sign-mode");
670 /* Used to implement write masking. This and most of the other instructions
671 * here would be easier to implement if there had been a translation
672 * to a 2 argument format (dst/arg0, arg1) at the shader level before
673 * attempting to translate to x86/sse code.
675 static void store_dest( struct aos_compilation
*cp
,
676 const struct tgsi_full_dst_register
*reg
,
677 struct x86_reg result
)
681 switch (reg
->DstRegister
.WriteMask
) {
685 case TGSI_WRITEMASK_XYZW
:
686 aos_adopt_xmm_reg(cp
,
687 get_xmm_writable(cp
, result
),
688 reg
->DstRegister
.File
,
689 reg
->DstRegister
.Index
,
696 dst
= aos_get_shader_reg_xmm(cp
,
697 reg
->DstRegister
.File
,
698 reg
->DstRegister
.Index
);
700 switch (reg
->DstRegister
.WriteMask
) {
701 case TGSI_WRITEMASK_X
:
702 sse_movss(cp
->func
, dst
, get_xmm(cp
, result
));
705 case TGSI_WRITEMASK_ZW
:
706 sse_shufps(cp
->func
, dst
, get_xmm(cp
, result
), SHUF(X
, Y
, Z
, W
));
709 case TGSI_WRITEMASK_XY
:
710 result
= get_xmm_writable(cp
, result
);
711 sse_shufps(cp
->func
, result
, dst
, SHUF(X
, Y
, Z
, W
));
715 case TGSI_WRITEMASK_YZW
:
716 result
= get_xmm_writable(cp
, result
);
717 sse_movss(cp
->func
, result
, dst
);
722 mask_write(cp
, dst
, result
, reg
->DstRegister
.WriteMask
);
726 aos_adopt_xmm_reg(cp
,
728 reg
->DstRegister
.File
,
729 reg
->DstRegister
.Index
,
734 static void inject_scalar( struct aos_compilation
*cp
,
736 struct x86_reg result
,
739 sse_shufps(cp
->func
, dst
, dst
, swizzle
);
740 sse_movss(cp
->func
, dst
, result
);
741 sse_shufps(cp
->func
, dst
, dst
, swizzle
);
745 static void store_scalar_dest( struct aos_compilation
*cp
,
746 const struct tgsi_full_dst_register
*reg
,
747 struct x86_reg result
)
749 unsigned writemask
= reg
->DstRegister
.WriteMask
;
752 if (writemask
!= TGSI_WRITEMASK_X
&&
753 writemask
!= TGSI_WRITEMASK_Y
&&
754 writemask
!= TGSI_WRITEMASK_Z
&&
755 writemask
!= TGSI_WRITEMASK_W
&&
758 result
= get_xmm_writable(cp
, result
); /* already true, right? */
759 sse_shufps(cp
->func
, result
, result
, SHUF(X
,X
,X
,X
));
760 store_dest(cp
, reg
, result
);
764 result
= get_xmm(cp
, result
);
765 dst
= aos_get_shader_reg_xmm(cp
,
766 reg
->DstRegister
.File
,
767 reg
->DstRegister
.Index
);
771 switch (reg
->DstRegister
.WriteMask
) {
772 case TGSI_WRITEMASK_X
:
773 sse_movss(cp
->func
, dst
, result
);
776 case TGSI_WRITEMASK_Y
:
777 inject_scalar(cp
, dst
, result
, SHUF(Y
, X
, Z
, W
));
780 case TGSI_WRITEMASK_Z
:
781 inject_scalar(cp
, dst
, result
, SHUF(Z
, Y
, X
, W
));
784 case TGSI_WRITEMASK_W
:
785 inject_scalar(cp
, dst
, result
, SHUF(W
, Y
, Z
, X
));
792 aos_adopt_xmm_reg(cp
,
794 reg
->DstRegister
.File
,
795 reg
->DstRegister
.Index
,
801 static void x87_fst_or_nop( struct x86_function
*func
,
806 assert(ptr
.file
== file_REG32
);
807 if (writemask
& (1<<channel
))
808 x87_fst( func
, x86_make_disp(ptr
, channel
* sizeof(float)) );
811 static void x87_fstp_or_pop( struct x86_function
*func
,
816 assert(ptr
.file
== file_REG32
);
817 if (writemask
& (1<<channel
))
818 x87_fstp( func
, x86_make_disp(ptr
, channel
* sizeof(float)) );
820 x87_fstp( func
, x86_make_reg( file_x87
, 0 ));
827 static void x87_fstp_dest4( struct aos_compilation
*cp
,
828 const struct tgsi_full_dst_register
*dst
)
830 struct x86_reg ptr
= get_dst_ptr(cp
, dst
);
831 unsigned writemask
= dst
->DstRegister
.WriteMask
;
833 x87_fst_or_nop(cp
->func
, writemask
, 0, ptr
);
834 x87_fst_or_nop(cp
->func
, writemask
, 1, ptr
);
835 x87_fst_or_nop(cp
->func
, writemask
, 2, ptr
);
836 x87_fstp_or_pop(cp
->func
, writemask
, 3, ptr
);
839 /* Save current x87 state and put it into single precision mode.
841 static void save_fpu_state( struct aos_compilation
*cp
)
843 x87_fnstcw( cp
->func
, x86_make_disp(cp
->machine_EDX
,
844 Offset(struct aos_machine
, fpu_restore
)));
847 static void restore_fpu_state( struct aos_compilation
*cp
)
849 x87_fnclex(cp
->func
);
850 x87_fldcw( cp
->func
, x86_make_disp(cp
->machine_EDX
,
851 Offset(struct aos_machine
, fpu_restore
)));
854 static void set_fpu_round_neg_inf( struct aos_compilation
*cp
)
856 if (cp
->fpucntl
!= FPU_RND_NEG
) {
857 cp
->fpucntl
= FPU_RND_NEG
;
858 x87_fnclex(cp
->func
);
859 x87_fldcw( cp
->func
, x86_make_disp(cp
->machine_EDX
,
860 Offset(struct aos_machine
, fpu_rnd_neg_inf
)));
864 static void set_fpu_round_nearest( struct aos_compilation
*cp
)
866 if (cp
->fpucntl
!= FPU_RND_NEAREST
) {
867 cp
->fpucntl
= FPU_RND_NEAREST
;
868 x87_fnclex(cp
->func
);
869 x87_fldcw( cp
->func
, x86_make_disp(cp
->machine_EDX
,
870 Offset(struct aos_machine
, fpu_rnd_nearest
)));
875 static void x87_emit_ex2( struct aos_compilation
*cp
)
877 struct x86_reg st0
= x86_make_reg(file_x87
, 0);
878 struct x86_reg st1
= x86_make_reg(file_x87
, 1);
879 int stack
= cp
->func
->x87_stack
;
881 // set_fpu_round_neg_inf( cp );
883 x87_fld(cp
->func
, st0
); /* a a */
884 x87_fprndint( cp
->func
); /* int(a) a*/
885 x87_fsubr(cp
->func
, st1
, st0
); /* int(a) frc(a) */
886 x87_fxch(cp
->func
, st1
); /* frc(a) int(a) */
887 x87_f2xm1(cp
->func
); /* (2^frc(a))-1 int(a) */
888 x87_fld1(cp
->func
); /* 1 (2^frc(a))-1 int(a) */
889 x87_faddp(cp
->func
, st1
); /* 2^frac(a) int(a) */
890 x87_fscale(cp
->func
); /* (2^frac(a)*2^int(int(a))) int(a) */
892 x87_fstp(cp
->func
, st1
); /* 2^a */
894 assert( stack
== cp
->func
->x87_stack
);
898 static void PIPE_CDECL
print_reg( const char *msg
,
901 debug_printf("%s: %f %f %f %f\n", msg
, reg
[0], reg
[1], reg
[2], reg
[3]);
904 static void emit_print( struct aos_compilation
*cp
,
905 const char *message
, /* must point to a static string! */
909 struct x86_reg ecx
= x86_make_reg( file_REG32
, reg_CX
);
910 struct x86_reg arg
= aos_get_shader_reg_ptr( cp
, file
, idx
);
913 /* There shouldn't be anything on the x87 stack. Can add this
914 * capacity later if need be.
916 assert(cp
->func
->x87_stack
== 0);
918 /* For absolute correctness, need to spill/invalidate all XMM regs
919 * too. We're obviously not concerned about performance on this
920 * debug path, so here goes:
922 for (i
= 0; i
< 8; i
++) {
923 if (cp
->xmm
[i
].dirty
)
926 aos_release_xmm_reg(cp
, i
);
929 /* Push caller-save (ie scratch) regs.
931 x86_cdecl_caller_push_regs( cp
->func
);
934 /* Push the arguments:
936 x86_lea( cp
->func
, ecx
, arg
);
937 x86_push( cp
->func
, ecx
);
938 x86_push_imm32( cp
->func
, (int)message
);
940 /* Call the helper. Could call debug_printf directly, but
941 * print_reg is a nice place to put a breakpoint if need be.
943 x86_mov_reg_imm( cp
->func
, ecx
, (int)print_reg
);
944 x86_call( cp
->func
, ecx
);
945 x86_pop( cp
->func
, ecx
);
946 x86_pop( cp
->func
, ecx
);
948 /* Pop caller-save regs
950 x86_cdecl_caller_pop_regs( cp
->func
);
957 * The traditional instructions. All operate on internal registers
958 * and ignore write masks and swizzling issues.
961 static boolean
emit_ABS( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
963 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
964 struct x86_reg neg
= aos_get_internal(cp
, IMM_NEGS
);
965 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
967 sse_movaps(cp
->func
, tmp
, arg0
);
968 sse_mulps(cp
->func
, tmp
, neg
);
969 sse_maxps(cp
->func
, tmp
, arg0
);
971 store_dest(cp
, &op
->FullDstRegisters
[0], tmp
);
975 static boolean
emit_ADD( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
977 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
978 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
979 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
981 sse_addps(cp
->func
, dst
, arg1
);
983 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
987 static boolean
emit_COS( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
989 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0);
991 x87_fstp_dest4(cp
, &op
->FullDstRegisters
[0]);
995 /* The dotproduct instructions don't really do that well in sse:
996 * XXX: produces wrong results -- disabled.
998 static boolean
emit_DP3( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1000 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1001 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1002 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
1003 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1005 sse_mulps(cp
->func
, dst
, arg1
);
1006 /* Now the hard bit: sum the first 3 values:
1008 sse_movhlps(cp
->func
, tmp
, dst
);
1009 sse_addss(cp
->func
, dst
, tmp
); /* a*x+c*z, b*y, ?, ? */
1010 emit_pshufd(cp
, tmp
, dst
, SHUF(Y
,X
,W
,Z
));
1011 sse_addss(cp
->func
, dst
, tmp
);
1013 aos_release_xmm_reg(cp
, tmp
.idx
);
1014 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1018 static boolean
emit_DP4( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1020 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1021 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1022 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
1023 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1025 sse_mulps(cp
->func
, dst
, arg1
);
1027 /* Now the hard bit: sum the values:
1029 sse_movhlps(cp
->func
, tmp
, dst
);
1030 sse_addps(cp
->func
, dst
, tmp
); /* a*x+c*z, b*y+d*w, a*x+c*z, b*y+d*w */
1031 emit_pshufd(cp
, tmp
, dst
, SHUF(Y
,X
,W
,Z
));
1032 sse_addss(cp
->func
, dst
, tmp
);
1034 aos_release_xmm_reg(cp
, tmp
.idx
);
1035 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1039 static boolean
emit_DPH( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1041 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1042 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1043 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
1044 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1046 sse_mulps(cp
->func
, dst
, arg1
);
1048 /* Now the hard bit: sum the values (from DP3):
1050 sse_movhlps(cp
->func
, tmp
, dst
);
1051 sse_addss(cp
->func
, dst
, tmp
); /* a*x+c*z, b*y, ?, ? */
1052 emit_pshufd(cp
, tmp
, dst
, SHUF(Y
,X
,W
,Z
));
1053 sse_addss(cp
->func
, dst
, tmp
);
1054 emit_pshufd(cp
, tmp
, arg1
, SHUF(W
,W
,W
,W
));
1055 sse_addss(cp
->func
, dst
, tmp
);
1057 aos_release_xmm_reg(cp
, tmp
.idx
);
1058 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1062 static boolean
emit_DST( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1064 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1065 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1066 struct x86_reg dst
= aos_get_xmm_reg(cp
);
1067 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
1068 struct x86_reg ones
= aos_get_internal(cp
, IMM_ONES
);
1070 /* dst[0] = 1.0 * 1.0F; */
1071 /* dst[1] = arg0[1] * arg1[1]; */
1072 /* dst[2] = arg0[2] * 1.0; */
1073 /* dst[3] = 1.0 * arg1[3]; */
1075 emit_shuf_copy2(cp
, dst
, arg0
, ones
, SHUF(X
,W
,Z
,Y
));
1076 emit_shuf_copy2(cp
, tmp
, arg1
, ones
, SHUF(X
,Z
,Y
,W
));
1077 sse_mulps(cp
->func
, dst
, tmp
);
1079 aos_release_xmm_reg(cp
, tmp
.idx
);
1080 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1084 static boolean
emit_LG2( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1086 x87_fld1(cp
->func
); /* 1 */
1087 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0); /* a0 1 */
1088 x87_fyl2x(cp
->func
); /* log2(a0) */
1089 x87_fstp_dest4(cp
, &op
->FullDstRegisters
[0]);
1094 static boolean
emit_EX2( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1096 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0);
1098 x87_fstp_dest4(cp
, &op
->FullDstRegisters
[0]);
1103 static boolean
emit_FLR( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1105 struct x86_reg dst
= get_dst_ptr(cp
, &op
->FullDstRegisters
[0]);
1106 unsigned writemask
= op
->FullDstRegisters
[0].DstRegister
.WriteMask
;
1109 set_fpu_round_neg_inf( cp
);
1111 /* Load all sources first to avoid aliasing
1113 for (i
= 3; i
>= 0; i
--) {
1114 if (writemask
& (1<<i
)) {
1115 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], i
);
1119 for (i
= 0; i
< 4; i
++) {
1120 if (writemask
& (1<<i
)) {
1121 x87_fprndint( cp
->func
);
1122 x87_fstp(cp
->func
, x86_make_disp(dst
, i
*4));
1130 static boolean
emit_RND( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1132 struct x86_reg dst
= get_dst_ptr(cp
, &op
->FullDstRegisters
[0]);
1133 unsigned writemask
= op
->FullDstRegisters
[0].DstRegister
.WriteMask
;
1136 set_fpu_round_nearest( cp
);
1138 /* Load all sources first to avoid aliasing
1140 for (i
= 3; i
>= 0; i
--) {
1141 if (writemask
& (1<<i
)) {
1142 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], i
);
1146 for (i
= 0; i
< 4; i
++) {
1147 if (writemask
& (1<<i
)) {
1148 x87_fprndint( cp
->func
);
1149 x87_fstp(cp
->func
, x86_make_disp(dst
, i
*4));
1157 static boolean
emit_FRC( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1159 struct x86_reg dst
= get_dst_ptr(cp
, &op
->FullDstRegisters
[0]);
1160 struct x86_reg st0
= x86_make_reg(file_x87
, 0);
1161 struct x86_reg st1
= x86_make_reg(file_x87
, 1);
1162 unsigned writemask
= op
->FullDstRegisters
[0].DstRegister
.WriteMask
;
1165 set_fpu_round_neg_inf( cp
);
1167 /* suck all the source values onto the stack before writing out any
1168 * dst, which may alias...
1170 for (i
= 3; i
>= 0; i
--) {
1171 if (writemask
& (1<<i
)) {
1172 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], i
);
1176 for (i
= 0; i
< 4; i
++) {
1177 if (writemask
& (1<<i
)) {
1178 x87_fld(cp
->func
, st0
); /* a a */
1179 x87_fprndint( cp
->func
); /* flr(a) a */
1180 x87_fsubp(cp
->func
, st1
); /* frc(a) */
1181 x87_fstp(cp
->func
, x86_make_disp(dst
, i
*4));
1193 static boolean
emit_LIT( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1195 struct x86_reg ecx
= x86_make_reg( file_REG32
, reg_CX
);
1196 unsigned writemask
= op
->FullDstRegisters
[0].DstRegister
.WriteMask
;
1197 unsigned lit_count
= cp
->lit_count
++;
1198 struct x86_reg result
, arg0
;
1202 /* For absolute correctness, need to spill/invalidate all XMM regs
1205 for (i
= 0; i
< 8; i
++) {
1206 if (cp
->xmm
[i
].dirty
)
1208 aos_release_xmm_reg(cp
, i
);
1212 if (writemask
!= TGSI_WRITEMASK_XYZW
)
1213 result
= x86_make_disp(cp
->machine_EDX
, Offset(struct aos_machine
, tmp
[0]));
1215 result
= get_dst_ptr(cp
, &op
->FullDstRegisters
[0]);
1218 arg0
= fetch_src( cp
, &op
->FullSrcRegisters
[0] );
1219 if (arg0
.file
== file_XMM
) {
1220 struct x86_reg tmp
= x86_make_disp(cp
->machine_EDX
,
1221 Offset(struct aos_machine
, tmp
[1]));
1222 sse_movaps( cp
->func
, tmp
, arg0
);
1228 /* Push caller-save (ie scratch) regs.
1230 x86_cdecl_caller_push_regs( cp
->func
);
1232 /* Push the arguments:
1234 x86_push_imm32( cp
->func
, lit_count
);
1236 x86_lea( cp
->func
, ecx
, arg0
);
1237 x86_push( cp
->func
, ecx
);
1239 x86_lea( cp
->func
, ecx
, result
);
1240 x86_push( cp
->func
, ecx
);
1242 x86_push( cp
->func
, cp
->machine_EDX
);
1244 if (lit_count
< MAX_LIT_INFO
) {
1245 x86_mov( cp
->func
, ecx
, x86_make_disp( cp
->machine_EDX
,
1246 Offset(struct aos_machine
, lit_info
) +
1247 lit_count
* sizeof(struct lit_info
) +
1248 Offset(struct lit_info
, func
)));
1251 x86_mov_reg_imm( cp
->func
, ecx
, (int)aos_do_lit
);
1254 x86_call( cp
->func
, ecx
);
1256 x86_pop( cp
->func
, ecx
); /* fixme... */
1257 x86_pop( cp
->func
, ecx
);
1258 x86_pop( cp
->func
, ecx
);
1259 x86_pop( cp
->func
, ecx
);
1261 x86_cdecl_caller_pop_regs( cp
->func
);
1263 if (writemask
!= TGSI_WRITEMASK_XYZW
) {
1265 &op
->FullDstRegisters
[0],
1266 get_xmm_writable( cp
, result
) );
1273 static boolean
emit_inline_LIT( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1275 struct x86_reg dst
= get_dst_ptr(cp
, &op
->FullDstRegisters
[0]);
1276 unsigned writemask
= op
->FullDstRegisters
[0].DstRegister
.WriteMask
;
1278 if (writemask
& TGSI_WRITEMASK_YZ
) {
1279 struct x86_reg st1
= x86_make_reg(file_x87
, 1);
1280 struct x86_reg st2
= x86_make_reg(file_x87
, 2);
1282 /* a1' = a1 <= 0 ? 1 : a1;
1284 x87_fldz(cp
->func
); /* 1 0 */
1286 x87_fld1(cp
->func
); /* 1 0 */
1288 /* Correct but slow due to fp exceptions generated in fyl2x - fix me.
1290 x87_fldz(cp
->func
); /* 1 0 */
1292 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 1); /* a1 1 0 */
1293 x87_fcomi(cp
->func
, st2
); /* a1 1 0 */
1294 x87_fcmovb(cp
->func
, st1
); /* a1' 1 0 */
1295 x87_fstp(cp
->func
, st1
); /* a1' 0 */
1296 x87_fstp(cp
->func
, st1
); /* a1' */
1298 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 3); /* a3 a1' */
1299 x87_fxch(cp
->func
, st1
); /* a1' a3 */
1302 /* Compute pow(a1, a3)
1304 x87_fyl2x(cp
->func
); /* a3*log2(a1) */
1305 x87_emit_ex2( cp
); /* 2^(a3*log2(a1)) */
1308 /* a0' = max2(a0, 0):
1310 x87_fldz(cp
->func
); /* 0 r2 */
1311 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0); /* a0 0 r2 */
1312 x87_fcomi(cp
->func
, st1
);
1313 x87_fcmovb(cp
->func
, st1
); /* a0' 0 r2 */
1315 x87_fst_or_nop(cp
->func
, writemask
, 1, dst
); /* result[1] = a0' */
1317 x87_fcomi(cp
->func
, st1
); /* a0' 0 r2 */
1318 x87_fcmovnbe(cp
->func
, st2
); /* r2' 0' r2 */
1320 x87_fstp_or_pop(cp
->func
, writemask
, 2, dst
); /* 0 r2 */
1321 x87_fpop(cp
->func
); /* r2 */
1325 if (writemask
& TGSI_WRITEMASK_XW
) {
1327 x87_fst_or_nop(cp
->func
, writemask
, 0, dst
);
1328 x87_fstp_or_pop(cp
->func
, writemask
, 3, dst
);
1337 static boolean
emit_MAX( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1339 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1340 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1341 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1343 sse_maxps(cp
->func
, dst
, arg1
);
1345 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1350 static boolean
emit_MIN( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1352 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1353 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1354 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1356 sse_minps(cp
->func
, dst
, arg1
);
1358 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1362 static boolean
emit_MOV( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1364 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1365 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1367 /* potentially nothing to do */
1369 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1373 static boolean
emit_MUL( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1375 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1376 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1377 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1379 sse_mulps(cp
->func
, dst
, arg1
);
1381 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1386 static boolean
emit_MAD( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1388 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1389 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1390 struct x86_reg arg2
= fetch_src(cp
, &op
->FullSrcRegisters
[2]);
1392 /* If we can't clobber old contents of arg0, get a temporary & copy
1393 * it there, then clobber it...
1395 arg0
= get_xmm_writable(cp
, arg0
);
1397 sse_mulps(cp
->func
, arg0
, arg1
);
1398 sse_addps(cp
->func
, arg0
, arg2
);
1399 store_dest(cp
, &op
->FullDstRegisters
[0], arg0
);
1405 /* A wrapper for powf().
1406 * Makes sure it is cdecl and operates on floats.
1408 static float PIPE_CDECL
_powerf( float x
, float y
)
1411 return util_fast_pow(x
, y
);
1413 return powf( x
, y
);
1418 static float PIPE_CDECL
_exp2(float x
)
1420 return util_fast_exp2(x
);
1425 /* Really not sufficient -- need to check for conditions that could
1426 * generate inf/nan values, which will slow things down hugely.
1428 static boolean
emit_POW( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1431 x87_fld_src(cp
, &op
->FullSrcRegisters
[1], 0); /* a1.x */
1432 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0); /* a0.x a1.x */
1433 x87_fyl2x(cp
->func
); /* a1*log2(a0) */
1435 x87_emit_ex2( cp
); /* 2^(a1*log2(a0)) */
1437 x87_fstp_dest4(cp
, &op
->FullDstRegisters
[0]);
1441 /* For absolute correctness, need to spill/invalidate all XMM regs
1444 for (i
= 0; i
< 8; i
++) {
1445 if (cp
->xmm
[i
].dirty
)
1447 aos_release_xmm_reg(cp
, i
);
1450 /* Push caller-save (ie scratch) regs.
1452 x86_cdecl_caller_push_regs( cp
->func
);
1454 x86_lea( cp
->func
, cp
->stack_ESP
, x86_make_disp(cp
->stack_ESP
, -8) );
1456 x87_fld_src( cp
, &op
->FullSrcRegisters
[1], 0 );
1457 x87_fstp( cp
->func
, x86_make_disp( cp
->stack_ESP
, 4 ) );
1458 x87_fld_src( cp
, &op
->FullSrcRegisters
[0], 0 );
1459 x87_fstp( cp
->func
, x86_make_disp( cp
->stack_ESP
, 0 ) );
1461 /* tmp_EAX has been pushed & will be restored below */
1462 x86_mov_reg_imm( cp
->func
, cp
->tmp_EAX
, (unsigned long) _powerf
);
1463 x86_call( cp
->func
, cp
->tmp_EAX
);
1465 x86_lea( cp
->func
, cp
->stack_ESP
, x86_make_disp(cp
->stack_ESP
, 8) );
1467 x86_cdecl_caller_pop_regs( cp
->func
);
1469 /* Note retval on x87 stack:
1471 cp
->func
->x87_stack
++;
1473 x87_fstp_dest4( cp
, &op
->FullDstRegisters
[0] );
1480 static boolean
emit_EXPBASE2( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1484 /* For absolute correctness, need to spill/invalidate all XMM regs
1487 for (i
= 0; i
< 8; i
++) {
1488 if (cp
->xmm
[i
].dirty
)
1490 aos_release_xmm_reg(cp
, i
);
1493 /* Push caller-save (ie scratch) regs.
1495 x86_cdecl_caller_push_regs( cp
->func
);
1497 x86_lea( cp
->func
, cp
->stack_ESP
, x86_make_disp(cp
->stack_ESP
, -4) );
1499 x87_fld_src( cp
, &op
->FullSrcRegisters
[0], 0 );
1500 x87_fstp( cp
->func
, x86_make_disp( cp
->stack_ESP
, 0 ) );
1502 /* tmp_EAX has been pushed & will be restored below */
1503 x86_mov_reg_imm( cp
->func
, cp
->tmp_EAX
, (unsigned long) _exp2
);
1504 x86_call( cp
->func
, cp
->tmp_EAX
);
1506 x86_lea( cp
->func
, cp
->stack_ESP
, x86_make_disp(cp
->stack_ESP
, 4) );
1508 x86_cdecl_caller_pop_regs( cp
->func
);
1510 /* Note retval on x87 stack:
1512 cp
->func
->x87_stack
++;
1514 x87_fstp_dest4( cp
, &op
->FullDstRegisters
[0] );
1521 static boolean
emit_RCP( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1523 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1524 struct x86_reg dst
= aos_get_xmm_reg(cp
);
1526 if (cp
->have_sse2
) {
1527 sse2_rcpss(cp
->func
, dst
, arg0
);
1528 /* extend precision here...
1532 struct x86_reg ones
= aos_get_internal(cp
, IMM_ONES
);
1533 sse_movss(cp
->func
, dst
, ones
);
1534 sse_divss(cp
->func
, dst
, arg0
);
1537 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1542 /* Although rsqrtps() and rcpps() are low precision on some/all SSE
1543 * implementations, it is possible to improve its precision at
1544 * fairly low cost, using a newton/raphson step, as below:
1546 * x1 = 2 * rcpps(a) - a * rcpps(a) * rcpps(a)
1547 * x1 = 0.5 * rsqrtps(a) * [3.0 - (a * rsqrtps(a))* rsqrtps(a)]
1549 * x1 = rsqrtps(a) * [1.5 - .5 * a * rsqrtps(a) * rsqrtps(a)]
1552 * See: http://softwarecommunity.intel.com/articles/eng/1818.htm
1554 static boolean
emit_RSQ( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1558 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1559 struct x86_reg r
= aos_get_xmm_reg(cp
);
1560 sse_rsqrtss(cp
->func
, r
, arg0
);
1561 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], r
);
1565 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1566 struct x86_reg r
= aos_get_xmm_reg(cp
);
1568 struct x86_reg neg_half
= get_reg_ptr( cp
, AOS_FILE_INTERNAL
, IMM_RSQ
);
1569 struct x86_reg one_point_five
= x86_make_disp( neg_half
, 4 );
1570 struct x86_reg src
= get_xmm_writable( cp
, arg0
);
1572 sse_rsqrtss( cp
->func
, r
, src
); /* rsqrtss(a) */
1573 sse_mulss( cp
->func
, src
, neg_half
); /* -.5 * a */
1574 sse_mulss( cp
->func
, src
, r
); /* -.5 * a * r */
1575 sse_mulss( cp
->func
, src
, r
); /* -.5 * a * r * r */
1576 sse_addss( cp
->func
, src
, one_point_five
); /* 1.5 - .5 * a * r * r */
1577 sse_mulss( cp
->func
, r
, src
); /* r * (1.5 - .5 * a * r * r) */
1579 store_scalar_dest(cp
, &op
->FullDstRegisters
[0], r
);
1585 static boolean
emit_SGE( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1587 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1588 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1589 struct x86_reg ones
= aos_get_internal(cp
, IMM_ONES
);
1590 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1592 sse_cmpps(cp
->func
, dst
, arg1
, cc_NotLessThan
);
1593 sse_andps(cp
->func
, dst
, ones
);
1595 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1599 static boolean
emit_SIN( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1601 x87_fld_src(cp
, &op
->FullSrcRegisters
[0], 0);
1603 x87_fstp_dest4(cp
, &op
->FullDstRegisters
[0]);
1609 static boolean
emit_SLT( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1611 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1612 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1613 struct x86_reg ones
= aos_get_internal(cp
, IMM_ONES
);
1614 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1616 sse_cmpps(cp
->func
, dst
, arg1
, cc_LessThan
);
1617 sse_andps(cp
->func
, dst
, ones
);
1619 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1623 static boolean
emit_SUB( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1625 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1626 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1627 struct x86_reg dst
= get_xmm_writable(cp
, arg0
);
1629 sse_subps(cp
->func
, dst
, arg1
);
1631 store_dest(cp
, &op
->FullDstRegisters
[0], dst
);
1635 static boolean
emit_TRUNC( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1637 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1638 struct x86_reg tmp0
= aos_get_xmm_reg(cp
);
1640 sse2_cvttps2dq(cp
->func
, tmp0
, arg0
);
1641 sse2_cvtdq2ps(cp
->func
, tmp0
, tmp0
);
1643 store_dest(cp
, &op
->FullDstRegisters
[0], tmp0
);
1647 static boolean
emit_XPD( struct aos_compilation
*cp
, const struct tgsi_full_instruction
*op
)
1649 struct x86_reg arg0
= fetch_src(cp
, &op
->FullSrcRegisters
[0]);
1650 struct x86_reg arg1
= fetch_src(cp
, &op
->FullSrcRegisters
[1]);
1651 struct x86_reg tmp0
= aos_get_xmm_reg(cp
);
1652 struct x86_reg tmp1
= aos_get_xmm_reg(cp
);
1654 emit_pshufd(cp
, tmp1
, arg1
, SHUF(Y
, Z
, X
, W
));
1655 sse_mulps(cp
->func
, tmp1
, arg0
);
1656 emit_pshufd(cp
, tmp0
, arg0
, SHUF(Y
, Z
, X
, W
));
1657 sse_mulps(cp
->func
, tmp0
, arg1
);
1658 sse_subps(cp
->func
, tmp1
, tmp0
);
1659 sse_shufps(cp
->func
, tmp1
, tmp1
, SHUF(Y
, Z
, X
, W
));
1661 /* dst[2] = arg0[0] * arg1[1] - arg0[1] * arg1[0]; */
1662 /* dst[0] = arg0[1] * arg1[2] - arg0[2] * arg1[1]; */
1663 /* dst[1] = arg0[2] * arg1[0] - arg0[0] * arg1[2]; */
1664 /* dst[3] is undef */
1667 aos_release_xmm_reg(cp
, tmp0
.idx
);
1668 store_dest(cp
, &op
->FullDstRegisters
[0], tmp1
);
1675 emit_instruction( struct aos_compilation
*cp
,
1676 struct tgsi_full_instruction
*inst
)
1678 x87_assert_stack_empty(cp
->func
);
1680 switch( inst
->Instruction
.Opcode
) {
1681 case TGSI_OPCODE_MOV
:
1682 return emit_MOV( cp
, inst
);
1684 case TGSI_OPCODE_LIT
:
1685 return emit_LIT(cp
, inst
);
1687 case TGSI_OPCODE_RCP
:
1688 return emit_RCP(cp
, inst
);
1690 case TGSI_OPCODE_RSQ
:
1691 return emit_RSQ(cp
, inst
);
1693 case TGSI_OPCODE_EXP
:
1694 /*return emit_EXP(cp, inst);*/
1697 case TGSI_OPCODE_LOG
:
1698 /*return emit_LOG(cp, inst);*/
1701 case TGSI_OPCODE_MUL
:
1702 return emit_MUL(cp
, inst
);
1704 case TGSI_OPCODE_ADD
:
1705 return emit_ADD(cp
, inst
);
1707 case TGSI_OPCODE_DP3
:
1708 return emit_DP3(cp
, inst
);
1710 case TGSI_OPCODE_DP4
:
1711 return emit_DP4(cp
, inst
);
1713 case TGSI_OPCODE_DST
:
1714 return emit_DST(cp
, inst
);
1716 case TGSI_OPCODE_MIN
:
1717 return emit_MIN(cp
, inst
);
1719 case TGSI_OPCODE_MAX
:
1720 return emit_MAX(cp
, inst
);
1722 case TGSI_OPCODE_SLT
:
1723 return emit_SLT(cp
, inst
);
1725 case TGSI_OPCODE_SGE
:
1726 return emit_SGE(cp
, inst
);
1728 case TGSI_OPCODE_MAD
:
1729 return emit_MAD(cp
, inst
);
1731 case TGSI_OPCODE_SUB
:
1732 return emit_SUB(cp
, inst
);
1734 case TGSI_OPCODE_LERP
:
1735 // return emit_LERP(cp, inst);
1738 case TGSI_OPCODE_FRAC
:
1739 return emit_FRC(cp
, inst
);
1741 case TGSI_OPCODE_CLAMP
:
1742 // return emit_CLAMP(cp, inst);
1745 case TGSI_OPCODE_FLOOR
:
1746 return emit_FLR(cp
, inst
);
1748 case TGSI_OPCODE_ROUND
:
1749 return emit_RND(cp
, inst
);
1751 case TGSI_OPCODE_EXPBASE2
:
1753 return emit_EXPBASE2(cp
, inst
);
1755 /* this seems to fail for "larger" exponents.
1756 * See glean tvertProg1's EX2 test.
1758 return emit_EX2(cp
, inst
);
1763 case TGSI_OPCODE_LOGBASE2
:
1764 return emit_LG2(cp
, inst
);
1766 case TGSI_OPCODE_POWER
:
1767 return emit_POW(cp
, inst
);
1769 case TGSI_OPCODE_CROSSPRODUCT
:
1770 return emit_XPD(cp
, inst
);
1772 case TGSI_OPCODE_ABS
:
1773 return emit_ABS(cp
, inst
);
1775 case TGSI_OPCODE_DPH
:
1776 return emit_DPH(cp
, inst
);
1778 case TGSI_OPCODE_COS
:
1779 return emit_COS(cp
, inst
);
1781 case TGSI_OPCODE_SIN
:
1782 return emit_SIN(cp
, inst
);
1784 case TGSI_OPCODE_TRUNC
:
1785 return emit_TRUNC(cp
, inst
);
1787 case TGSI_OPCODE_END
:
1796 static boolean
emit_viewport( struct aos_compilation
*cp
)
1798 struct x86_reg pos
= aos_get_shader_reg_xmm(cp
,
1800 cp
->vaos
->draw
->vs
.position_output
);
1802 struct x86_reg scale
= x86_make_disp(cp
->machine_EDX
,
1803 Offset(struct aos_machine
, scale
));
1805 struct x86_reg translate
= x86_make_disp(cp
->machine_EDX
,
1806 Offset(struct aos_machine
, translate
));
1808 sse_mulps(cp
->func
, pos
, scale
);
1809 sse_addps(cp
->func
, pos
, translate
);
1811 aos_adopt_xmm_reg( cp
,
1814 cp
->vaos
->draw
->vs
.position_output
,
1820 /* This is useful to be able to see the results on softpipe. Doesn't
1821 * do proper clipping, just assumes the backend can do it during
1822 * rasterization -- for debug only...
1824 static boolean
emit_rhw_viewport( struct aos_compilation
*cp
)
1826 struct x86_reg tmp
= aos_get_xmm_reg(cp
);
1827 struct x86_reg pos
= aos_get_shader_reg_xmm(cp
,
1829 cp
->vaos
->draw
->vs
.position_output
);
1831 struct x86_reg scale
= x86_make_disp(cp
->machine_EDX
,
1832 Offset(struct aos_machine
, scale
));
1834 struct x86_reg translate
= x86_make_disp(cp
->machine_EDX
,
1835 Offset(struct aos_machine
, translate
));
1839 emit_pshufd(cp
, tmp
, pos
, SHUF(W
, W
, W
, W
));
1840 sse2_rcpss(cp
->func
, tmp
, tmp
);
1841 sse_shufps(cp
->func
, tmp
, tmp
, SHUF(X
, X
, X
, X
));
1843 sse_mulps(cp
->func
, pos
, scale
);
1844 sse_mulps(cp
->func
, pos
, tmp
);
1845 sse_addps(cp
->func
, pos
, translate
);
1849 mask_write(cp
, pos
, tmp
, TGSI_WRITEMASK_W
);
1851 aos_adopt_xmm_reg( cp
,
1854 cp
->vaos
->draw
->vs
.position_output
,
1861 static boolean
note_immediate( struct aos_compilation
*cp
,
1862 struct tgsi_full_immediate
*imm
)
1864 unsigned pos
= cp
->num_immediates
++;
1867 for (j
= 0; j
< imm
->Immediate
.Size
; j
++) {
1868 cp
->vaos
->machine
->immediate
[pos
][j
] = imm
->u
.ImmediateFloat32
[j
].Float
;
1878 static void find_last_write_outputs( struct aos_compilation
*cp
)
1880 struct tgsi_parse_context parse
;
1881 unsigned this_instruction
= 0;
1884 tgsi_parse_init( &parse
, cp
->vaos
->base
.vs
->state
.tokens
);
1886 while (!tgsi_parse_end_of_tokens( &parse
)) {
1888 tgsi_parse_token( &parse
);
1890 if (parse
.FullToken
.Token
.Type
!= TGSI_TOKEN_TYPE_INSTRUCTION
)
1893 for (i
= 0; i
< TGSI_FULL_MAX_DST_REGISTERS
; i
++) {
1894 if (parse
.FullToken
.FullInstruction
.FullDstRegisters
[i
].DstRegister
.File
==
1897 unsigned idx
= parse
.FullToken
.FullInstruction
.FullDstRegisters
[i
].DstRegister
.Index
;
1898 cp
->output_last_write
[idx
] = this_instruction
;
1905 tgsi_parse_free( &parse
);
1909 #define ARG_MACHINE 1
1910 #define ARG_START_ELTS 2
1912 #define ARG_OUTBUF 4
1915 static boolean
build_vertex_program( struct draw_vs_varient_aos_sse
*varient
,
1918 struct tgsi_parse_context parse
;
1919 struct aos_compilation cp
;
1920 unsigned fixup
, label
;
1924 tgsi_parse_init( &parse
, varient
->base
.vs
->state
.tokens
);
1926 memset(&cp
, 0, sizeof(cp
));
1928 cp
.insn_counter
= 1;
1931 cp
.func
= &varient
->func
[ linear
? 0 : 1 ];
1933 cp
.tmp_EAX
= x86_make_reg(file_REG32
, reg_AX
);
1934 cp
.idx_EBX
= x86_make_reg(file_REG32
, reg_BX
);
1935 cp
.outbuf_ECX
= x86_make_reg(file_REG32
, reg_CX
);
1936 cp
.machine_EDX
= x86_make_reg(file_REG32
, reg_DX
);
1937 cp
.count_ESI
= x86_make_reg(file_REG32
, reg_SI
);
1938 cp
.temp_EBP
= x86_make_reg(file_REG32
, reg_BP
);
1939 cp
.stack_ESP
= x86_make_reg( file_REG32
, reg_SP
);
1941 x86_init_func(cp
.func
);
1943 find_last_write_outputs(&cp
);
1945 x86_push(cp
.func
, cp
.idx_EBX
);
1946 x86_push(cp
.func
, cp
.count_ESI
);
1947 x86_push(cp
.func
, cp
.temp_EBP
);
1950 /* Load arguments into regs:
1952 x86_mov(cp
.func
, cp
.machine_EDX
, x86_fn_arg(cp
.func
, ARG_MACHINE
));
1953 x86_mov(cp
.func
, cp
.idx_EBX
, x86_fn_arg(cp
.func
, ARG_START_ELTS
));
1954 x86_mov(cp
.func
, cp
.count_ESI
, x86_fn_arg(cp
.func
, ARG_COUNT
));
1955 x86_mov(cp
.func
, cp
.outbuf_ECX
, x86_fn_arg(cp
.func
, ARG_OUTBUF
));
1958 /* Compare count to zero and possibly bail.
1960 x86_xor(cp
.func
, cp
.tmp_EAX
, cp
.tmp_EAX
);
1961 x86_cmp(cp
.func
, cp
.count_ESI
, cp
.tmp_EAX
);
1962 fixup
= x86_jcc_forward(cp
.func
, cc_E
);
1965 save_fpu_state( &cp
);
1966 set_fpu_round_nearest( &cp
);
1968 aos_init_inputs( &cp
, linear
);
1973 /* Note address for loop jump
1975 label
= x86_get_label(cp
.func
);
1977 /* Fetch inputs... TODO: fetch lazily...
1979 if (!aos_fetch_inputs( &cp
, linear
))
1984 while( !tgsi_parse_end_of_tokens( &parse
) && !cp
.error
)
1986 tgsi_parse_token( &parse
);
1988 switch (parse
.FullToken
.Token
.Type
) {
1989 case TGSI_TOKEN_TYPE_IMMEDIATE
:
1991 if (!note_immediate( &cp
, &parse
.FullToken
.FullImmediate
))
1996 case TGSI_TOKEN_TYPE_INSTRUCTION
:
1998 tgsi_dump_instruction( &parse
.FullToken
.FullInstruction
, cp
.insn_counter
);
2000 if (!emit_instruction( &cp
, &parse
.FullToken
.FullInstruction
))
2005 x87_assert_stack_empty(cp
.func
);
2015 for (i
= 0; i
< 8; i
++) {
2016 if (cp
.xmm
[i
].file
!= TGSI_FILE_OUTPUT
) {
2017 cp
.xmm
[i
].file
= TGSI_FILE_NULL
;
2018 cp
.xmm
[i
].dirty
= 0;
2026 if (cp
.vaos
->base
.key
.clip
) {
2027 /* not really handling clipping, just do the rhw so we can
2028 * see the results...
2030 emit_rhw_viewport(&cp
);
2032 else if (cp
.vaos
->base
.key
.viewport
) {
2036 /* Emit output... TODO: do this eagerly after the last write to a
2039 if (!aos_emit_outputs( &cp
))
2047 x86_make_disp(cp
.outbuf_ECX
,
2048 cp
.vaos
->base
.key
.output_stride
));
2052 aos_incr_inputs( &cp
, linear
);
2054 /* decr count, loop if not zero
2056 x86_dec(cp
.func
, cp
.count_ESI
);
2057 x86_jcc(cp
.func
, cc_NZ
, label
);
2059 restore_fpu_state(&cp
);
2061 /* Land forward jump here:
2063 x86_fixup_fwd_jump(cp
.func
, fixup
);
2067 if (cp
.func
->need_emms
)
2070 x86_pop(cp
.func
, cp
.temp_EBP
);
2071 x86_pop(cp
.func
, cp
.count_ESI
);
2072 x86_pop(cp
.func
, cp
.idx_EBX
);
2074 x87_assert_stack_empty(cp
.func
);
2077 tgsi_parse_free( &parse
);
2081 tgsi_parse_free( &parse
);
2087 static void vaos_set_buffer( struct draw_vs_varient
*varient
,
2092 struct draw_vs_varient_aos_sse
*vaos
= (struct draw_vs_varient_aos_sse
*)varient
;
2094 if (buf
< vaos
->nr_vb
) {
2095 vaos
->buffer
[buf
].base_ptr
= (char *)ptr
;
2096 vaos
->buffer
[buf
].stride
= stride
;
2099 if (0) debug_printf("%s %d/%d: %p %d\n", __FUNCTION__
, buf
, vaos
->nr_vb
, ptr
, stride
);
2104 static void PIPE_CDECL
vaos_run_elts( struct draw_vs_varient
*varient
,
2105 const unsigned *elts
,
2107 void *output_buffer
)
2109 struct draw_vs_varient_aos_sse
*vaos
= (struct draw_vs_varient_aos_sse
*)varient
;
2110 struct aos_machine
*machine
= vaos
->draw
->vs
.aos_machine
;
2112 if (0) debug_printf("%s %d\n", __FUNCTION__
, count
);
2114 machine
->internal
[IMM_PSIZE
][0] = vaos
->draw
->rasterizer
->point_size
;
2115 machine
->constants
= vaos
->draw
->vs
.aligned_constants
;
2116 machine
->immediates
= vaos
->base
.vs
->immediates
;
2117 machine
->buffer
= vaos
->buffer
;
2119 vaos
->gen_run_elts( machine
,
2125 static void PIPE_CDECL
vaos_run_linear( struct draw_vs_varient
*varient
,
2128 void *output_buffer
)
2130 struct draw_vs_varient_aos_sse
*vaos
= (struct draw_vs_varient_aos_sse
*)varient
;
2131 struct aos_machine
*machine
= vaos
->draw
->vs
.aos_machine
;
2133 if (0) debug_printf("%s %d %d const: %x\n", __FUNCTION__
, start
, count
,
2134 vaos
->base
.key
.const_vbuffers
);
2136 machine
->internal
[IMM_PSIZE
][0] = vaos
->draw
->rasterizer
->point_size
;
2137 machine
->constants
= vaos
->draw
->vs
.aligned_constants
;
2138 machine
->immediates
= vaos
->base
.vs
->immediates
;
2139 machine
->buffer
= vaos
->buffer
;
2141 vaos
->gen_run_linear( machine
,
2146 /* Sanity spot checks to make sure we didn't trash our constants */
2147 assert(machine
->internal
[IMM_ONES
][0] == 1.0f
);
2148 assert(machine
->internal
[IMM_IDENTITY
][0] == 0.0f
);
2149 assert(machine
->internal
[IMM_NEGS
][0] == -1.0f
);
2154 static void vaos_destroy( struct draw_vs_varient
*varient
)
2156 struct draw_vs_varient_aos_sse
*vaos
= (struct draw_vs_varient_aos_sse
*)varient
;
2158 FREE( vaos
->buffer
);
2160 x86_release_func( &vaos
->func
[0] );
2161 x86_release_func( &vaos
->func
[1] );
2168 static struct draw_vs_varient
*varient_aos_sse( struct draw_vertex_shader
*vs
,
2169 const struct draw_vs_varient_key
*key
)
2172 struct draw_vs_varient_aos_sse
*vaos
= CALLOC_STRUCT(draw_vs_varient_aos_sse
);
2177 vaos
->base
.key
= *key
;
2179 vaos
->base
.set_buffer
= vaos_set_buffer
;
2180 vaos
->base
.destroy
= vaos_destroy
;
2181 vaos
->base
.run_linear
= vaos_run_linear
;
2182 vaos
->base
.run_elts
= vaos_run_elts
;
2184 vaos
->draw
= vs
->draw
;
2186 for (i
= 0; i
< key
->nr_inputs
; i
++)
2187 vaos
->nr_vb
= MAX2( vaos
->nr_vb
, key
->element
[i
].in
.buffer
+ 1 );
2189 vaos
->buffer
= MALLOC( vaos
->nr_vb
* sizeof(vaos
->buffer
[0]) );
2194 debug_printf("nr_vb: %d const: %x\n", vaos
->nr_vb
, vaos
->base
.key
.const_vbuffers
);
2197 tgsi_dump(vs
->state
.tokens
, 0);
2200 if (!build_vertex_program( vaos
, TRUE
))
2203 if (!build_vertex_program( vaos
, FALSE
))
2206 vaos
->gen_run_linear
= (vaos_run_linear_func
)x86_get_func(&vaos
->func
[0]);
2207 if (!vaos
->gen_run_linear
)
2210 vaos
->gen_run_elts
= (vaos_run_elts_func
)x86_get_func(&vaos
->func
[1]);
2211 if (!vaos
->gen_run_elts
)
2217 if (vaos
&& vaos
->buffer
)
2221 x86_release_func( &vaos
->func
[0] );
2224 x86_release_func( &vaos
->func
[1] );
2232 struct draw_vs_varient
*draw_vs_varient_aos_sse( struct draw_vertex_shader
*vs
,
2233 const struct draw_vs_varient_key
*key
)
2235 struct draw_vs_varient
*varient
= varient_aos_sse( vs
, key
);
2237 if (varient
== NULL
) {
2238 varient
= draw_vs_varient_generic( vs
, key
);
2246 #endif /* PIPE_ARCH_X86 */