X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fauxiliary%2Ftgsi%2Ftgsi_exec.c;h=48d91af3e16e9e2b095f270076b9715b57a49895;hb=e6469ec43b25898e99766a30aa8f54cc64c3bc04;hp=1220478f9d24fe474ee9c10a5ab329cef2d874e9;hpb=ad30e4545b4f8b9cbcd0223266a2a461290f4143;p=mesa.git diff --git a/src/gallium/auxiliary/tgsi/tgsi_exec.c b/src/gallium/auxiliary/tgsi/tgsi_exec.c index 1220478f9d2..48d91af3e16 100644 --- a/src/gallium/auxiliary/tgsi/tgsi_exec.c +++ b/src/gallium/auxiliary/tgsi/tgsi_exec.c @@ -1,6 +1,6 @@ /************************************************************************** * - * Copyright 2007-2008 Tungsten Graphics, Inc., Cedar Park, Texas. + * Copyright 2007-2008 VMware, Inc. * All Rights Reserved. * Copyright 2009-2010 VMware, Inc. All rights Reserved. * @@ -19,7 +19,7 @@ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. - * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR + * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. @@ -58,8 +58,13 @@ #include "tgsi/tgsi_parse.h" #include "tgsi/tgsi_util.h" #include "tgsi_exec.h" +#include "util/u_half.h" #include "util/u_memory.h" #include "util/u_math.h" +#include "util/rounding.h" + + +#define DEBUG_EXECUTION 0 #define FAST_MATH 0 @@ -69,6 +74,18 @@ #define TILE_BOTTOM_LEFT 2 #define TILE_BOTTOM_RIGHT 3 +union tgsi_double_channel { + double d[TGSI_QUAD_SIZE]; + unsigned u[TGSI_QUAD_SIZE][2]; + uint64_t u64[TGSI_QUAD_SIZE]; + int64_t i64[TGSI_QUAD_SIZE]; +}; + +struct tgsi_double_vector { + union tgsi_double_channel xy; + union tgsi_double_channel zw; +}; + static void micro_abs(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -109,18 +126,6 @@ micro_ceil(union tgsi_exec_channel *dst, dst->f[3] = ceilf(src->f[3]); } -static void -micro_clamp(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1, - const union tgsi_exec_channel *src2) -{ - dst->f[0] = src0->f[0] < src1->f[0] ? src1->f[0] : src0->f[0] > src2->f[0] ? src2->f[0] : src0->f[0]; - dst->f[1] = src0->f[1] < src1->f[1] ? src1->f[1] : src0->f[1] > src2->f[1] ? src2->f[1] : src0->f[1]; - dst->f[2] = src0->f[2] < src1->f[2] ? src1->f[2] : src0->f[2] > src2->f[2] ? src2->f[2] : src0->f[2]; - dst->f[3] = src0->f[3] < src1->f[3] ? src1->f[3] : src0->f[3] > src2->f[3] ? src2->f[3] : src0->f[3]; -} - static void micro_cmp(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, @@ -133,18 +138,6 @@ micro_cmp(union tgsi_exec_channel *dst, dst->f[3] = src0->f[3] < 0.0f ? src1->f[3] : src2->f[3]; } -static void -micro_cnd(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1, - const union tgsi_exec_channel *src2) -{ - dst->f[0] = src2->f[0] > 0.5f ? src0->f[0] : src1->f[0]; - dst->f[1] = src2->f[1] > 0.5f ? src0->f[1] : src1->f[1]; - dst->f[2] = src2->f[2] > 0.5f ? src0->f[2] : src1->f[2]; - dst->f[3] = src2->f[3] > 0.5f ? src0->f[3] : src1->f[3]; -} - static void micro_cos(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -155,6 +148,65 @@ micro_cos(union tgsi_exec_channel *dst, dst->f[3] = cosf(src->f[3]); } +static void +micro_d2f(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->f[0] = (float)src->d[0]; + dst->f[1] = (float)src->d[1]; + dst->f[2] = (float)src->d[2]; + dst->f[3] = (float)src->d[3]; +} + +static void +micro_d2i(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i[0] = (int)src->d[0]; + dst->i[1] = (int)src->d[1]; + dst->i[2] = (int)src->d[2]; + dst->i[3] = (int)src->d[3]; +} + +static void +micro_d2u(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0] = (unsigned)src->d[0]; + dst->u[1] = (unsigned)src->d[1]; + dst->u[2] = (unsigned)src->d[2]; + dst->u[3] = (unsigned)src->d[3]; +} +static void +micro_dabs(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src->d[0] >= 0.0 ? src->d[0] : -src->d[0]; + dst->d[1] = src->d[1] >= 0.0 ? src->d[1] : -src->d[1]; + dst->d[2] = src->d[2] >= 0.0 ? src->d[2] : -src->d[2]; + dst->d[3] = src->d[3] >= 0.0 ? src->d[3] : -src->d[3]; +} + +static void +micro_dadd(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] + src[1].d[0]; + dst->d[1] = src[0].d[1] + src[1].d[1]; + dst->d[2] = src[0].d[2] + src[1].d[2]; + dst->d[3] = src[0].d[3] + src[1].d[3]; +} + +static void +micro_ddiv(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] / src[1].d[0]; + dst->d[1] = src[0].d[1] / src[1].d[1]; + dst->d[2] = src[0].d[2] / src[1].d[2]; + dst->d[3] = src[0].d[3] / src[1].d[3]; +} + static void micro_ddx(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -175,6 +227,158 @@ micro_ddy(union tgsi_exec_channel *dst, dst->f[3] = src->f[TILE_BOTTOM_LEFT] - src->f[TILE_TOP_LEFT]; } +static void +micro_dmul(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] * src[1].d[0]; + dst->d[1] = src[0].d[1] * src[1].d[1]; + dst->d[2] = src[0].d[2] * src[1].d[2]; + dst->d[3] = src[0].d[3] * src[1].d[3]; +} + +static void +micro_dmax(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] > src[1].d[0] ? src[0].d[0] : src[1].d[0]; + dst->d[1] = src[0].d[1] > src[1].d[1] ? src[0].d[1] : src[1].d[1]; + dst->d[2] = src[0].d[2] > src[1].d[2] ? src[0].d[2] : src[1].d[2]; + dst->d[3] = src[0].d[3] > src[1].d[3] ? src[0].d[3] : src[1].d[3]; +} + +static void +micro_dmin(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] < src[1].d[0] ? src[0].d[0] : src[1].d[0]; + dst->d[1] = src[0].d[1] < src[1].d[1] ? src[0].d[1] : src[1].d[1]; + dst->d[2] = src[0].d[2] < src[1].d[2] ? src[0].d[2] : src[1].d[2]; + dst->d[3] = src[0].d[3] < src[1].d[3] ? src[0].d[3] : src[1].d[3]; +} + +static void +micro_dneg(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = -src->d[0]; + dst->d[1] = -src->d[1]; + dst->d[2] = -src->d[2]; + dst->d[3] = -src->d[3]; +} + +static void +micro_dslt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].d[0] < src[1].d[0] ? ~0U : 0U; + dst->u[1][0] = src[0].d[1] < src[1].d[1] ? ~0U : 0U; + dst->u[2][0] = src[0].d[2] < src[1].d[2] ? ~0U : 0U; + dst->u[3][0] = src[0].d[3] < src[1].d[3] ? ~0U : 0U; +} + +static void +micro_dsne(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].d[0] != src[1].d[0] ? ~0U : 0U; + dst->u[1][0] = src[0].d[1] != src[1].d[1] ? ~0U : 0U; + dst->u[2][0] = src[0].d[2] != src[1].d[2] ? ~0U : 0U; + dst->u[3][0] = src[0].d[3] != src[1].d[3] ? ~0U : 0U; +} + +static void +micro_dsge(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].d[0] >= src[1].d[0] ? ~0U : 0U; + dst->u[1][0] = src[0].d[1] >= src[1].d[1] ? ~0U : 0U; + dst->u[2][0] = src[0].d[2] >= src[1].d[2] ? ~0U : 0U; + dst->u[3][0] = src[0].d[3] >= src[1].d[3] ? ~0U : 0U; +} + +static void +micro_dseq(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].d[0] == src[1].d[0] ? ~0U : 0U; + dst->u[1][0] = src[0].d[1] == src[1].d[1] ? ~0U : 0U; + dst->u[2][0] = src[0].d[2] == src[1].d[2] ? ~0U : 0U; + dst->u[3][0] = src[0].d[3] == src[1].d[3] ? ~0U : 0U; +} + +static void +micro_drcp(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = 1.0 / src->d[0]; + dst->d[1] = 1.0 / src->d[1]; + dst->d[2] = 1.0 / src->d[2]; + dst->d[3] = 1.0 / src->d[3]; +} + +static void +micro_dsqrt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = sqrt(src->d[0]); + dst->d[1] = sqrt(src->d[1]); + dst->d[2] = sqrt(src->d[2]); + dst->d[3] = sqrt(src->d[3]); +} + +static void +micro_drsq(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = 1.0 / sqrt(src->d[0]); + dst->d[1] = 1.0 / sqrt(src->d[1]); + dst->d[2] = 1.0 / sqrt(src->d[2]); + dst->d[3] = 1.0 / sqrt(src->d[3]); +} + +static void +micro_dmad(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] * src[1].d[0] + src[2].d[0]; + dst->d[1] = src[0].d[1] * src[1].d[1] + src[2].d[1]; + dst->d[2] = src[0].d[2] * src[1].d[2] + src[2].d[2]; + dst->d[3] = src[0].d[3] * src[1].d[3] + src[2].d[3]; +} + +static void +micro_dfrac(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src->d[0] - floor(src->d[0]); + dst->d[1] = src->d[1] - floor(src->d[1]); + dst->d[2] = src->d[2] - floor(src->d[2]); + dst->d[3] = src->d[3] - floor(src->d[3]); +} + +static void +micro_dldexp(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + dst->d[0] = ldexp(src0->d[0], src1->i[0]); + dst->d[1] = ldexp(src0->d[1], src1->i[1]); + dst->d[2] = ldexp(src0->d[2], src1->i[2]); + dst->d[3] = ldexp(src0->d[3], src1->i[3]); +} + +static void +micro_dfracexp(union tgsi_double_channel *dst, + union tgsi_exec_channel *dst_exp, + const union tgsi_double_channel *src) +{ + dst->d[0] = frexp(src->d[0], &dst_exp->i[0]); + dst->d[1] = frexp(src->d[1], &dst_exp->i[1]); + dst->d[2] = frexp(src->d[2], &dst_exp->i[2]); + dst->d[3] = frexp(src->d[3], &dst_exp->i[3]); +} + static void micro_exp2(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -209,6 +413,16 @@ micro_exp2(union tgsi_exec_channel *dst, #endif /* FAST_MATH */ } +static void +micro_f2d(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->d[0] = (double)src->f[0]; + dst->d[1] = (double)src->f[1]; + dst->d[2] = (double)src->f[2]; + dst->d[3] = (double)src->f[3]; +} + static void micro_flr(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -229,6 +443,16 @@ micro_frc(union tgsi_exec_channel *dst, dst->f[3] = src->f[3] - floorf(src->f[3]); } +static void +micro_i2d(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->d[0] = (double)src->i[0]; + dst->d[1] = (double)src->i[1]; + dst->d[2] = (double)src->i[2]; + dst->d[3] = (double)src->i[3]; +} + static void micro_iabs(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -320,10 +544,10 @@ static void micro_rnd(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) { - dst->f[0] = floorf(src->f[0] + 0.5f); - dst->f[1] = floorf(src->f[1] + 0.5f); - dst->f[2] = floorf(src->f[2] + 0.5f); - dst->f[3] = floorf(src->f[3] + 0.5f); + dst->f[0] = _mesa_roundevenf(src->f[0]); + dst->f[1] = _mesa_roundevenf(src->f[1]); + dst->f[2] = _mesa_roundevenf(src->f[2]); + dst->f[3] = _mesa_roundevenf(src->f[3]); } static void @@ -336,20 +560,20 @@ micro_rsq(union tgsi_exec_channel *dst, assert(src->f[2] != 0.0f); assert(src->f[3] != 0.0f); #endif - dst->f[0] = 1.0f / sqrtf(fabsf(src->f[0])); - dst->f[1] = 1.0f / sqrtf(fabsf(src->f[1])); - dst->f[2] = 1.0f / sqrtf(fabsf(src->f[2])); - dst->f[3] = 1.0f / sqrtf(fabsf(src->f[3])); + dst->f[0] = 1.0f / sqrtf(src->f[0]); + dst->f[1] = 1.0f / sqrtf(src->f[1]); + dst->f[2] = 1.0f / sqrtf(src->f[2]); + dst->f[3] = 1.0f / sqrtf(src->f[3]); } static void micro_sqrt(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) { - dst->f[0] = sqrtf(fabsf(src->f[0])); - dst->f[1] = sqrtf(fabsf(src->f[1])); - dst->f[2] = sqrtf(fabsf(src->f[2])); - dst->f[3] = sqrtf(fabsf(src->f[3])); + dst->f[0] = sqrtf(src->f[0]); + dst->f[1] = sqrtf(src->f[1]); + dst->f[2] = sqrtf(src->f[2]); + dst->f[3] = sqrtf(src->f[3]); } static void @@ -449,38 +673,270 @@ micro_sne(union tgsi_exec_channel *dst, } static void -micro_sfl(union tgsi_exec_channel *dst) +micro_trunc(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->f[0] = truncf(src->f[0]); + dst->f[1] = truncf(src->f[1]); + dst->f[2] = truncf(src->f[2]); + dst->f[3] = truncf(src->f[3]); +} + +static void +micro_u2d(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) { - dst->f[0] = 0.0f; - dst->f[1] = 0.0f; - dst->f[2] = 0.0f; - dst->f[3] = 0.0f; + dst->d[0] = (double)src->u[0]; + dst->d[1] = (double)src->u[1]; + dst->d[2] = (double)src->u[2]; + dst->d[3] = (double)src->u[3]; } static void -micro_str(union tgsi_exec_channel *dst) +micro_i64abs(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) { - dst->f[0] = 1.0f; - dst->f[1] = 1.0f; - dst->f[2] = 1.0f; - dst->f[3] = 1.0f; + dst->i64[0] = src->i64[0] >= 0.0 ? src->i64[0] : -src->i64[0]; + dst->i64[1] = src->i64[1] >= 0.0 ? src->i64[1] : -src->i64[1]; + dst->i64[2] = src->i64[2] >= 0.0 ? src->i64[2] : -src->i64[2]; + dst->i64[3] = src->i64[3] >= 0.0 ? src->i64[3] : -src->i64[3]; } static void -micro_trunc(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src) +micro_i64sgn(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src->i64[0] < 0 ? -1 : src->i64[0] > 0 ? 1 : 0; + dst->i64[1] = src->i64[1] < 0 ? -1 : src->i64[1] > 0 ? 1 : 0; + dst->i64[2] = src->i64[2] < 0 ? -1 : src->i64[2] > 0 ? 1 : 0; + dst->i64[3] = src->i64[3] < 0 ? -1 : src->i64[3] > 0 ? 1 : 0; +} + +static void +micro_i64neg(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = -src->i64[0]; + dst->i64[1] = -src->i64[1]; + dst->i64[2] = -src->i64[2]; + dst->i64[3] = -src->i64[3]; +} + +static void +micro_u64seq(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) { - dst->f[0] = (float)(int)src->f[0]; - dst->f[1] = (float)(int)src->f[1]; - dst->f[2] = (float)(int)src->f[2]; - dst->f[3] = (float)(int)src->f[3]; + dst->u[0][0] = src[0].u64[0] == src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] == src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] == src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] == src[1].u64[3] ? ~0U : 0U; } +static void +micro_u64sne(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] != src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] != src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] != src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] != src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_i64slt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].i64[0] < src[1].i64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].i64[1] < src[1].i64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].i64[2] < src[1].i64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].i64[3] < src[1].i64[3] ? ~0U : 0U; +} + +static void +micro_u64slt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] < src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] < src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] < src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] < src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_i64sge(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].i64[0] >= src[1].i64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].i64[1] >= src[1].i64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].i64[2] >= src[1].i64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].i64[3] >= src[1].i64[3] ? ~0U : 0U; +} + +static void +micro_u64sge(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] >= src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] >= src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] >= src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] >= src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_u64max(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] > src[1].u64[0] ? src[0].u64[0] : src[1].u64[0]; + dst->u64[1] = src[0].u64[1] > src[1].u64[1] ? src[0].u64[1] : src[1].u64[1]; + dst->u64[2] = src[0].u64[2] > src[1].u64[2] ? src[0].u64[2] : src[1].u64[2]; + dst->u64[3] = src[0].u64[3] > src[1].u64[3] ? src[0].u64[3] : src[1].u64[3]; +} + +static void +micro_i64max(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] > src[1].i64[0] ? src[0].i64[0] : src[1].i64[0]; + dst->i64[1] = src[0].i64[1] > src[1].i64[1] ? src[0].i64[1] : src[1].i64[1]; + dst->i64[2] = src[0].i64[2] > src[1].i64[2] ? src[0].i64[2] : src[1].i64[2]; + dst->i64[3] = src[0].i64[3] > src[1].i64[3] ? src[0].i64[3] : src[1].i64[3]; +} + +static void +micro_u64min(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] < src[1].u64[0] ? src[0].u64[0] : src[1].u64[0]; + dst->u64[1] = src[0].u64[1] < src[1].u64[1] ? src[0].u64[1] : src[1].u64[1]; + dst->u64[2] = src[0].u64[2] < src[1].u64[2] ? src[0].u64[2] : src[1].u64[2]; + dst->u64[3] = src[0].u64[3] < src[1].u64[3] ? src[0].u64[3] : src[1].u64[3]; +} + +static void +micro_i64min(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] < src[1].i64[0] ? src[0].i64[0] : src[1].i64[0]; + dst->i64[1] = src[0].i64[1] < src[1].i64[1] ? src[0].i64[1] : src[1].i64[1]; + dst->i64[2] = src[0].i64[2] < src[1].i64[2] ? src[0].i64[2] : src[1].i64[2]; + dst->i64[3] = src[0].i64[3] < src[1].i64[3] ? src[0].i64[3] : src[1].i64[3]; +} + +static void +micro_u64add(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] + src[1].u64[0]; + dst->u64[1] = src[0].u64[1] + src[1].u64[1]; + dst->u64[2] = src[0].u64[2] + src[1].u64[2]; + dst->u64[3] = src[0].u64[3] + src[1].u64[3]; +} + +static void +micro_u64mul(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] * src[1].u64[0]; + dst->u64[1] = src[0].u64[1] * src[1].u64[1]; + dst->u64[2] = src[0].u64[2] * src[1].u64[2]; + dst->u64[3] = src[0].u64[3] * src[1].u64[3]; +} + +static void +micro_u64div(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] / src[1].u64[0]; + dst->u64[1] = src[0].u64[1] / src[1].u64[1]; + dst->u64[2] = src[0].u64[2] / src[1].u64[2]; + dst->u64[3] = src[0].u64[3] / src[1].u64[3]; +} + +static void +micro_i64div(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] / src[1].i64[0]; + dst->i64[1] = src[0].i64[1] / src[1].i64[1]; + dst->i64[2] = src[0].i64[2] / src[1].i64[2]; + dst->i64[3] = src[0].i64[3] / src[1].i64[3]; +} + +static void +micro_u64mod(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] % src[1].u64[0]; + dst->u64[1] = src[0].u64[1] % src[1].u64[1]; + dst->u64[2] = src[0].u64[2] % src[1].u64[2]; + dst->u64[3] = src[0].u64[3] % src[1].u64[3]; +} + +static void +micro_i64mod(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] % src[1].i64[0]; + dst->i64[1] = src[0].i64[1] % src[1].i64[1]; + dst->i64[2] = src[0].i64[2] % src[1].i64[2]; + dst->i64[3] = src[0].i64[3] % src[1].i64[3]; +} + +static void +micro_u64shl(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->u64[0] = src0->u64[0] << masked_count; + masked_count = src1->u[1] & 0x3f; + dst->u64[1] = src0->u64[1] << masked_count; + masked_count = src1->u[2] & 0x3f; + dst->u64[2] = src0->u64[2] << masked_count; + masked_count = src1->u[3] & 0x3f; + dst->u64[3] = src0->u64[3] << masked_count; +} + +static void +micro_i64shr(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->i64[0] = src0->i64[0] >> masked_count; + masked_count = src1->u[1] & 0x3f; + dst->i64[1] = src0->i64[1] >> masked_count; + masked_count = src1->u[2] & 0x3f; + dst->i64[2] = src0->i64[2] >> masked_count; + masked_count = src1->u[3] & 0x3f; + dst->i64[3] = src0->i64[3] >> masked_count; +} + +static void +micro_u64shr(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->u64[0] = src0->u64[0] >> masked_count; + masked_count = src1->u[1] & 0x3f; + dst->u64[1] = src0->u64[1] >> masked_count; + masked_count = src1->u[2] & 0x3f; + dst->u64[2] = src0->u64[2] >> masked_count; + masked_count = src1->u[3] & 0x3f; + dst->u64[3] = src0->u64[3] >> masked_count; +} enum tgsi_exec_datatype { TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_INT, - TGSI_EXEC_DATA_UINT + TGSI_EXEC_DATA_UINT, + TGSI_EXEC_DATA_DOUBLE, + TGSI_EXEC_DATA_INT64, + TGSI_EXEC_DATA_UINT64, }; /* @@ -521,7 +977,7 @@ static const union tgsi_exec_channel M128Vec = { * not lead to crashes, etc. But when debugging, it's helpful to catch * them. */ -static INLINE void +static inline void check_inf_or_nan(const union tgsi_exec_channel *chan) { assert(!util_is_inf_or_nan((chan)->f[0])); @@ -638,8 +1094,9 @@ void tgsi_exec_machine_bind_shader( struct tgsi_exec_machine *mach, const struct tgsi_token *tokens, - uint numSamplers, - struct tgsi_sampler **samplers) + struct tgsi_sampler *sampler, + struct tgsi_image *image, + struct tgsi_buffer *buffer) { uint k; struct tgsi_parse_context parse; @@ -654,12 +1111,11 @@ tgsi_exec_machine_bind_shader( util_init_math(); - if (numSamplers) { - assert(samplers); - } mach->Tokens = tokens; - mach->Samplers = samplers; + mach->Sampler = sampler; + mach->Image = image; + mach->Buffer = buffer; if (!tokens) { /* unbind and free all */ @@ -680,16 +1136,19 @@ tgsi_exec_machine_bind_shader( return; } - mach->Processor = parse.FullHeader.Processor.Processor; mach->ImmLimit = 0; + mach->NumOutputs = 0; - if (mach->Processor == TGSI_PROCESSOR_GEOMETRY && + for (k = 0; k < TGSI_SEMANTIC_COUNT; k++) + mach->SysSemanticToIndex[k] = -1; + + if (mach->ShaderType == PIPE_SHADER_GEOMETRY && !mach->UsedGeometryShader) { struct tgsi_exec_vector *inputs; struct tgsi_exec_vector *outputs; inputs = align_malloc(sizeof(struct tgsi_exec_vector) * - TGSI_MAX_PRIM_VERTICES * PIPE_MAX_ATTRIBS, + TGSI_MAX_PRIM_VERTICES * PIPE_MAX_SHADER_INPUTS, 16); if (!inputs) @@ -749,19 +1208,11 @@ tgsi_exec_machine_bind_shader( ++mach->NumOutputs; } } - if (parse.FullToken.FullDeclaration.Declaration.File == - TGSI_FILE_IMMEDIATE_ARRAY) { - unsigned reg; - struct tgsi_full_declaration *decl = - &parse.FullToken.FullDeclaration; - debug_assert(decl->Range.Last < TGSI_EXEC_NUM_IMMEDIATES); - for (reg = decl->Range.First; reg <= decl->Range.Last; ++reg) { - for( i = 0; i < 4; i++ ) { - int idx = reg * 4 + i; - mach->ImmArray[reg][i] = decl->ImmediateData.u[idx].Float; - } - } + else if (parse.FullToken.FullDeclaration.Declaration.File == TGSI_FILE_SYSTEM_VALUE) { + const struct tgsi_full_declaration *decl = &parse.FullToken.FullDeclaration; + mach->SysSemanticToIndex[decl->Semantic.Name] = decl->Range.First; } + memcpy(declarations + numDeclarations, &parse.FullToken.FullDeclaration, sizeof(declarations[0])); @@ -802,6 +1253,11 @@ tgsi_exec_machine_bind_shader( break; case TGSI_TOKEN_TYPE_PROPERTY: + if (mach->ShaderType == PIPE_SHADER_GEOMETRY) { + if (parse.FullToken.FullProperty.Property.PropertyName == TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES) { + mach->MaxOutputVertices = parse.FullToken.FullProperty.u[0].Data; + } + } break; default: @@ -821,7 +1277,7 @@ tgsi_exec_machine_bind_shader( struct tgsi_exec_machine * -tgsi_exec_machine_create( void ) +tgsi_exec_machine_create(enum pipe_shader_type shader_type) { struct tgsi_exec_machine *mach; uint i; @@ -832,14 +1288,17 @@ tgsi_exec_machine_create( void ) memset(mach, 0, sizeof(*mach)); + mach->ShaderType = shader_type; mach->Addrs = &mach->Temps[TGSI_EXEC_TEMP_ADDR]; mach->MaxGeometryShaderOutputs = TGSI_MAX_TOTAL_VERTICES; mach->Predicates = &mach->Temps[TGSI_EXEC_TEMP_P0]; - mach->Inputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_ATTRIBS, 16); - mach->Outputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_ATTRIBS, 16); - if (!mach->Inputs || !mach->Outputs) - goto fail; + if (shader_type != PIPE_SHADER_COMPUTE) { + mach->Inputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_INPUTS, 16); + mach->Outputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_OUTPUTS, 16); + if (!mach->Inputs || !mach->Outputs) + goto fail; + } /* Setup constants needed by the SSE2 executor. */ for( i = 0; i < 4; i++ ) { @@ -918,40 +1377,6 @@ micro_div( } } -static void -micro_rcc(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src) -{ - uint i; - - for (i = 0; i < 4; i++) { - float recip = 1.0f / src->f[i]; - - if (recip > 0.0f) { - if (recip > 1.884467e+019f) { - dst->f[i] = 1.884467e+019f; - } - else if (recip < 5.42101e-020f) { - dst->f[i] = 5.42101e-020f; - } - else { - dst->f[i] = recip; - } - } - else { - if (recip < -1.884467e+019f) { - dst->f[i] = -1.884467e+019f; - } - else if (recip > -5.42101e-020f) { - dst->f[i] = -5.42101e-020f; - } - else { - dst->f[i] = recip; - } - } - } -} - static void micro_lt( union tgsi_exec_channel *dst, @@ -1086,7 +1511,7 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, case TGSI_FILE_INPUT: for (i = 0; i < TGSI_QUAD_SIZE; i++) { /* - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { debug_printf("Fetching Input[%d] (2d=%d, 1d=%d)\n", index2D->i[i] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index->i[i], index2D->i[i], index->i[i]); @@ -1103,7 +1528,7 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, * gl_FragCoord, for example, in a sys value register. */ for (i = 0; i < TGSI_QUAD_SIZE; i++) { - chan->u[i] = mach->SystemValue[index->i[i]].u[i]; + chan->u[i] = mach->SystemValue[index->i[i]].xyzw[swizzle].u[i]; } break; @@ -1116,16 +1541,6 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, } break; - case TGSI_FILE_TEMPORARY_ARRAY: - for (i = 0; i < TGSI_QUAD_SIZE; i++) { - assert(index->i[i] < TGSI_EXEC_NUM_TEMPS); - assert(index2D->i[i] < TGSI_EXEC_NUM_TEMP_ARRAYS); - - chan->u[i] = - mach->TempArray[index2D->i[i]][index->i[i]].xyzw[swizzle].u[i]; - } - break; - case TGSI_FILE_IMMEDIATE: for (i = 0; i < TGSI_QUAD_SIZE; i++) { assert(index->i[i] >= 0 && index->i[i] < (int)mach->ImmLimit); @@ -1135,14 +1550,6 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, } break; - case TGSI_FILE_IMMEDIATE_ARRAY: - for (i = 0; i < TGSI_QUAD_SIZE; i++) { - assert(index2D->i[i] == 0); - - chan->f[i] = mach->ImmArray[index->i[i]][swizzle]; - } - break; - case TGSI_FILE_ADDRESS: for (i = 0; i < TGSI_QUAD_SIZE; i++) { assert(index->i[i] >= 0); @@ -1180,11 +1587,11 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, } static void -fetch_source(const struct tgsi_exec_machine *mach, - union tgsi_exec_channel *chan, - const struct tgsi_full_src_register *reg, - const uint chan_index, - enum tgsi_exec_datatype src_datatype) +fetch_source_d(const struct tgsi_exec_machine *mach, + union tgsi_exec_channel *chan, + const struct tgsi_full_src_register *reg, + const uint chan_index, + enum tgsi_exec_datatype src_datatype) { union tgsi_exec_channel index; union tgsi_exec_channel index2D; @@ -1224,7 +1631,7 @@ fetch_source(const struct tgsi_exec_machine *mach, index2.i[2] = index2.i[3] = reg->Indirect.Index; /* get current value of address register[swizzle] */ - swizzle = tgsi_util_get_src_register_swizzle( ®->Indirect, TGSI_CHAN_X ); + swizzle = reg->Indirect.Swizzle; fetch_src_file_channel(mach, chan_index, reg->Indirect.File, @@ -1284,7 +1691,7 @@ fetch_source(const struct tgsi_exec_machine *mach, index2.i[2] = index2.i[3] = reg->DimIndirect.Index; - swizzle = tgsi_util_get_src_register_swizzle( ®->DimIndirect, TGSI_CHAN_X ); + swizzle = reg->DimIndirect.Swizzle; fetch_src_file_channel(mach, chan_index, reg->DimIndirect.File, @@ -1327,6 +1734,16 @@ fetch_source(const struct tgsi_exec_machine *mach, &index, &index2D, chan); +} + +static void +fetch_source(const struct tgsi_exec_machine *mach, + union tgsi_exec_channel *chan, + const struct tgsi_full_src_register *reg, + const uint chan_index, + enum tgsi_exec_datatype src_datatype) +{ + fetch_source_d(mach, chan, reg, chan_index, src_datatype); if (reg->Register.Absolute) { if (src_datatype == TGSI_EXEC_DATA_FLOAT) { @@ -1345,16 +1762,16 @@ fetch_source(const struct tgsi_exec_machine *mach, } } -static void -store_dest(struct tgsi_exec_machine *mach, - const union tgsi_exec_channel *chan, - const struct tgsi_full_dst_register *reg, - const struct tgsi_full_instruction *inst, - uint chan_index, - enum tgsi_exec_datatype dst_datatype) +static union tgsi_exec_channel * +store_dest_dstret(struct tgsi_exec_machine *mach, + const union tgsi_exec_channel *chan, + const struct tgsi_full_dst_register *reg, + const struct tgsi_full_instruction *inst, + uint chan_index, + enum tgsi_exec_datatype dst_datatype) { uint i; - union tgsi_exec_channel null; + static union tgsi_exec_channel null; union tgsi_exec_channel *dst; union tgsi_exec_channel index2D; uint execmask = mach->ExecMask; @@ -1388,7 +1805,7 @@ store_dest(struct tgsi_exec_machine *mach, index.i[3] = reg->Indirect.Index; /* get current value of address register[swizzle] */ - swizzle = tgsi_util_get_src_register_swizzle( ®->Indirect, TGSI_CHAN_X ); + swizzle = reg->Indirect.Swizzle; /* fetch values from the address/indirection register */ fetch_src_file_channel(mach, @@ -1440,7 +1857,7 @@ store_dest(struct tgsi_exec_machine *mach, index2.i[2] = index2.i[3] = reg->DimIndirect.Index; - swizzle = tgsi_util_get_src_register_swizzle( ®->DimIndirect, TGSI_CHAN_X ); + swizzle = reg->DimIndirect.Swizzle; fetch_src_file_channel(mach, chan_index, reg->DimIndirect.File, @@ -1485,12 +1902,15 @@ store_dest(struct tgsi_exec_machine *mach, + reg->Register.Index; dst = &mach->Outputs[offset + index].xyzw[chan_index]; #if 0 - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { - fprintf(stderr, "STORING OUT[%d] mask(%d), = (", offset + index, execmask); + debug_printf("NumOutputs = %d, TEMP_O_C/I = %d, redindex = %d\n", + mach->NumOutputs, mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0], + reg->Register.Index); + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { + debug_printf("STORING OUT[%d] mask(%d), = (", offset + index, execmask); for (i = 0; i < TGSI_QUAD_SIZE; i++) if (execmask & (1 << i)) - fprintf(stderr, "%f, ", chan->f[i]); - fprintf(stderr, ")\n"); + debug_printf("%f, ", chan->f[i]); + debug_printf(")\n"); } #endif break; @@ -1501,16 +1921,6 @@ store_dest(struct tgsi_exec_machine *mach, dst = &mach->Temps[offset + index].xyzw[chan_index]; break; - case TGSI_FILE_TEMPORARY_ARRAY: - index = reg->Register.Index; - assert( index < TGSI_EXEC_NUM_TEMPS ); - assert( index2D.i[0] < TGSI_EXEC_NUM_TEMP_ARRAYS ); - /* XXX we use index2D.i[0] here but somehow we might - * end up with someone trying to store indirectly in - * different buffers */ - dst = &mach->TempArray[index2D.i[0]][offset + index].xyzw[chan_index]; - break; - case TGSI_FILE_ADDRESS: index = reg->Register.Index; dst = &mach->Addrs[index].xyzw[chan_index]; @@ -1524,7 +1934,7 @@ store_dest(struct tgsi_exec_machine *mach, default: assert( 0 ); - return; + return NULL; } if (inst->Instruction.Predicate) { @@ -1546,7 +1956,7 @@ store_dest(struct tgsi_exec_machine *mach, break; default: assert(0); - return; + return NULL; } assert(inst->Predicate.Index == 0); @@ -1568,14 +1978,55 @@ store_dest(struct tgsi_exec_machine *mach, } } - switch (inst->Instruction.Saturate) { - case TGSI_SAT_NONE: + return dst; +} + +static void +store_dest_double(struct tgsi_exec_machine *mach, + const union tgsi_exec_channel *chan, + const struct tgsi_full_dst_register *reg, + const struct tgsi_full_instruction *inst, + uint chan_index, + enum tgsi_exec_datatype dst_datatype) +{ + union tgsi_exec_channel *dst; + const uint execmask = mach->ExecMask; + int i; + + dst = store_dest_dstret(mach, chan, reg, inst, chan_index, + dst_datatype); + if (!dst) + return; + + /* doubles path */ + for (i = 0; i < TGSI_QUAD_SIZE; i++) + if (execmask & (1 << i)) + dst->i[i] = chan->i[i]; +} + +static void +store_dest(struct tgsi_exec_machine *mach, + const union tgsi_exec_channel *chan, + const struct tgsi_full_dst_register *reg, + const struct tgsi_full_instruction *inst, + uint chan_index, + enum tgsi_exec_datatype dst_datatype) +{ + union tgsi_exec_channel *dst; + const uint execmask = mach->ExecMask; + int i; + + dst = store_dest_dstret(mach, chan, reg, inst, chan_index, + dst_datatype); + if (!dst) + return; + + if (!inst->Instruction.Saturate) { for (i = 0; i < TGSI_QUAD_SIZE; i++) if (execmask & (1 << i)) dst->i[i] = chan->i[i]; - break; - - case TGSI_SAT_ZERO_ONE: + } + else { for (i = 0; i < TGSI_QUAD_SIZE; i++) if (execmask & (1 << i)) { if (chan->f[i] < 0.0f) @@ -1585,22 +2036,6 @@ store_dest(struct tgsi_exec_machine *mach, else dst->i[i] = chan->i[i]; } - break; - - case TGSI_SAT_MINUS_PLUS_ONE: - for (i = 0; i < TGSI_QUAD_SIZE; i++) - if (execmask & (1 << i)) { - if (chan->f[i] < -1.0f) - dst->f[i] = -1.0f; - else if (chan->f[i] > 1.0f) - dst->f[i] = 1.0f; - else - dst->i[i] = chan->i[i]; - } - break; - - default: - assert( 0 ); } } @@ -1616,8 +2051,8 @@ store_dest(struct tgsi_exec_machine *mach, * Kill fragment if any of the four values is less than zero. */ static void -exec_kil(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_kill_if(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { uint uniquemask; uint chan_index; @@ -1648,20 +2083,22 @@ exec_kil(struct tgsi_exec_machine *mach, kilmask |= 1 << i; } + /* restrict to fragments currently executing */ + kilmask &= mach->ExecMask; + mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0] |= kilmask; } /** - * Execute NVIDIA-style KIL which is predicated by a condition code. - * Kill fragment if the condition code is TRUE. + * Unconditional fragment kill/discard. */ static void -exec_kilp(struct tgsi_exec_machine *mach, +exec_kill(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { uint kilmask; /* bit 0 = pixel 0, bit 1 = pixel 1, etc */ - /* "unconditional" kil */ + /* kill fragment for all fragments currently executing */ kilmask = mach->ExecMask; mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0] |= kilmask; } @@ -1675,6 +2112,9 @@ emit_vertex(struct tgsi_exec_machine *mach) if ((mach->ExecMask & (1 << i))) */ if (mach->ExecMask) { + if (mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]] >= mach->MaxOutputVertices) + return; + mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0] += mach->NumOutputs; mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]]++; } @@ -1699,7 +2139,7 @@ emit_primitive(struct tgsi_exec_machine *mach) static void conditional_emit_primitive(struct tgsi_exec_machine *mach) { - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { int emitted_verts = mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]]; if (emitted_verts) { @@ -1714,11 +2154,15 @@ conditional_emit_primitive(struct tgsi_exec_machine *mach) */ static void fetch_texel( struct tgsi_sampler *sampler, + const unsigned sview_idx, + const unsigned sampler_idx, const union tgsi_exec_channel *s, const union tgsi_exec_channel *t, const union tgsi_exec_channel *p, const union tgsi_exec_channel *c0, const union tgsi_exec_channel *c1, + float derivs[3][2][TGSI_QUAD_SIZE], + const int8_t offset[3], enum tgsi_sampler_control control, union tgsi_exec_channel *r, union tgsi_exec_channel *g, @@ -1728,7 +2172,9 @@ fetch_texel( struct tgsi_sampler *sampler, uint j; float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; - sampler->get_samples(sampler, s->f, t->f, p->f, c0->f, c1->f, control, rgba); + /* FIXME: handle explicit derivs, offsets */ + sampler->get_samples(sampler, sview_idx, sampler_idx, + s->f, t->f, p->f, c0->f, c1->f, derivs, offset, control, rgba); for (j = 0; j < 4; j++) { r->f[j] = rgba[0][j]; @@ -1743,11 +2189,101 @@ fetch_texel( struct tgsi_sampler *sampler, #define TEX_MODIFIER_PROJECTED 1 #define TEX_MODIFIER_LOD_BIAS 2 #define TEX_MODIFIER_EXPLICIT_LOD 3 +#define TEX_MODIFIER_LEVEL_ZERO 4 +#define TEX_MODIFIER_GATHER 5 + +/* + * Fetch all 3 (for s,t,r coords) texel offsets, put them into int array. + */ +static void +fetch_texel_offsets(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + int8_t offsets[3]) +{ + if (inst->Texture.NumOffsets == 1) { + union tgsi_exec_channel index; + union tgsi_exec_channel offset[3]; + index.i[0] = index.i[1] = index.i[2] = index.i[3] = inst->TexOffsets[0].Index; + fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + inst->TexOffsets[0].SwizzleX, &index, &ZeroVec, &offset[0]); + fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + inst->TexOffsets[0].SwizzleY, &index, &ZeroVec, &offset[1]); + fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + inst->TexOffsets[0].SwizzleZ, &index, &ZeroVec, &offset[2]); + offsets[0] = offset[0].i[0]; + offsets[1] = offset[1].i[0]; + offsets[2] = offset[2].i[0]; + } else { + assert(inst->Texture.NumOffsets == 0); + offsets[0] = offsets[1] = offsets[2] = 0; + } +} + + +/* + * Fetch dx and dy values for one channel (s, t or r). + * Put dx values into one float array, dy values into another. + */ +static void +fetch_assign_deriv_channel(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + unsigned regdsrcx, + unsigned chan, + float derivs[2][TGSI_QUAD_SIZE]) +{ + union tgsi_exec_channel d; + FETCH(&d, regdsrcx, chan); + derivs[0][0] = d.f[0]; + derivs[0][1] = d.f[1]; + derivs[0][2] = d.f[2]; + derivs[0][3] = d.f[3]; + FETCH(&d, regdsrcx + 1, chan); + derivs[1][0] = d.f[0]; + derivs[1][1] = d.f[1]; + derivs[1][2] = d.f[2]; + derivs[1][3] = d.f[3]; +} + +static uint +fetch_sampler_unit(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + uint sampler) +{ + uint unit = 0; + int i; + if (inst->Src[sampler].Register.Indirect) { + const struct tgsi_full_src_register *reg = &inst->Src[sampler]; + union tgsi_exec_channel indir_index, index2; + const uint execmask = mach->ExecMask; + index2.i[0] = + index2.i[1] = + index2.i[2] = + index2.i[3] = reg->Indirect.Index; + + fetch_src_file_channel(mach, + 0, + reg->Indirect.File, + reg->Indirect.Swizzle, + &index2, + &ZeroVec, + &indir_index); + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + if (execmask & (1 << i)) { + unit = inst->Src[sampler].Register.Index + indir_index.i[i]; + break; + } + } + + } else { + unit = inst->Src[sampler].Register.Index; + } + return unit; +} /* * execute a texture instruction. * - * modifier is used to control the channel routing for the\ + * modifier is used to control the channel routing for the * instruction variants like proj, lod, and texture with lod bias. * sampler indicates which src register the sampler is contained in. */ @@ -1756,179 +2292,91 @@ exec_tex(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst, uint modifier, uint sampler) { - const uint unit = inst->Src[sampler].Register.Index; - union tgsi_exec_channel r[4], cubearraycomp, cubelod; - const union tgsi_exec_channel *lod = &ZeroVec; - enum tgsi_sampler_control control; + const union tgsi_exec_channel *args[5], *proj = NULL; + union tgsi_exec_channel r[5]; + enum tgsi_sampler_control control = TGSI_SAMPLER_LOD_NONE; uint chan; + uint unit; + int8_t offsets[3]; + int dim, shadow_ref, i; - if (modifier != TEX_MODIFIER_NONE && (sampler == 1)) { - FETCH(&r[3], 0, TGSI_CHAN_W); - if (modifier != TEX_MODIFIER_PROJECTED) { - lod = &r[3]; - } - } - - if (modifier == TEX_MODIFIER_EXPLICIT_LOD) { - control = tgsi_sampler_lod_explicit; - } else { - control = tgsi_sampler_lod_bias; - } - - switch (inst->Texture.Texture) { - case TGSI_TEXTURE_1D: - FETCH(&r[0], 0, TGSI_CHAN_X); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - } + unit = fetch_sampler_unit(mach, inst, sampler); + /* always fetch all 3 offsets, overkill but keeps code simple */ + fetch_texel_offsets(mach, inst, offsets); - fetch_texel(mach->Samplers[unit], - &r[0], &ZeroVec, &ZeroVec, lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ - break; - case TGSI_TEXTURE_SHADOW1D: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[2], 0, TGSI_CHAN_Z); + assert(modifier != TEX_MODIFIER_LEVEL_ZERO); + assert(inst->Texture.Texture != TGSI_TEXTURE_BUFFER); - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - } + dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture); + shadow_ref = tgsi_util_get_shadow_ref_src_index(inst->Texture.Texture); - fetch_texel(mach->Samplers[unit], - &r[0], &ZeroVec, &r[2], lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ - break; + assert(dim <= 4); + if (shadow_ref >= 0) + assert(shadow_ref >= dim && shadow_ref < ARRAY_SIZE(args)); - case TGSI_TEXTURE_2D: - case TGSI_TEXTURE_RECT: - case TGSI_TEXTURE_SHADOW2D: - case TGSI_TEXTURE_SHADOWRECT: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); + /* fetch modifier to the last argument */ + if (modifier != TEX_MODIFIER_NONE) { + const int last = ARRAY_SIZE(args) - 1; - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - micro_div(&r[1], &r[1], &r[3]); - micro_div(&r[2], &r[2], &r[3]); + /* fetch modifier from src0.w or src1.x */ + if (sampler == 1) { + assert(dim <= TGSI_CHAN_W && shadow_ref != TGSI_CHAN_W); + FETCH(&r[last], 0, TGSI_CHAN_W); } - - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - - case TGSI_TEXTURE_1D_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); + else { + assert(shadow_ref != 4); + FETCH(&r[last], 1, TGSI_CHAN_X); } - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &ZeroVec, lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - case TGSI_TEXTURE_SHADOW1D_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); + if (modifier != TEX_MODIFIER_PROJECTED) { + args[last] = &r[last]; } - - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - - case TGSI_TEXTURE_2D_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - micro_div(&r[1], &r[1], &r[3]); + else { + proj = &r[last]; + args[last] = &ZeroVec; } - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - case TGSI_TEXTURE_SHADOW2D_ARRAY: - case TGSI_TEXTURE_SHADOWCUBE: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - FETCH(&r[3], 0, TGSI_CHAN_W); - - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &r[3], &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - case TGSI_TEXTURE_CUBE_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - FETCH(&r[3], 0, TGSI_CHAN_W); + /* point unused arguments to zero vector */ + for (i = dim; i < last; i++) + args[i] = &ZeroVec; - if (modifier == TEX_MODIFIER_EXPLICIT_LOD || - modifier == TEX_MODIFIER_LOD_BIAS) - FETCH(&cubelod, 1, TGSI_CHAN_X); - else - cubelod = ZeroVec; + if (modifier == TEX_MODIFIER_EXPLICIT_LOD) + control = TGSI_SAMPLER_LOD_EXPLICIT; + else if (modifier == TEX_MODIFIER_LOD_BIAS) + control = TGSI_SAMPLER_LOD_BIAS; + else if (modifier == TEX_MODIFIER_GATHER) + control = TGSI_SAMPLER_GATHER; + } + else { + for (i = dim; i < ARRAY_SIZE(args); i++) + args[i] = &ZeroVec; + } - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &r[3], &cubelod, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - case TGSI_TEXTURE_3D: - case TGSI_TEXTURE_CUBE: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); + /* fetch coordinates */ + for (i = 0; i < dim; i++) { + FETCH(&r[i], 0, TGSI_CHAN_X + i); - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - micro_div(&r[1], &r[1], &r[3]); - micro_div(&r[2], &r[2], &r[3]); - } + if (proj) + micro_div(&r[i], &r[i], proj); - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, - control, - &r[0], &r[1], &r[2], &r[3]); - break; + args[i] = &r[i]; + } - case TGSI_TEXTURE_SHADOWCUBE_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); - FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - FETCH(&r[3], 0, TGSI_CHAN_W); + /* fetch reference value */ + if (shadow_ref >= 0) { + FETCH(&r[shadow_ref], shadow_ref / 4, TGSI_CHAN_X + (shadow_ref % 4)); - FETCH(&cubearraycomp, 1, TGSI_CHAN_X); + if (proj) + micro_div(&r[shadow_ref], &r[shadow_ref], proj); - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &r[3], &cubearraycomp, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ - break; - default: - assert(0); + args[shadow_ref] = &r[shadow_ref]; } + fetch_texel(mach->Sampler, unit, unit, + args[0], args[1], args[2], args[3], args[4], + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ + #if 0 debug_printf("fetch r: %g %g %g %g\n", r[0].f[0], r[0].f[1], r[0].f[2], r[0].f[3]); @@ -1947,72 +2395,141 @@ exec_tex(struct tgsi_exec_machine *mach, } } +static void +exec_lodq(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + uint unit; + int dim; + int i; + union tgsi_exec_channel coords[4]; + const union tgsi_exec_channel *args[ARRAY_SIZE(coords)]; + union tgsi_exec_channel r[2]; + + unit = fetch_sampler_unit(mach, inst, 1); + dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture); + assert(dim <= ARRAY_SIZE(coords)); + /* fetch coordinates */ + for (i = 0; i < dim; i++) { + FETCH(&coords[i], 0, TGSI_CHAN_X + i); + args[i] = &coords[i]; + } + for (i = dim; i < ARRAY_SIZE(coords); i++) { + args[i] = &ZeroVec; + } + mach->Sampler->query_lod(mach->Sampler, unit, unit, + args[0]->f, + args[1]->f, + args[2]->f, + args[3]->f, + TGSI_SAMPLER_LOD_NONE, + r[0].f, + r[1].f); + + if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { + store_dest(mach, &r[0], &inst->Dst[0], inst, TGSI_CHAN_X, + TGSI_EXEC_DATA_FLOAT); + } + if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { + store_dest(mach, &r[1], &inst->Dst[0], inst, TGSI_CHAN_Y, + TGSI_EXEC_DATA_FLOAT); + } +} + static void exec_txd(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { - const uint unit = inst->Src[3].Register.Index; union tgsi_exec_channel r[4]; + float derivs[3][2][TGSI_QUAD_SIZE]; uint chan; + uint unit; + int8_t offsets[3]; - /* - * XXX: This is fake TXD -- the derivatives are not taken into account, yet. - */ + unit = fetch_sampler_unit(mach, inst, 3); + /* always fetch all 3 offsets, overkill but keeps code simple */ + fetch_texel_offsets(mach, inst, offsets); switch (inst->Texture.Texture) { case TGSI_TEXTURE_1D: - case TGSI_TEXTURE_SHADOW1D: - FETCH(&r[0], 0, TGSI_CHAN_X); - fetch_texel(mach->Samplers[unit], - &r[0], &ZeroVec, &ZeroVec, &ZeroVec, &ZeroVec, /* S, T, P, BIAS */ - tgsi_sampler_lod_bias, + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_X, derivs[0]); + + fetch_texel(mach->Sampler, unit, unit, + &r[0], &ZeroVec, &ZeroVec, &ZeroVec, &ZeroVec, /* S, T, P, C, LOD */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ break; + case TGSI_TEXTURE_SHADOW1D: case TGSI_TEXTURE_1D_ARRAY: - case TGSI_TEXTURE_2D: - case TGSI_TEXTURE_RECT: case TGSI_TEXTURE_SHADOW1D_ARRAY: - case TGSI_TEXTURE_SHADOW2D: - case TGSI_TEXTURE_SHADOWRECT: - + /* SHADOW1D/1D_ARRAY would not need Y/Z respectively, but don't bother */ FETCH(&r[0], 0, TGSI_CHAN_X); FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec, /* inputs */ - tgsi_sampler_lod_bias, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_X, derivs[0]); + + fetch_texel(mach->Sampler, unit, unit, + &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec, /* S, T, P, C, LOD */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, + &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ break; - case TGSI_TEXTURE_2D_ARRAY: - case TGSI_TEXTURE_3D: - case TGSI_TEXTURE_CUBE: - case TGSI_TEXTURE_CUBE_ARRAY: + case TGSI_TEXTURE_2D: + case TGSI_TEXTURE_RECT: FETCH(&r[0], 0, TGSI_CHAN_X); FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec, - tgsi_sampler_lod_bias, - &r[0], &r[1], &r[2], &r[3]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_X, derivs[0]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_Y, derivs[1]); + + fetch_texel(mach->Sampler, unit, unit, + &r[0], &r[1], &ZeroVec, &ZeroVec, &ZeroVec, /* S, T, P, C, LOD */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, + &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ break; + + case TGSI_TEXTURE_SHADOW2D: + case TGSI_TEXTURE_SHADOWRECT: + case TGSI_TEXTURE_2D_ARRAY: case TGSI_TEXTURE_SHADOW2D_ARRAY: + /* only SHADOW2D_ARRAY actually needs W */ + FETCH(&r[0], 0, TGSI_CHAN_X); + FETCH(&r[1], 0, TGSI_CHAN_Y); + FETCH(&r[2], 0, TGSI_CHAN_Z); + FETCH(&r[3], 0, TGSI_CHAN_W); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_X, derivs[0]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_Y, derivs[1]); + + fetch_texel(mach->Sampler, unit, unit, + &r[0], &r[1], &r[2], &r[3], &ZeroVec, /* inputs */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, + &r[0], &r[1], &r[2], &r[3]); /* outputs */ + break; + + case TGSI_TEXTURE_3D: + case TGSI_TEXTURE_CUBE: + case TGSI_TEXTURE_CUBE_ARRAY: + case TGSI_TEXTURE_SHADOWCUBE: + /* only TEXTURE_CUBE_ARRAY and TEXTURE_SHADOWCUBE actually need W */ FETCH(&r[0], 0, TGSI_CHAN_X); FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); FETCH(&r[3], 0, TGSI_CHAN_W); - fetch_texel(mach->Samplers[unit], - &r[0], &r[1], &r[2], &r[3], &ZeroVec, - tgsi_sampler_lod_bias, - &r[0], &r[1], &r[2], &r[3]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_X, derivs[0]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_Y, derivs[1]); + fetch_assign_deriv_channel(mach, inst, 1, TGSI_CHAN_Z, derivs[2]); + + fetch_texel(mach->Sampler, unit, unit, + &r[0], &r[1], &r[2], &r[3], &ZeroVec, /* inputs */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, + &r[0], &r[1], &r[2], &r[3]); /* outputs */ break; default: @@ -2029,38 +2546,34 @@ exec_txd(struct tgsi_exec_machine *mach, static void exec_txf(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) + const struct tgsi_full_instruction *inst) { - struct tgsi_sampler *sampler; - const uint unit = inst->Src[2].Register.Index; union tgsi_exec_channel r[4]; - union tgsi_exec_channel offset[3]; uint chan; + uint unit; float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; int j; int8_t offsets[3]; + unsigned target; - if (inst->Texture.NumOffsets == 1) { - union tgsi_exec_channel index; - index.i[0] = index.i[1] = index.i[2] = index.i[3] = inst->TexOffsets[0].Index; - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, - inst->TexOffsets[0].SwizzleX, &index, &ZeroVec, &offset[0]); - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, - inst->TexOffsets[0].SwizzleY, &index, &ZeroVec, &offset[1]); - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, - inst->TexOffsets[0].SwizzleZ, &index, &ZeroVec, &offset[2]); - offsets[0] = offset[0].i[0]; - offsets[1] = offset[1].i[0]; - offsets[2] = offset[2].i[0]; - } else - offsets[0] = offsets[1] = offsets[2] = 0; + unit = fetch_sampler_unit(mach, inst, 1); + /* always fetch all 3 offsets, overkill but keeps code simple */ + fetch_texel_offsets(mach, inst, offsets); IFETCH(&r[3], 0, TGSI_CHAN_W); - switch(inst->Texture.Texture) { + if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I || + inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I_MS) { + target = mach->SamplerViews[unit].Resource; + } + else { + target = inst->Texture.Texture; + } + switch(target) { case TGSI_TEXTURE_3D: case TGSI_TEXTURE_2D_ARRAY: case TGSI_TEXTURE_SHADOW2D_ARRAY: + case TGSI_TEXTURE_2D_ARRAY_MSAA: IFETCH(&r[2], 0, TGSI_CHAN_Z); /* fallthrough */ case TGSI_TEXTURE_2D: @@ -2069,6 +2582,7 @@ exec_txf(struct tgsi_exec_machine *mach, case TGSI_TEXTURE_SHADOW2D: case TGSI_TEXTURE_SHADOWRECT: case TGSI_TEXTURE_1D_ARRAY: + case TGSI_TEXTURE_2D_MSAA: IFETCH(&r[1], 0, TGSI_CHAN_Y); /* fallthrough */ case TGSI_TEXTURE_BUFFER: @@ -2081,9 +2595,8 @@ exec_txf(struct tgsi_exec_machine *mach, break; } - sampler = mach->Samplers[unit]; - sampler->get_texel(sampler, r[0].i, r[1].i, r[2].i, r[3].i, - offsets, rgba); + mach->Sampler->get_texel(mach->Sampler, unit, r[0].i, r[1].i, r[2].i, r[3].i, + offsets, rgba); for (j = 0; j < TGSI_QUAD_SIZE; j++) { r[0].f[j] = rgba[0][j]; @@ -2092,9 +2605,26 @@ exec_txf(struct tgsi_exec_machine *mach, r[3].f[j] = rgba[3][j]; } - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I || + inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I_MS) { + unsigned char swizzles[4]; + swizzles[0] = inst->Src[1].Register.SwizzleX; + swizzles[1] = inst->Src[1].Register.SwizzleY; + swizzles[2] = inst->Src[1].Register.SwizzleZ; + swizzles[3] = inst->Src[1].Register.SwizzleW; + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[swizzles[chan]], + &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } + } + else { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } } } } @@ -2103,28 +2633,29 @@ static void exec_txq(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { - struct tgsi_sampler *sampler; - const uint unit = inst->Src[1].Register.Index; int result[4]; union tgsi_exec_channel r[4], src; uint chan; + uint unit; int i,j; + unit = fetch_sampler_unit(mach, inst, 1); + fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); - sampler = mach->Samplers[unit]; - sampler->get_dims(sampler, src.i[0], result); + /* XXX: This interface can't return per-pixel values */ + mach->Sampler->get_dims(mach->Sampler, unit, src.i[0], result); for (i = 0; i < TGSI_QUAD_SIZE; i++) { for (j = 0; j < 4; j++) { - r[j].i[i] = result[j]; + r[j].i[i] = result[j]; } } for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, - TGSI_EXEC_DATA_INT); + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_INT); } } } @@ -2132,111 +2663,130 @@ exec_txq(struct tgsi_exec_machine *mach, static void exec_sample(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst, - uint modifier) + uint modifier, boolean compare) { const uint resource_unit = inst->Src[1].Register.Index; const uint sampler_unit = inst->Src[2].Register.Index; - union tgsi_exec_channel r[4]; + union tgsi_exec_channel r[5], c1; const union tgsi_exec_channel *lod = &ZeroVec; - enum tgsi_sampler_control control; + enum tgsi_sampler_control control = TGSI_SAMPLER_LOD_NONE; uint chan; + unsigned char swizzles[4]; + int8_t offsets[3]; - if (modifier != TEX_MODIFIER_NONE) { - if (modifier == TEX_MODIFIER_LOD_BIAS) - FETCH(&r[3], 3, TGSI_CHAN_X); - else /*TEX_MODIFIER_LOD*/ - FETCH(&r[3], 0, TGSI_CHAN_W); + /* always fetch all 3 offsets, overkill but keeps code simple */ + fetch_texel_offsets(mach, inst, offsets); - if (modifier != TEX_MODIFIER_PROJECTED) { - lod = &r[3]; + assert(modifier != TEX_MODIFIER_PROJECTED); + + if (modifier != TEX_MODIFIER_NONE) { + if (modifier == TEX_MODIFIER_LOD_BIAS) { + FETCH(&c1, 3, TGSI_CHAN_X); + lod = &c1; + control = TGSI_SAMPLER_LOD_BIAS; + } + else if (modifier == TEX_MODIFIER_EXPLICIT_LOD) { + FETCH(&c1, 3, TGSI_CHAN_X); + lod = &c1; + control = TGSI_SAMPLER_LOD_EXPLICIT; + } + else { + assert(modifier == TEX_MODIFIER_LEVEL_ZERO); + control = TGSI_SAMPLER_LOD_ZERO; } } - if (modifier == TEX_MODIFIER_EXPLICIT_LOD) { - control = tgsi_sampler_lod_explicit; - } else { - control = tgsi_sampler_lod_bias; - } + FETCH(&r[0], 0, TGSI_CHAN_X); switch (mach->SamplerViews[resource_unit].Resource) { case TGSI_TEXTURE_1D: - case TGSI_TEXTURE_SHADOW1D: - FETCH(&r[0], 0, TGSI_CHAN_X); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); + if (compare) { + FETCH(&r[2], 3, TGSI_CHAN_X); + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &ZeroVec, &r[2], &ZeroVec, lod, /* S, T, P, C, LOD */ + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ + } + else { + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &ZeroVec, &ZeroVec, &ZeroVec, lod, /* S, T, P, C, LOD */ + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ } - - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &ZeroVec, &ZeroVec, lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ break; case TGSI_TEXTURE_1D_ARRAY: case TGSI_TEXTURE_2D: case TGSI_TEXTURE_RECT: - case TGSI_TEXTURE_SHADOW1D_ARRAY: - case TGSI_TEXTURE_SHADOW2D: - case TGSI_TEXTURE_SHADOWRECT: - FETCH(&r[0], 0, TGSI_CHAN_X); FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[2], 0, TGSI_CHAN_Z); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - micro_div(&r[1], &r[1], &r[3]); - micro_div(&r[2], &r[2], &r[3]); + if (compare) { + FETCH(&r[2], 3, TGSI_CHAN_X); + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &ZeroVec, lod, /* S, T, P, C, LOD */ + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); /* outputs */ + } + else { + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &ZeroVec, &ZeroVec, lod, /* S, T, P, C, LOD */ + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); /* outputs */ } - - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, /* S, T, P, LOD */ - control, - &r[0], &r[1], &r[2], &r[3]); /* outputs */ break; case TGSI_TEXTURE_2D_ARRAY: case TGSI_TEXTURE_3D: case TGSI_TEXTURE_CUBE: - case TGSI_TEXTURE_CUBE_ARRAY: - FETCH(&r[0], 0, TGSI_CHAN_X); FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); - - if (modifier == TEX_MODIFIER_PROJECTED) { - micro_div(&r[0], &r[0], &r[3]); - micro_div(&r[1], &r[1], &r[3]); - micro_div(&r[2], &r[2], &r[3]); + if(compare) { + FETCH(&r[3], 3, TGSI_CHAN_X); + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &r[3], lod, + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); + } + else { + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &ZeroVec, lod, + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); } - - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &r[1], &r[2], lod, &ZeroVec, - control, - &r[0], &r[1], &r[2], &r[3]); break; - case TGSI_TEXTURE_SHADOW2D_ARRAY: - case TGSI_TEXTURE_SHADOWCUBE: - FETCH(&r[0], 0, TGSI_CHAN_X); + case TGSI_TEXTURE_CUBE_ARRAY: FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); FETCH(&r[3], 0, TGSI_CHAN_W); - - assert(modifier != TEX_MODIFIER_PROJECTED); - - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &r[1], &r[2], &r[3], &ZeroVec, - control, - &r[0], &r[1], &r[2], &r[3]); + if(compare) { + FETCH(&r[4], 3, TGSI_CHAN_X); + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &r[3], &r[4], + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); + } + else { + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &r[3], lod, + NULL, offsets, control, + &r[0], &r[1], &r[2], &r[3]); + } break; + default: assert(0); } + swizzles[0] = inst->Src[1].Register.SwizzleX; + swizzles[1] = inst->Src[1].Register.SwizzleY; + swizzles[2] = inst->Src[1].Register.SwizzleZ; + swizzles[3] = inst->Src[1].Register.SwizzleW; + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + store_dest(mach, &r[swizzles[chan]], + &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); } } } @@ -2248,49 +2798,61 @@ exec_sample_d(struct tgsi_exec_machine *mach, const uint resource_unit = inst->Src[1].Register.Index; const uint sampler_unit = inst->Src[2].Register.Index; union tgsi_exec_channel r[4]; + float derivs[3][2][TGSI_QUAD_SIZE]; uint chan; - /* - * XXX: This is fake SAMPLE_D -- the derivatives are not taken into account, yet. - */ + unsigned char swizzles[4]; + int8_t offsets[3]; + + /* always fetch all 3 offsets, overkill but keeps code simple */ + fetch_texel_offsets(mach, inst, offsets); + + FETCH(&r[0], 0, TGSI_CHAN_X); switch (mach->SamplerViews[resource_unit].Resource) { case TGSI_TEXTURE_1D: - case TGSI_TEXTURE_SHADOW1D: + case TGSI_TEXTURE_1D_ARRAY: + /* only 1D array actually needs Y */ + FETCH(&r[1], 0, TGSI_CHAN_Y); - FETCH(&r[0], 0, TGSI_CHAN_X); + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_X, derivs[0]); - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &ZeroVec, &ZeroVec, &ZeroVec, &ZeroVec, /* S, T, P, BIAS */ - tgsi_sampler_lod_bias, + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &ZeroVec, &ZeroVec, &ZeroVec, /* S, T, P, C, LOD */ + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, &r[0], &r[1], &r[2], &r[3]); /* R, G, B, A */ break; case TGSI_TEXTURE_2D: case TGSI_TEXTURE_RECT: - case TGSI_TEXTURE_SHADOW2D: - case TGSI_TEXTURE_SHADOWRECT: - - FETCH(&r[0], 0, TGSI_CHAN_X); + case TGSI_TEXTURE_2D_ARRAY: + /* only 2D array actually needs Z */ FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); - fetch_texel(mach->Samplers[sampler_unit], + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_X, derivs[0]); + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_Y, derivs[1]); + + fetch_texel(mach->Sampler, resource_unit, sampler_unit, &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec, /* inputs */ - tgsi_sampler_lod_bias, + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, &r[0], &r[1], &r[2], &r[3]); /* outputs */ break; case TGSI_TEXTURE_3D: case TGSI_TEXTURE_CUBE: case TGSI_TEXTURE_CUBE_ARRAY: - - FETCH(&r[0], 0, TGSI_CHAN_X); + /* only cube array actually needs W */ FETCH(&r[1], 0, TGSI_CHAN_Y); FETCH(&r[2], 0, TGSI_CHAN_Z); + FETCH(&r[3], 0, TGSI_CHAN_W); + + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_X, derivs[0]); + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_Y, derivs[1]); + fetch_assign_deriv_channel(mach, inst, 3, TGSI_CHAN_Z, derivs[2]); - fetch_texel(mach->Samplers[sampler_unit], - &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec, - tgsi_sampler_lod_bias, + fetch_texel(mach->Sampler, resource_unit, sampler_unit, + &r[0], &r[1], &r[2], &r[3], &ZeroVec, + derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT, &r[0], &r[1], &r[2], &r[3]); break; @@ -2298,9 +2860,15 @@ exec_sample_d(struct tgsi_exec_machine *mach, assert(0); } + swizzles[0] = inst->Src[1].Register.SwizzleX; + swizzles[1] = inst->Src[1].Register.SwizzleY; + swizzles[2] = inst->Src[1].Register.SwizzleZ; + swizzles[3] = inst->Src[1].Register.SwizzleW; + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + store_dest(mach, &r[swizzles[chan]], + &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); } } } @@ -2382,7 +2950,7 @@ exec_declaration(struct tgsi_exec_machine *mach, return; } - if (mach->Processor == TGSI_PROCESSOR_FRAGMENT) { + if (mach->ShaderType == PIPE_SHADER_FRAGMENT) { if (decl->Declaration.File == TGSI_FILE_INPUT) { uint first, last, mask; @@ -2440,33 +3008,26 @@ exec_declaration(struct tgsi_exec_machine *mach, } } } - } - } - - if (decl->Declaration.File == TGSI_FILE_SYSTEM_VALUE) { - mach->SysSemanticToIndex[decl->Declaration.Semantic] = decl->Range.First; - } -} - - -typedef void (* micro_op)(union tgsi_exec_channel *dst); - -static void -exec_vector(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst, - micro_op op, - enum tgsi_exec_datatype dst_datatype) -{ - unsigned int chan; - - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - union tgsi_exec_channel dst; - op(&dst); - store_dest(mach, &dst, &inst->Dst[0], inst, chan, dst_datatype); + if (DEBUG_EXECUTION) { + uint i, j; + for (i = first; i <= last; ++i) { + debug_printf("IN[%2u] = ", i); + for (j = 0; j < TGSI_NUM_CHANNELS; j++) { + if (j > 0) { + debug_printf(" "); + } + debug_printf("(%6f %u, %6f %u, %6f %u, %6f %u)\n", + mach->Inputs[i].xyzw[0].f[j], mach->Inputs[i].xyzw[0].u[j], + mach->Inputs[i].xyzw[1].f[j], mach->Inputs[i].xyzw[1].u[j], + mach->Inputs[i].xyzw[2].f[j], mach->Inputs[i].xyzw[2].u[j], + mach->Inputs[i].xyzw[3].f[j], mach->Inputs[i].xyzw[3].u[j]); + } + } + } } } + } typedef void (* micro_unary_op)(union tgsi_exec_channel *dst, @@ -2533,7 +3094,7 @@ exec_scalar_binary(struct tgsi_exec_machine *mach, union tgsi_exec_channel dst; fetch_source(mach, &src[0], &inst->Src[0], TGSI_CHAN_X, src_datatype); - fetch_source(mach, &src[1], &inst->Src[1], TGSI_CHAN_Y, src_datatype); + fetch_source(mach, &src[1], &inst->Src[1], TGSI_CHAN_X, src_datatype); op(&dst, &src[0], &src[1]); for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { if (inst->Dst[0].Register.WriteMask & (1 << chan)) { @@ -2600,6 +3161,40 @@ exec_vector_trinary(struct tgsi_exec_machine *mach, } } +typedef void (* micro_quaternary_op)(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2, + const union tgsi_exec_channel *src3); + +static void +exec_vector_quaternary(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_quaternary_op op, + enum tgsi_exec_datatype dst_datatype, + enum tgsi_exec_datatype src_datatype) +{ + unsigned int chan; + struct tgsi_exec_vector dst; + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + union tgsi_exec_channel src[4]; + + fetch_source(mach, &src[0], &inst->Src[0], chan, src_datatype); + fetch_source(mach, &src[1], &inst->Src[1], chan, src_datatype); + fetch_source(mach, &src[2], &inst->Src[2], chan, src_datatype); + fetch_source(mach, &src[3], &inst->Src[3], chan, src_datatype); + op(&dst.xyzw[chan], &src[0], &src[1], &src[2], &src[3]); + } + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &dst.xyzw[chan], &inst->Dst[0], inst, chan, dst_datatype); + } + } +} + static void exec_dp3(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) @@ -2725,66 +3320,81 @@ exec_dp2(struct tgsi_exec_machine *mach, } static void -exec_nrm4(struct tgsi_exec_machine *mach, +exec_pk2h(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { - unsigned int chan; - union tgsi_exec_channel arg[4]; - union tgsi_exec_channel scale; + unsigned chan; + union tgsi_exec_channel arg[2], dst; fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&scale, &arg[0], &arg[0]); - - for (chan = TGSI_CHAN_Y; chan <= TGSI_CHAN_W; chan++) { - union tgsi_exec_channel product; - - fetch_source(mach, &arg[chan], &inst->Src[0], chan, TGSI_EXEC_DATA_FLOAT); - micro_mul(&product, &arg[chan], &arg[chan]); - micro_add(&scale, &scale, &product); + fetch_source(mach, &arg[1], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); + for (chan = 0; chan < TGSI_QUAD_SIZE; chan++) { + dst.u[chan] = util_float_to_half(arg[0].f[chan]) | + (util_float_to_half(arg[1].f[chan]) << 16); } - - micro_rsq(&scale, &scale); - - for (chan = TGSI_CHAN_X; chan <= TGSI_CHAN_W; chan++) { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - micro_mul(&arg[chan], &arg[chan], &scale); - store_dest(mach, &arg[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + store_dest(mach, &dst, &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_UINT); } } } static void -exec_nrm3(struct tgsi_exec_machine *mach, +exec_up2h(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XYZ) { - unsigned int chan; - union tgsi_exec_channel arg[3]; - union tgsi_exec_channel scale; + unsigned chan; + union tgsi_exec_channel arg, dst[2]; - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&scale, &arg[0], &arg[0]); + fetch_source(mach, &arg, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_UINT); + for (chan = 0; chan < TGSI_QUAD_SIZE; chan++) { + dst[0].f[chan] = util_half_to_float(arg.u[chan] & 0xffff); + dst[1].f[chan] = util_half_to_float(arg.u[chan] >> 16); + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &dst[chan & 1], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} - for (chan = TGSI_CHAN_Y; chan <= TGSI_CHAN_Z; chan++) { - union tgsi_exec_channel product; +static void +micro_ucmp(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2) +{ + dst->f[0] = src0->u[0] ? src1->f[0] : src2->f[0]; + dst->f[1] = src0->u[1] ? src1->f[1] : src2->f[1]; + dst->f[2] = src0->u[2] ? src1->f[2] : src2->f[2]; + dst->f[3] = src0->u[3] ? src1->f[3] : src2->f[3]; +} - fetch_source(mach, &arg[chan], &inst->Src[0], chan, TGSI_EXEC_DATA_FLOAT); - micro_mul(&product, &arg[chan], &arg[chan]); - micro_add(&scale, &scale, &product); - } +static void +exec_ucmp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + unsigned int chan; + struct tgsi_exec_vector dst; - micro_rsq(&scale, &scale); + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + union tgsi_exec_channel src[3]; - for (chan = TGSI_CHAN_X; chan <= TGSI_CHAN_Z; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - micro_mul(&arg[chan], &arg[chan], &scale); - store_dest(mach, &arg[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); - } + fetch_source(mach, &src[0], &inst->Src[0], chan, + TGSI_EXEC_DATA_UINT); + fetch_source(mach, &src[1], &inst->Src[1], chan, + TGSI_EXEC_DATA_FLOAT); + fetch_source(mach, &src[2], &inst->Src[2], chan, + TGSI_EXEC_DATA_FLOAT); + micro_ucmp(&dst.xyzw[chan], &src[0], &src[1], &src[2]); } } - - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_W) { - store_dest(mach, &OneVec, &inst->Dst[0], inst, TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &dst.xyzw[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_FLOAT); + } } } @@ -2815,99 +3425,6 @@ exec_scs(struct tgsi_exec_machine *mach, } } -static void -exec_x2d(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - union tgsi_exec_channel r[4]; - union tgsi_exec_channel d[2]; - - fetch_source(mach, &r[0], &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &r[1], &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XZ) { - fetch_source(mach, &r[2], &inst->Src[2], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[2], &r[2], &r[0]); - fetch_source(mach, &r[3], &inst->Src[2], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[3], &r[3], &r[1]); - micro_add(&r[2], &r[2], &r[3]); - fetch_source(mach, &r[3], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_add(&d[0], &r[2], &r[3]); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_YW) { - fetch_source(mach, &r[2], &inst->Src[2], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[2], &r[2], &r[0]); - fetch_source(mach, &r[3], &inst->Src[2], TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[3], &r[3], &r[1]); - micro_add(&r[2], &r[2], &r[3]); - fetch_source(mach, &r[3], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_add(&d[1], &r[2], &r[3]); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { - store_dest(mach, &d[0], &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { - store_dest(mach, &d[1], &inst->Dst[0], inst, TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Z) { - store_dest(mach, &d[0], &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_W) { - store_dest(mach, &d[1], &inst->Dst[0], inst, TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); - } -} - -static void -exec_rfl(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - union tgsi_exec_channel r[9]; - - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XYZ) { - /* r0 = dp3(src0, src0) */ - fetch_source(mach, &r[2], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[0], &r[2], &r[2]); - fetch_source(mach, &r[4], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[8], &r[4], &r[4]); - micro_add(&r[0], &r[0], &r[8]); - fetch_source(mach, &r[6], &inst->Src[0], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[8], &r[6], &r[6]); - micro_add(&r[0], &r[0], &r[8]); - - /* r1 = dp3(src0, src1) */ - fetch_source(mach, &r[3], &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[1], &r[2], &r[3]); - fetch_source(mach, &r[5], &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[8], &r[4], &r[5]); - micro_add(&r[1], &r[1], &r[8]); - fetch_source(mach, &r[7], &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - micro_mul(&r[8], &r[6], &r[7]); - micro_add(&r[1], &r[1], &r[8]); - - /* r1 = 2 * r1 / r0 */ - micro_add(&r[1], &r[1], &r[1]); - micro_div(&r[1], &r[1], &r[0]); - - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { - micro_mul(&r[2], &r[2], &r[1]); - micro_sub(&r[2], &r[2], &r[3]); - store_dest(mach, &r[2], &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { - micro_mul(&r[4], &r[4], &r[1]); - micro_sub(&r[4], &r[4], &r[5]); - store_dest(mach, &r[4], &inst->Dst[0], inst, TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Z) { - micro_mul(&r[6], &r[6], &r[1]); - micro_sub(&r[6], &r[6], &r[7]); - store_dest(mach, &r[6], &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - } - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_W) { - store_dest(mach, &OneVec, &inst->Dst[0], inst, TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); - } -} - static void exec_xpd(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) @@ -3136,6 +3653,7 @@ exec_case(struct tgsi_exec_machine *mach, UPDATE_EXEC_MASK(mach); } +/* FIXME: this will only work if default is last */ static void exec_default(struct tgsi_exec_machine *mach) { @@ -3155,40 +3673,1010 @@ exec_endswitch(struct tgsi_exec_machine *mach) UPDATE_EXEC_MASK(mach); } +typedef void (* micro_dop)(union tgsi_double_channel *dst, + const union tgsi_double_channel *src); + +typedef void (* micro_dop_sop)(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1); + +typedef void (* micro_dop_s)(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src); + +typedef void (* micro_sop_d)(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src); + static void -micro_i2f(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src) +fetch_double_channel(struct tgsi_exec_machine *mach, + union tgsi_double_channel *chan, + const struct tgsi_full_src_register *reg, + uint chan_0, + uint chan_1) { - dst->f[0] = (float)src->i[0]; - dst->f[1] = (float)src->i[1]; - dst->f[2] = (float)src->i[2]; - dst->f[3] = (float)src->i[3]; + union tgsi_exec_channel src[2]; + uint i; + + fetch_source_d(mach, &src[0], reg, chan_0, TGSI_EXEC_DATA_UINT); + fetch_source_d(mach, &src[1], reg, chan_1, TGSI_EXEC_DATA_UINT); + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + chan->u[i][0] = src[0].u[i]; + chan->u[i][1] = src[1].u[i]; + } + if (reg->Register.Absolute) { + micro_dabs(chan, chan); + } + if (reg->Register.Negate) { + micro_dneg(chan, chan); + } } static void -micro_not(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src) -{ - dst->u[0] = ~src->u[0]; - dst->u[1] = ~src->u[1]; - dst->u[2] = ~src->u[2]; - dst->u[3] = ~src->u[3]; +store_double_channel(struct tgsi_exec_machine *mach, + const union tgsi_double_channel *chan, + const struct tgsi_full_dst_register *reg, + const struct tgsi_full_instruction *inst, + uint chan_0, + uint chan_1) +{ + union tgsi_exec_channel dst[2]; + uint i; + union tgsi_double_channel temp; + const uint execmask = mach->ExecMask; + + if (!inst->Instruction.Saturate) { + for (i = 0; i < TGSI_QUAD_SIZE; i++) + if (execmask & (1 << i)) { + dst[0].u[i] = chan->u[i][0]; + dst[1].u[i] = chan->u[i][1]; + } + } + else { + for (i = 0; i < TGSI_QUAD_SIZE; i++) + if (execmask & (1 << i)) { + if (chan->d[i] < 0.0) + temp.d[i] = 0.0; + else if (chan->d[i] > 1.0) + temp.d[i] = 1.0; + else + temp.d[i] = chan->d[i]; + + dst[0].u[i] = temp.u[i][0]; + dst[1].u[i] = temp.u[i][1]; + } + } + + store_dest_double(mach, &dst[0], reg, inst, chan_0, TGSI_EXEC_DATA_UINT); + if (chan_1 != -1) + store_dest_double(mach, &dst[1], reg, inst, chan_1, TGSI_EXEC_DATA_UINT); } static void -micro_shl(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1) +exec_double_unary(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop op) { - dst->u[0] = src0->u[0] << src1->u[0]; - dst->u[1] = src0->u[1] << src1->u[1]; - dst->u[2] = src0->u[2] << src1->u[2]; - dst->u[3] = src0->u[3] << src1->u[3]; + union tgsi_double_channel src; + union tgsi_double_channel dst; + + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + op(&dst, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + } + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + op(&dst, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + } } static void -micro_and(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, +exec_double_binary(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop op, + enum tgsi_exec_datatype dst_datatype) +{ + union tgsi_double_channel src[2]; + union tgsi_double_channel dst; + int first_dest_chan, second_dest_chan; + int wmask; + + wmask = inst->Dst[0].Register.WriteMask; + /* these are & because of the way DSLT etc store their destinations */ + if (wmask & TGSI_WRITEMASK_XY) { + first_dest_chan = TGSI_CHAN_X; + second_dest_chan = TGSI_CHAN_Y; + if (dst_datatype == TGSI_EXEC_DATA_UINT) { + first_dest_chan = (wmask & TGSI_WRITEMASK_X) ? TGSI_CHAN_X : TGSI_CHAN_Y; + second_dest_chan = -1; + } + + fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_X, TGSI_CHAN_Y); + op(&dst, src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, first_dest_chan, second_dest_chan); + } + + if (wmask & TGSI_WRITEMASK_ZW) { + first_dest_chan = TGSI_CHAN_Z; + second_dest_chan = TGSI_CHAN_W; + if (dst_datatype == TGSI_EXEC_DATA_UINT) { + first_dest_chan = (wmask & TGSI_WRITEMASK_Z) ? TGSI_CHAN_Z : TGSI_CHAN_W; + second_dest_chan = -1; + } + + fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_CHAN_W); + op(&dst, src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, first_dest_chan, second_dest_chan); + } +} + +static void +exec_double_trinary(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop op) +{ + union tgsi_double_channel src[3]; + union tgsi_double_channel dst; + + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_double_channel(mach, &src[2], &inst->Src[2], TGSI_CHAN_X, TGSI_CHAN_Y); + op(&dst, src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + } + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_double_channel(mach, &src[2], &inst->Src[2], TGSI_CHAN_Z, TGSI_CHAN_W); + op(&dst, src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + } +} + +static void +exec_dldexp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_double_channel src0; + union tgsi_exec_channel src1; + union tgsi_double_channel dst; + int wmask; + + wmask = inst->Dst[0].Register.WriteMask; + if (wmask & TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); + micro_dldexp(&dst, &src0, &src1); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + } + + if (wmask & TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT); + micro_dldexp(&dst, &src0, &src1); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + } +} + +static void +exec_dfracexp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_double_channel src; + union tgsi_double_channel dst; + union tgsi_exec_channel dst_exp; + + if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY)) { + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + micro_dfracexp(&dst, &dst_exp, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT); + } + if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW)) { + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + micro_dfracexp(&dst, &dst_exp, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT); + } +} + +static void +exec_arg0_64_arg1_32(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop_sop op) +{ + union tgsi_double_channel src0; + union tgsi_exec_channel src1; + union tgsi_double_channel dst; + int wmask; + + wmask = inst->Dst[0].Register.WriteMask; + if (wmask & TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); + op(&dst, &src0, &src1); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + } + + if (wmask & TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT); + op(&dst, &src0, &src1); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + } +} + +static int +get_image_coord_dim(unsigned tgsi_tex) +{ + int dim; + switch (tgsi_tex) { + case TGSI_TEXTURE_BUFFER: + case TGSI_TEXTURE_1D: + dim = 1; + break; + case TGSI_TEXTURE_2D: + case TGSI_TEXTURE_RECT: + case TGSI_TEXTURE_1D_ARRAY: + case TGSI_TEXTURE_2D_MSAA: + dim = 2; + break; + case TGSI_TEXTURE_3D: + case TGSI_TEXTURE_CUBE: + case TGSI_TEXTURE_2D_ARRAY: + case TGSI_TEXTURE_2D_ARRAY_MSAA: + case TGSI_TEXTURE_CUBE_ARRAY: + dim = 3; + break; + default: + assert(!"unknown texture target"); + dim = 0; + break; + } + + return dim; +} + +static int +get_image_coord_sample(unsigned tgsi_tex) +{ + int sample = 0; + switch (tgsi_tex) { + case TGSI_TEXTURE_2D_MSAA: + sample = 3; + break; + case TGSI_TEXTURE_2D_ARRAY_MSAA: + sample = 4; + break; + default: + break; + } + return sample; +} + +static void +exec_load_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4], sample_r; + uint unit; + int sample; + int i, j; + int dim; + uint chan; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 1, TGSI_CHAN_X + i); + } + + if (sample) + IFETCH(&sample_r, 1, TGSI_CHAN_X + sample); + + mach->Image->load(mach->Image, ¶ms, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba); + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + uint unit; + int j; + uint chan; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + IFETCH(&r[0], 1, TGSI_CHAN_X); + + mach->Buffer->load(mach->Buffer, ¶ms, + r[0].i, rgba); + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + uint chan; + char *ptr = mach->LocalMem; + uint32_t offset; + int j; + + IFETCH(&r[0], 1, TGSI_CHAN_X); + if (r[0].u[0] >= mach->LocalMemSize) + return; + + offset = r[0].u[0]; + ptr += offset; + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + memcpy(&r[chan].u[j], ptr + (4 * chan), 4); + } + } + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_load_img(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) + exec_load_buf(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) + exec_load_mem(mach, inst); +} + +static void +exec_store_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3], sample_r; + union tgsi_exec_channel value[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int dim; + int sample; + int i, j; + uint unit; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + unit = inst->Dst[0].Register.Index; + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 0, TGSI_CHAN_X + i); + } + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + if (sample) + IFETCH(&sample_r, 0, TGSI_CHAN_X + sample); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + + mach->Image->store(mach->Image, ¶ms, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba); +} + +static void +exec_store_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3]; + union tgsi_exec_channel value[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int i, j; + uint unit; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = inst->Dst[0].Register.Index; + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.writemask = inst->Dst[0].Register.WriteMask; + + IFETCH(&r[0], 0, TGSI_CHAN_X); + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + + mach->Buffer->store(mach->Buffer, ¶ms, + r[0].i, + rgba); +} + +static void +exec_store_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3]; + union tgsi_exec_channel value[4]; + uint i, chan; + char *ptr = mach->LocalMem; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + int execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + + IFETCH(&r[0], 0, TGSI_CHAN_X); + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + + if (r[0].u[0] >= mach->LocalMemSize) + return; + ptr += r[0].u[0]; + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + if (execmask & (1 << i)) { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + memcpy(ptr + (chan * 4), &value[chan].u[0], 4); + } + } + } + } +} + +static void +exec_store(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Dst[0].Register.File == TGSI_FILE_IMAGE) + exec_store_img(mach, inst); + else if (inst->Dst[0].Register.File == TGSI_FILE_BUFFER) + exec_store_buf(mach, inst); + else if (inst->Dst[0].Register.File == TGSI_FILE_MEMORY) + exec_store_mem(mach, inst); +} + +static void +exec_atomop_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4], sample_r; + union tgsi_exec_channel value[4], value2[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int dim; + int sample; + int i, j; + uint unit, chan; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + unit = fetch_sampler_unit(mach, inst, 0); + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 1, TGSI_CHAN_X + i); + } + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + if (sample) + IFETCH(&sample_r, 1, TGSI_CHAN_X + sample); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) { + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba2[0][j] = value2[0].f[j]; + rgba2[1][j] = value2[1].f[j]; + rgba2[2][j] = value2[2].f[j]; + rgba2[3][j] = value2[3].f[j]; + } + } + + mach->Image->op(mach->Image, ¶ms, inst->Instruction.Opcode, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba, rgba2); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + union tgsi_exec_channel value[4], value2[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int i, j; + uint unit, chan; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.writemask = inst->Dst[0].Register.WriteMask; + + IFETCH(&r[0], 1, TGSI_CHAN_X); + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) { + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba2[0][j] = value2[0].f[j]; + rgba2[1][j] = value2[1].f[j]; + rgba2[2][j] = value2[2].f[j]; + rgba2[3][j] = value2[3].f[j]; + } + } + + mach->Buffer->op(mach->Buffer, ¶ms, inst->Instruction.Opcode, + r[0].i, + rgba, rgba2); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + union tgsi_exec_channel value[4], value2[4]; + char *ptr = mach->LocalMem; + uint32_t val; + uint chan, i; + uint32_t offset; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + int execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + IFETCH(&r[0], 1, TGSI_CHAN_X); + + if (r[0].u[0] >= mach->LocalMemSize) + return; + + offset = r[0].u[0]; + ptr += offset; + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + + memcpy(&r[0].u[0], ptr, 4); + val = r[0].u[0]; + switch (inst->Instruction.Opcode) { + case TGSI_OPCODE_ATOMUADD: + val += value[0].u[0]; + break; + case TGSI_OPCODE_ATOMXOR: + val ^= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMOR: + val |= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMAND: + val &= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMUMIN: + val = MIN2(val, value[0].u[0]); + break; + case TGSI_OPCODE_ATOMUMAX: + val = MAX2(val, value[0].u[0]); + break; + case TGSI_OPCODE_ATOMIMIN: + val = MIN2(r[0].i[0], value[0].i[0]); + break; + case TGSI_OPCODE_ATOMIMAX: + val = MAX2(r[0].i[0], value[0].i[0]); + break; + case TGSI_OPCODE_ATOMXCHG: + val = value[0].i[0]; + break; + case TGSI_OPCODE_ATOMCAS: + if (val == value[0].u[0]) + val = value2[0].u[0]; + break; + default: + break; + } + for (i = 0; i < TGSI_QUAD_SIZE; i++) + if (execmask & (1 << i)) + memcpy(ptr, &val, 4); + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_atomop_img(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) + exec_atomop_buf(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) + exec_atomop_mem(mach, inst); +} + +static void +exec_resq_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + int result[4]; + union tgsi_exec_channel r[4]; + uint unit; + int i, chan, j; + struct tgsi_image_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + mach->Image->get_dims(mach->Image, ¶ms, result); + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + for (j = 0; j < 4; j++) { + r[j].i[i] = result[j]; + } + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_INT); + } + } +} + +static void +exec_resq_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + int result; + union tgsi_exec_channel r[4]; + uint unit; + int i, chan; + struct tgsi_buffer_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + + mach->Buffer->get_dims(mach->Buffer, ¶ms, &result); + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + r[0].i[i] = result; + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_INT); + } + } +} + +static void +exec_resq(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_resq_img(mach, inst); + else + exec_resq_buf(mach, inst); +} + +static void +micro_f2u64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u64[0] = (uint64_t)src->f[0]; + dst->u64[1] = (uint64_t)src->f[1]; + dst->u64[2] = (uint64_t)src->f[2]; + dst->u64[3] = (uint64_t)src->f[3]; +} + +static void +micro_f2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->i64[0] = (int64_t)src->f[0]; + dst->i64[1] = (int64_t)src->f[1]; + dst->i64[2] = (int64_t)src->f[2]; + dst->i64[3] = (int64_t)src->f[3]; +} + +static void +micro_u2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u64[0] = (uint64_t)src->u[0]; + dst->u64[1] = (uint64_t)src->u[1]; + dst->u64[2] = (uint64_t)src->u[2]; + dst->u64[3] = (uint64_t)src->u[3]; +} + +static void +micro_i2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->i64[0] = (int64_t)src->i[0]; + dst->i64[1] = (int64_t)src->i[1]; + dst->i64[2] = (int64_t)src->i[2]; + dst->i64[3] = (int64_t)src->i[3]; +} + +static void +micro_d2u64(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = (uint64_t)src->d[0]; + dst->u64[1] = (uint64_t)src->d[1]; + dst->u64[2] = (uint64_t)src->d[2]; + dst->u64[3] = (uint64_t)src->d[3]; +} + +static void +micro_d2i64(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = (int64_t)src->d[0]; + dst->i64[1] = (int64_t)src->d[1]; + dst->i64[2] = (int64_t)src->d[2]; + dst->i64[3] = (int64_t)src->d[3]; +} + +static void +micro_u642d(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = (double)src->u64[0]; + dst->d[1] = (double)src->u64[1]; + dst->d[2] = (double)src->u64[2]; + dst->d[3] = (double)src->u64[3]; +} + +static void +micro_i642d(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = (double)src->i64[0]; + dst->d[1] = (double)src->i64[1]; + dst->d[2] = (double)src->i64[2]; + dst->d[3] = (double)src->i64[3]; +} + +static void +micro_u642f(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->f[0] = (float)src->u64[0]; + dst->f[1] = (float)src->u64[1]; + dst->f[2] = (float)src->u64[2]; + dst->f[3] = (float)src->u64[3]; +} + +static void +micro_i642f(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->f[0] = (float)src->i64[0]; + dst->f[1] = (float)src->i64[1]; + dst->f[2] = (float)src->i64[2]; + dst->f[3] = (float)src->i64[3]; +} + +static void +exec_t_2_64(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop_s op, + enum tgsi_exec_datatype src_datatype) +{ + union tgsi_exec_channel src; + union tgsi_double_channel dst; + + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { + fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, src_datatype); + op(&dst, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + } + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { + fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, src_datatype); + op(&dst, &src); + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + } +} + +static void +exec_64_2_t(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_sop_d op, + enum tgsi_exec_datatype dst_datatype) +{ + union tgsi_double_channel src; + union tgsi_exec_channel dst; + int wm = inst->Dst[0].Register.WriteMask; + int i; + int bit; + for (i = 0; i < 2; i++) { + bit = ffs(wm); + if (bit) { + wm &= ~(1 << (bit - 1)); + if (i == 0) + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + else + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + op(&dst, &src); + store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, dst_datatype); + } + } +} + +static void +micro_i2f(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->f[0] = (float)src->i[0]; + dst->f[1] = (float)src->i[1]; + dst->f[2] = (float)src->i[2]; + dst->f[3] = (float)src->i[3]; +} + +static void +micro_not(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u[0] = ~src->u[0]; + dst->u[1] = ~src->u[1]; + dst->u[2] = ~src->u[2]; + dst->u[3] = ~src->u[3]; +} + +static void +micro_shl(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x1f; + dst->u[0] = src0->u[0] << masked_count; + masked_count = src1->u[1] & 0x1f; + dst->u[1] = src0->u[1] << masked_count; + masked_count = src1->u[2] & 0x1f; + dst->u[2] = src0->u[2] << masked_count; + masked_count = src1->u[3] & 0x1f; + dst->u[3] = src0->u[3] << masked_count; +} + +static void +micro_and(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { dst->u[0] = src0->u[0] & src1->u[0]; @@ -3240,15 +4728,59 @@ micro_f2i(union tgsi_exec_channel *dst, dst->i[3] = (int)src->f[3]; } +static void +micro_fseq(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->u[0] = src0->f[0] == src1->f[0] ? ~0 : 0; + dst->u[1] = src0->f[1] == src1->f[1] ? ~0 : 0; + dst->u[2] = src0->f[2] == src1->f[2] ? ~0 : 0; + dst->u[3] = src0->f[3] == src1->f[3] ? ~0 : 0; +} + +static void +micro_fsge(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->u[0] = src0->f[0] >= src1->f[0] ? ~0 : 0; + dst->u[1] = src0->f[1] >= src1->f[1] ? ~0 : 0; + dst->u[2] = src0->f[2] >= src1->f[2] ? ~0 : 0; + dst->u[3] = src0->f[3] >= src1->f[3] ? ~0 : 0; +} + +static void +micro_fslt(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->u[0] = src0->f[0] < src1->f[0] ? ~0 : 0; + dst->u[1] = src0->f[1] < src1->f[1] ? ~0 : 0; + dst->u[2] = src0->f[2] < src1->f[2] ? ~0 : 0; + dst->u[3] = src0->f[3] < src1->f[3] ? ~0 : 0; +} + +static void +micro_fsne(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->u[0] = src0->f[0] != src1->f[0] ? ~0 : 0; + dst->u[1] = src0->f[1] != src1->f[1] ? ~0 : 0; + dst->u[2] = src0->f[2] != src1->f[2] ? ~0 : 0; + dst->u[3] = src0->f[3] != src1->f[3] ? ~0 : 0; +} + static void micro_idiv(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->i[0] = src0->i[0] / src1->i[0]; - dst->i[1] = src0->i[1] / src1->i[1]; - dst->i[2] = src0->i[2] / src1->i[2]; - dst->i[3] = src0->i[3] / src1->i[3]; + dst->i[0] = src1->i[0] ? src0->i[0] / src1->i[0] : 0; + dst->i[1] = src1->i[1] ? src0->i[1] / src1->i[1] : 0; + dst->i[2] = src1->i[2] ? src0->i[2] / src1->i[2] : 0; + dst->i[3] = src1->i[3] ? src0->i[3] / src1->i[3] : 0; } static void @@ -3289,10 +4821,15 @@ micro_ishr(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->i[0] = src0->i[0] >> src1->i[0]; - dst->i[1] = src0->i[1] >> src1->i[1]; - dst->i[2] = src0->i[2] >> src1->i[2]; - dst->i[3] = src0->i[3] >> src1->i[3]; + unsigned masked_count; + masked_count = src1->i[0] & 0x1f; + dst->i[0] = src0->i[0] >> masked_count; + masked_count = src1->i[1] & 0x1f; + dst->i[1] = src0->i[1] >> masked_count; + masked_count = src1->i[2] & 0x1f; + dst->i[2] = src0->i[2] >> masked_count; + masked_count = src1->i[3] & 0x1f; + dst->i[3] = src0->i[3] >> masked_count; } static void @@ -3342,10 +4879,10 @@ micro_udiv(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->u[0] = src0->u[0] / src1->u[0]; - dst->u[1] = src0->u[1] / src1->u[1]; - dst->u[2] = src0->u[2] / src1->u[2]; - dst->u[3] = src0->u[3] / src1->u[3]; + dst->u[0] = src1->u[0] ? src0->u[0] / src1->u[0] : ~0u; + dst->u[1] = src1->u[1] ? src0->u[1] / src1->u[1] : ~0u; + dst->u[2] = src1->u[2] ? src0->u[2] / src1->u[2] : ~0u; + dst->u[3] = src1->u[3] ? src0->u[3] / src1->u[3] : ~0u; } static void @@ -3387,10 +4924,10 @@ micro_umod(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->u[0] = src0->u[0] % src1->u[0]; - dst->u[1] = src0->u[1] % src1->u[1]; - dst->u[2] = src0->u[2] % src1->u[2]; - dst->u[3] = src0->u[3] % src1->u[3]; + dst->u[0] = src1->u[0] ? src0->u[0] % src1->u[0] : ~0u; + dst->u[1] = src1->u[1] ? src0->u[1] % src1->u[1] : ~0u; + dst->u[2] = src1->u[2] ? src0->u[2] % src1->u[2] : ~0u; + dst->u[3] = src1->u[3] ? src0->u[3] % src1->u[3] : ~0u; } static void @@ -3404,6 +4941,32 @@ micro_umul(union tgsi_exec_channel *dst, dst->u[3] = src0->u[3] * src1->u[3]; } +static void +micro_imul_hi(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ +#define I64M(x, y) ((((int64_t)x) * ((int64_t)y)) >> 32) + dst->i[0] = I64M(src0->i[0], src1->i[0]); + dst->i[1] = I64M(src0->i[1], src1->i[1]); + dst->i[2] = I64M(src0->i[2], src1->i[2]); + dst->i[3] = I64M(src0->i[3], src1->i[3]); +#undef I64M +} + +static void +micro_umul_hi(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ +#define U64M(x, y) ((((uint64_t)x) * ((uint64_t)y)) >> 32) + dst->u[0] = U64M(src0->u[0], src1->u[0]); + dst->u[1] = U64M(src0->u[1], src1->u[1]); + dst->u[2] = U64M(src0->u[2], src1->u[2]); + dst->u[3] = U64M(src0->u[3], src1->u[3]); +#undef U64M +} + static void micro_useq(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, @@ -3431,10 +4994,15 @@ micro_ushr(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->u[0] = src0->u[0] >> src1->u[0]; - dst->u[1] = src0->u[1] >> src1->u[1]; - dst->u[2] = src0->u[2] >> src1->u[2]; - dst->u[3] = src0->u[3] >> src1->u[3]; + unsigned masked_count; + masked_count = src1->u[0] & 0x1f; + dst->u[0] = src0->u[0] >> masked_count; + masked_count = src1->u[1] & 0x1f; + dst->u[1] = src0->u[1] >> masked_count; + masked_count = src1->u[2] & 0x1f; + dst->u[2] = src0->u[2] >> masked_count; + masked_count = src1->u[3] & 0x1f; + dst->u[3] = src0->u[3] >> masked_count; } static void @@ -3442,46 +5010,152 @@ micro_uslt(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->u[0] = src0->u[0] < src1->u[0] ? ~0 : 0; - dst->u[1] = src0->u[1] < src1->u[1] ? ~0 : 0; - dst->u[2] = src0->u[2] < src1->u[2] ? ~0 : 0; - dst->u[3] = src0->u[3] < src1->u[3] ? ~0 : 0; + dst->u[0] = src0->u[0] < src1->u[0] ? ~0 : 0; + dst->u[1] = src0->u[1] < src1->u[1] ? ~0 : 0; + dst->u[2] = src0->u[2] < src1->u[2] ? ~0 : 0; + dst->u[3] = src0->u[3] < src1->u[3] ? ~0 : 0; +} + +static void +micro_usne(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->u[0] = src0->u[0] != src1->u[0] ? ~0 : 0; + dst->u[1] = src0->u[1] != src1->u[1] ? ~0 : 0; + dst->u[2] = src0->u[2] != src1->u[2] ? ~0 : 0; + dst->u[3] = src0->u[3] != src1->u[3] ? ~0 : 0; +} + +static void +micro_uarl(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->i[0] = src->u[0]; + dst->i[1] = src->u[1]; + dst->i[2] = src->u[2]; + dst->i[3] = src->u[3]; +} + +/** + * Signed bitfield extract (i.e. sign-extend the extracted bits) + */ +static void +micro_ibfe(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2) +{ + int i; + for (i = 0; i < 4; i++) { + int width = src2->i[i] & 0x1f; + int offset = src1->i[i] & 0x1f; + if (width == 0) + dst->i[i] = 0; + else if (width + offset < 32) + dst->i[i] = (src0->i[i] << (32 - width - offset)) >> (32 - width); + else + dst->i[i] = src0->i[i] >> offset; + } +} + +/** + * Unsigned bitfield extract + */ +static void +micro_ubfe(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2) +{ + int i; + for (i = 0; i < 4; i++) { + int width = src2->u[i] & 0x1f; + int offset = src1->u[i] & 0x1f; + if (width == 0) + dst->u[i] = 0; + else if (width + offset < 32) + dst->u[i] = (src0->u[i] << (32 - width - offset)) >> (32 - width); + else + dst->u[i] = src0->u[i] >> offset; + } +} + +/** + * Bitfield insert: copy low bits from src1 into a region of src0. + */ +static void +micro_bfi(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2, + const union tgsi_exec_channel *src3) +{ + int i; + for (i = 0; i < 4; i++) { + int width = src3->u[i] & 0x1f; + int offset = src2->u[i] & 0x1f; + int bitmask = ((1 << width) - 1) << offset; + dst->u[i] = ((src1->u[i] << offset) & bitmask) | (src0->u[i] & ~bitmask); + } +} + +static void +micro_brev(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u[0] = util_bitreverse(src->u[0]); + dst->u[1] = util_bitreverse(src->u[1]); + dst->u[2] = util_bitreverse(src->u[2]); + dst->u[3] = util_bitreverse(src->u[3]); +} + +static void +micro_popc(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u[0] = util_bitcount(src->u[0]); + dst->u[1] = util_bitcount(src->u[1]); + dst->u[2] = util_bitcount(src->u[2]); + dst->u[3] = util_bitcount(src->u[3]); } static void -micro_usne(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1) +micro_lsb(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) { - dst->u[0] = src0->u[0] != src1->u[0] ? ~0 : 0; - dst->u[1] = src0->u[1] != src1->u[1] ? ~0 : 0; - dst->u[2] = src0->u[2] != src1->u[2] ? ~0 : 0; - dst->u[3] = src0->u[3] != src1->u[3] ? ~0 : 0; + dst->i[0] = ffs(src->u[0]) - 1; + dst->i[1] = ffs(src->u[1]) - 1; + dst->i[2] = ffs(src->u[2]) - 1; + dst->i[3] = ffs(src->u[3]) - 1; } static void -micro_uarl(union tgsi_exec_channel *dst, +micro_imsb(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) { - dst->i[0] = src->u[0]; - dst->i[1] = src->u[1]; - dst->i[2] = src->u[2]; - dst->i[3] = src->u[3]; + dst->i[0] = util_last_bit_signed(src->i[0]) - 1; + dst->i[1] = util_last_bit_signed(src->i[1]) - 1; + dst->i[2] = util_last_bit_signed(src->i[2]) - 1; + dst->i[3] = util_last_bit_signed(src->i[3]) - 1; } static void -micro_ucmp(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1, - const union tgsi_exec_channel *src2) +micro_umsb(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) { - dst->u[0] = src0->u[0] ? src1->u[0] : src2->u[0]; - dst->u[1] = src0->u[1] ? src1->u[1] : src2->u[1]; - dst->u[2] = src0->u[2] ? src1->u[2] : src2->u[2]; - dst->u[3] = src0->u[3] ? src1->u[3] : src2->u[3]; + dst->i[0] = util_last_bit(src->u[0]) - 1; + dst->i[1] = util_last_bit(src->u[1]) - 1; + dst->i[2] = util_last_bit(src->u[2]) - 1; + dst->i[3] = util_last_bit(src->u[3]) - 1; } -static void +/** + * Execute a TGSI instruction. + * Returns TRUE if a barrier instruction is hit, + * otherwise FALSE. + */ +static boolean exec_instruction( struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst, @@ -3560,20 +5234,12 @@ exec_instruction( exec_vector_trinary(mach, inst, micro_mad, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_SUB: - exec_vector_binary(mach, inst, micro_sub, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_LRP: exec_vector_trinary(mach, inst, micro_lrp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_CND: - exec_vector_trinary(mach, inst, micro_cnd, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_SQRT: - exec_vector_unary(mach, inst, micro_sqrt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); + exec_scalar_unary(mach, inst, micro_sqrt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; case TGSI_OPCODE_DP2A: @@ -3584,10 +5250,6 @@ exec_instruction( exec_vector_unary(mach, inst, micro_frc, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_CLAMP: - exec_vector_trinary(mach, inst, micro_clamp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_FLR: exec_vector_unary(mach, inst, micro_flr, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -3612,14 +5274,6 @@ exec_instruction( exec_xpd(mach, inst); break; - case TGSI_OPCODE_ABS: - exec_vector_unary(mach, inst, micro_abs, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - - case TGSI_OPCODE_RCC: - exec_scalar_unary(mach, inst, micro_rcc, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_DPH: exec_dph(mach, inst); break; @@ -3636,16 +5290,16 @@ exec_instruction( exec_vector_unary(mach, inst, micro_ddy, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_KILP: - exec_kilp (mach, inst); + case TGSI_OPCODE_KILL: + exec_kill (mach, inst); break; - case TGSI_OPCODE_KIL: - exec_kil (mach, inst); + case TGSI_OPCODE_KILL_IF: + exec_kill_if (mach, inst); break; case TGSI_OPCODE_PK2H: - assert (0); + exec_pk2h(mach, inst); break; case TGSI_OPCODE_PK2US: @@ -3660,18 +5314,10 @@ exec_instruction( assert (0); break; - case TGSI_OPCODE_RFL: - exec_rfl(mach, inst); - break; - case TGSI_OPCODE_SEQ: exec_vector_binary(mach, inst, micro_seq, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_SFL: - exec_vector(mach, inst, micro_sfl, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_SGT: exec_vector_binary(mach, inst, micro_sgt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -3688,10 +5334,6 @@ exec_instruction( exec_vector_binary(mach, inst, micro_sne, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_STR: - exec_vector(mach, inst, micro_str, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_TEX: /* simple texture lookup */ /* src[0] = texcoord */ @@ -3729,8 +5371,21 @@ exec_instruction( exec_tex(mach, inst, TEX_MODIFIER_PROJECTED, 1); break; + case TGSI_OPCODE_TG4: + /* src[0] = texcoord */ + /* src[1] = component */ + /* src[2] = sampler unit */ + exec_tex(mach, inst, TEX_MODIFIER_GATHER, 2); + break; + + case TGSI_OPCODE_LODQ: + /* src[0] = texcoord */ + /* src[1] = sampler unit */ + exec_lodq(mach, inst); + break; + case TGSI_OPCODE_UP2H: - assert (0); + exec_up2h(mach, inst); break; case TGSI_OPCODE_UP2US: @@ -3745,22 +5400,10 @@ exec_instruction( assert (0); break; - case TGSI_OPCODE_X2D: - exec_x2d(mach, inst); - break; - - case TGSI_OPCODE_ARA: - assert (0); - break; - case TGSI_OPCODE_ARR: exec_vector_unary(mach, inst, micro_arr, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_BRA: - assert (0); - break; - case TGSI_OPCODE_CAL: /* skip the call if no execution channels are enabled */ if (mach->ExecMask) { @@ -3798,8 +5441,12 @@ exec_instruction( mach->BreakStack[mach->BreakStackTop++] = mach->BreakType; mach->FuncStack[mach->FuncStackTop++] = mach->FuncMask; - /* Finally, jump to the subroutine */ + /* Finally, jump to the subroutine. The label is a pointer + * (an instruction number) to the BGNSUB instruction. + */ *pc = inst->Label.Label; + assert(mach->Instructions[*pc].Instruction.Opcode + == TGSI_OPCODE_BGNSUB); } break; @@ -3814,8 +5461,12 @@ exec_instruction( /* returning from main() */ mach->CondStackTop = 0; mach->LoopStackTop = 0; + mach->ContStackTop = 0; + mach->LoopLabelStackTop = 0; + mach->SwitchStackTop = 0; + mach->BreakStackTop = 0; *pc = -1; - return; + return FALSE; } assert(mach->CallStackTop > 0); @@ -3857,14 +5508,6 @@ exec_instruction( exec_scs(mach, inst); break; - case TGSI_OPCODE_NRM: - exec_nrm3(mach, inst); - break; - - case TGSI_OPCODE_NRM4: - exec_nrm4(mach, inst); - break; - case TGSI_OPCODE_DIV: exec_vector_binary(mach, inst, micro_div, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -3879,6 +5522,28 @@ exec_instruction( mach->CondStack[mach->CondStackTop++] = mach->CondMask; FETCH( &r[0], 0, TGSI_CHAN_X ); /* update CondMask */ + if( ! r[0].f[0] ) { + mach->CondMask &= ~0x1; + } + if( ! r[0].f[1] ) { + mach->CondMask &= ~0x2; + } + if( ! r[0].f[2] ) { + mach->CondMask &= ~0x4; + } + if( ! r[0].f[3] ) { + mach->CondMask &= ~0x8; + } + UPDATE_EXEC_MASK(mach); + /* Todo: If CondMask==0, jump to ELSE */ + break; + + case TGSI_OPCODE_UIF: + /* push CondMask */ + assert(mach->CondStackTop < TGSI_EXEC_MAX_COND_NESTING); + mach->CondStack[mach->CondStackTop++] = mach->CondMask; + IFETCH( &r[0], 0, TGSI_CHAN_X ); + /* update CondMask */ if( ! r[0].u[0] ) { mach->CondMask &= ~0x1; } @@ -4075,7 +5740,7 @@ exec_instruction( break; case TGSI_OPCODE_BREAKC: - FETCH(&r[0], 0, TGSI_CHAN_X); + IFETCH(&r[0], 0, TGSI_CHAN_X); /* update CondMask */ if (r[0].u[0] && (mach->ExecMask & 0x1)) { mach->LoopMask &= ~0x1; @@ -4097,6 +5762,22 @@ exec_instruction( exec_vector_unary(mach, inst, micro_f2i, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_FLOAT); break; + case TGSI_OPCODE_FSEQ: + exec_vector_binary(mach, inst, micro_fseq, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_FSGE: + exec_vector_binary(mach, inst, micro_fsge, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_FSLT: + exec_vector_binary(mach, inst, micro_fslt, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_FSNE: + exec_vector_binary(mach, inst, micro_fsne, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_FLOAT); + break; + case TGSI_OPCODE_IDIV: exec_vector_binary(mach, inst, micro_idiv, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_INT); break; @@ -4134,7 +5815,7 @@ exec_instruction( break; case TGSI_OPCODE_UADD: - exec_vector_binary(mach, inst, micro_uadd, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + exec_vector_binary(mach, inst, micro_uadd, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_INT); break; case TGSI_OPCODE_UDIV: @@ -4161,6 +5842,14 @@ exec_instruction( exec_vector_binary(mach, inst, micro_umul, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); break; + case TGSI_OPCODE_IMUL_HI: + exec_vector_binary(mach, inst, micro_imul_hi, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_UMUL_HI: + exec_vector_binary(mach, inst, micro_umul_hi, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_USEQ: exec_vector_binary(mach, inst, micro_useq, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); break; @@ -4198,27 +5887,27 @@ exec_instruction( break; case TGSI_OPCODE_SAMPLE_I: - assert(0); + exec_txf(mach, inst); break; case TGSI_OPCODE_SAMPLE_I_MS: - assert(0); + exec_txf(mach, inst); break; case TGSI_OPCODE_SAMPLE: - exec_sample(mach, inst, TEX_MODIFIER_NONE); + exec_sample(mach, inst, TEX_MODIFIER_NONE, FALSE); break; case TGSI_OPCODE_SAMPLE_B: - exec_sample(mach, inst, TEX_MODIFIER_LOD_BIAS); + exec_sample(mach, inst, TEX_MODIFIER_LOD_BIAS, FALSE); break; case TGSI_OPCODE_SAMPLE_C: - exec_sample(mach, inst, TEX_MODIFIER_NONE); + exec_sample(mach, inst, TEX_MODIFIER_NONE, TRUE); break; case TGSI_OPCODE_SAMPLE_C_LZ: - exec_sample(mach, inst, TEX_MODIFIER_LOD_BIAS); + exec_sample(mach, inst, TEX_MODIFIER_LEVEL_ZERO, TRUE); break; case TGSI_OPCODE_SAMPLE_D: @@ -4226,7 +5915,7 @@ exec_instruction( break; case TGSI_OPCODE_SAMPLE_L: - exec_sample(mach, inst, TEX_MODIFIER_EXPLICIT_LOD); + exec_sample(mach, inst, TEX_MODIFIER_EXPLICIT_LOD, FALSE); break; case TGSI_OPCODE_GATHER4: @@ -4234,7 +5923,7 @@ exec_instruction( break; case TGSI_OPCODE_SVIEWINFO: - assert(0); + exec_txq(mach, inst); break; case TGSI_OPCODE_SAMPLE_POS: @@ -4250,7 +5939,7 @@ exec_instruction( break; case TGSI_OPCODE_UCMP: - exec_vector_trinary(mach, inst, micro_ucmp, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + exec_ucmp(mach, inst); break; case TGSI_OPCODE_IABS: @@ -4282,32 +5971,298 @@ exec_instruction( /* src[2] = sampler unit */ exec_tex(mach, inst, TEX_MODIFIER_EXPLICIT_LOD, 2); break; + + case TGSI_OPCODE_IBFE: + exec_vector_trinary(mach, inst, micro_ibfe, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_INT); + break; + case TGSI_OPCODE_UBFE: + exec_vector_trinary(mach, inst, micro_ubfe, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_BFI: + exec_vector_quaternary(mach, inst, micro_bfi, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_BREV: + exec_vector_unary(mach, inst, micro_brev, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_POPC: + exec_vector_unary(mach, inst, micro_popc, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_LSB: + exec_vector_unary(mach, inst, micro_lsb, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_IMSB: + exec_vector_unary(mach, inst, micro_imsb, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_INT); + break; + case TGSI_OPCODE_UMSB: + exec_vector_unary(mach, inst, micro_umsb, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_F2D: + exec_t_2_64(mach, inst, micro_f2d, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_D2F: + exec_64_2_t(mach, inst, micro_d2f, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_DABS: + exec_double_unary(mach, inst, micro_dabs); + break; + + case TGSI_OPCODE_DNEG: + exec_double_unary(mach, inst, micro_dneg); + break; + + case TGSI_OPCODE_DADD: + exec_double_binary(mach, inst, micro_dadd, TGSI_EXEC_DATA_DOUBLE); + break; + + case TGSI_OPCODE_DDIV: + exec_double_binary(mach, inst, micro_ddiv, TGSI_EXEC_DATA_DOUBLE); + break; + + case TGSI_OPCODE_DMUL: + exec_double_binary(mach, inst, micro_dmul, TGSI_EXEC_DATA_DOUBLE); + break; + + case TGSI_OPCODE_DMAX: + exec_double_binary(mach, inst, micro_dmax, TGSI_EXEC_DATA_DOUBLE); + break; + + case TGSI_OPCODE_DMIN: + exec_double_binary(mach, inst, micro_dmin, TGSI_EXEC_DATA_DOUBLE); + break; + + case TGSI_OPCODE_DSLT: + exec_double_binary(mach, inst, micro_dslt, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_DSGE: + exec_double_binary(mach, inst, micro_dsge, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_DSEQ: + exec_double_binary(mach, inst, micro_dseq, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_DSNE: + exec_double_binary(mach, inst, micro_dsne, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_DRCP: + exec_double_unary(mach, inst, micro_drcp); + break; + + case TGSI_OPCODE_DSQRT: + exec_double_unary(mach, inst, micro_dsqrt); + break; + + case TGSI_OPCODE_DRSQ: + exec_double_unary(mach, inst, micro_drsq); + break; + + case TGSI_OPCODE_DMAD: + exec_double_trinary(mach, inst, micro_dmad); + break; + + case TGSI_OPCODE_DFRAC: + exec_double_unary(mach, inst, micro_dfrac); + break; + + case TGSI_OPCODE_DLDEXP: + exec_dldexp(mach, inst); + break; + + case TGSI_OPCODE_DFRACEXP: + exec_dfracexp(mach, inst); + break; + + case TGSI_OPCODE_I2D: + exec_t_2_64(mach, inst, micro_i2d, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_D2I: + exec_64_2_t(mach, inst, micro_d2i, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_U2D: + exec_t_2_64(mach, inst, micro_u2d, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_D2U: + exec_64_2_t(mach, inst, micro_d2u, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_LOAD: + exec_load(mach, inst); + break; + + case TGSI_OPCODE_STORE: + exec_store(mach, inst); + break; + + case TGSI_OPCODE_ATOMUADD: + case TGSI_OPCODE_ATOMXCHG: + case TGSI_OPCODE_ATOMCAS: + case TGSI_OPCODE_ATOMAND: + case TGSI_OPCODE_ATOMOR: + case TGSI_OPCODE_ATOMXOR: + case TGSI_OPCODE_ATOMUMIN: + case TGSI_OPCODE_ATOMUMAX: + case TGSI_OPCODE_ATOMIMIN: + case TGSI_OPCODE_ATOMIMAX: + exec_atomop(mach, inst); + break; + + case TGSI_OPCODE_RESQ: + exec_resq(mach, inst); + break; + case TGSI_OPCODE_BARRIER: + case TGSI_OPCODE_MEMBAR: + return TRUE; + break; + + case TGSI_OPCODE_I64ABS: + exec_double_unary(mach, inst, micro_i64abs); + break; + + case TGSI_OPCODE_I64SSG: + exec_double_unary(mach, inst, micro_i64sgn); + break; + + case TGSI_OPCODE_I64NEG: + exec_double_unary(mach, inst, micro_i64neg); + break; + + case TGSI_OPCODE_U64SEQ: + exec_double_binary(mach, inst, micro_u64seq, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_U64SNE: + exec_double_binary(mach, inst, micro_u64sne, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64SLT: + exec_double_binary(mach, inst, micro_i64slt, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_U64SLT: + exec_double_binary(mach, inst, micro_u64slt, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64SGE: + exec_double_binary(mach, inst, micro_i64sge, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_U64SGE: + exec_double_binary(mach, inst, micro_u64sge, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64MIN: + exec_double_binary(mach, inst, micro_i64min, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MIN: + exec_double_binary(mach, inst, micro_u64min, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64MAX: + exec_double_binary(mach, inst, micro_i64max, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MAX: + exec_double_binary(mach, inst, micro_u64max, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64ADD: + exec_double_binary(mach, inst, micro_u64add, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64MUL: + exec_double_binary(mach, inst, micro_u64mul, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64SHL: + exec_arg0_64_arg1_32(mach, inst, micro_u64shl); + break; + case TGSI_OPCODE_I64SHR: + exec_arg0_64_arg1_32(mach, inst, micro_i64shr); + break; + case TGSI_OPCODE_U64SHR: + exec_arg0_64_arg1_32(mach, inst, micro_u64shr); + break; + case TGSI_OPCODE_U64DIV: + exec_double_binary(mach, inst, micro_u64div, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64DIV: + exec_double_binary(mach, inst, micro_i64div, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MOD: + exec_double_binary(mach, inst, micro_u64mod, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64MOD: + exec_double_binary(mach, inst, micro_i64mod, TGSI_EXEC_DATA_INT64); + break; + + case TGSI_OPCODE_F2U64: + exec_t_2_64(mach, inst, micro_f2u64, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_F2I64: + exec_t_2_64(mach, inst, micro_f2i64, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_U2I64: + exec_t_2_64(mach, inst, micro_u2i64, TGSI_EXEC_DATA_INT); + break; + case TGSI_OPCODE_I2I64: + exec_t_2_64(mach, inst, micro_i2i64, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_D2U64: + exec_double_unary(mach, inst, micro_d2u64); + break; + + case TGSI_OPCODE_D2I64: + exec_double_unary(mach, inst, micro_d2i64); + break; + + case TGSI_OPCODE_U642F: + exec_64_2_t(mach, inst, micro_u642f, TGSI_EXEC_DATA_FLOAT); + break; + case TGSI_OPCODE_I642F: + exec_64_2_t(mach, inst, micro_i642f, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_U642D: + exec_double_unary(mach, inst, micro_u642d); + break; + case TGSI_OPCODE_I642D: + exec_double_unary(mach, inst, micro_i642d); + break; + default: assert( 0 ); } + return FALSE; } +static void +tgsi_exec_machine_setup_masks(struct tgsi_exec_machine *mach) +{ + uint default_mask = 0xf; -#define DEBUG_EXECUTION 0 - + mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0] = 0; + mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0] = 0; -/** - * Run TGSI interpreter. - * \return bitmask of "alive" quad components - */ -uint -tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) -{ - uint i; - int pc = 0; + if (mach->ShaderType == PIPE_SHADER_GEOMETRY) { + mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0] = 0; + mach->Primitives[0] = 0; + /* GS runs on a single primitive for now */ + default_mask = 0x1; + } - mach->CondMask = 0xf; - mach->LoopMask = 0xf; - mach->ContMask = 0xf; - mach->FuncMask = 0xf; - mach->ExecMask = 0xf; + if (mach->NonHelperMask == 0) + mach->NonHelperMask = default_mask; + mach->CondMask = default_mask; + mach->LoopMask = default_mask; + mach->ContMask = default_mask; + mach->FuncMask = default_mask; + mach->ExecMask = default_mask; - mach->Switch.mask = 0xf; + mach->Switch.mask = default_mask; assert(mach->CondStackTop == 0); assert(mach->LoopStackTop == 0); @@ -4315,18 +6270,26 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) assert(mach->SwitchStackTop == 0); assert(mach->BreakStackTop == 0); assert(mach->CallStackTop == 0); +} - mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0] = 0; - mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0] = 0; +/** + * Run TGSI interpreter. + * \return bitmask of "alive" quad components + */ +uint +tgsi_exec_machine_run( struct tgsi_exec_machine *mach, int start_pc ) +{ + uint i; - if( mach->Processor == TGSI_PROCESSOR_GEOMETRY ) { - mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0] = 0; - mach->Primitives[0] = 0; - } + mach->pc = start_pc; + + if (!start_pc) { + tgsi_exec_machine_setup_masks(mach); - /* execute declarations (interpolants) */ - for (i = 0; i < mach->NumDeclarations; i++) { - exec_declaration( mach, mach->Declarations+i ); + /* execute declarations (interpolants) */ + for (i = 0; i < mach->NumDeclarations; i++) { + exec_declaration( mach, mach->Declarations+i ); + } } { @@ -4335,21 +6298,30 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) struct tgsi_exec_vector outputs[PIPE_MAX_ATTRIBS]; uint inst = 1; - memcpy(temps, mach->Temps, sizeof(temps)); - memcpy(outputs, mach->Outputs, sizeof(outputs)); + if (!start_pc) { + memset(mach->Temps, 0, sizeof(temps)); + if (mach->Outputs) + memset(mach->Outputs, 0, sizeof(outputs)); + memset(temps, 0, sizeof(temps)); + memset(outputs, 0, sizeof(outputs)); + } #endif /* execute instructions, until pc is set to -1 */ - while (pc != -1) { - + while (mach->pc != -1) { + boolean barrier_hit; #if DEBUG_EXECUTION uint i; - tgsi_dump_instruction(&mach->Instructions[pc], inst++); + tgsi_dump_instruction(&mach->Instructions[mach->pc], inst++); #endif - assert(pc < (int) mach->NumInstructions); - exec_instruction(mach, mach->Instructions + pc, &pc); + assert(mach->pc < (int) mach->NumInstructions); + barrier_hit = exec_instruction(mach, mach->Instructions + mach->pc, &mach->pc); + + /* for compute shaders if we hit a barrier return now for later rescheduling */ + if (barrier_hit && mach->ShaderType == PIPE_SHADER_COMPUTE) + return 0; #if DEBUG_EXECUTION for (i = 0; i < TGSI_EXEC_NUM_TEMPS + TGSI_EXEC_NUM_TEMP_EXTRAS; i++) { @@ -4370,21 +6342,23 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) } } } - for (i = 0; i < PIPE_MAX_ATTRIBS; i++) { - if (memcmp(&outputs[i], &mach->Outputs[i], sizeof(outputs[i]))) { - uint j; - - memcpy(&outputs[i], &mach->Outputs[i], sizeof(outputs[i])); - debug_printf("OUT[%2u] = ", i); - for (j = 0; j < 4; j++) { - if (j > 0) { - debug_printf(" "); + if (mach->Outputs) { + for (i = 0; i < PIPE_MAX_ATTRIBS; i++) { + if (memcmp(&outputs[i], &mach->Outputs[i], sizeof(outputs[i]))) { + uint j; + + memcpy(&outputs[i], &mach->Outputs[i], sizeof(outputs[i])); + debug_printf("OUT[%2u] = ", i); + for (j = 0; j < 4; j++) { + if (j > 0) { + debug_printf(" "); + } + debug_printf("(%6f %u, %6f %u, %6f %u, %6f %u)\n", + outputs[i].xyzw[0].f[j], outputs[i].xyzw[0].u[j], + outputs[i].xyzw[1].f[j], outputs[i].xyzw[1].u[j], + outputs[i].xyzw[2].f[j], outputs[i].xyzw[2].u[j], + outputs[i].xyzw[3].f[j], outputs[i].xyzw[3].u[j]); } - debug_printf("(%6f %u, %6f %u, %6f %u, %6f %u)\n", - outputs[i].xyzw[0].f[j], outputs[i].xyzw[0].u[j], - outputs[i].xyzw[1].f[j], outputs[i].xyzw[1].u[j], - outputs[i].xyzw[2].f[j], outputs[i].xyzw[2].u[j], - outputs[i].xyzw[3].f[j], outputs[i].xyzw[3].u[j]); } } } @@ -4394,7 +6368,7 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) #if 0 /* we scale from floats in [0,1] to Zbuffer ints in sp_quad_depth_test.c */ - if (mach->Processor == TGSI_PROCESSOR_FRAGMENT) { + if (mach->ShaderType == PIPE_SHADER_FRAGMENT) { /* * Scale back depth component. */