X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fauxiliary%2Ftgsi%2Ftgsi_exec.c;h=e0ff9476cb20f17edfa3ff0bcc3377d0dbc6a3f2;hb=092f6226ea677812f0292cc96d0424067fa0c94e;hp=126259fc0f86612c5a990a8df94fde7c210c93c8;hpb=d335b6abc0eaa7506203df7c99898645214b4c72;p=mesa.git diff --git a/src/gallium/auxiliary/tgsi/tgsi_exec.c b/src/gallium/auxiliary/tgsi/tgsi_exec.c index 126259fc0f8..e0ff9476cb2 100644 --- a/src/gallium/auxiliary/tgsi/tgsi_exec.c +++ b/src/gallium/auxiliary/tgsi/tgsi_exec.c @@ -61,6 +61,7 @@ #include "util/u_half.h" #include "util/u_memory.h" #include "util/u_math.h" +#include "util/rounding.h" #define DEBUG_EXECUTION 0 @@ -76,6 +77,8 @@ union tgsi_double_channel { double d[TGSI_QUAD_SIZE]; unsigned u[TGSI_QUAD_SIZE][2]; + uint64_t u64[TGSI_QUAD_SIZE]; + int64_t i64[TGSI_QUAD_SIZE]; }; struct tgsi_double_vector { @@ -123,18 +126,6 @@ micro_ceil(union tgsi_exec_channel *dst, dst->f[3] = ceilf(src->f[3]); } -static void -micro_clamp(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1, - const union tgsi_exec_channel *src2) -{ - dst->f[0] = src0->f[0] < src1->f[0] ? src1->f[0] : src0->f[0] > src2->f[0] ? src2->f[0] : src0->f[0]; - dst->f[1] = src0->f[1] < src1->f[1] ? src1->f[1] : src0->f[1] > src2->f[1] ? src2->f[1] : src0->f[1]; - dst->f[2] = src0->f[2] < src1->f[2] ? src1->f[2] : src0->f[2] > src2->f[2] ? src2->f[2] : src0->f[2]; - dst->f[3] = src0->f[3] < src1->f[3] ? src1->f[3] : src0->f[3] > src2->f[3] ? src2->f[3] : src0->f[3]; -} - static void micro_cmp(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, @@ -206,6 +197,16 @@ micro_dadd(union tgsi_double_channel *dst, dst->d[3] = src[0].d[3] + src[1].d[3]; } +static void +micro_ddiv(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = src[0].d[0] / src[1].d[0]; + dst->d[1] = src[0].d[1] / src[1].d[1]; + dst->d[2] = src[0].d[2] / src[1].d[2]; + dst->d[3] = src[0].d[3] / src[1].d[3]; +} + static void micro_ddx(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -216,6 +217,17 @@ micro_ddx(union tgsi_exec_channel *dst, dst->f[3] = src->f[TILE_BOTTOM_RIGHT] - src->f[TILE_BOTTOM_LEFT]; } +static void +micro_ddx_fine(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->f[0] = + dst->f[1] = src->f[TILE_TOP_RIGHT] - src->f[TILE_TOP_LEFT]; + dst->f[2] = + dst->f[3] = src->f[TILE_BOTTOM_RIGHT] - src->f[TILE_BOTTOM_LEFT]; +} + + static void micro_ddy(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -226,6 +238,16 @@ micro_ddy(union tgsi_exec_channel *dst, dst->f[3] = src->f[TILE_BOTTOM_LEFT] - src->f[TILE_TOP_LEFT]; } +static void +micro_ddy_fine(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->f[0] = + dst->f[2] = src->f[TILE_BOTTOM_LEFT] - src->f[TILE_TOP_LEFT]; + dst->f[1] = + dst->f[3] = src->f[TILE_BOTTOM_RIGHT] - src->f[TILE_TOP_RIGHT]; +} + static void micro_dmul(union tgsi_double_channel *dst, const union tgsi_double_channel *src) @@ -543,10 +565,10 @@ static void micro_rnd(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) { - dst->f[0] = floorf(src->f[0] + 0.5f); - dst->f[1] = floorf(src->f[1] + 0.5f); - dst->f[2] = floorf(src->f[2] + 0.5f); - dst->f[3] = floorf(src->f[3] + 0.5f); + dst->f[0] = _mesa_roundevenf(src->f[0]); + dst->f[1] = _mesa_roundevenf(src->f[1]); + dst->f[2] = _mesa_roundevenf(src->f[2]); + dst->f[3] = _mesa_roundevenf(src->f[3]); } static void @@ -675,10 +697,10 @@ static void micro_trunc(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) { - dst->f[0] = (float)(int)src->f[0]; - dst->f[1] = (float)(int)src->f[1]; - dst->f[2] = (float)(int)src->f[2]; - dst->f[3] = (float)(int)src->f[3]; + dst->f[0] = truncf(src->f[0]); + dst->f[1] = truncf(src->f[1]); + dst->f[2] = truncf(src->f[2]); + dst->f[3] = truncf(src->f[3]); } static void @@ -691,11 +713,251 @@ micro_u2d(union tgsi_double_channel *dst, dst->d[3] = (double)src->u[3]; } +static void +micro_i64abs(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src->i64[0] >= 0.0 ? src->i64[0] : -src->i64[0]; + dst->i64[1] = src->i64[1] >= 0.0 ? src->i64[1] : -src->i64[1]; + dst->i64[2] = src->i64[2] >= 0.0 ? src->i64[2] : -src->i64[2]; + dst->i64[3] = src->i64[3] >= 0.0 ? src->i64[3] : -src->i64[3]; +} + +static void +micro_i64sgn(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src->i64[0] < 0 ? -1 : src->i64[0] > 0 ? 1 : 0; + dst->i64[1] = src->i64[1] < 0 ? -1 : src->i64[1] > 0 ? 1 : 0; + dst->i64[2] = src->i64[2] < 0 ? -1 : src->i64[2] > 0 ? 1 : 0; + dst->i64[3] = src->i64[3] < 0 ? -1 : src->i64[3] > 0 ? 1 : 0; +} + +static void +micro_i64neg(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = -src->i64[0]; + dst->i64[1] = -src->i64[1]; + dst->i64[2] = -src->i64[2]; + dst->i64[3] = -src->i64[3]; +} + +static void +micro_u64seq(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] == src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] == src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] == src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] == src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_u64sne(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] != src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] != src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] != src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] != src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_i64slt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].i64[0] < src[1].i64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].i64[1] < src[1].i64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].i64[2] < src[1].i64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].i64[3] < src[1].i64[3] ? ~0U : 0U; +} + +static void +micro_u64slt(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] < src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] < src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] < src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] < src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_i64sge(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].i64[0] >= src[1].i64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].i64[1] >= src[1].i64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].i64[2] >= src[1].i64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].i64[3] >= src[1].i64[3] ? ~0U : 0U; +} + +static void +micro_u64sge(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u[0][0] = src[0].u64[0] >= src[1].u64[0] ? ~0U : 0U; + dst->u[1][0] = src[0].u64[1] >= src[1].u64[1] ? ~0U : 0U; + dst->u[2][0] = src[0].u64[2] >= src[1].u64[2] ? ~0U : 0U; + dst->u[3][0] = src[0].u64[3] >= src[1].u64[3] ? ~0U : 0U; +} + +static void +micro_u64max(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] > src[1].u64[0] ? src[0].u64[0] : src[1].u64[0]; + dst->u64[1] = src[0].u64[1] > src[1].u64[1] ? src[0].u64[1] : src[1].u64[1]; + dst->u64[2] = src[0].u64[2] > src[1].u64[2] ? src[0].u64[2] : src[1].u64[2]; + dst->u64[3] = src[0].u64[3] > src[1].u64[3] ? src[0].u64[3] : src[1].u64[3]; +} + +static void +micro_i64max(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] > src[1].i64[0] ? src[0].i64[0] : src[1].i64[0]; + dst->i64[1] = src[0].i64[1] > src[1].i64[1] ? src[0].i64[1] : src[1].i64[1]; + dst->i64[2] = src[0].i64[2] > src[1].i64[2] ? src[0].i64[2] : src[1].i64[2]; + dst->i64[3] = src[0].i64[3] > src[1].i64[3] ? src[0].i64[3] : src[1].i64[3]; +} + +static void +micro_u64min(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] < src[1].u64[0] ? src[0].u64[0] : src[1].u64[0]; + dst->u64[1] = src[0].u64[1] < src[1].u64[1] ? src[0].u64[1] : src[1].u64[1]; + dst->u64[2] = src[0].u64[2] < src[1].u64[2] ? src[0].u64[2] : src[1].u64[2]; + dst->u64[3] = src[0].u64[3] < src[1].u64[3] ? src[0].u64[3] : src[1].u64[3]; +} + +static void +micro_i64min(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[0].i64[0] < src[1].i64[0] ? src[0].i64[0] : src[1].i64[0]; + dst->i64[1] = src[0].i64[1] < src[1].i64[1] ? src[0].i64[1] : src[1].i64[1]; + dst->i64[2] = src[0].i64[2] < src[1].i64[2] ? src[0].i64[2] : src[1].i64[2]; + dst->i64[3] = src[0].i64[3] < src[1].i64[3] ? src[0].i64[3] : src[1].i64[3]; +} + +static void +micro_u64add(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] + src[1].u64[0]; + dst->u64[1] = src[0].u64[1] + src[1].u64[1]; + dst->u64[2] = src[0].u64[2] + src[1].u64[2]; + dst->u64[3] = src[0].u64[3] + src[1].u64[3]; +} + +static void +micro_u64mul(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[0].u64[0] * src[1].u64[0]; + dst->u64[1] = src[0].u64[1] * src[1].u64[1]; + dst->u64[2] = src[0].u64[2] * src[1].u64[2]; + dst->u64[3] = src[0].u64[3] * src[1].u64[3]; +} + +static void +micro_u64div(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[1].u64[0] ? src[0].u64[0] / src[1].u64[0] : ~0ull; + dst->u64[1] = src[1].u64[1] ? src[0].u64[1] / src[1].u64[1] : ~0ull; + dst->u64[2] = src[1].u64[2] ? src[0].u64[2] / src[1].u64[2] : ~0ull; + dst->u64[3] = src[1].u64[3] ? src[0].u64[3] / src[1].u64[3] : ~0ull; +} + +static void +micro_i64div(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[1].i64[0] ? src[0].i64[0] / src[1].i64[0] : 0; + dst->i64[1] = src[1].i64[1] ? src[0].i64[1] / src[1].i64[1] : 0; + dst->i64[2] = src[1].i64[2] ? src[0].i64[2] / src[1].i64[2] : 0; + dst->i64[3] = src[1].i64[3] ? src[0].i64[3] / src[1].i64[3] : 0; +} + +static void +micro_u64mod(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = src[1].u64[0] ? src[0].u64[0] % src[1].u64[0] : ~0ull; + dst->u64[1] = src[1].u64[1] ? src[0].u64[1] % src[1].u64[1] : ~0ull; + dst->u64[2] = src[1].u64[2] ? src[0].u64[2] % src[1].u64[2] : ~0ull; + dst->u64[3] = src[1].u64[3] ? src[0].u64[3] % src[1].u64[3] : ~0ull; +} + +static void +micro_i64mod(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = src[1].i64[0] ? src[0].i64[0] % src[1].i64[0] : ~0ll; + dst->i64[1] = src[1].i64[1] ? src[0].i64[1] % src[1].i64[1] : ~0ll; + dst->i64[2] = src[1].i64[2] ? src[0].i64[2] % src[1].i64[2] : ~0ll; + dst->i64[3] = src[1].i64[3] ? src[0].i64[3] % src[1].i64[3] : ~0ll; +} + +static void +micro_u64shl(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->u64[0] = src0->u64[0] << masked_count; + masked_count = src1->u[1] & 0x3f; + dst->u64[1] = src0->u64[1] << masked_count; + masked_count = src1->u[2] & 0x3f; + dst->u64[2] = src0->u64[2] << masked_count; + masked_count = src1->u[3] & 0x3f; + dst->u64[3] = src0->u64[3] << masked_count; +} + +static void +micro_i64shr(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->i64[0] = src0->i64[0] >> masked_count; + masked_count = src1->u[1] & 0x3f; + dst->i64[1] = src0->i64[1] >> masked_count; + masked_count = src1->u[2] & 0x3f; + dst->i64[2] = src0->i64[2] >> masked_count; + masked_count = src1->u[3] & 0x3f; + dst->i64[3] = src0->i64[3] >> masked_count; +} + +static void +micro_u64shr(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1) +{ + unsigned masked_count; + masked_count = src1->u[0] & 0x3f; + dst->u64[0] = src0->u64[0] >> masked_count; + masked_count = src1->u[1] & 0x3f; + dst->u64[1] = src0->u64[1] >> masked_count; + masked_count = src1->u[2] & 0x3f; + dst->u64[2] = src0->u64[2] >> masked_count; + masked_count = src1->u[3] & 0x3f; + dst->u64[3] = src0->u64[3] >> masked_count; +} + enum tgsi_exec_datatype { TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_UINT, - TGSI_EXEC_DATA_DOUBLE + TGSI_EXEC_DATA_DOUBLE, + TGSI_EXEC_DATA_INT64, + TGSI_EXEC_DATA_UINT64, }; /* @@ -707,7 +969,22 @@ enum tgsi_exec_datatype { #define TEMP_OUTPUT_C TGSI_EXEC_TEMP_OUTPUT_C #define TEMP_PRIMITIVE_I TGSI_EXEC_TEMP_PRIMITIVE_I #define TEMP_PRIMITIVE_C TGSI_EXEC_TEMP_PRIMITIVE_C - +#define TEMP_PRIMITIVE_S1_I TGSI_EXEC_TEMP_PRIMITIVE_S1_I +#define TEMP_PRIMITIVE_S1_C TGSI_EXEC_TEMP_PRIMITIVE_S1_C +#define TEMP_PRIMITIVE_S2_I TGSI_EXEC_TEMP_PRIMITIVE_S2_I +#define TEMP_PRIMITIVE_S2_C TGSI_EXEC_TEMP_PRIMITIVE_S2_C +#define TEMP_PRIMITIVE_S3_I TGSI_EXEC_TEMP_PRIMITIVE_S3_I +#define TEMP_PRIMITIVE_S3_C TGSI_EXEC_TEMP_PRIMITIVE_S3_C + +static const struct { + int idx; + int chan; +} temp_prim_idxs[] = { + { TEMP_PRIMITIVE_I, TEMP_PRIMITIVE_C }, + { TEMP_PRIMITIVE_S1_I, TEMP_PRIMITIVE_S1_C }, + { TEMP_PRIMITIVE_S2_I, TEMP_PRIMITIVE_S2_C }, + { TEMP_PRIMITIVE_S3_I, TEMP_PRIMITIVE_S3_C }, +}; /** The execution mask depends on the conditional mask and the loop mask */ #define UPDATE_EXEC_MASK(MACH) \ @@ -789,61 +1066,6 @@ tgsi_exec_set_constant_buffers(struct tgsi_exec_machine *mach, } } - -/** - * Check if there's a potential src/dst register data dependency when - * using SOA execution. - * Example: - * MOV T, T.yxwz; - * This would expand into: - * MOV t0, t1; - * MOV t1, t0; - * MOV t2, t3; - * MOV t3, t2; - * The second instruction will have the wrong value for t0 if executed as-is. - */ -boolean -tgsi_check_soa_dependencies(const struct tgsi_full_instruction *inst) -{ - uint i, chan; - - uint writemask = inst->Dst[0].Register.WriteMask; - if (writemask == TGSI_WRITEMASK_X || - writemask == TGSI_WRITEMASK_Y || - writemask == TGSI_WRITEMASK_Z || - writemask == TGSI_WRITEMASK_W || - writemask == TGSI_WRITEMASK_NONE) { - /* no chance of data dependency */ - return FALSE; - } - - /* loop over src regs */ - for (i = 0; i < inst->Instruction.NumSrcRegs; i++) { - if ((inst->Src[i].Register.File == - inst->Dst[0].Register.File) && - ((inst->Src[i].Register.Index == - inst->Dst[0].Register.Index) || - inst->Src[i].Register.Indirect || - inst->Dst[0].Register.Indirect)) { - /* loop over dest channels */ - uint channelsWritten = 0x0; - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - /* check if we're reading a channel that's been written */ - uint swizzle = tgsi_util_get_full_src_register_swizzle(&inst->Src[i], chan); - if (channelsWritten & (1 << swizzle)) { - return TRUE; - } - - channelsWritten |= (1 << chan); - } - } - } - } - return FALSE; -} - - /** * Initialize machine state by expanding tokens to full instructions, * allocating temporary storage, setting up constants, etc. @@ -853,7 +1075,9 @@ void tgsi_exec_machine_bind_shader( struct tgsi_exec_machine *mach, const struct tgsi_token *tokens, - struct tgsi_sampler *sampler) + struct tgsi_sampler *sampler, + struct tgsi_image *image, + struct tgsi_buffer *buffer) { uint k; struct tgsi_parse_context parse; @@ -871,6 +1095,8 @@ tgsi_exec_machine_bind_shader( mach->Tokens = tokens; mach->Sampler = sampler; + mach->Image = image; + mach->Buffer = buffer; if (!tokens) { /* unbind and free all */ @@ -891,11 +1117,13 @@ tgsi_exec_machine_bind_shader( return; } - mach->Processor = parse.FullHeader.Processor.Processor; mach->ImmLimit = 0; mach->NumOutputs = 0; - if (mach->Processor == TGSI_PROCESSOR_GEOMETRY && + for (k = 0; k < TGSI_SEMANTIC_COUNT; k++) + mach->SysSemanticToIndex[k] = -1; + + if (mach->ShaderType == PIPE_SHADER_GEOMETRY && !mach->UsedGeometryShader) { struct tgsi_exec_vector *inputs; struct tgsi_exec_vector *outputs; @@ -961,6 +1189,11 @@ tgsi_exec_machine_bind_shader( ++mach->NumOutputs; } } + else if (parse.FullToken.FullDeclaration.Declaration.File == TGSI_FILE_SYSTEM_VALUE) { + const struct tgsi_full_declaration *decl = &parse.FullToken.FullDeclaration; + mach->SysSemanticToIndex[decl->Semantic.Name] = decl->Range.First; + } + memcpy(declarations + numDeclarations, &parse.FullToken.FullDeclaration, sizeof(declarations[0])); @@ -971,7 +1204,17 @@ tgsi_exec_machine_bind_shader( { uint size = parse.FullToken.FullImmediate.Immediate.NrTokens - 1; assert( size <= 4 ); - assert( mach->ImmLimit + 1 <= TGSI_EXEC_NUM_IMMEDIATES ); + if (mach->ImmLimit >= mach->ImmsReserved) { + unsigned newReserved = mach->ImmsReserved ? 2 * mach->ImmsReserved : 128; + float4 *imms = REALLOC(mach->Imms, mach->ImmsReserved, newReserved * sizeof(float4)); + if (imms) { + mach->ImmsReserved = newReserved; + mach->Imms = imms; + } else { + debug_printf("Unable to (re)allocate space for immidiate constants\n"); + break; + } + } for( i = 0; i < size; i++ ) { mach->Imms[mach->ImmLimit][i] = @@ -1001,7 +1244,7 @@ tgsi_exec_machine_bind_shader( break; case TGSI_TOKEN_TYPE_PROPERTY: - if (mach->Processor == TGSI_PROCESSOR_GEOMETRY) { + if (mach->ShaderType == PIPE_SHADER_GEOMETRY) { if (parse.FullToken.FullProperty.Property.PropertyName == TGSI_PROPERTY_GS_MAX_OUTPUT_VERTICES) { mach->MaxOutputVertices = parse.FullToken.FullProperty.u[0].Data; } @@ -1025,10 +1268,9 @@ tgsi_exec_machine_bind_shader( struct tgsi_exec_machine * -tgsi_exec_machine_create( void ) +tgsi_exec_machine_create(enum pipe_shader_type shader_type) { struct tgsi_exec_machine *mach; - uint i; mach = align_malloc( sizeof *mach, 16 ); if (!mach) @@ -1036,27 +1278,21 @@ tgsi_exec_machine_create( void ) memset(mach, 0, sizeof(*mach)); + mach->ShaderType = shader_type; mach->Addrs = &mach->Temps[TGSI_EXEC_TEMP_ADDR]; mach->MaxGeometryShaderOutputs = TGSI_MAX_TOTAL_VERTICES; - mach->Predicates = &mach->Temps[TGSI_EXEC_TEMP_P0]; - mach->Inputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_INPUTS, 16); - mach->Outputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_OUTPUTS, 16); - if (!mach->Inputs || !mach->Outputs) - goto fail; + if (shader_type != PIPE_SHADER_COMPUTE) { + mach->Inputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_INPUTS, 16); + mach->Outputs = align_malloc(sizeof(struct tgsi_exec_vector) * PIPE_MAX_SHADER_OUTPUTS, 16); + if (!mach->Inputs || !mach->Outputs) + goto fail; + } - /* Setup constants needed by the SSE2 executor. */ - for( i = 0; i < 4; i++ ) { - mach->Temps[TGSI_EXEC_TEMP_00000000_I].xyzw[TGSI_EXEC_TEMP_00000000_C].u[i] = 0x00000000; - mach->Temps[TGSI_EXEC_TEMP_7FFFFFFF_I].xyzw[TGSI_EXEC_TEMP_7FFFFFFF_C].u[i] = 0x7FFFFFFF; - mach->Temps[TGSI_EXEC_TEMP_80000000_I].xyzw[TGSI_EXEC_TEMP_80000000_C].u[i] = 0x80000000; - mach->Temps[TGSI_EXEC_TEMP_FFFFFFFF_I].xyzw[TGSI_EXEC_TEMP_FFFFFFFF_C].u[i] = 0xFFFFFFFF; /* not used */ - mach->Temps[TGSI_EXEC_TEMP_ONE_I].xyzw[TGSI_EXEC_TEMP_ONE_C].f[i] = 1.0f; - mach->Temps[TGSI_EXEC_TEMP_TWO_I].xyzw[TGSI_EXEC_TEMP_TWO_C].f[i] = 2.0f; /* not used */ - mach->Temps[TGSI_EXEC_TEMP_128_I].xyzw[TGSI_EXEC_TEMP_128_C].f[i] = 128.0f; - mach->Temps[TGSI_EXEC_TEMP_MINUS_128_I].xyzw[TGSI_EXEC_TEMP_MINUS_128_C].f[i] = -128.0f; - mach->Temps[TGSI_EXEC_TEMP_THREE_I].xyzw[TGSI_EXEC_TEMP_THREE_C].f[i] = 3.0f; - mach->Temps[TGSI_EXEC_TEMP_HALF_I].xyzw[TGSI_EXEC_TEMP_HALF_C].f[i] = 0.5f; + if (shader_type == PIPE_SHADER_FRAGMENT) { + mach->InputSampleOffsetApply = align_malloc(sizeof(apply_sample_offset_func) * PIPE_MAX_SHADER_INPUTS, 16); + if (!mach->InputSampleOffsetApply) + goto fail; } #ifdef DEBUG @@ -1069,6 +1305,7 @@ tgsi_exec_machine_create( void ) fail: if (mach) { + align_free(mach->InputSampleOffsetApply); align_free(mach->Inputs); align_free(mach->Outputs); align_free(mach); @@ -1083,7 +1320,9 @@ tgsi_exec_machine_destroy(struct tgsi_exec_machine *mach) if (mach) { FREE(mach->Instructions); FREE(mach->Declarations); + FREE(mach->Imms); + align_free(mach->InputSampleOffsetApply); align_free(mach->Inputs); align_free(mach->Outputs); @@ -1199,6 +1438,17 @@ micro_pow( #endif } +static void +micro_ldexp(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1) +{ + dst->f[0] = ldexpf(src0->f[0], src1->i[0]); + dst->f[1] = ldexpf(src0->f[1], src1->i[1]); + dst->f[2] = ldexpf(src0->f[2], src1->i[2]); + dst->f[3] = ldexpf(src0->f[3], src1->i[3]); +} + static void micro_sub(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, @@ -1212,7 +1462,6 @@ micro_sub(union tgsi_exec_channel *dst, static void fetch_src_file_channel(const struct tgsi_exec_machine *mach, - const uint chan_index, const uint file, const uint swizzle, const union tgsi_exec_channel *index, @@ -1256,7 +1505,7 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, case TGSI_FILE_INPUT: for (i = 0; i < TGSI_QUAD_SIZE; i++) { /* - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { debug_printf("Fetching Input[%d] (2d=%d, 1d=%d)\n", index2D->i[i] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index->i[i], index2D->i[i], index->i[i]); @@ -1269,11 +1518,8 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, break; case TGSI_FILE_SYSTEM_VALUE: - /* XXX no swizzling at this point. Will be needed if we put - * gl_FragCoord, for example, in a sys value register. - */ for (i = 0; i < TGSI_QUAD_SIZE; i++) { - chan->u[i] = mach->SystemValue[index->i[i]].u[i]; + chan->u[i] = mach->SystemValue[index->i[i]].xyzw[swizzle].u[i]; } break; @@ -1304,15 +1550,6 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, } break; - case TGSI_FILE_PREDICATE: - for (i = 0; i < TGSI_QUAD_SIZE; i++) { - assert(index->i[i] >= 0 && index->i[i] < TGSI_EXEC_NUM_PREDS); - assert(index2D->i[i] == 0); - - chan->u[i] = mach->Predicates[0].xyzw[swizzle].u[i]; - } - break; - case TGSI_FILE_OUTPUT: /* vertex/fragment output vars can be read too */ for (i = 0; i < TGSI_QUAD_SIZE; i++) { @@ -1332,14 +1569,11 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach, } static void -fetch_source_d(const struct tgsi_exec_machine *mach, - union tgsi_exec_channel *chan, - const struct tgsi_full_src_register *reg, - const uint chan_index, - enum tgsi_exec_datatype src_datatype) +get_index_registers(const struct tgsi_exec_machine *mach, + const struct tgsi_full_src_register *reg, + union tgsi_exec_channel *index, + union tgsi_exec_channel *index2D) { - union tgsi_exec_channel index; - union tgsi_exec_channel index2D; uint swizzle; /* We start with a direct index into a register file. @@ -1349,10 +1583,10 @@ fetch_source_d(const struct tgsi_exec_machine *mach, * file = Register.File * [1] = Register.Index */ - index.i[0] = - index.i[1] = - index.i[2] = - index.i[3] = reg->Register.Index; + index->i[0] = + index->i[1] = + index->i[2] = + index->i[3] = reg->Register.Index; /* There is an extra source register that indirectly subscripts * a register file. The direct index now becomes an offset @@ -1378,7 +1612,6 @@ fetch_source_d(const struct tgsi_exec_machine *mach, /* get current value of address register[swizzle] */ swizzle = reg->Indirect.Swizzle; fetch_src_file_channel(mach, - chan_index, reg->Indirect.File, swizzle, &index2, @@ -1386,17 +1619,17 @@ fetch_source_d(const struct tgsi_exec_machine *mach, &indir_index); /* add value of address register to the offset */ - index.i[0] += indir_index.i[0]; - index.i[1] += indir_index.i[1]; - index.i[2] += indir_index.i[2]; - index.i[3] += indir_index.i[3]; + index->i[0] += indir_index.i[0]; + index->i[1] += indir_index.i[1]; + index->i[2] += indir_index.i[2]; + index->i[3] += indir_index.i[3]; /* for disabled execution channels, zero-out the index to * avoid using a potential garbage value. */ for (i = 0; i < TGSI_QUAD_SIZE; i++) { if ((execmask & (1 << i)) == 0) - index.i[i] = 0; + index->i[i] = 0; } } @@ -1409,10 +1642,10 @@ fetch_source_d(const struct tgsi_exec_machine *mach, * [3] = Dimension.Index */ if (reg->Register.Dimension) { - index2D.i[0] = - index2D.i[1] = - index2D.i[2] = - index2D.i[3] = reg->Dimension.Index; + index2D->i[0] = + index2D->i[1] = + index2D->i[2] = + index2D->i[3] = reg->Dimension.Index; /* Again, the second subscript index can be addressed indirectly * identically to the first one. @@ -1438,24 +1671,23 @@ fetch_source_d(const struct tgsi_exec_machine *mach, swizzle = reg->DimIndirect.Swizzle; fetch_src_file_channel(mach, - chan_index, reg->DimIndirect.File, swizzle, &index2, &ZeroVec, &indir_index); - index2D.i[0] += indir_index.i[0]; - index2D.i[1] += indir_index.i[1]; - index2D.i[2] += indir_index.i[2]; - index2D.i[3] += indir_index.i[3]; + index2D->i[0] += indir_index.i[0]; + index2D->i[1] += indir_index.i[1]; + index2D->i[2] += indir_index.i[2]; + index2D->i[3] += indir_index.i[3]; /* for disabled execution channels, zero-out the index to * avoid using a potential garbage value. */ for (i = 0; i < TGSI_QUAD_SIZE; i++) { if ((execmask & (1 << i)) == 0) { - index2D.i[i] = 0; + index2D->i[i] = 0; } } } @@ -1465,15 +1697,29 @@ fetch_source_d(const struct tgsi_exec_machine *mach, * by a dimension register and continue the saga. */ } else { - index2D.i[0] = - index2D.i[1] = - index2D.i[2] = - index2D.i[3] = 0; + index2D->i[0] = + index2D->i[1] = + index2D->i[2] = + index2D->i[3] = 0; } +} + + +static void +fetch_source_d(const struct tgsi_exec_machine *mach, + union tgsi_exec_channel *chan, + const struct tgsi_full_src_register *reg, + const uint chan_index) +{ + union tgsi_exec_channel index; + union tgsi_exec_channel index2D; + uint swizzle; + + get_index_registers(mach, reg, &index, &index2D); + swizzle = tgsi_util_get_full_src_register_swizzle( reg, chan_index ); fetch_src_file_channel(mach, - chan_index, reg->Register.File, swizzle, &index, @@ -1488,7 +1734,7 @@ fetch_source(const struct tgsi_exec_machine *mach, const uint chan_index, enum tgsi_exec_datatype src_datatype) { - fetch_source_d(mach, chan, reg, chan_index, src_datatype); + fetch_source_d(mach, chan, reg, chan_index); if (reg->Register.Absolute) { if (src_datatype == TGSI_EXEC_DATA_FLOAT) { @@ -1511,15 +1757,12 @@ static union tgsi_exec_channel * store_dest_dstret(struct tgsi_exec_machine *mach, const union tgsi_exec_channel *chan, const struct tgsi_full_dst_register *reg, - const struct tgsi_full_instruction *inst, uint chan_index, enum tgsi_exec_datatype dst_datatype) { - uint i; static union tgsi_exec_channel null; union tgsi_exec_channel *dst; union tgsi_exec_channel index2D; - uint execmask = mach->ExecMask; int offset = 0; /* indirection offset */ int index; @@ -1554,7 +1797,6 @@ store_dest_dstret(struct tgsi_exec_machine *mach, /* fetch values from the address/indirection register */ fetch_src_file_channel(mach, - chan_index, reg->Indirect.File, swizzle, &index, @@ -1604,7 +1846,6 @@ store_dest_dstret(struct tgsi_exec_machine *mach, swizzle = reg->DimIndirect.Swizzle; fetch_src_file_channel(mach, - chan_index, reg->DimIndirect.File, swizzle, &index2, @@ -1650,7 +1891,7 @@ store_dest_dstret(struct tgsi_exec_machine *mach, debug_printf("NumOutputs = %d, TEMP_O_C/I = %d, redindex = %d\n", mach->NumOutputs, mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0], reg->Register.Index); - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { debug_printf("STORING OUT[%d] mask(%d), = (", offset + index, execmask); for (i = 0; i < TGSI_QUAD_SIZE; i++) if (execmask & (1 << i)) @@ -1671,58 +1912,11 @@ store_dest_dstret(struct tgsi_exec_machine *mach, dst = &mach->Addrs[index].xyzw[chan_index]; break; - case TGSI_FILE_PREDICATE: - index = reg->Register.Index; - assert(index < TGSI_EXEC_NUM_PREDS); - dst = &mach->Predicates[index].xyzw[chan_index]; - break; - default: assert( 0 ); return NULL; } - if (inst->Instruction.Predicate) { - uint swizzle; - union tgsi_exec_channel *pred; - - switch (chan_index) { - case TGSI_CHAN_X: - swizzle = inst->Predicate.SwizzleX; - break; - case TGSI_CHAN_Y: - swizzle = inst->Predicate.SwizzleY; - break; - case TGSI_CHAN_Z: - swizzle = inst->Predicate.SwizzleZ; - break; - case TGSI_CHAN_W: - swizzle = inst->Predicate.SwizzleW; - break; - default: - assert(0); - return NULL; - } - - assert(inst->Predicate.Index == 0); - - pred = &mach->Predicates[inst->Predicate.Index].xyzw[swizzle]; - - if (inst->Predicate.Negate) { - for (i = 0; i < TGSI_QUAD_SIZE; i++) { - if (pred->u[i]) { - execmask &= ~(1 << i); - } - } - } else { - for (i = 0; i < TGSI_QUAD_SIZE; i++) { - if (!pred->u[i]) { - execmask &= ~(1 << i); - } - } - } - } - return dst; } @@ -1730,7 +1924,6 @@ static void store_dest_double(struct tgsi_exec_machine *mach, const union tgsi_exec_channel *chan, const struct tgsi_full_dst_register *reg, - const struct tgsi_full_instruction *inst, uint chan_index, enum tgsi_exec_datatype dst_datatype) { @@ -1738,8 +1931,7 @@ store_dest_double(struct tgsi_exec_machine *mach, const uint execmask = mach->ExecMask; int i; - dst = store_dest_dstret(mach, chan, reg, inst, chan_index, - dst_datatype); + dst = store_dest_dstret(mach, chan, reg, chan_index, dst_datatype); if (!dst) return; @@ -1761,8 +1953,7 @@ store_dest(struct tgsi_exec_machine *mach, const uint execmask = mach->ExecMask; int i; - dst = store_dest_dstret(mach, chan, reg, inst, chan_index, - dst_datatype); + dst = store_dest_dstret(mach, chan, reg, chan_index, dst_datatype); if (!dst) return; @@ -1838,8 +2029,7 @@ exec_kill_if(struct tgsi_exec_machine *mach, * Unconditional fragment kill/discard. */ static void -exec_kill(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_kill(struct tgsi_exec_machine *mach) { uint kilmask; /* bit 0 = pixel 0, bit 1 = pixel 1, etc */ @@ -1849,46 +2039,63 @@ exec_kill(struct tgsi_exec_machine *mach, } static void -emit_vertex(struct tgsi_exec_machine *mach) +emit_vertex(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { + union tgsi_exec_channel r[1]; + unsigned stream_id; + unsigned *prim_count; /* FIXME: check for exec mask correctly unsigned i; for (i = 0; i < TGSI_QUAD_SIZE; ++i) { if ((mach->ExecMask & (1 << i))) */ + IFETCH(&r[0], 0, TGSI_CHAN_X); + stream_id = r[0].u[0]; + prim_count = &mach->Temps[temp_prim_idxs[stream_id].idx].xyzw[temp_prim_idxs[stream_id].chan].u[0]; if (mach->ExecMask) { - if (mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]] >= mach->MaxOutputVertices) + if (mach->Primitives[stream_id][*prim_count] >= mach->MaxOutputVertices) return; + if (mach->Primitives[stream_id][*prim_count] == 0) + mach->PrimitiveOffsets[stream_id][*prim_count] = mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0]; mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0] += mach->NumOutputs; - mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]]++; + mach->Primitives[stream_id][*prim_count]++; } } static void -emit_primitive(struct tgsi_exec_machine *mach) +emit_primitive(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { - unsigned *prim_count = &mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]; + unsigned *prim_count; + union tgsi_exec_channel r[1]; + unsigned stream_id = 0; /* FIXME: check for exec mask correctly unsigned i; for (i = 0; i < TGSI_QUAD_SIZE; ++i) { if ((mach->ExecMask & (1 << i))) */ + if (inst) { + IFETCH(&r[0], 0, TGSI_CHAN_X); + stream_id = r[0].u[0]; + } + prim_count = &mach->Temps[temp_prim_idxs[stream_id].idx].xyzw[temp_prim_idxs[stream_id].chan].u[0]; if (mach->ExecMask) { ++(*prim_count); debug_assert((*prim_count * mach->NumOutputs) < mach->MaxGeometryShaderOutputs); - mach->Primitives[*prim_count] = 0; + mach->Primitives[stream_id][*prim_count] = 0; } } static void conditional_emit_primitive(struct tgsi_exec_machine *mach) { - if (TGSI_PROCESSOR_GEOMETRY == mach->Processor) { + if (PIPE_SHADER_GEOMETRY == mach->ShaderType) { int emitted_verts = - mach->Primitives[mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0]]; + mach->Primitives[0][mach->Temps[temp_prim_idxs[0].idx].xyzw[temp_prim_idxs[0].chan].u[0]]; if (emitted_verts) { - emit_primitive(mach); + emit_primitive(mach, NULL); } } } @@ -1949,11 +2156,11 @@ fetch_texel_offsets(struct tgsi_exec_machine *mach, union tgsi_exec_channel index; union tgsi_exec_channel offset[3]; index.i[0] = index.i[1] = index.i[2] = index.i[3] = inst->TexOffsets[0].Index; - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + fetch_src_file_channel(mach, inst->TexOffsets[0].File, inst->TexOffsets[0].SwizzleX, &index, &ZeroVec, &offset[0]); - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + fetch_src_file_channel(mach, inst->TexOffsets[0].File, inst->TexOffsets[0].SwizzleY, &index, &ZeroVec, &offset[1]); - fetch_src_file_channel(mach, 0, inst->TexOffsets[0].File, + fetch_src_file_channel(mach, inst->TexOffsets[0].File, inst->TexOffsets[0].SwizzleZ, &index, &ZeroVec, &offset[2]); offsets[0] = offset[0].i[0]; offsets[1] = offset[1].i[0]; @@ -1994,25 +2201,30 @@ fetch_sampler_unit(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst, uint sampler) { - uint unit; - + uint unit = 0; + int i; if (inst->Src[sampler].Register.Indirect) { const struct tgsi_full_src_register *reg = &inst->Src[sampler]; union tgsi_exec_channel indir_index, index2; - + const uint execmask = mach->ExecMask; index2.i[0] = index2.i[1] = index2.i[2] = index2.i[3] = reg->Indirect.Index; fetch_src_file_channel(mach, - 0, reg->Indirect.File, reg->Indirect.Swizzle, &index2, &ZeroVec, &indir_index); - unit = inst->Src[sampler].Register.Index + indir_index.i[0]; + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + if (execmask & (1 << i)) { + unit = inst->Src[sampler].Register.Index + indir_index.i[i]; + break; + } + } + } else { unit = inst->Src[sampler].Register.Index; } @@ -2046,15 +2258,16 @@ exec_tex(struct tgsi_exec_machine *mach, assert(modifier != TEX_MODIFIER_LEVEL_ZERO); assert(inst->Texture.Texture != TGSI_TEXTURE_BUFFER); - dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture, &shadow_ref); + dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture); + shadow_ref = tgsi_util_get_shadow_ref_src_index(inst->Texture.Texture); assert(dim <= 4); if (shadow_ref >= 0) - assert(shadow_ref >= dim && shadow_ref < Elements(args)); + assert(shadow_ref >= dim && shadow_ref < (int)ARRAY_SIZE(args)); /* fetch modifier to the last argument */ if (modifier != TEX_MODIFIER_NONE) { - const int last = Elements(args) - 1; + const int last = ARRAY_SIZE(args) - 1; /* fetch modifier from src0.w or src1.x */ if (sampler == 1) { @@ -2062,7 +2275,6 @@ exec_tex(struct tgsi_exec_machine *mach, FETCH(&r[last], 0, TGSI_CHAN_W); } else { - assert(shadow_ref != 4); FETCH(&r[last], 1, TGSI_CHAN_X); } @@ -2086,7 +2298,7 @@ exec_tex(struct tgsi_exec_machine *mach, control = TGSI_SAMPLER_GATHER; } else { - for (i = dim; i < Elements(args); i++) + for (i = dim; i < (int)ARRAY_SIZE(args); i++) args[i] = &ZeroVec; } @@ -2137,25 +2349,32 @@ static void exec_lodq(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) { - uint unit; - int dim; - int i; + uint resource_unit, sampler_unit; + unsigned dim; + unsigned i; union tgsi_exec_channel coords[4]; - const union tgsi_exec_channel *args[Elements(coords)]; + const union tgsi_exec_channel *args[ARRAY_SIZE(coords)]; union tgsi_exec_channel r[2]; - unit = fetch_sampler_unit(mach, inst, 1); - dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture, NULL); - assert(dim <= Elements(coords)); + resource_unit = fetch_sampler_unit(mach, inst, 1); + if (inst->Instruction.Opcode == TGSI_OPCODE_LOD) { + uint target = mach->SamplerViews[resource_unit].Resource; + dim = tgsi_util_get_texture_coord_dim(target); + sampler_unit = fetch_sampler_unit(mach, inst, 2); + } else { + dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture); + sampler_unit = resource_unit; + } + assert(dim <= ARRAY_SIZE(coords)); /* fetch coordinates */ for (i = 0; i < dim; i++) { FETCH(&coords[i], 0, TGSI_CHAN_X + i); args[i] = &coords[i]; } - for (i = dim; i < Elements(coords); i++) { + for (i = dim; i < ARRAY_SIZE(coords); i++) { args[i] = &ZeroVec; } - mach->Sampler->query_lod(mach->Sampler, unit, unit, + mach->Sampler->query_lod(mach->Sampler, resource_unit, sampler_unit, args[0]->f, args[1]->f, args[2]->f, @@ -2172,14 +2391,43 @@ exec_lodq(struct tgsi_exec_machine *mach, store_dest(mach, &r[1], &inst->Dst[0], inst, TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); } -} + if (inst->Instruction.Opcode == TGSI_OPCODE_LOD) { + unsigned char swizzles[4]; + unsigned chan; + swizzles[0] = inst->Src[1].Register.SwizzleX; + swizzles[1] = inst->Src[1].Register.SwizzleY; + swizzles[2] = inst->Src[1].Register.SwizzleZ; + swizzles[3] = inst->Src[1].Register.SwizzleW; -static void -exec_txd(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - union tgsi_exec_channel r[4]; - float derivs[3][2][TGSI_QUAD_SIZE]; + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + if (swizzles[chan] >= 2) { + store_dest(mach, &ZeroVec, + &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } else { + store_dest(mach, &r[swizzles[chan]], + &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } + } + } else { + if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { + store_dest(mach, &r[0], &inst->Dst[0], inst, TGSI_CHAN_X, + TGSI_EXEC_DATA_FLOAT); + } + if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { + store_dest(mach, &r[1], &inst->Dst[0], inst, TGSI_CHAN_Y, + TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_txd(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + float derivs[3][2][TGSI_QUAD_SIZE]; uint chan; uint unit; int8_t offsets[3]; @@ -2428,6 +2676,9 @@ exec_sample(struct tgsi_exec_machine *mach, lod = &c1; control = TGSI_SAMPLER_LOD_EXPLICIT; } + else if (modifier == TEX_MODIFIER_GATHER) { + control = TGSI_SAMPLER_GATHER; + } else { assert(modifier == TEX_MODIFIER_LEVEL_ZERO); control = TGSI_SAMPLER_LOD_ZERO; @@ -2629,21 +2880,50 @@ eval_constant_coef( } } +static void +interp_constant_offset( + UNUSED const struct tgsi_exec_machine *mach, + UNUSED unsigned attrib, + UNUSED unsigned chan, + UNUSED float ofs_x, + UNUSED float ofs_y, + UNUSED union tgsi_exec_channel *out_chan) +{ +} + /** * Evaluate a linear-valued coefficient at the position of the * current quad. */ static void -eval_linear_coef( - struct tgsi_exec_machine *mach, - unsigned attrib, - unsigned chan ) +interp_linear_offset( + const struct tgsi_exec_machine *mach, + unsigned attrib, + unsigned chan, + float ofs_x, + float ofs_y, + union tgsi_exec_channel *out_chan) +{ + const float dadx = mach->InterpCoefs[attrib].dadx[chan]; + const float dady = mach->InterpCoefs[attrib].dady[chan]; + const float delta = ofs_x * dadx + ofs_y * dady; + out_chan->f[0] += delta; + out_chan->f[1] += delta; + out_chan->f[2] += delta; + out_chan->f[3] += delta; +} + +static void +eval_linear_coef(struct tgsi_exec_machine *mach, + unsigned attrib, + unsigned chan) { const float x = mach->QuadPos.xyzw[0].f[0]; const float y = mach->QuadPos.xyzw[1].f[0]; const float dadx = mach->InterpCoefs[attrib].dadx[chan]; const float dady = mach->InterpCoefs[attrib].dady[chan]; const float a0 = mach->InterpCoefs[attrib].a0[chan] + dadx * x + dady * y; + mach->Inputs[attrib].xyzw[chan].f[0] = a0; mach->Inputs[attrib].xyzw[chan].f[1] = a0 + dadx; mach->Inputs[attrib].xyzw[chan].f[2] = a0 + dady; @@ -2654,6 +2934,26 @@ eval_linear_coef( * Evaluate a perspective-valued coefficient at the position of the * current quad. */ + +static void +interp_perspective_offset( + const struct tgsi_exec_machine *mach, + unsigned attrib, + unsigned chan, + float ofs_x, + float ofs_y, + union tgsi_exec_channel *out_chan) +{ + const float dadx = mach->InterpCoefs[attrib].dadx[chan]; + const float dady = mach->InterpCoefs[attrib].dady[chan]; + const float *w = mach->QuadPos.xyzw[3].f; + const float delta = ofs_x * dadx + ofs_y * dady; + out_chan->f[0] += delta / w[0]; + out_chan->f[1] += delta / w[1]; + out_chan->f[2] += delta / w[2]; + out_chan->f[3] += delta / w[3]; +} + static void eval_perspective_coef( struct tgsi_exec_machine *mach, @@ -2688,7 +2988,7 @@ exec_declaration(struct tgsi_exec_machine *mach, return; } - if (mach->Processor == TGSI_PROCESSOR_FRAGMENT) { + if (mach->ShaderType == PIPE_SHADER_FRAGMENT) { if (decl->Declaration.File == TGSI_FILE_INPUT) { uint first, last, mask; @@ -2714,23 +3014,28 @@ exec_declaration(struct tgsi_exec_machine *mach, } } else { eval_coef_func eval; + apply_sample_offset_func interp; uint i, j; switch (decl->Interp.Interpolate) { case TGSI_INTERPOLATE_CONSTANT: eval = eval_constant_coef; + interp = interp_constant_offset; break; case TGSI_INTERPOLATE_LINEAR: eval = eval_linear_coef; + interp = interp_linear_offset; break; case TGSI_INTERPOLATE_PERSPECTIVE: eval = eval_perspective_coef; + interp = interp_perspective_offset; break; case TGSI_INTERPOLATE_COLOR: eval = mach->flatshade_color ? eval_constant_coef : eval_perspective_coef; + interp = mach->flatshade_color ? interp_constant_offset : interp_perspective_offset; break; default: @@ -2738,6 +3043,9 @@ exec_declaration(struct tgsi_exec_machine *mach, return; } + for (i = first; i <= last; i++) + mach->InputSampleOffsetApply[i] = interp; + for (j = 0; j < TGSI_NUM_CHANNELS; j++) { if (mask & (1 << j)) { for (i = first; i <= last; i++) { @@ -2766,9 +3074,6 @@ exec_declaration(struct tgsi_exec_machine *mach, } } - if (decl->Declaration.File == TGSI_FILE_SYSTEM_VALUE) { - mach->SysSemanticToIndex[decl->Declaration.Semantic] = decl->Range.First; - } } typedef void (* micro_unary_op)(union tgsi_exec_channel *dst, @@ -2984,60 +3289,6 @@ exec_dp4(struct tgsi_exec_machine *mach, } } -static void -exec_dp2a(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - unsigned int chan; - union tgsi_exec_channel arg[3]; - - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&arg[2], &arg[0], &arg[1]); - - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_mad(&arg[0], &arg[0], &arg[1], &arg[2]); - - fetch_source(mach, &arg[1], &inst->Src[2], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_add(&arg[0], &arg[0], &arg[1]); - - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &arg[0], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); - } - } -} - -static void -exec_dph(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - unsigned int chan; - union tgsi_exec_channel arg[3]; - - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_mul(&arg[2], &arg[0], &arg[1]); - - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_mad(&arg[2], &arg[0], &arg[1], &arg[2]); - - fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - micro_mad(&arg[0], &arg[0], &arg[1], &arg[2]); - - fetch_source(mach, &arg[1], &inst->Src[1], TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); - micro_add(&arg[0], &arg[0], &arg[1]); - - for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { - if (inst->Dst[0].Register.WriteMask & (1 << chan)) { - store_dest(mach, &arg[0], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); - } - } -} - static void exec_dp2(struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst) @@ -3100,74 +3351,42 @@ exec_up2h(struct tgsi_exec_machine *mach, } static void -exec_scs(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +micro_ucmp(union tgsi_exec_channel *dst, + const union tgsi_exec_channel *src0, + const union tgsi_exec_channel *src1, + const union tgsi_exec_channel *src2) { - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) { - union tgsi_exec_channel arg; - union tgsi_exec_channel result; - - fetch_source(mach, &arg, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { - micro_cos(&result, &arg); - store_dest(mach, &result, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { - micro_sin(&result, &arg); - store_dest(mach, &result, &inst->Dst[0], inst, TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - } - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Z) { - store_dest(mach, &ZeroVec, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_W) { - store_dest(mach, &OneVec, &inst->Dst[0], inst, TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); - } + dst->f[0] = src0->u[0] ? src1->f[0] : src2->f[0]; + dst->f[1] = src0->u[1] ? src1->f[1] : src2->f[1]; + dst->f[2] = src0->u[2] ? src1->f[2] : src2->f[2]; + dst->f[3] = src0->u[3] ? src1->f[3] : src2->f[3]; } static void -exec_xpd(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_ucmp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { - union tgsi_exec_channel r[6]; - union tgsi_exec_channel d[3]; - - fetch_source(mach, &r[0], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &r[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - - micro_mul(&r[2], &r[0], &r[1]); - - fetch_source(mach, &r[3], &inst->Src[0], TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); - fetch_source(mach, &r[4], &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - - micro_mul(&r[5], &r[3], &r[4] ); - micro_sub(&d[TGSI_CHAN_X], &r[2], &r[5]); - - fetch_source(mach, &r[2], &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - - micro_mul(&r[3], &r[3], &r[2]); - - fetch_source(mach, &r[5], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - - micro_mul(&r[1], &r[1], &r[5]); - micro_sub(&d[TGSI_CHAN_Y], &r[3], &r[1]); + unsigned int chan; + struct tgsi_exec_vector dst; - micro_mul(&r[5], &r[5], &r[4]); - micro_mul(&r[0], &r[0], &r[2]); - micro_sub(&d[TGSI_CHAN_Z], &r[5], &r[0]); + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + union tgsi_exec_channel src[3]; - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) { - store_dest(mach, &d[TGSI_CHAN_X], &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) { - store_dest(mach, &d[TGSI_CHAN_Y], &inst->Dst[0], inst, TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Z) { - store_dest(mach, &d[TGSI_CHAN_Z], &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_EXEC_DATA_FLOAT); + fetch_source(mach, &src[0], &inst->Src[0], chan, + TGSI_EXEC_DATA_UINT); + fetch_source(mach, &src[1], &inst->Src[1], chan, + TGSI_EXEC_DATA_FLOAT); + fetch_source(mach, &src[2], &inst->Src[2], chan, + TGSI_EXEC_DATA_FLOAT); + micro_ucmp(&dst.xyzw[chan], &src[0], &src[1], &src[2]); + } } - if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_W) { - store_dest(mach, &OneVec, &inst->Dst[0], inst, TGSI_CHAN_W, TGSI_EXEC_DATA_FLOAT); + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &dst.xyzw[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_FLOAT); + } } } @@ -3377,6 +3596,16 @@ exec_endswitch(struct tgsi_exec_machine *mach) typedef void (* micro_dop)(union tgsi_double_channel *dst, const union tgsi_double_channel *src); +typedef void (* micro_dop_sop)(union tgsi_double_channel *dst, + const union tgsi_double_channel *src0, + union tgsi_exec_channel *src1); + +typedef void (* micro_dop_s)(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src); + +typedef void (* micro_sop_d)(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src); + static void fetch_double_channel(struct tgsi_exec_machine *mach, union tgsi_double_channel *chan, @@ -3387,8 +3616,8 @@ fetch_double_channel(struct tgsi_exec_machine *mach, union tgsi_exec_channel src[2]; uint i; - fetch_source_d(mach, &src[0], reg, chan_0, TGSI_EXEC_DATA_UINT); - fetch_source_d(mach, &src[1], reg, chan_1, TGSI_EXEC_DATA_UINT); + fetch_source_d(mach, &src[0], reg, chan_0); + fetch_source_d(mach, &src[1], reg, chan_1); for (i = 0; i < TGSI_QUAD_SIZE; i++) { chan->u[i][0] = src[0].u[i]; @@ -3437,9 +3666,9 @@ store_double_channel(struct tgsi_exec_machine *mach, } } - store_dest_double(mach, &dst[0], reg, inst, chan_0, TGSI_EXEC_DATA_UINT); - if (chan_1 != -1) - store_dest_double(mach, &dst[1], reg, inst, chan_1, TGSI_EXEC_DATA_UINT); + store_dest_double(mach, &dst[0], reg, chan_0, TGSI_EXEC_DATA_UINT); + if (chan_1 != (unsigned)-1) + store_dest_double(mach, &dst[1], reg, chan_1, TGSI_EXEC_DATA_UINT); } static void @@ -3529,110 +3758,819 @@ exec_double_trinary(struct tgsi_exec_machine *mach, } static void -exec_f2d(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_dldexp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { - union tgsi_exec_channel src; + union tgsi_double_channel src0; + union tgsi_exec_channel src1; union tgsi_double_channel dst; + int wmask; - if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); - micro_f2d(&dst, &src); + wmask = inst->Dst[0].Register.WriteMask; + if (wmask & TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); + micro_dldexp(&dst, &src0, &src1); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); } - if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); - micro_f2d(&dst, &src); + + if (wmask & TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT); + micro_dldexp(&dst, &src0, &src1); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); } } static void -exec_d2f(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_dfracexp(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { union tgsi_double_channel src; - union tgsi_exec_channel dst; - int wm = inst->Dst[0].Register.WriteMask; - int i; - int bit; - for (i = 0; i < 2; i++) { - bit = ffs(wm); - if (bit) { - wm &= ~(1 << (bit - 1)); - if (i == 0) - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); - else - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); - micro_d2f(&dst, &src); - store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_FLOAT); - } + union tgsi_double_channel dst; + union tgsi_exec_channel dst_exp; + + fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + micro_dfracexp(&dst, &dst_exp, &src); + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); + if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) + store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); + for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[1].Register.WriteMask & (1 << chan)) + store_dest(mach, &dst_exp, &inst->Dst[1], inst, chan, TGSI_EXEC_DATA_INT); } } static void -exec_i2d(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_arg0_64_arg1_32(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop_sop op) { - union tgsi_exec_channel src; + union tgsi_double_channel src0; + union tgsi_exec_channel src1; union tgsi_double_channel dst; + int wmask; - if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); - micro_i2d(&dst, &src); + wmask = inst->Dst[0].Register.WriteMask; + if (wmask & TGSI_WRITEMASK_XY) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); + op(&dst, &src0, &src1); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); } - if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_INT); - micro_i2d(&dst, &src); + + if (wmask & TGSI_WRITEMASK_ZW) { + fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); + fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT); + op(&dst, &src0, &src1); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); } } +static int +get_image_coord_dim(unsigned tgsi_tex) +{ + int dim; + switch (tgsi_tex) { + case TGSI_TEXTURE_BUFFER: + case TGSI_TEXTURE_1D: + dim = 1; + break; + case TGSI_TEXTURE_2D: + case TGSI_TEXTURE_RECT: + case TGSI_TEXTURE_1D_ARRAY: + case TGSI_TEXTURE_2D_MSAA: + dim = 2; + break; + case TGSI_TEXTURE_3D: + case TGSI_TEXTURE_CUBE: + case TGSI_TEXTURE_2D_ARRAY: + case TGSI_TEXTURE_2D_ARRAY_MSAA: + case TGSI_TEXTURE_CUBE_ARRAY: + dim = 3; + break; + default: + assert(!"unknown texture target"); + dim = 0; + break; + } + + return dim; +} + +static int +get_image_coord_sample(unsigned tgsi_tex) +{ + int sample = 0; + switch (tgsi_tex) { + case TGSI_TEXTURE_2D_MSAA: + sample = 3; + break; + case TGSI_TEXTURE_2D_ARRAY_MSAA: + sample = 4; + break; + default: + break; + } + return sample; +} + static void -exec_d2i(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_load_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) { - union tgsi_double_channel src; - union tgsi_exec_channel dst; - int wm = inst->Dst[0].Register.WriteMask; - int i; - int bit; - for (i = 0; i < 2; i++) { - bit = ffs(wm); - if (bit) { - wm &= ~(1 << (bit - 1)); - if (i == 0) - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); - else - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); - micro_d2i(&dst, &src); - store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_INT); - } + union tgsi_exec_channel r[4], sample_r; + uint unit; + int sample; + int i, j; + int dim; + uint chan; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 1, TGSI_CHAN_X + i); } + + if (sample) + IFETCH(&sample_r, 1, TGSI_CHAN_X + sample); + + mach->Image->load(mach->Image, ¶ms, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba); + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + uint unit; + int j; + uint chan; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + IFETCH(&r[0], 1, TGSI_CHAN_X); + + mach->Buffer->load(mach->Buffer, ¶ms, + r[0].i, rgba); + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + uint chan; + char *ptr = mach->LocalMem; + uint32_t offset; + int j; + + IFETCH(&r[0], 1, TGSI_CHAN_X); + if (r[0].u[0] >= mach->LocalMemSize) + return; + + offset = r[0].u[0]; + ptr += offset; + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + memcpy(&r[chan].u[j], ptr + (4 * chan), 4); + } + } + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_load(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_load_img(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) + exec_load_buf(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) + exec_load_mem(mach, inst); +} + +static uint +fetch_store_img_unit(struct tgsi_exec_machine *mach, + const struct tgsi_full_dst_register *dst) +{ + uint unit = 0; + int i; + if (dst->Register.Indirect) { + union tgsi_exec_channel indir_index, index2; + const uint execmask = mach->ExecMask; + index2.i[0] = + index2.i[1] = + index2.i[2] = + index2.i[3] = dst->Indirect.Index; + + fetch_src_file_channel(mach, + dst->Indirect.File, + dst->Indirect.Swizzle, + &index2, + &ZeroVec, + &indir_index); + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + if (execmask & (1 << i)) { + unit = dst->Register.Index + indir_index.i[i]; + break; + } + } + } else { + unit = dst->Register.Index; + } + return unit; +} + +static void +exec_store_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3], sample_r; + union tgsi_exec_channel value[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int dim; + int sample; + int i, j; + uint unit; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + unit = fetch_store_img_unit(mach, &inst->Dst[0]); + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 0, TGSI_CHAN_X + i); + } + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + if (sample) + IFETCH(&sample_r, 0, TGSI_CHAN_X + sample); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + + mach->Image->store(mach->Image, ¶ms, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba); +} + +static void +exec_store_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3]; + union tgsi_exec_channel value[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int i, j; + uint unit; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_store_img_unit(mach, &inst->Dst[0]); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.writemask = inst->Dst[0].Register.WriteMask; + + IFETCH(&r[0], 0, TGSI_CHAN_X); + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + + mach->Buffer->store(mach->Buffer, ¶ms, + r[0].i, + rgba); +} + +static void +exec_store_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[3]; + union tgsi_exec_channel value[4]; + uint i, chan; + char *ptr = mach->LocalMem; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + int execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + + IFETCH(&r[0], 0, TGSI_CHAN_X); + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 1, TGSI_CHAN_X + i); + } + + if (r[0].u[0] >= mach->LocalMemSize) + return; + ptr += r[0].u[0]; + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + if (execmask & (1 << i)) { + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + memcpy(ptr + (chan * 4), &value[chan].u[0], 4); + } + } + } + } +} + +static void +exec_store(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Dst[0].Register.File == TGSI_FILE_IMAGE) + exec_store_img(mach, inst); + else if (inst->Dst[0].Register.File == TGSI_FILE_BUFFER) + exec_store_buf(mach, inst); + else if (inst->Dst[0].Register.File == TGSI_FILE_MEMORY) + exec_store_mem(mach, inst); +} + +static void +exec_atomop_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4], sample_r; + union tgsi_exec_channel value[4], value2[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_image_params params; + int dim; + int sample; + int i, j; + uint unit, chan; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + unit = fetch_sampler_unit(mach, inst, 0); + dim = get_image_coord_dim(inst->Memory.Texture); + sample = get_image_coord_sample(inst->Memory.Texture); + assert(dim <= 3); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + for (i = 0; i < dim; i++) { + IFETCH(&r[i], 1, TGSI_CHAN_X + i); + } + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + if (sample) + IFETCH(&sample_r, 1, TGSI_CHAN_X + sample); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) { + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba2[0][j] = value2[0].f[j]; + rgba2[1][j] = value2[1].f[j]; + rgba2[2][j] = value2[2].f[j]; + rgba2[3][j] = value2[3].f[j]; + } + } + + mach->Image->op(mach->Image, ¶ms, inst->Instruction.Opcode, + r[0].i, r[1].i, r[2].i, sample_r.i, + rgba, rgba2); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + union tgsi_exec_channel value[4], value2[4]; + float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE]; + struct tgsi_buffer_params params; + int i, j; + uint unit, chan; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.writemask = inst->Dst[0].Register.WriteMask; + + IFETCH(&r[0], 1, TGSI_CHAN_X); + + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba[0][j] = value[0].f[j]; + rgba[1][j] = value[1].f[j]; + rgba[2][j] = value[2].f[j]; + rgba[3][j] = value[3].f[j]; + } + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) { + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + rgba2[0][j] = value2[0].f[j]; + rgba2[1][j] = value2[1].f[j]; + rgba2[2][j] = value2[2].f[j]; + rgba2[3][j] = value2[3].f[j]; + } + } + + mach->Buffer->op(mach->Buffer, ¶ms, inst->Instruction.Opcode, + r[0].i, + rgba, rgba2); + + for (j = 0; j < TGSI_QUAD_SIZE; j++) { + r[0].f[j] = rgba[0][j]; + r[1].f[j] = rgba[1][j]; + r[2].f[j] = rgba[2][j]; + r[3].f[j] = rgba[3][j]; + } + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop_mem(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel r[4]; + union tgsi_exec_channel value[4], value2[4]; + char *ptr = mach->LocalMem; + uint32_t val; + uint chan, i; + uint32_t offset; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + int execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + IFETCH(&r[0], 1, TGSI_CHAN_X); + + if (r[0].u[0] >= mach->LocalMemSize) + return; + + offset = r[0].u[0]; + ptr += offset; + for (i = 0; i < 4; i++) { + FETCH(&value[i], 2, TGSI_CHAN_X + i); + if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) + FETCH(&value2[i], 3, TGSI_CHAN_X + i); + } + + memcpy(&r[0].u[0], ptr, 4); + val = r[0].u[0]; + switch (inst->Instruction.Opcode) { + case TGSI_OPCODE_ATOMUADD: + val += value[0].u[0]; + break; + case TGSI_OPCODE_ATOMXOR: + val ^= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMOR: + val |= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMAND: + val &= value[0].u[0]; + break; + case TGSI_OPCODE_ATOMUMIN: + val = MIN2(val, value[0].u[0]); + break; + case TGSI_OPCODE_ATOMUMAX: + val = MAX2(val, value[0].u[0]); + break; + case TGSI_OPCODE_ATOMIMIN: + val = MIN2(r[0].i[0], value[0].i[0]); + break; + case TGSI_OPCODE_ATOMIMAX: + val = MAX2(r[0].i[0], value[0].i[0]); + break; + case TGSI_OPCODE_ATOMXCHG: + val = value[0].i[0]; + break; + case TGSI_OPCODE_ATOMCAS: + if (val == value[0].u[0]) + val = value2[0].u[0]; + break; + case TGSI_OPCODE_ATOMFADD: + val = fui(r[0].f[0] + value[0].f[0]); + break; + default: + break; + } + for (i = 0; i < TGSI_QUAD_SIZE; i++) + if (execmask & (1 << i)) + memcpy(ptr, &val, 4); + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + } +} + +static void +exec_atomop(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_atomop_img(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_BUFFER) + exec_atomop_buf(mach, inst); + else if (inst->Src[0].Register.File == TGSI_FILE_MEMORY) + exec_atomop_mem(mach, inst); +} + +static void +exec_resq_img(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + int result[4]; + union tgsi_exec_channel r[4]; + uint unit; + int i, chan, j; + struct tgsi_image_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + params.tgsi_tex_instr = inst->Memory.Texture; + params.format = inst->Memory.Format; + + mach->Image->get_dims(mach->Image, ¶ms, result); + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + for (j = 0; j < 4; j++) { + r[j].i[i] = result[j]; + } + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_INT); + } + } +} + +static void +exec_resq_buf(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + int result; + union tgsi_exec_channel r[4]; + uint unit; + int i, chan; + struct tgsi_buffer_params params; + int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0]; + + unit = fetch_sampler_unit(mach, inst, 0); + + params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask; + params.unit = unit; + + mach->Buffer->get_dims(mach->Buffer, ¶ms, &result); + + for (i = 0; i < TGSI_QUAD_SIZE; i++) { + r[0].i[i] = result; + } + + for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (inst->Dst[0].Register.WriteMask & (1 << chan)) { + store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, + TGSI_EXEC_DATA_INT); + } + } +} + +static void +exec_resq(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + if (inst->Src[0].Register.File == TGSI_FILE_IMAGE) + exec_resq_img(mach, inst); + else + exec_resq_buf(mach, inst); +} + +static void +micro_f2u64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u64[0] = (uint64_t)src->f[0]; + dst->u64[1] = (uint64_t)src->f[1]; + dst->u64[2] = (uint64_t)src->f[2]; + dst->u64[3] = (uint64_t)src->f[3]; +} + +static void +micro_f2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->i64[0] = (int64_t)src->f[0]; + dst->i64[1] = (int64_t)src->f[1]; + dst->i64[2] = (int64_t)src->f[2]; + dst->i64[3] = (int64_t)src->f[3]; +} + +static void +micro_u2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->u64[0] = (uint64_t)src->u[0]; + dst->u64[1] = (uint64_t)src->u[1]; + dst->u64[2] = (uint64_t)src->u[2]; + dst->u64[3] = (uint64_t)src->u[3]; +} + +static void +micro_i2i64(union tgsi_double_channel *dst, + const union tgsi_exec_channel *src) +{ + dst->i64[0] = (int64_t)src->i[0]; + dst->i64[1] = (int64_t)src->i[1]; + dst->i64[2] = (int64_t)src->i[2]; + dst->i64[3] = (int64_t)src->i[3]; +} + +static void +micro_d2u64(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->u64[0] = (uint64_t)src->d[0]; + dst->u64[1] = (uint64_t)src->d[1]; + dst->u64[2] = (uint64_t)src->d[2]; + dst->u64[3] = (uint64_t)src->d[3]; +} + +static void +micro_d2i64(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->i64[0] = (int64_t)src->d[0]; + dst->i64[1] = (int64_t)src->d[1]; + dst->i64[2] = (int64_t)src->d[2]; + dst->i64[3] = (int64_t)src->d[3]; +} + +static void +micro_u642d(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = (double)src->u64[0]; + dst->d[1] = (double)src->u64[1]; + dst->d[2] = (double)src->u64[2]; + dst->d[3] = (double)src->u64[3]; +} + +static void +micro_i642d(union tgsi_double_channel *dst, + const union tgsi_double_channel *src) +{ + dst->d[0] = (double)src->i64[0]; + dst->d[1] = (double)src->i64[1]; + dst->d[2] = (double)src->i64[2]; + dst->d[3] = (double)src->i64[3]; } + static void -exec_u2d(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +micro_u642f(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->f[0] = (float)src->u64[0]; + dst->f[1] = (float)src->u64[1]; + dst->f[2] = (float)src->u64[2]; + dst->f[3] = (float)src->u64[3]; +} + +static void +micro_i642f(union tgsi_exec_channel *dst, + const union tgsi_double_channel *src) +{ + dst->f[0] = (float)src->i64[0]; + dst->f[1] = (float)src->i64[1]; + dst->f[2] = (float)src->i64[2]; + dst->f[3] = (float)src->i64[3]; +} + +static void +exec_t_2_64(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_dop_s op, + enum tgsi_exec_datatype src_datatype) { union tgsi_exec_channel src; union tgsi_double_channel dst; if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_UINT); - micro_u2d(&dst, &src); + fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, src_datatype); + op(&dst, &src); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); } if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) { - fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_UINT); - micro_u2d(&dst, &src); + fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, src_datatype); + op(&dst, &src); store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); } } static void -exec_d2u(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) +exec_64_2_t(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst, + micro_sop_d op, + enum tgsi_exec_datatype dst_datatype) { union tgsi_double_channel src; union tgsi_exec_channel dst; @@ -3647,60 +4585,12 @@ exec_d2u(struct tgsi_exec_machine *mach, fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); else fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); - micro_d2u(&dst, &src); - store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_UINT); + op(&dst, &src); + store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, dst_datatype); } } } -static void -exec_dldexp(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - union tgsi_double_channel src0; - union tgsi_exec_channel src1; - union tgsi_double_channel dst; - int wmask; - - wmask = inst->Dst[0].Register.WriteMask; - if (wmask & TGSI_WRITEMASK_XY) { - fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); - fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT); - micro_dldexp(&dst, &src0, &src1); - store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); - } - - if (wmask & TGSI_WRITEMASK_ZW) { - fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); - fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT); - micro_dldexp(&dst, &src0, &src1); - store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); - } -} - -static void -exec_dfracexp(struct tgsi_exec_machine *mach, - const struct tgsi_full_instruction *inst) -{ - union tgsi_double_channel src; - union tgsi_double_channel dst; - union tgsi_exec_channel dst_exp; - - if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY)) { - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y); - micro_dfracexp(&dst, &dst_exp, &src); - store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y); - store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT); - } - if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW)) { - fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W); - micro_dfracexp(&dst, &dst_exp, &src); - store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W); - store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT); - } -} - - static void micro_i2f(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src) @@ -3775,10 +4665,10 @@ micro_mod(union tgsi_exec_channel *dst, const union tgsi_exec_channel *src0, const union tgsi_exec_channel *src1) { - dst->i[0] = src0->i[0] % src1->i[0]; - dst->i[1] = src0->i[1] % src1->i[1]; - dst->i[2] = src0->i[2] % src1->i[2]; - dst->i[3] = src0->i[3] % src1->i[3]; + dst->i[0] = src1->i[0] ? src0->i[0] % src1->i[0] : ~0; + dst->i[1] = src1->i[1] ? src0->i[1] % src1->i[1] : ~0; + dst->i[2] = src1->i[2] ? src0->i[2] % src1->i[2] : ~0; + dst->i[3] = src1->i[3] ? src0->i[3] % src1->i[3] : ~0; } static void @@ -4100,18 +4990,6 @@ micro_uarl(union tgsi_exec_channel *dst, dst->i[3] = src->u[3]; } -static void -micro_ucmp(union tgsi_exec_channel *dst, - const union tgsi_exec_channel *src0, - const union tgsi_exec_channel *src1, - const union tgsi_exec_channel *src2) -{ - dst->u[0] = src0->u[0] ? src1->u[0] : src2->u[0]; - dst->u[1] = src0->u[1] ? src1->u[1] : src2->u[1]; - dst->u[2] = src0->u[2] ? src1->u[2] : src2->u[2]; - dst->u[3] = src0->u[3] ? src1->u[3] : src2->u[3]; -} - /** * Signed bitfield extract (i.e. sign-extend the extracted bits) */ @@ -4123,8 +5001,13 @@ micro_ibfe(union tgsi_exec_channel *dst, { int i; for (i = 0; i < 4; i++) { - int width = src2->i[i] & 0x1f; + int width = src2->i[i]; int offset = src1->i[i] & 0x1f; + if (width == 32 && offset == 0) { + dst->i[i] = src0->i[i]; + continue; + } + width &= 0x1f; if (width == 0) dst->i[i] = 0; else if (width + offset < 32) @@ -4145,8 +5028,13 @@ micro_ubfe(union tgsi_exec_channel *dst, { int i; for (i = 0; i < 4; i++) { - int width = src2->u[i] & 0x1f; + int width = src2->u[i]; int offset = src1->u[i] & 0x1f; + if (width == 32 && offset == 0) { + dst->u[i] = src0->u[i]; + continue; + } + width &= 0x1f; if (width == 0) dst->u[i] = 0; else if (width + offset < 32) @@ -4168,10 +5056,14 @@ micro_bfi(union tgsi_exec_channel *dst, { int i; for (i = 0; i < 4; i++) { - int width = src3->u[i] & 0x1f; + int width = src3->u[i]; int offset = src2->u[i] & 0x1f; - int bitmask = ((1 << width) - 1) << offset; - dst->u[i] = ((src1->u[i] << offset) & bitmask) | (src0->u[i] & ~bitmask); + if (width == 32) { + dst->u[i] = src1->u[i]; + } else { + int bitmask = ((1 << width) - 1) << offset; + dst->u[i] = ((src1->u[i] << offset) & bitmask) | (src0->u[i] & ~bitmask); + } } } @@ -4225,7 +5117,117 @@ micro_umsb(union tgsi_exec_channel *dst, dst->i[3] = util_last_bit(src->u[3]) - 1; } + +static void +exec_interp_at_sample(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel index; + union tgsi_exec_channel index2D; + union tgsi_exec_channel result[TGSI_NUM_CHANNELS]; + const struct tgsi_full_src_register *reg = &inst->Src[0]; + + assert(reg->Register.File == TGSI_FILE_INPUT); + assert(inst->Src[1].Register.File == TGSI_FILE_IMMEDIATE); + + get_index_registers(mach, reg, &index, &index2D); + float sample = mach->Imms[inst->Src[1].Register.Index][inst->Src[1].Register.SwizzleX]; + + /* Short cut: sample 0 is like a normal fetch */ + for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (!(inst->Dst[0].Register.WriteMask & (1 << chan))) + continue; + + fetch_src_file_channel(mach, TGSI_FILE_INPUT, chan, &index, &index2D, + &result[chan]); + if (sample != 0.0f) { + + /* TODO: define the samples > 0, but so far we only do fake MSAA */ + float x = 0; + float y = 0; + + unsigned pos = index2D.i[chan] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index.i[chan]; + assert(pos >= 0); + assert(pos < TGSI_MAX_PRIM_VERTICES * PIPE_MAX_ATTRIBS); + mach->InputSampleOffsetApply[pos](mach, pos, chan, x, y, &result[chan]); + } + store_dest(mach, &result[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } +} + + +static void +exec_interp_at_offset(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel index; + union tgsi_exec_channel index2D; + union tgsi_exec_channel ofsx; + union tgsi_exec_channel ofsy; + const struct tgsi_full_src_register *reg = &inst->Src[0]; + + assert(reg->Register.File == TGSI_FILE_INPUT); + + get_index_registers(mach, reg, &index, &index2D); + unsigned pos = index2D.i[0] * TGSI_EXEC_MAX_INPUT_ATTRIBS + index.i[0]; + + fetch_source(mach, &ofsx, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT); + fetch_source(mach, &ofsy, &inst->Src[1], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT); + + for (int chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (!(inst->Dst[0].Register.WriteMask & (1 << chan))) + continue; + union tgsi_exec_channel result; + fetch_src_file_channel(mach, TGSI_FILE_INPUT, chan, &index, &index2D, &result); + mach->InputSampleOffsetApply[pos](mach, pos, chan, ofsx.f[chan], ofsy.f[chan], &result); + store_dest(mach, &result, &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } +} + + static void +exec_interp_at_centroid(struct tgsi_exec_machine *mach, + const struct tgsi_full_instruction *inst) +{ + union tgsi_exec_channel index; + union tgsi_exec_channel index2D; + union tgsi_exec_channel result[TGSI_NUM_CHANNELS]; + const struct tgsi_full_src_register *reg = &inst->Src[0]; + + assert(reg->Register.File == TGSI_FILE_INPUT); + get_index_registers(mach, reg, &index, &index2D); + + for (unsigned chan = 0; chan < TGSI_NUM_CHANNELS; chan++) { + if (!(inst->Dst[0].Register.WriteMask & (1 << chan))) + continue; + + /* Here we should add the change to use a sample that lies within the + * primitive (Section 15.2): + * + * "When interpolating variables declared using centroid in , + * the variable is sampled at a location within the pixel covered + * by the primitive generating the fragment. + * ... + * The built-in functions interpolateAtCentroid ... will sample + * variables as though they were declared with the centroid ... + * qualifier[s]." + * + * Since we only support 1 sample currently, this is just a pass-through. + */ + fetch_src_file_channel(mach, TGSI_FILE_INPUT, chan, &index, &index2D, + &result[chan]); + store_dest(mach, &result[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT); + } + +} + + +/** + * Execute a TGSI instruction. + * Returns TRUE if a barrier instruction is hit, + * otherwise FALSE. + */ +static boolean exec_instruction( struct tgsi_exec_machine *mach, const struct tgsi_full_instruction *inst, @@ -4304,10 +5306,6 @@ exec_instruction( exec_vector_trinary(mach, inst, micro_mad, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_SUB: - exec_vector_binary(mach, inst, micro_sub, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_LRP: exec_vector_trinary(mach, inst, micro_lrp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -4316,18 +5314,10 @@ exec_instruction( exec_scalar_unary(mach, inst, micro_sqrt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_DP2A: - exec_dp2a(mach, inst); - break; - case TGSI_OPCODE_FRC: exec_vector_unary(mach, inst, micro_frc, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_CLAMP: - exec_vector_trinary(mach, inst, micro_clamp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - case TGSI_OPCODE_FLR: exec_vector_unary(mach, inst, micro_flr, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -4348,32 +5338,32 @@ exec_instruction( exec_scalar_binary(mach, inst, micro_pow, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_XPD: - exec_xpd(mach, inst); - break; - - case TGSI_OPCODE_ABS: - exec_vector_unary(mach, inst, micro_abs, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); - break; - - case TGSI_OPCODE_DPH: - exec_dph(mach, inst); + case TGSI_OPCODE_LDEXP: + exec_vector_binary(mach, inst, micro_ldexp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; case TGSI_OPCODE_COS: exec_scalar_unary(mach, inst, micro_cos, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; + case TGSI_OPCODE_DDX_FINE: + exec_vector_unary(mach, inst, micro_ddx_fine, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); + break; + case TGSI_OPCODE_DDX: exec_vector_unary(mach, inst, micro_ddx, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; + case TGSI_OPCODE_DDY_FINE: + exec_vector_unary(mach, inst, micro_ddy_fine, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); + break; + case TGSI_OPCODE_DDY: exec_vector_unary(mach, inst, micro_ddy, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; case TGSI_OPCODE_KILL: - exec_kill (mach, inst); + exec_kill (mach); break; case TGSI_OPCODE_KILL_IF: @@ -4543,8 +5533,12 @@ exec_instruction( /* returning from main() */ mach->CondStackTop = 0; mach->LoopStackTop = 0; + mach->ContStackTop = 0; + mach->LoopLabelStackTop = 0; + mach->SwitchStackTop = 0; + mach->BreakStackTop = 0; *pc = -1; - return; + return FALSE; } assert(mach->CallStackTop > 0); @@ -4582,10 +5576,6 @@ exec_instruction( exec_vector_trinary(mach, inst, micro_cmp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; - case TGSI_OPCODE_SCS: - exec_scs(mach, inst); - break; - case TGSI_OPCODE_DIV: exec_vector_binary(mach, inst, micro_div, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -4665,14 +5655,6 @@ exec_instruction( *pc = -1; break; - case TGSI_OPCODE_PUSHA: - assert (0); - break; - - case TGSI_OPCODE_POPA: - assert (0); - break; - case TGSI_OPCODE_CEIL: exec_vector_unary(mach, inst, micro_ceil, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT); break; @@ -4709,10 +5691,6 @@ exec_instruction( exec_vector_binary(mach, inst, micro_xor, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); break; - case TGSI_OPCODE_SAD: - assert (0); - break; - case TGSI_OPCODE_TXF: exec_txf(mach, inst); break; @@ -4722,11 +5700,11 @@ exec_instruction( break; case TGSI_OPCODE_EMIT: - emit_vertex(mach); + emit_vertex(mach, inst); break; case TGSI_OPCODE_ENDPRIM: - emit_primitive(mach); + emit_primitive(mach, inst); break; case TGSI_OPCODE_BGNLOOP: @@ -4817,25 +5795,6 @@ exec_instruction( case TGSI_OPCODE_NOP: break; - case TGSI_OPCODE_BREAKC: - IFETCH(&r[0], 0, TGSI_CHAN_X); - /* update CondMask */ - if (r[0].u[0] && (mach->ExecMask & 0x1)) { - mach->LoopMask &= ~0x1; - } - if (r[0].u[1] && (mach->ExecMask & 0x2)) { - mach->LoopMask &= ~0x2; - } - if (r[0].u[2] && (mach->ExecMask & 0x4)) { - mach->LoopMask &= ~0x4; - } - if (r[0].u[3] && (mach->ExecMask & 0x8)) { - mach->LoopMask &= ~0x8; - } - /* Todo: if mach->LoopMask == 0, jump to end of loop */ - UPDATE_EXEC_MASK(mach); - break; - case TGSI_OPCODE_F2I: exec_vector_unary(mach, inst, micro_f2i, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_FLOAT); break; @@ -4997,7 +5956,7 @@ exec_instruction( break; case TGSI_OPCODE_GATHER4: - assert(0); + exec_sample(mach, inst, TEX_MODIFIER_GATHER, FALSE); break; case TGSI_OPCODE_SVIEWINFO: @@ -5012,12 +5971,16 @@ exec_instruction( assert(0); break; + case TGSI_OPCODE_LOD: + exec_lodq(mach, inst); + break; + case TGSI_OPCODE_UARL: exec_vector_unary(mach, inst, micro_uarl, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_UINT); break; case TGSI_OPCODE_UCMP: - exec_vector_trinary(mach, inst, micro_ucmp, TGSI_EXEC_DATA_UINT, TGSI_EXEC_DATA_UINT); + exec_ucmp(mach, inst); break; case TGSI_OPCODE_IABS: @@ -5076,11 +6039,11 @@ exec_instruction( break; case TGSI_OPCODE_F2D: - exec_f2d(mach, inst); + exec_t_2_64(mach, inst, micro_f2d, TGSI_EXEC_DATA_FLOAT); break; case TGSI_OPCODE_D2F: - exec_d2f(mach, inst); + exec_64_2_t(mach, inst, micro_d2f, TGSI_EXEC_DATA_FLOAT); break; case TGSI_OPCODE_DABS: @@ -5095,6 +6058,10 @@ exec_instruction( exec_double_binary(mach, inst, micro_dadd, TGSI_EXEC_DATA_DOUBLE); break; + case TGSI_OPCODE_DDIV: + exec_double_binary(mach, inst, micro_ddiv, TGSI_EXEC_DATA_DOUBLE); + break; + case TGSI_OPCODE_DMUL: exec_double_binary(mach, inst, micro_dmul, TGSI_EXEC_DATA_DOUBLE); break; @@ -5152,47 +6119,195 @@ exec_instruction( break; case TGSI_OPCODE_I2D: - exec_i2d(mach, inst); + exec_t_2_64(mach, inst, micro_i2d, TGSI_EXEC_DATA_INT); break; case TGSI_OPCODE_D2I: - exec_d2i(mach, inst); + exec_64_2_t(mach, inst, micro_d2i, TGSI_EXEC_DATA_INT); break; case TGSI_OPCODE_U2D: - exec_u2d(mach, inst); + exec_t_2_64(mach, inst, micro_u2d, TGSI_EXEC_DATA_UINT); break; case TGSI_OPCODE_D2U: - exec_d2u(mach, inst); + exec_64_2_t(mach, inst, micro_d2u, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_LOAD: + exec_load(mach, inst); + break; + + case TGSI_OPCODE_STORE: + exec_store(mach, inst); + break; + + case TGSI_OPCODE_ATOMUADD: + case TGSI_OPCODE_ATOMXCHG: + case TGSI_OPCODE_ATOMCAS: + case TGSI_OPCODE_ATOMAND: + case TGSI_OPCODE_ATOMOR: + case TGSI_OPCODE_ATOMXOR: + case TGSI_OPCODE_ATOMUMIN: + case TGSI_OPCODE_ATOMUMAX: + case TGSI_OPCODE_ATOMIMIN: + case TGSI_OPCODE_ATOMIMAX: + case TGSI_OPCODE_ATOMFADD: + exec_atomop(mach, inst); + break; + + case TGSI_OPCODE_RESQ: + exec_resq(mach, inst); + break; + case TGSI_OPCODE_BARRIER: + case TGSI_OPCODE_MEMBAR: + return TRUE; + break; + + case TGSI_OPCODE_I64ABS: + exec_double_unary(mach, inst, micro_i64abs); + break; + + case TGSI_OPCODE_I64SSG: + exec_double_unary(mach, inst, micro_i64sgn); + break; + + case TGSI_OPCODE_I64NEG: + exec_double_unary(mach, inst, micro_i64neg); + break; + + case TGSI_OPCODE_U64SEQ: + exec_double_binary(mach, inst, micro_u64seq, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_U64SNE: + exec_double_binary(mach, inst, micro_u64sne, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64SLT: + exec_double_binary(mach, inst, micro_i64slt, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_U64SLT: + exec_double_binary(mach, inst, micro_u64slt, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64SGE: + exec_double_binary(mach, inst, micro_i64sge, TGSI_EXEC_DATA_UINT); + break; + case TGSI_OPCODE_U64SGE: + exec_double_binary(mach, inst, micro_u64sge, TGSI_EXEC_DATA_UINT); + break; + + case TGSI_OPCODE_I64MIN: + exec_double_binary(mach, inst, micro_i64min, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MIN: + exec_double_binary(mach, inst, micro_u64min, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64MAX: + exec_double_binary(mach, inst, micro_i64max, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MAX: + exec_double_binary(mach, inst, micro_u64max, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64ADD: + exec_double_binary(mach, inst, micro_u64add, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64MUL: + exec_double_binary(mach, inst, micro_u64mul, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_U64SHL: + exec_arg0_64_arg1_32(mach, inst, micro_u64shl); + break; + case TGSI_OPCODE_I64SHR: + exec_arg0_64_arg1_32(mach, inst, micro_i64shr); + break; + case TGSI_OPCODE_U64SHR: + exec_arg0_64_arg1_32(mach, inst, micro_u64shr); + break; + case TGSI_OPCODE_U64DIV: + exec_double_binary(mach, inst, micro_u64div, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64DIV: + exec_double_binary(mach, inst, micro_i64div, TGSI_EXEC_DATA_INT64); + break; + case TGSI_OPCODE_U64MOD: + exec_double_binary(mach, inst, micro_u64mod, TGSI_EXEC_DATA_UINT64); + break; + case TGSI_OPCODE_I64MOD: + exec_double_binary(mach, inst, micro_i64mod, TGSI_EXEC_DATA_INT64); + break; + + case TGSI_OPCODE_F2U64: + exec_t_2_64(mach, inst, micro_f2u64, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_F2I64: + exec_t_2_64(mach, inst, micro_f2i64, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_U2I64: + exec_t_2_64(mach, inst, micro_u2i64, TGSI_EXEC_DATA_INT); + break; + case TGSI_OPCODE_I2I64: + exec_t_2_64(mach, inst, micro_i2i64, TGSI_EXEC_DATA_INT); + break; + + case TGSI_OPCODE_D2U64: + exec_double_unary(mach, inst, micro_d2u64); + break; + + case TGSI_OPCODE_D2I64: + exec_double_unary(mach, inst, micro_d2i64); + break; + + case TGSI_OPCODE_U642F: + exec_64_2_t(mach, inst, micro_u642f, TGSI_EXEC_DATA_FLOAT); + break; + case TGSI_OPCODE_I642F: + exec_64_2_t(mach, inst, micro_i642f, TGSI_EXEC_DATA_FLOAT); + break; + + case TGSI_OPCODE_U642D: + exec_double_unary(mach, inst, micro_u642d); + break; + case TGSI_OPCODE_I642D: + exec_double_unary(mach, inst, micro_i642d); + break; + case TGSI_OPCODE_INTERP_SAMPLE: + exec_interp_at_sample(mach, inst); + break; + case TGSI_OPCODE_INTERP_OFFSET: + exec_interp_at_offset(mach, inst); + break; + case TGSI_OPCODE_INTERP_CENTROID: + exec_interp_at_centroid(mach, inst); break; default: assert( 0 ); } + return FALSE; } - -/** - * Run TGSI interpreter. - * \return bitmask of "alive" quad components - */ -uint -tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) +static void +tgsi_exec_machine_setup_masks(struct tgsi_exec_machine *mach) { - uint i; - int pc = 0; uint default_mask = 0xf; mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0] = 0; mach->Temps[TEMP_OUTPUT_I].xyzw[TEMP_OUTPUT_C].u[0] = 0; - if( mach->Processor == TGSI_PROCESSOR_GEOMETRY ) { - mach->Temps[TEMP_PRIMITIVE_I].xyzw[TEMP_PRIMITIVE_C].u[0] = 0; - mach->Primitives[0] = 0; + if (mach->ShaderType == PIPE_SHADER_GEOMETRY) { + for (unsigned i = 0; i < TGSI_MAX_VERTEX_STREAMS; i++) { + mach->Temps[temp_prim_idxs[i].idx].xyzw[temp_prim_idxs[i].chan].u[0] = 0; + mach->Primitives[i][0] = 0; + } /* GS runs on a single primitive for now */ default_mask = 0x1; } + if (mach->NonHelperMask == 0) + mach->NonHelperMask = default_mask; mach->CondMask = default_mask; mach->LoopMask = default_mask; mach->ContMask = default_mask; @@ -5207,11 +6322,26 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) assert(mach->SwitchStackTop == 0); assert(mach->BreakStackTop == 0); assert(mach->CallStackTop == 0); +} + +/** + * Run TGSI interpreter. + * \return bitmask of "alive" quad components + */ +uint +tgsi_exec_machine_run( struct tgsi_exec_machine *mach, int start_pc ) +{ + uint i; + + mach->pc = start_pc; + if (!start_pc) { + tgsi_exec_machine_setup_masks(mach); - /* execute declarations (interpolants) */ - for (i = 0; i < mach->NumDeclarations; i++) { - exec_declaration( mach, mach->Declarations+i ); + /* execute declarations (interpolants) */ + for (i = 0; i < mach->NumDeclarations; i++) { + exec_declaration( mach, mach->Declarations+i ); + } } { @@ -5220,23 +6350,30 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) struct tgsi_exec_vector outputs[PIPE_MAX_ATTRIBS]; uint inst = 1; - memset(mach->Temps, 0, sizeof(temps)); - memset(mach->Outputs, 0, sizeof(outputs)); - memset(temps, 0, sizeof(temps)); - memset(outputs, 0, sizeof(outputs)); + if (!start_pc) { + memset(mach->Temps, 0, sizeof(temps)); + if (mach->Outputs) + memset(mach->Outputs, 0, sizeof(outputs)); + memset(temps, 0, sizeof(temps)); + memset(outputs, 0, sizeof(outputs)); + } #endif /* execute instructions, until pc is set to -1 */ - while (pc != -1) { - + while (mach->pc != -1) { + boolean barrier_hit; #if DEBUG_EXECUTION uint i; - tgsi_dump_instruction(&mach->Instructions[pc], inst++); + tgsi_dump_instruction(&mach->Instructions[mach->pc], inst++); #endif - assert(pc < (int) mach->NumInstructions); - exec_instruction(mach, mach->Instructions + pc, &pc); + assert(mach->pc < (int) mach->NumInstructions); + barrier_hit = exec_instruction(mach, mach->Instructions + mach->pc, &mach->pc); + + /* for compute shaders if we hit a barrier return now for later rescheduling */ + if (barrier_hit && mach->ShaderType == PIPE_SHADER_COMPUTE) + return 0; #if DEBUG_EXECUTION for (i = 0; i < TGSI_EXEC_NUM_TEMPS + TGSI_EXEC_NUM_TEMP_EXTRAS; i++) { @@ -5257,21 +6394,23 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) } } } - for (i = 0; i < PIPE_MAX_ATTRIBS; i++) { - if (memcmp(&outputs[i], &mach->Outputs[i], sizeof(outputs[i]))) { - uint j; - - memcpy(&outputs[i], &mach->Outputs[i], sizeof(outputs[i])); - debug_printf("OUT[%2u] = ", i); - for (j = 0; j < 4; j++) { - if (j > 0) { - debug_printf(" "); + if (mach->Outputs) { + for (i = 0; i < PIPE_MAX_ATTRIBS; i++) { + if (memcmp(&outputs[i], &mach->Outputs[i], sizeof(outputs[i]))) { + uint j; + + memcpy(&outputs[i], &mach->Outputs[i], sizeof(outputs[i])); + debug_printf("OUT[%2u] = ", i); + for (j = 0; j < 4; j++) { + if (j > 0) { + debug_printf(" "); + } + debug_printf("(%6f %u, %6f %u, %6f %u, %6f %u)\n", + outputs[i].xyzw[0].f[j], outputs[i].xyzw[0].u[j], + outputs[i].xyzw[1].f[j], outputs[i].xyzw[1].u[j], + outputs[i].xyzw[2].f[j], outputs[i].xyzw[2].u[j], + outputs[i].xyzw[3].f[j], outputs[i].xyzw[3].u[j]); } - debug_printf("(%6f %u, %6f %u, %6f %u, %6f %u)\n", - outputs[i].xyzw[0].f[j], outputs[i].xyzw[0].u[j], - outputs[i].xyzw[1].f[j], outputs[i].xyzw[1].u[j], - outputs[i].xyzw[2].f[j], outputs[i].xyzw[2].u[j], - outputs[i].xyzw[3].f[j], outputs[i].xyzw[3].u[j]); } } } @@ -5281,7 +6420,7 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach ) #if 0 /* we scale from floats in [0,1] to Zbuffer ints in sp_quad_depth_test.c */ - if (mach->Processor == TGSI_PROCESSOR_FRAGMENT) { + if (mach->ShaderType == PIPE_SHADER_FRAGMENT) { /* * Scale back depth component. */