X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fgallium%2Fauxiliary%2Ftgsi%2Ftgsi_exec.c;h=41dd0f0466a28285e78321004aefe9f0a6b6b26e;hb=b89708f95fafc458cc79bc210407b723a0f0f78c;hp=ab13c133c0720ed30e055755b01fe9356429f209;hpb=dc00b382b58bb3eb94ca393d32bd7eb3bb07d021;p=mesa.git

diff --git a/src/gallium/auxiliary/tgsi/tgsi_exec.c b/src/gallium/auxiliary/tgsi/tgsi_exec.c
index ab13c133c07..41dd0f0466a 100644
--- a/src/gallium/auxiliary/tgsi/tgsi_exec.c
+++ b/src/gallium/auxiliary/tgsi/tgsi_exec.c
@@ -58,6 +58,7 @@
 #include "tgsi/tgsi_parse.h"
 #include "tgsi/tgsi_util.h"
 #include "tgsi_exec.h"
+#include "util/u_half.h"
 #include "util/u_memory.h"
 #include "util/u_math.h"
 
@@ -72,6 +73,16 @@
 #define TILE_BOTTOM_LEFT  2
 #define TILE_BOTTOM_RIGHT 3
 
+union tgsi_double_channel {
+   double d[TGSI_QUAD_SIZE];
+   unsigned u[TGSI_QUAD_SIZE][2];
+};
+
+struct tgsi_double_vector {
+   union tgsi_double_channel xy;
+   union tgsi_double_channel zw;
+};
+
 static void
 micro_abs(union tgsi_exec_channel *dst,
           const union tgsi_exec_channel *src)
@@ -136,18 +147,6 @@ micro_cmp(union tgsi_exec_channel *dst,
    dst->f[3] = src0->f[3] < 0.0f ? src1->f[3] : src2->f[3];
 }
 
-static void
-micro_cnd(union tgsi_exec_channel *dst,
-          const union tgsi_exec_channel *src0,
-          const union tgsi_exec_channel *src1,
-          const union tgsi_exec_channel *src2)
-{
-   dst->f[0] = src2->f[0] > 0.5f ? src0->f[0] : src1->f[0];
-   dst->f[1] = src2->f[1] > 0.5f ? src0->f[1] : src1->f[1];
-   dst->f[2] = src2->f[2] > 0.5f ? src0->f[2] : src1->f[2];
-   dst->f[3] = src2->f[3] > 0.5f ? src0->f[3] : src1->f[3];
-}
-
 static void
 micro_cos(union tgsi_exec_channel *dst,
           const union tgsi_exec_channel *src)
@@ -158,6 +157,55 @@ micro_cos(union tgsi_exec_channel *dst,
    dst->f[3] = cosf(src->f[3]);
 }
 
+static void
+micro_d2f(union tgsi_exec_channel *dst,
+          const union tgsi_double_channel *src)
+{
+   dst->f[0] = (float)src->d[0];
+   dst->f[1] = (float)src->d[1];
+   dst->f[2] = (float)src->d[2];
+   dst->f[3] = (float)src->d[3];
+}
+
+static void
+micro_d2i(union tgsi_exec_channel *dst,
+          const union tgsi_double_channel *src)
+{
+   dst->i[0] = (int)src->d[0];
+   dst->i[1] = (int)src->d[1];
+   dst->i[2] = (int)src->d[2];
+   dst->i[3] = (int)src->d[3];
+}
+
+static void
+micro_d2u(union tgsi_exec_channel *dst,
+          const union tgsi_double_channel *src)
+{
+   dst->u[0] = (unsigned)src->d[0];
+   dst->u[1] = (unsigned)src->d[1];
+   dst->u[2] = (unsigned)src->d[2];
+   dst->u[3] = (unsigned)src->d[3];
+}
+static void
+micro_dabs(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = src->d[0] >= 0.0 ? src->d[0] : -src->d[0];
+   dst->d[1] = src->d[1] >= 0.0 ? src->d[1] : -src->d[1];
+   dst->d[2] = src->d[2] >= 0.0 ? src->d[2] : -src->d[2];
+   dst->d[3] = src->d[3] >= 0.0 ? src->d[3] : -src->d[3];
+}
+
+static void
+micro_dadd(union tgsi_double_channel *dst,
+          const union tgsi_double_channel *src)
+{
+   dst->d[0] = src[0].d[0] + src[1].d[0];
+   dst->d[1] = src[0].d[1] + src[1].d[1];
+   dst->d[2] = src[0].d[2] + src[1].d[2];
+   dst->d[3] = src[0].d[3] + src[1].d[3];
+}
+
 static void
 micro_ddx(union tgsi_exec_channel *dst,
           const union tgsi_exec_channel *src)
@@ -178,6 +226,158 @@ micro_ddy(union tgsi_exec_channel *dst,
    dst->f[3] = src->f[TILE_BOTTOM_LEFT] - src->f[TILE_TOP_LEFT];
 }
 
+static void
+micro_dmul(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = src[0].d[0] * src[1].d[0];
+   dst->d[1] = src[0].d[1] * src[1].d[1];
+   dst->d[2] = src[0].d[2] * src[1].d[2];
+   dst->d[3] = src[0].d[3] * src[1].d[3];
+}
+
+static void
+micro_dmax(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = src[0].d[0] > src[1].d[0] ? src[0].d[0] : src[1].d[0];
+   dst->d[1] = src[0].d[1] > src[1].d[1] ? src[0].d[1] : src[1].d[1];
+   dst->d[2] = src[0].d[2] > src[1].d[2] ? src[0].d[2] : src[1].d[2];
+   dst->d[3] = src[0].d[3] > src[1].d[3] ? src[0].d[3] : src[1].d[3];
+}
+
+static void
+micro_dmin(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = src[0].d[0] < src[1].d[0] ? src[0].d[0] : src[1].d[0];
+   dst->d[1] = src[0].d[1] < src[1].d[1] ? src[0].d[1] : src[1].d[1];
+   dst->d[2] = src[0].d[2] < src[1].d[2] ? src[0].d[2] : src[1].d[2];
+   dst->d[3] = src[0].d[3] < src[1].d[3] ? src[0].d[3] : src[1].d[3];
+}
+
+static void
+micro_dneg(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = -src->d[0];
+   dst->d[1] = -src->d[1];
+   dst->d[2] = -src->d[2];
+   dst->d[3] = -src->d[3];
+}
+
+static void
+micro_dslt(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->u[0][0] = src[0].d[0] < src[1].d[0] ? ~0U : 0U;
+   dst->u[1][0] = src[0].d[1] < src[1].d[1] ? ~0U : 0U;
+   dst->u[2][0] = src[0].d[2] < src[1].d[2] ? ~0U : 0U;
+   dst->u[3][0] = src[0].d[3] < src[1].d[3] ? ~0U : 0U;
+}
+
+static void
+micro_dsne(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->u[0][0] = src[0].d[0] != src[1].d[0] ? ~0U : 0U;
+   dst->u[1][0] = src[0].d[1] != src[1].d[1] ? ~0U : 0U;
+   dst->u[2][0] = src[0].d[2] != src[1].d[2] ? ~0U : 0U;
+   dst->u[3][0] = src[0].d[3] != src[1].d[3] ? ~0U : 0U;
+}
+
+static void
+micro_dsge(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->u[0][0] = src[0].d[0] >= src[1].d[0] ? ~0U : 0U;
+   dst->u[1][0] = src[0].d[1] >= src[1].d[1] ? ~0U : 0U;
+   dst->u[2][0] = src[0].d[2] >= src[1].d[2] ? ~0U : 0U;
+   dst->u[3][0] = src[0].d[3] >= src[1].d[3] ? ~0U : 0U;
+}
+
+static void
+micro_dseq(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->u[0][0] = src[0].d[0] == src[1].d[0] ? ~0U : 0U;
+   dst->u[1][0] = src[0].d[1] == src[1].d[1] ? ~0U : 0U;
+   dst->u[2][0] = src[0].d[2] == src[1].d[2] ? ~0U : 0U;
+   dst->u[3][0] = src[0].d[3] == src[1].d[3] ? ~0U : 0U;
+}
+
+static void
+micro_drcp(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = 1.0 / src->d[0];
+   dst->d[1] = 1.0 / src->d[1];
+   dst->d[2] = 1.0 / src->d[2];
+   dst->d[3] = 1.0 / src->d[3];
+}
+
+static void
+micro_dsqrt(union tgsi_double_channel *dst,
+            const union tgsi_double_channel *src)
+{
+   dst->d[0] = sqrt(src->d[0]);
+   dst->d[1] = sqrt(src->d[1]);
+   dst->d[2] = sqrt(src->d[2]);
+   dst->d[3] = sqrt(src->d[3]);
+}
+
+static void
+micro_drsq(union tgsi_double_channel *dst,
+          const union tgsi_double_channel *src)
+{
+   dst->d[0] = 1.0 / sqrt(src->d[0]);
+   dst->d[1] = 1.0 / sqrt(src->d[1]);
+   dst->d[2] = 1.0 / sqrt(src->d[2]);
+   dst->d[3] = 1.0 / sqrt(src->d[3]);
+}
+
+static void
+micro_dmad(union tgsi_double_channel *dst,
+           const union tgsi_double_channel *src)
+{
+   dst->d[0] = src[0].d[0] * src[1].d[0] + src[2].d[0];
+   dst->d[1] = src[0].d[1] * src[1].d[1] + src[2].d[1];
+   dst->d[2] = src[0].d[2] * src[1].d[2] + src[2].d[2];
+   dst->d[3] = src[0].d[3] * src[1].d[3] + src[2].d[3];
+}
+
+static void
+micro_dfrac(union tgsi_double_channel *dst,
+            const union tgsi_double_channel *src)
+{
+   dst->d[0] = src->d[0] - floor(src->d[0]);
+   dst->d[1] = src->d[1] - floor(src->d[1]);
+   dst->d[2] = src->d[2] - floor(src->d[2]);
+   dst->d[3] = src->d[3] - floor(src->d[3]);
+}
+
+static void
+micro_dldexp(union tgsi_double_channel *dst,
+             const union tgsi_double_channel *src0,
+             union tgsi_exec_channel *src1)
+{
+   dst->d[0] = ldexp(src0->d[0], src1->i[0]);
+   dst->d[1] = ldexp(src0->d[1], src1->i[1]);
+   dst->d[2] = ldexp(src0->d[2], src1->i[2]);
+   dst->d[3] = ldexp(src0->d[3], src1->i[3]);
+}
+
+static void
+micro_dfracexp(union tgsi_double_channel *dst,
+               union tgsi_exec_channel *dst_exp,
+               const union tgsi_double_channel *src)
+{
+   dst->d[0] = frexp(src->d[0], &dst_exp->i[0]);
+   dst->d[1] = frexp(src->d[1], &dst_exp->i[1]);
+   dst->d[2] = frexp(src->d[2], &dst_exp->i[2]);
+   dst->d[3] = frexp(src->d[3], &dst_exp->i[3]);
+}
+
 static void
 micro_exp2(union tgsi_exec_channel *dst,
            const union tgsi_exec_channel *src)
@@ -212,6 +412,16 @@ micro_exp2(union tgsi_exec_channel *dst,
 #endif /* FAST_MATH */
 }
 
+static void
+micro_f2d(union tgsi_double_channel *dst,
+          const union tgsi_exec_channel *src)
+{
+   dst->d[0] = (double)src->f[0];
+   dst->d[1] = (double)src->f[1];
+   dst->d[2] = (double)src->f[2];
+   dst->d[3] = (double)src->f[3];
+}
+
 static void
 micro_flr(union tgsi_exec_channel *dst,
           const union tgsi_exec_channel *src)
@@ -232,6 +442,16 @@ micro_frc(union tgsi_exec_channel *dst,
    dst->f[3] = src->f[3] - floorf(src->f[3]);
 }
 
+static void
+micro_i2d(union tgsi_double_channel *dst,
+          const union tgsi_exec_channel *src)
+{
+   dst->d[0] = (double)src->i[0];
+   dst->d[1] = (double)src->i[1];
+   dst->d[2] = (double)src->i[2];
+   dst->d[3] = (double)src->i[3];
+}
+
 static void
 micro_iabs(union tgsi_exec_channel *dst,
            const union tgsi_exec_channel *src)
@@ -451,24 +671,6 @@ micro_sne(union tgsi_exec_channel *dst,
    dst->f[3] = src0->f[3] != src1->f[3] ? 1.0f : 0.0f;
 }
 
-static void
-micro_sfl(union tgsi_exec_channel *dst)
-{
-   dst->f[0] = 0.0f;
-   dst->f[1] = 0.0f;
-   dst->f[2] = 0.0f;
-   dst->f[3] = 0.0f;
-}
-
-static void
-micro_str(union tgsi_exec_channel *dst)
-{
-   dst->f[0] = 1.0f;
-   dst->f[1] = 1.0f;
-   dst->f[2] = 1.0f;
-   dst->f[3] = 1.0f;
-}
-
 static void
 micro_trunc(union tgsi_exec_channel *dst,
             const union tgsi_exec_channel *src)
@@ -479,11 +681,21 @@ micro_trunc(union tgsi_exec_channel *dst,
    dst->f[3] = (float)(int)src->f[3];
 }
 
+static void
+micro_u2d(union tgsi_double_channel *dst,
+          const union tgsi_exec_channel *src)
+{
+   dst->d[0] = (double)src->u[0];
+   dst->d[1] = (double)src->u[1];
+   dst->d[2] = (double)src->u[2];
+   dst->d[3] = (double)src->u[3];
+}
 
 enum tgsi_exec_datatype {
    TGSI_EXEC_DATA_FLOAT,
    TGSI_EXEC_DATA_INT,
-   TGSI_EXEC_DATA_UINT
+   TGSI_EXEC_DATA_UINT,
+   TGSI_EXEC_DATA_DOUBLE
 };
 
 /*
@@ -524,7 +736,7 @@ static const union tgsi_exec_channel M128Vec = {
  * not lead to crashes, etc.  But when debugging, it's helpful to catch
  * them.
  */
-static INLINE void
+static inline void
 check_inf_or_nan(const union tgsi_exec_channel *chan)
 {
    assert(!util_is_inf_or_nan((chan)->f[0]));
@@ -641,7 +853,9 @@ void
 tgsi_exec_machine_bind_shader(
    struct tgsi_exec_machine *mach,
    const struct tgsi_token *tokens,
-   struct tgsi_sampler *sampler)
+   struct tgsi_sampler *sampler,
+   struct tgsi_image *image,
+   struct tgsi_buffer *buffer)
 {
    uint k;
    struct tgsi_parse_context parse;
@@ -659,6 +873,8 @@ tgsi_exec_machine_bind_shader(
 
    mach->Tokens = tokens;
    mach->Sampler = sampler;
+   mach->Image = image;
+   mach->Buffer = buffer;
 
    if (!tokens) {
       /* unbind and free all */
@@ -1120,11 +1336,11 @@ fetch_src_file_channel(const struct tgsi_exec_machine *mach,
 }
 
 static void
-fetch_source(const struct tgsi_exec_machine *mach,
-             union tgsi_exec_channel *chan,
-             const struct tgsi_full_src_register *reg,
-             const uint chan_index,
-             enum tgsi_exec_datatype src_datatype)
+fetch_source_d(const struct tgsi_exec_machine *mach,
+               union tgsi_exec_channel *chan,
+               const struct tgsi_full_src_register *reg,
+               const uint chan_index,
+               enum tgsi_exec_datatype src_datatype)
 {
    union tgsi_exec_channel index;
    union tgsi_exec_channel index2D;
@@ -1267,6 +1483,16 @@ fetch_source(const struct tgsi_exec_machine *mach,
                           &index,
                           &index2D,
                           chan);
+}
+
+static void
+fetch_source(const struct tgsi_exec_machine *mach,
+             union tgsi_exec_channel *chan,
+             const struct tgsi_full_src_register *reg,
+             const uint chan_index,
+             enum tgsi_exec_datatype src_datatype)
+{
+   fetch_source_d(mach, chan, reg, chan_index, src_datatype);
 
    if (reg->Register.Absolute) {
       if (src_datatype == TGSI_EXEC_DATA_FLOAT) {
@@ -1285,16 +1511,16 @@ fetch_source(const struct tgsi_exec_machine *mach,
    }
 }
 
-static void
-store_dest(struct tgsi_exec_machine *mach,
-           const union tgsi_exec_channel *chan,
-           const struct tgsi_full_dst_register *reg,
-           const struct tgsi_full_instruction *inst,
-           uint chan_index,
-           enum tgsi_exec_datatype dst_datatype)
+static union tgsi_exec_channel *
+store_dest_dstret(struct tgsi_exec_machine *mach,
+                 const union tgsi_exec_channel *chan,
+                 const struct tgsi_full_dst_register *reg,
+                 const struct tgsi_full_instruction *inst,
+                 uint chan_index,
+                 enum tgsi_exec_datatype dst_datatype)
 {
    uint i;
-   union tgsi_exec_channel null;
+   static union tgsi_exec_channel null;
    union tgsi_exec_channel *dst;
    union tgsi_exec_channel index2D;
    uint execmask = mach->ExecMask;
@@ -1457,7 +1683,7 @@ store_dest(struct tgsi_exec_machine *mach,
 
    default:
       assert( 0 );
-      return;
+      return NULL;
    }
 
    if (inst->Instruction.Predicate) {
@@ -1479,7 +1705,7 @@ store_dest(struct tgsi_exec_machine *mach,
          break;
       default:
          assert(0);
-         return;
+         return NULL;
       }
 
       assert(inst->Predicate.Index == 0);
@@ -1501,14 +1727,55 @@ store_dest(struct tgsi_exec_machine *mach,
       }
    }
 
-   switch (inst->Instruction.Saturate) {
-   case TGSI_SAT_NONE:
+   return dst;
+}
+
+static void
+store_dest_double(struct tgsi_exec_machine *mach,
+                 const union tgsi_exec_channel *chan,
+                 const struct tgsi_full_dst_register *reg,
+                 const struct tgsi_full_instruction *inst,
+                 uint chan_index,
+                 enum tgsi_exec_datatype dst_datatype)
+{
+   union tgsi_exec_channel *dst;
+   const uint execmask = mach->ExecMask;
+   int i;
+
+   dst = store_dest_dstret(mach, chan, reg, inst, chan_index,
+			   dst_datatype);
+   if (!dst)
+      return;
+
+   /* doubles path */
+   for (i = 0; i < TGSI_QUAD_SIZE; i++)
+      if (execmask & (1 << i))
+         dst->i[i] = chan->i[i];
+}
+
+static void
+store_dest(struct tgsi_exec_machine *mach,
+           const union tgsi_exec_channel *chan,
+           const struct tgsi_full_dst_register *reg,
+           const struct tgsi_full_instruction *inst,
+           uint chan_index,
+           enum tgsi_exec_datatype dst_datatype)
+{
+   union tgsi_exec_channel *dst;
+   const uint execmask = mach->ExecMask;
+   int i;
+
+   dst = store_dest_dstret(mach, chan, reg, inst, chan_index,
+                    dst_datatype);
+   if (!dst)
+      return;
+
+   if (!inst->Instruction.Saturate) {
       for (i = 0; i < TGSI_QUAD_SIZE; i++)
          if (execmask & (1 << i))
             dst->i[i] = chan->i[i];
-      break;
-
-   case TGSI_SAT_ZERO_ONE:
+   }
+   else {
       for (i = 0; i < TGSI_QUAD_SIZE; i++)
          if (execmask & (1 << i)) {
             if (chan->f[i] < 0.0f)
@@ -1518,22 +1785,6 @@ store_dest(struct tgsi_exec_machine *mach,
             else
                dst->i[i] = chan->i[i];
          }
-      break;
-
-   case TGSI_SAT_MINUS_PLUS_ONE:
-      for (i = 0; i < TGSI_QUAD_SIZE; i++)
-         if (execmask & (1 << i)) {
-            if (chan->f[i] < -1.0f)
-               dst->f[i] = -1.0f;
-            else if (chan->f[i] > 1.0f)
-               dst->f[i] = 1.0f;
-            else
-               dst->i[i] = chan->i[i];
-         }
-      break;
-
-   default:
-      assert( 0 );
    }
 }
 
@@ -1688,7 +1939,7 @@ fetch_texel( struct tgsi_sampler *sampler,
 #define TEX_MODIFIER_LOD_BIAS       2
 #define TEX_MODIFIER_EXPLICIT_LOD   3
 #define TEX_MODIFIER_LEVEL_ZERO     4
-
+#define TEX_MODIFIER_GATHER         5
 
 /*
  * Fetch all 3 (for s,t,r coords) texel offsets, put them into int array.
@@ -1742,11 +1993,46 @@ fetch_assign_deriv_channel(struct tgsi_exec_machine *mach,
    derivs[1][3] = d.f[3];
 }
 
+static uint
+fetch_sampler_unit(struct tgsi_exec_machine *mach,
+                   const struct tgsi_full_instruction *inst,
+                   uint sampler)
+{
+   uint unit = 0;
+   int i;
+   if (inst->Src[sampler].Register.Indirect) {
+      const struct tgsi_full_src_register *reg = &inst->Src[sampler];
+      union tgsi_exec_channel indir_index, index2;
+      const uint execmask = mach->ExecMask;
+      index2.i[0] =
+      index2.i[1] =
+      index2.i[2] =
+      index2.i[3] = reg->Indirect.Index;
+
+      fetch_src_file_channel(mach,
+                             0,
+                             reg->Indirect.File,
+                             reg->Indirect.Swizzle,
+                             &index2,
+                             &ZeroVec,
+                             &indir_index);
+      for (i = 0; i < TGSI_QUAD_SIZE; i++) {
+         if (execmask & (1 << i)) {
+            unit = inst->Src[sampler].Register.Index + indir_index.i[i];
+            break;
+         }
+      }
+
+   } else {
+      unit = inst->Src[sampler].Register.Index;
+   }
+   return unit;
+}
 
 /*
  * execute a texture instruction.
  *
- * modifier is used to control the channel routing for the\
+ * modifier is used to control the channel routing for the
  * instruction variants like proj, lod, and texture with lod bias.
  * sampler indicates which src register the sampler is contained in.
  */
@@ -1755,21 +2041,23 @@ exec_tex(struct tgsi_exec_machine *mach,
          const struct tgsi_full_instruction *inst,
          uint modifier, uint sampler)
 {
-   const uint unit = inst->Src[sampler].Register.Index;
    const union tgsi_exec_channel *args[5], *proj = NULL;
    union tgsi_exec_channel r[5];
-   enum tgsi_sampler_control control =  tgsi_sampler_lod_none;
+   enum tgsi_sampler_control control = TGSI_SAMPLER_LOD_NONE;
    uint chan;
+   uint unit;
    int8_t offsets[3];
    int dim, shadow_ref, i;
 
+   unit = fetch_sampler_unit(mach, inst, sampler);
    /* always fetch all 3 offsets, overkill but keeps code simple */
    fetch_texel_offsets(mach, inst, offsets);
 
    assert(modifier != TEX_MODIFIER_LEVEL_ZERO);
    assert(inst->Texture.Texture != TGSI_TEXTURE_BUFFER);
 
-   dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture, &shadow_ref);
+   dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture);
+   shadow_ref = tgsi_util_get_shadow_ref_src_index(inst->Texture.Texture);
 
    assert(dim <= 4);
    if (shadow_ref >= 0)
@@ -1802,9 +2090,11 @@ exec_tex(struct tgsi_exec_machine *mach,
          args[i] = &ZeroVec;
 
       if (modifier == TEX_MODIFIER_EXPLICIT_LOD)
-         control = tgsi_sampler_lod_explicit;
+         control = TGSI_SAMPLER_LOD_EXPLICIT;
       else if (modifier == TEX_MODIFIER_LOD_BIAS)
-         control = tgsi_sampler_lod_bias;
+         control = TGSI_SAMPLER_LOD_BIAS;
+      else if (modifier == TEX_MODIFIER_GATHER)
+         control = TGSI_SAMPLER_GATHER;
    }
    else {
       for (i = dim; i < Elements(args); i++)
@@ -1854,17 +2144,58 @@ exec_tex(struct tgsi_exec_machine *mach,
    }
 }
 
+static void
+exec_lodq(struct tgsi_exec_machine *mach,
+          const struct tgsi_full_instruction *inst)
+{
+   uint unit;
+   int dim;
+   int i;
+   union tgsi_exec_channel coords[4];
+   const union tgsi_exec_channel *args[Elements(coords)];
+   union tgsi_exec_channel r[2];
+
+   unit = fetch_sampler_unit(mach, inst, 1);
+   dim = tgsi_util_get_texture_coord_dim(inst->Texture.Texture);
+   assert(dim <= Elements(coords));
+   /* fetch coordinates */
+   for (i = 0; i < dim; i++) {
+      FETCH(&coords[i], 0, TGSI_CHAN_X + i);
+      args[i] = &coords[i];
+   }
+   for (i = dim; i < Elements(coords); i++) {
+      args[i] = &ZeroVec;
+   }
+   mach->Sampler->query_lod(mach->Sampler, unit, unit,
+                            args[0]->f,
+                            args[1]->f,
+                            args[2]->f,
+                            args[3]->f,
+                            TGSI_SAMPLER_LOD_NONE,
+                            r[0].f,
+                            r[1].f);
+
+   if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_X) {
+      store_dest(mach, &r[0], &inst->Dst[0], inst, TGSI_CHAN_X,
+                 TGSI_EXEC_DATA_FLOAT);
+   }
+   if (inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_Y) {
+      store_dest(mach, &r[1], &inst->Dst[0], inst, TGSI_CHAN_Y,
+                 TGSI_EXEC_DATA_FLOAT);
+   }
+}
 
 static void
 exec_txd(struct tgsi_exec_machine *mach,
          const struct tgsi_full_instruction *inst)
 {
-   const uint unit = inst->Src[3].Register.Index;
    union tgsi_exec_channel r[4];
    float derivs[3][2][TGSI_QUAD_SIZE];
    uint chan;
+   uint unit;
    int8_t offsets[3];
 
+   unit = fetch_sampler_unit(mach, inst, 3);
    /* always fetch all 3 offsets, overkill but keeps code simple */
    fetch_texel_offsets(mach, inst, offsets);
 
@@ -1876,7 +2207,7 @@ exec_txd(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, unit, unit,
                   &r[0], &ZeroVec, &ZeroVec, &ZeroVec, &ZeroVec,   /* S, T, P, C, LOD */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);           /* R, G, B, A */
       break;
 
@@ -1892,7 +2223,7 @@ exec_txd(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, unit, unit,
                   &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec,   /* S, T, P, C, LOD */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);           /* R, G, B, A */
       break;
 
@@ -1906,7 +2237,7 @@ exec_txd(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, unit, unit,
                   &r[0], &r[1], &ZeroVec, &ZeroVec, &ZeroVec,   /* S, T, P, C, LOD */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);           /* R, G, B, A */
       break;
 
@@ -1926,7 +2257,7 @@ exec_txd(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, unit, unit,
                   &r[0], &r[1], &r[2], &r[3], &ZeroVec,   /* inputs */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);     /* outputs */
       break;
 
@@ -1946,7 +2277,7 @@ exec_txd(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, unit, unit,
                   &r[0], &r[1], &r[2], &r[3], &ZeroVec,   /* inputs */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);     /* outputs */
       break;
 
@@ -1966,20 +2297,22 @@ static void
 exec_txf(struct tgsi_exec_machine *mach,
          const struct tgsi_full_instruction *inst)
 {
-   const uint unit = inst->Src[1].Register.Index;
    union tgsi_exec_channel r[4];
    uint chan;
+   uint unit;
    float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
    int j;
    int8_t offsets[3];
    unsigned target;
 
+   unit = fetch_sampler_unit(mach, inst, 1);
    /* always fetch all 3 offsets, overkill but keeps code simple */
    fetch_texel_offsets(mach, inst, offsets);
 
    IFETCH(&r[3], 0, TGSI_CHAN_W);
 
-   if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I) {
+   if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I ||
+       inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I_MS) {
       target = mach->SamplerViews[unit].Resource;
    }
    else {
@@ -2021,7 +2354,8 @@ exec_txf(struct tgsi_exec_machine *mach,
       r[3].f[j] = rgba[3][j];
    }
 
-   if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I) {
+   if (inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I ||
+       inst->Instruction.Opcode == TGSI_OPCODE_SAMPLE_I_MS) {
       unsigned char swizzles[4];
       swizzles[0] = inst->Src[1].Register.SwizzleX;
       swizzles[1] = inst->Src[1].Register.SwizzleY;
@@ -2048,12 +2382,14 @@ static void
 exec_txq(struct tgsi_exec_machine *mach,
          const struct tgsi_full_instruction *inst)
 {
-   const uint unit = inst->Src[1].Register.Index;
    int result[4];
    union tgsi_exec_channel r[4], src;
    uint chan;
+   uint unit;
    int i,j;
 
+   unit = fetch_sampler_unit(mach, inst, 1);
+
    fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_INT);
 
    /* XXX: This interface can't return per-pixel values */
@@ -2080,9 +2416,9 @@ exec_sample(struct tgsi_exec_machine *mach,
 {
    const uint resource_unit = inst->Src[1].Register.Index;
    const uint sampler_unit = inst->Src[2].Register.Index;
-   union tgsi_exec_channel r[4], c1;
+   union tgsi_exec_channel r[5], c1;
    const union tgsi_exec_channel *lod = &ZeroVec;
-   enum tgsi_sampler_control control = tgsi_sampler_lod_none;
+   enum tgsi_sampler_control control = TGSI_SAMPLER_LOD_NONE;
    uint chan;
    unsigned char swizzles[4];
    int8_t offsets[3];
@@ -2096,16 +2432,16 @@ exec_sample(struct tgsi_exec_machine *mach,
       if (modifier == TEX_MODIFIER_LOD_BIAS) {
          FETCH(&c1, 3, TGSI_CHAN_X);
          lod = &c1;
-         control = tgsi_sampler_lod_bias;
+         control = TGSI_SAMPLER_LOD_BIAS;
       }
       else if (modifier == TEX_MODIFIER_EXPLICIT_LOD) {
          FETCH(&c1, 3, TGSI_CHAN_X);
          lod = &c1;
-         control = tgsi_sampler_lod_explicit;
+         control = TGSI_SAMPLER_LOD_EXPLICIT;
       }
       else {
          assert(modifier == TEX_MODIFIER_LEVEL_ZERO);
-         control = tgsi_sampler_lod_zero;
+         control = TGSI_SAMPLER_LOD_ZERO;
       }
    }
 
@@ -2231,7 +2567,7 @@ exec_sample_d(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, resource_unit, sampler_unit,
                   &r[0], &r[1], &ZeroVec, &ZeroVec, &ZeroVec,   /* S, T, P, C, LOD */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);           /* R, G, B, A */
       break;
 
@@ -2247,7 +2583,7 @@ exec_sample_d(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, resource_unit, sampler_unit,
                   &r[0], &r[1], &r[2], &ZeroVec, &ZeroVec,   /* inputs */
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);     /* outputs */
       break;
 
@@ -2265,7 +2601,7 @@ exec_sample_d(struct tgsi_exec_machine *mach,
 
       fetch_texel(mach->Sampler, resource_unit, sampler_unit,
                   &r[0], &r[1], &r[2], &r[3], &ZeroVec,
-                  derivs, offsets, tgsi_sampler_derivs_explicit,
+                  derivs, offsets, TGSI_SAMPLER_DERIVS_EXPLICIT,
                   &r[0], &r[1], &r[2], &r[3]);
       break;
 
@@ -2446,27 +2782,6 @@ exec_declaration(struct tgsi_exec_machine *mach,
    }
 }
 
-
-typedef void (* micro_op)(union tgsi_exec_channel *dst);
-
-static void
-exec_vector(struct tgsi_exec_machine *mach,
-            const struct tgsi_full_instruction *inst,
-            micro_op op,
-            enum tgsi_exec_datatype dst_datatype)
-{
-   unsigned int chan;
-
-   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
-      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
-         union tgsi_exec_channel dst;
-
-         op(&dst);
-         store_dest(mach, &dst, &inst->Dst[0], inst, chan, dst_datatype);
-      }
-   }
-}
-
 typedef void (* micro_unary_op)(union tgsi_exec_channel *dst,
                                 const union tgsi_exec_channel *src);
 
@@ -2756,6 +3071,45 @@ exec_dp2(struct tgsi_exec_machine *mach,
    }
 }
 
+static void
+exec_pk2h(struct tgsi_exec_machine *mach,
+          const struct tgsi_full_instruction *inst)
+{
+   unsigned chan;
+   union tgsi_exec_channel arg[2], dst;
+
+   fetch_source(mach, &arg[0], &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT);
+   fetch_source(mach, &arg[1], &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT);
+   for (chan = 0; chan < TGSI_QUAD_SIZE; chan++) {
+      dst.u[chan] = util_float_to_half(arg[0].f[chan]) |
+         (util_float_to_half(arg[1].f[chan]) << 16);
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &dst, &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_UINT);
+      }
+   }
+}
+
+static void
+exec_up2h(struct tgsi_exec_machine *mach,
+          const struct tgsi_full_instruction *inst)
+{
+   unsigned chan;
+   union tgsi_exec_channel arg, dst[2];
+
+   fetch_source(mach, &arg, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_UINT);
+   for (chan = 0; chan < TGSI_QUAD_SIZE; chan++) {
+      dst[0].f[chan] = util_half_to_float(arg.u[chan] & 0xffff);
+      dst[1].f[chan] = util_half_to_float(arg.u[chan] >> 16);
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &dst[chan & 1], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
 static void
 exec_scs(struct tgsi_exec_machine *mach,
          const struct tgsi_full_instruction *inst)
@@ -3031,8 +3385,773 @@ exec_endswitch(struct tgsi_exec_machine *mach)
    UPDATE_EXEC_MASK(mach);
 }
 
+typedef void (* micro_dop)(union tgsi_double_channel *dst,
+                           const union tgsi_double_channel *src);
+
 static void
-micro_i2f(union tgsi_exec_channel *dst,
+fetch_double_channel(struct tgsi_exec_machine *mach,
+                     union tgsi_double_channel *chan,
+                     const struct tgsi_full_src_register *reg,
+                     uint chan_0,
+                     uint chan_1)
+{
+   union tgsi_exec_channel src[2];
+   uint i;
+
+   fetch_source_d(mach, &src[0], reg, chan_0, TGSI_EXEC_DATA_UINT);
+   fetch_source_d(mach, &src[1], reg, chan_1, TGSI_EXEC_DATA_UINT);
+
+   for (i = 0; i < TGSI_QUAD_SIZE; i++) {
+      chan->u[i][0] = src[0].u[i];
+      chan->u[i][1] = src[1].u[i];
+   }
+   if (reg->Register.Absolute) {
+      micro_dabs(chan, chan);
+   }
+   if (reg->Register.Negate) {
+      micro_dneg(chan, chan);
+   }
+}
+
+static void
+store_double_channel(struct tgsi_exec_machine *mach,
+                     const union tgsi_double_channel *chan,
+                     const struct tgsi_full_dst_register *reg,
+                     const struct tgsi_full_instruction *inst,
+                     uint chan_0,
+                     uint chan_1)
+{
+   union tgsi_exec_channel dst[2];
+   uint i;
+   union tgsi_double_channel temp;
+   const uint execmask = mach->ExecMask;
+
+   if (!inst->Instruction.Saturate) {
+      for (i = 0; i < TGSI_QUAD_SIZE; i++)
+         if (execmask & (1 << i)) {
+            dst[0].u[i] = chan->u[i][0];
+            dst[1].u[i] = chan->u[i][1];
+         }
+   }
+   else {
+      for (i = 0; i < TGSI_QUAD_SIZE; i++)
+         if (execmask & (1 << i)) {
+            if (chan->d[i] < 0.0)
+               temp.d[i] = 0.0;
+            else if (chan->d[i] > 1.0)
+               temp.d[i] = 1.0;
+            else
+               temp.d[i] = chan->d[i];
+
+            dst[0].u[i] = temp.u[i][0];
+            dst[1].u[i] = temp.u[i][1];
+         }
+   }
+
+   store_dest_double(mach, &dst[0], reg, inst, chan_0, TGSI_EXEC_DATA_UINT);
+   if (chan_1 != -1)
+      store_dest_double(mach, &dst[1], reg, inst, chan_1, TGSI_EXEC_DATA_UINT);
+}
+
+static void
+exec_double_unary(struct tgsi_exec_machine *mach,
+                  const struct tgsi_full_instruction *inst,
+                  micro_dop op)
+{
+   union tgsi_double_channel src;
+   union tgsi_double_channel dst;
+
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) {
+      fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+      op(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) {
+      fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+      op(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_double_binary(struct tgsi_exec_machine *mach,
+                   const struct tgsi_full_instruction *inst,
+                   micro_dop op,
+                   enum tgsi_exec_datatype dst_datatype)
+{
+   union tgsi_double_channel src[2];
+   union tgsi_double_channel dst;
+   int first_dest_chan, second_dest_chan;
+   int wmask;
+
+   wmask = inst->Dst[0].Register.WriteMask;
+   /* these are & because of the way DSLT etc store their destinations */
+   if (wmask & TGSI_WRITEMASK_XY) {
+      first_dest_chan = TGSI_CHAN_X;
+      second_dest_chan = TGSI_CHAN_Y;
+      if (dst_datatype == TGSI_EXEC_DATA_UINT) {
+         first_dest_chan = (wmask & TGSI_WRITEMASK_X) ? TGSI_CHAN_X : TGSI_CHAN_Y;
+         second_dest_chan = -1;
+      }
+
+      fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+      fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_X, TGSI_CHAN_Y);
+      op(&dst, src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, first_dest_chan, second_dest_chan);
+   }
+
+   if (wmask & TGSI_WRITEMASK_ZW) {
+      first_dest_chan = TGSI_CHAN_Z;
+      second_dest_chan = TGSI_CHAN_W;
+      if (dst_datatype == TGSI_EXEC_DATA_UINT) {
+         first_dest_chan = (wmask & TGSI_WRITEMASK_Z) ? TGSI_CHAN_Z : TGSI_CHAN_W;
+         second_dest_chan = -1;
+      }
+
+      fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+      fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_CHAN_W);
+      op(&dst, src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, first_dest_chan, second_dest_chan);
+   }
+}
+
+static void
+exec_double_trinary(struct tgsi_exec_machine *mach,
+                    const struct tgsi_full_instruction *inst,
+                    micro_dop op)
+{
+   union tgsi_double_channel src[3];
+   union tgsi_double_channel dst;
+
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) {
+      fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+      fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_X, TGSI_CHAN_Y);
+      fetch_double_channel(mach, &src[2], &inst->Src[2], TGSI_CHAN_X, TGSI_CHAN_Y);
+      op(&dst, src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) {
+      fetch_double_channel(mach, &src[0], &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+      fetch_double_channel(mach, &src[1], &inst->Src[1], TGSI_CHAN_Z, TGSI_CHAN_W);
+      fetch_double_channel(mach, &src[2], &inst->Src[2], TGSI_CHAN_Z, TGSI_CHAN_W);
+      op(&dst, src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_f2d(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel src;
+   union tgsi_double_channel dst;
+
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_FLOAT);
+      micro_f2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_FLOAT);
+      micro_f2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_d2f(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_double_channel src;
+   union tgsi_exec_channel dst;
+   int wm = inst->Dst[0].Register.WriteMask;
+   int i;
+   int bit;
+   for (i = 0; i < 2; i++) {
+      bit = ffs(wm);
+      if (bit) {
+         wm &= ~(1 << (bit - 1));
+         if (i == 0)
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+         else
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+         micro_d2f(&dst, &src);
+         store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
+static void
+exec_i2d(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel src;
+   union tgsi_double_channel dst;
+
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_INT);
+      micro_i2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_INT);
+      micro_i2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_d2i(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_double_channel src;
+   union tgsi_exec_channel dst;
+   int wm = inst->Dst[0].Register.WriteMask;
+   int i;
+   int bit;
+   for (i = 0; i < 2; i++) {
+      bit = ffs(wm);
+      if (bit) {
+         wm &= ~(1 << (bit - 1));
+         if (i == 0)
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+         else
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+         micro_d2i(&dst, &src);
+         store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_INT);
+      }
+   }
+}
+static void
+exec_u2d(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel src;
+   union tgsi_double_channel dst;
+
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_EXEC_DATA_UINT);
+      micro_u2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+   if ((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW) {
+      fetch_source(mach, &src, &inst->Src[0], TGSI_CHAN_Y, TGSI_EXEC_DATA_UINT);
+      micro_u2d(&dst, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_d2u(struct tgsi_exec_machine *mach,
+         const struct tgsi_full_instruction *inst)
+{
+   union tgsi_double_channel src;
+   union tgsi_exec_channel dst;
+   int wm = inst->Dst[0].Register.WriteMask;
+   int i;
+   int bit;
+   for (i = 0; i < 2; i++) {
+      bit = ffs(wm);
+      if (bit) {
+         wm &= ~(1 << (bit - 1));
+         if (i == 0)
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+         else
+            fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+         micro_d2u(&dst, &src);
+         store_dest(mach, &dst, &inst->Dst[0], inst, bit - 1, TGSI_EXEC_DATA_UINT);
+      }
+   }
+}
+
+static void
+exec_dldexp(struct tgsi_exec_machine *mach,
+            const struct tgsi_full_instruction *inst)
+{
+   union tgsi_double_channel src0;
+   union tgsi_exec_channel src1;
+   union tgsi_double_channel dst;
+   int wmask;
+
+   wmask = inst->Dst[0].Register.WriteMask;
+   if (wmask & TGSI_WRITEMASK_XY) {
+      fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+      fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_X, TGSI_EXEC_DATA_INT);
+      micro_dldexp(&dst, &src0, &src1);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+   }
+
+   if (wmask & TGSI_WRITEMASK_ZW) {
+      fetch_double_channel(mach, &src0, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+      fetch_source(mach, &src1, &inst->Src[1], TGSI_CHAN_Z, TGSI_EXEC_DATA_INT);
+      micro_dldexp(&dst, &src0, &src1);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+   }
+}
+
+static void
+exec_dfracexp(struct tgsi_exec_machine *mach,
+              const struct tgsi_full_instruction *inst)
+{
+   union tgsi_double_channel src;
+   union tgsi_double_channel dst;
+   union tgsi_exec_channel dst_exp;
+
+   if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_XY) == TGSI_WRITEMASK_XY)) {
+      fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_X, TGSI_CHAN_Y);
+      micro_dfracexp(&dst, &dst_exp, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_X, TGSI_CHAN_Y);
+      store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT);
+   }
+   if (((inst->Dst[0].Register.WriteMask & TGSI_WRITEMASK_ZW) == TGSI_WRITEMASK_ZW)) {
+      fetch_double_channel(mach, &src, &inst->Src[0], TGSI_CHAN_Z, TGSI_CHAN_W);
+      micro_dfracexp(&dst, &dst_exp, &src);
+      store_double_channel(mach, &dst, &inst->Dst[0], inst, TGSI_CHAN_Z, TGSI_CHAN_W);
+      store_dest(mach, &dst_exp, &inst->Dst[1], inst, ffs(inst->Dst[1].Register.WriteMask) - 1, TGSI_EXEC_DATA_INT);
+   }
+}
+
+static int
+get_image_coord_dim(unsigned tgsi_tex)
+{
+   int dim;
+   switch (tgsi_tex) {
+   case TGSI_TEXTURE_BUFFER:
+   case TGSI_TEXTURE_1D:
+      dim = 1;
+      break;
+   case TGSI_TEXTURE_2D:
+   case TGSI_TEXTURE_RECT:
+   case TGSI_TEXTURE_1D_ARRAY:
+   case TGSI_TEXTURE_2D_MSAA:
+      dim = 2;
+      break;
+   case TGSI_TEXTURE_3D:
+   case TGSI_TEXTURE_CUBE:
+   case TGSI_TEXTURE_2D_ARRAY:
+   case TGSI_TEXTURE_2D_ARRAY_MSAA:
+   case TGSI_TEXTURE_CUBE_ARRAY:
+      dim = 3;
+      break;
+   default:
+      assert(!"unknown texture target");
+      dim = 0;
+      break;
+   }
+
+   return dim;
+}
+
+static int
+get_image_coord_sample(unsigned tgsi_tex)
+{
+   int sample = 0;
+   switch (tgsi_tex) {
+   case TGSI_TEXTURE_2D_MSAA:
+      sample = 3;
+      break;
+   case TGSI_TEXTURE_2D_ARRAY_MSAA:
+      sample = 4;
+      break;
+   default:
+      break;
+   }
+   return sample;
+}
+
+static void
+exec_load_img(struct tgsi_exec_machine *mach,
+              const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[4], sample_r;
+   uint unit;
+   int sample;
+   int i, j;
+   int dim;
+   uint chan;
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_image_params params;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = fetch_sampler_unit(mach, inst, 0);
+   dim = get_image_coord_dim(inst->Memory.Texture);
+   sample = get_image_coord_sample(inst->Memory.Texture);
+   assert(dim <= 3);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.tgsi_tex_instr = inst->Memory.Texture;
+   params.format = inst->Memory.Format;
+
+   for (i = 0; i < dim; i++) {
+      IFETCH(&r[i], 1, TGSI_CHAN_X + i);
+   }
+
+   if (sample)
+      IFETCH(&sample_r, 1, TGSI_CHAN_X + sample);
+
+   mach->Image->load(mach->Image, &params,
+                     r[0].i, r[1].i, r[2].i, sample_r.i,
+                     rgba);
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      r[0].f[j] = rgba[0][j];
+      r[1].f[j] = rgba[1][j];
+      r[2].f[j] = rgba[2][j];
+      r[3].f[j] = rgba[3][j];
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
+static void
+exec_load_buf(struct tgsi_exec_machine *mach,
+              const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[4];
+   uint unit;
+   int j;
+   uint chan;
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_buffer_params params;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = fetch_sampler_unit(mach, inst, 0);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   IFETCH(&r[0], 1, TGSI_CHAN_X);
+
+   mach->Buffer->load(mach->Buffer, &params,
+                      r[0].i, rgba);
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      r[0].f[j] = rgba[0][j];
+      r[1].f[j] = rgba[1][j];
+      r[2].f[j] = rgba[2][j];
+      r[3].f[j] = rgba[3][j];
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
+static void
+exec_load(struct tgsi_exec_machine *mach,
+          const struct tgsi_full_instruction *inst)
+{
+   if (inst->Src[0].Register.File == TGSI_FILE_IMAGE)
+      exec_load_img(mach, inst);
+   else
+      exec_load_buf(mach, inst);
+}
+
+static void
+exec_store_img(struct tgsi_exec_machine *mach,
+               const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[3], sample_r;
+   union tgsi_exec_channel value[4];
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_image_params params;
+   int dim;
+   int sample;
+   int i, j;
+   uint unit;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+   unit = inst->Dst[0].Register.Index;
+   dim = get_image_coord_dim(inst->Memory.Texture);
+   sample = get_image_coord_sample(inst->Memory.Texture);
+   assert(dim <= 3);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.tgsi_tex_instr = inst->Memory.Texture;
+   params.format = inst->Memory.Format;
+
+   for (i = 0; i < dim; i++) {
+      IFETCH(&r[i], 0, TGSI_CHAN_X + i);
+   }
+
+   for (i = 0; i < 4; i++) {
+      FETCH(&value[i], 1, TGSI_CHAN_X + i);
+   }
+   if (sample)
+      IFETCH(&sample_r, 0, TGSI_CHAN_X + sample);
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      rgba[0][j] = value[0].f[j];
+      rgba[1][j] = value[1].f[j];
+      rgba[2][j] = value[2].f[j];
+      rgba[3][j] = value[3].f[j];
+   }
+
+   mach->Image->store(mach->Image, &params,
+                      r[0].i, r[1].i, r[2].i, sample_r.i,
+                      rgba);
+}
+
+static void
+exec_store_buf(struct tgsi_exec_machine *mach,
+               const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[3];
+   union tgsi_exec_channel value[4];
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_buffer_params params;
+   int i, j;
+   uint unit;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = inst->Dst[0].Register.Index;
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.writemask = inst->Dst[0].Register.WriteMask;
+
+   IFETCH(&r[0], 0, TGSI_CHAN_X);
+   for (i = 0; i < 4; i++) {
+      FETCH(&value[i], 1, TGSI_CHAN_X + i);
+   }
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      rgba[0][j] = value[0].f[j];
+      rgba[1][j] = value[1].f[j];
+      rgba[2][j] = value[2].f[j];
+      rgba[3][j] = value[3].f[j];
+   }
+
+   mach->Buffer->store(mach->Buffer, &params,
+                      r[0].i,
+                      rgba);
+}
+
+static void
+exec_store(struct tgsi_exec_machine *mach,
+           const struct tgsi_full_instruction *inst)
+{
+   if (inst->Dst[0].Register.File == TGSI_FILE_IMAGE)
+      exec_store_img(mach, inst);
+   else
+      exec_store_buf(mach, inst);
+}
+
+static void
+exec_atomop_img(struct tgsi_exec_machine *mach,
+                const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[4], sample_r;
+   union tgsi_exec_channel value[4], value2[4];
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_image_params params;
+   int dim;
+   int sample;
+   int i, j;
+   uint unit, chan;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+   unit = fetch_sampler_unit(mach, inst, 0);
+   dim = get_image_coord_dim(inst->Memory.Texture);
+   sample = get_image_coord_sample(inst->Memory.Texture);
+   assert(dim <= 3);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.tgsi_tex_instr = inst->Memory.Texture;
+   params.format = inst->Memory.Format;
+
+   for (i = 0; i < dim; i++) {
+      IFETCH(&r[i], 1, TGSI_CHAN_X + i);
+   }
+
+   for (i = 0; i < 4; i++) {
+      FETCH(&value[i], 2, TGSI_CHAN_X + i);
+      if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS)
+         FETCH(&value2[i], 3, TGSI_CHAN_X + i);
+   }
+   if (sample)
+      IFETCH(&sample_r, 1, TGSI_CHAN_X + sample);
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      rgba[0][j] = value[0].f[j];
+      rgba[1][j] = value[1].f[j];
+      rgba[2][j] = value[2].f[j];
+      rgba[3][j] = value[3].f[j];
+   }
+   if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
+      for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+         rgba2[0][j] = value2[0].f[j];
+         rgba2[1][j] = value2[1].f[j];
+         rgba2[2][j] = value2[2].f[j];
+         rgba2[3][j] = value2[3].f[j];
+      }
+   }
+
+   mach->Image->op(mach->Image, &params, inst->Instruction.Opcode,
+                   r[0].i, r[1].i, r[2].i, sample_r.i,
+                   rgba, rgba2);
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      r[0].f[j] = rgba[0][j];
+      r[1].f[j] = rgba[1][j];
+      r[2].f[j] = rgba[2][j];
+      r[3].f[j] = rgba[3][j];
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
+static void
+exec_atomop_buf(struct tgsi_exec_machine *mach,
+                const struct tgsi_full_instruction *inst)
+{
+   union tgsi_exec_channel r[4];
+   union tgsi_exec_channel value[4], value2[4];
+   float rgba[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   float rgba2[TGSI_NUM_CHANNELS][TGSI_QUAD_SIZE];
+   struct tgsi_buffer_params params;
+   int i, j;
+   uint unit, chan;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = fetch_sampler_unit(mach, inst, 0);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.writemask = inst->Dst[0].Register.WriteMask;
+
+   IFETCH(&r[0], 1, TGSI_CHAN_X);
+
+   for (i = 0; i < 4; i++) {
+      FETCH(&value[i], 2, TGSI_CHAN_X + i);
+      if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS)
+         FETCH(&value2[i], 3, TGSI_CHAN_X + i);
+   }
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      rgba[0][j] = value[0].f[j];
+      rgba[1][j] = value[1].f[j];
+      rgba[2][j] = value[2].f[j];
+      rgba[3][j] = value[3].f[j];
+   }
+   if (inst->Instruction.Opcode == TGSI_OPCODE_ATOMCAS) {
+      for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+         rgba2[0][j] = value2[0].f[j];
+         rgba2[1][j] = value2[1].f[j];
+         rgba2[2][j] = value2[2].f[j];
+         rgba2[3][j] = value2[3].f[j];
+      }
+   }
+
+   mach->Buffer->op(mach->Buffer, &params, inst->Instruction.Opcode,
+                   r[0].i,
+                   rgba, rgba2);
+
+   for (j = 0; j < TGSI_QUAD_SIZE; j++) {
+      r[0].f[j] = rgba[0][j];
+      r[1].f[j] = rgba[1][j];
+      r[2].f[j] = rgba[2][j];
+      r[3].f[j] = rgba[3][j];
+   }
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan, TGSI_EXEC_DATA_FLOAT);
+      }
+   }
+}
+
+static void
+exec_atomop(struct tgsi_exec_machine *mach,
+            const struct tgsi_full_instruction *inst)
+{
+   if (inst->Src[0].Register.File == TGSI_FILE_IMAGE)
+      exec_atomop_img(mach, inst);
+   else
+      exec_atomop_buf(mach, inst);
+}
+
+static void
+exec_resq_img(struct tgsi_exec_machine *mach,
+              const struct tgsi_full_instruction *inst)
+{
+   int result[4];
+   union tgsi_exec_channel r[4];
+   uint unit;
+   int i, chan, j;
+   struct tgsi_image_params params;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = fetch_sampler_unit(mach, inst, 0);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+   params.tgsi_tex_instr = inst->Memory.Texture;
+   params.format = inst->Memory.Format;
+
+   mach->Image->get_dims(mach->Image, &params, result);
+
+   for (i = 0; i < TGSI_QUAD_SIZE; i++) {
+      for (j = 0; j < 4; j++) {
+         r[j].i[i] = result[j];
+      }
+   }
+
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan,
+                    TGSI_EXEC_DATA_INT);
+      }
+   }
+}
+
+static void
+exec_resq_buf(struct tgsi_exec_machine *mach,
+              const struct tgsi_full_instruction *inst)
+{
+   int result;
+   union tgsi_exec_channel r[4];
+   uint unit;
+   int i, chan;
+   struct tgsi_buffer_params params;
+   int kilmask = mach->Temps[TEMP_KILMASK_I].xyzw[TEMP_KILMASK_C].u[0];
+
+   unit = fetch_sampler_unit(mach, inst, 0);
+
+   params.execmask = mach->ExecMask & mach->NonHelperMask & ~kilmask;
+   params.unit = unit;
+
+   mach->Buffer->get_dims(mach->Buffer, &params, &result);
+
+   for (i = 0; i < TGSI_QUAD_SIZE; i++) {
+      r[0].i[i] = result;
+   }
+
+   for (chan = 0; chan < TGSI_NUM_CHANNELS; chan++) {
+      if (inst->Dst[0].Register.WriteMask & (1 << chan)) {
+         store_dest(mach, &r[chan], &inst->Dst[0], inst, chan,
+                    TGSI_EXEC_DATA_INT);
+      }
+   }
+}
+
+static void
+exec_resq(struct tgsi_exec_machine *mach,
+          const struct tgsi_full_instruction *inst)
+{
+   if (inst->Src[0].Register.File == TGSI_FILE_IMAGE)
+      exec_resq_img(mach, inst);
+   else
+      exec_resq_buf(mach, inst);
+}
+
+static void
+micro_i2f(union tgsi_exec_channel *dst,
           const union tgsi_exec_channel *src)
 {
    dst->f[0] = (float)src->i[0];
@@ -3642,10 +4761,6 @@ exec_instruction(
       exec_vector_trinary(mach, inst, micro_lrp, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
       break;
 
-   case TGSI_OPCODE_CND:
-      exec_vector_trinary(mach, inst, micro_cnd, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
-      break;
-
    case TGSI_OPCODE_SQRT:
       exec_scalar_unary(mach, inst, micro_sqrt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
       break;
@@ -3715,7 +4830,7 @@ exec_instruction(
       break;
 
    case TGSI_OPCODE_PK2H:
-      assert (0);
+      exec_pk2h(mach, inst);
       break;
 
    case TGSI_OPCODE_PK2US:
@@ -3734,10 +4849,6 @@ exec_instruction(
       exec_vector_binary(mach, inst, micro_seq, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
       break;
 
-   case TGSI_OPCODE_SFL:
-      exec_vector(mach, inst, micro_sfl, TGSI_EXEC_DATA_FLOAT);
-      break;
-
    case TGSI_OPCODE_SGT:
       exec_vector_binary(mach, inst, micro_sgt, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
       break;
@@ -3754,10 +4865,6 @@ exec_instruction(
       exec_vector_binary(mach, inst, micro_sne, TGSI_EXEC_DATA_FLOAT, TGSI_EXEC_DATA_FLOAT);
       break;
 
-   case TGSI_OPCODE_STR:
-      exec_vector(mach, inst, micro_str, TGSI_EXEC_DATA_FLOAT);
-      break;
-
    case TGSI_OPCODE_TEX:
       /* simple texture lookup */
       /* src[0] = texcoord */
@@ -3795,8 +4902,21 @@ exec_instruction(
       exec_tex(mach, inst, TEX_MODIFIER_PROJECTED, 1);
       break;
 
+   case TGSI_OPCODE_TG4:
+      /* src[0] = texcoord */
+      /* src[1] = component */
+      /* src[2] = sampler unit */
+      exec_tex(mach, inst, TEX_MODIFIER_GATHER, 2);
+      break;
+
+   case TGSI_OPCODE_LODQ:
+      /* src[0] = texcoord */
+      /* src[1] = sampler unit */
+      exec_lodq(mach, inst);
+      break;
+
    case TGSI_OPCODE_UP2H:
-      assert (0);
+      exec_up2h(mach, inst);
       break;
 
    case TGSI_OPCODE_UP2US:
@@ -3815,10 +4935,6 @@ exec_instruction(
       exec_vector_unary(mach, inst, micro_arr, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_FLOAT);
       break;
 
-   case TGSI_OPCODE_BRA:
-      assert (0);
-      break;
-
    case TGSI_OPCODE_CAL:
       /* skip the call if no execution channels are enabled */
       if (mach->ExecMask) {
@@ -3856,8 +4972,12 @@ exec_instruction(
          mach->BreakStack[mach->BreakStackTop++] = mach->BreakType;
          mach->FuncStack[mach->FuncStackTop++] = mach->FuncMask;
 
-         /* Finally, jump to the subroutine */
+         /* Finally, jump to the subroutine.  The label is a pointer
+          * (an instruction number) to the BGNSUB instruction.
+          */
          *pc = inst->Label.Label;
+         assert(mach->Instructions[*pc].Instruction.Opcode
+                == TGSI_OPCODE_BGNSUB);
       }
       break;
 
@@ -4298,7 +5418,7 @@ exec_instruction(
       break;
 
    case TGSI_OPCODE_SAMPLE_I_MS:
-      assert(0);
+      exec_txf(mach, inst);
       break;
 
    case TGSI_OPCODE_SAMPLE:
@@ -4403,6 +5523,126 @@ exec_instruction(
    case TGSI_OPCODE_UMSB:
       exec_vector_unary(mach, inst, micro_umsb, TGSI_EXEC_DATA_INT, TGSI_EXEC_DATA_UINT);
       break;
+
+   case TGSI_OPCODE_F2D:
+      exec_f2d(mach, inst);
+      break;
+
+   case TGSI_OPCODE_D2F:
+      exec_d2f(mach, inst);
+      break;
+
+   case TGSI_OPCODE_DABS:
+      exec_double_unary(mach, inst, micro_dabs);
+      break;
+
+   case TGSI_OPCODE_DNEG:
+      exec_double_unary(mach, inst, micro_dneg);
+      break;
+
+   case TGSI_OPCODE_DADD:
+      exec_double_binary(mach, inst, micro_dadd, TGSI_EXEC_DATA_DOUBLE);
+      break;
+
+   case TGSI_OPCODE_DMUL:
+      exec_double_binary(mach, inst, micro_dmul, TGSI_EXEC_DATA_DOUBLE);
+      break;
+
+   case TGSI_OPCODE_DMAX:
+      exec_double_binary(mach, inst, micro_dmax, TGSI_EXEC_DATA_DOUBLE);
+      break;
+
+   case TGSI_OPCODE_DMIN:
+      exec_double_binary(mach, inst, micro_dmin, TGSI_EXEC_DATA_DOUBLE);
+      break;
+
+   case TGSI_OPCODE_DSLT:
+      exec_double_binary(mach, inst, micro_dslt, TGSI_EXEC_DATA_UINT);
+      break;
+
+   case TGSI_OPCODE_DSGE:
+      exec_double_binary(mach, inst, micro_dsge, TGSI_EXEC_DATA_UINT);
+      break;
+
+   case TGSI_OPCODE_DSEQ:
+      exec_double_binary(mach, inst, micro_dseq, TGSI_EXEC_DATA_UINT);
+      break;
+
+   case TGSI_OPCODE_DSNE:
+      exec_double_binary(mach, inst, micro_dsne, TGSI_EXEC_DATA_UINT);
+      break;
+
+   case TGSI_OPCODE_DRCP:
+      exec_double_unary(mach, inst, micro_drcp);
+      break;
+
+   case TGSI_OPCODE_DSQRT:
+      exec_double_unary(mach, inst, micro_dsqrt);
+      break;
+
+   case TGSI_OPCODE_DRSQ:
+      exec_double_unary(mach, inst, micro_drsq);
+      break;
+
+   case TGSI_OPCODE_DMAD:
+      exec_double_trinary(mach, inst, micro_dmad);
+      break;
+
+   case TGSI_OPCODE_DFRAC:
+      exec_double_unary(mach, inst, micro_dfrac);
+      break;
+
+   case TGSI_OPCODE_DLDEXP:
+      exec_dldexp(mach, inst);
+      break;
+
+   case TGSI_OPCODE_DFRACEXP:
+      exec_dfracexp(mach, inst);
+      break;
+
+   case TGSI_OPCODE_I2D:
+      exec_i2d(mach, inst);
+      break;
+
+   case TGSI_OPCODE_D2I:
+      exec_d2i(mach, inst);
+      break;
+
+   case TGSI_OPCODE_U2D:
+      exec_u2d(mach, inst);
+      break;
+
+   case TGSI_OPCODE_D2U:
+      exec_d2u(mach, inst);
+      break;
+
+   case TGSI_OPCODE_LOAD:
+      exec_load(mach, inst);
+      break;
+
+   case TGSI_OPCODE_STORE:
+      exec_store(mach, inst);
+      break;
+
+   case TGSI_OPCODE_ATOMUADD:
+   case TGSI_OPCODE_ATOMXCHG:
+   case TGSI_OPCODE_ATOMCAS:
+   case TGSI_OPCODE_ATOMAND:
+   case TGSI_OPCODE_ATOMOR:
+   case TGSI_OPCODE_ATOMXOR:
+   case TGSI_OPCODE_ATOMUMIN:
+   case TGSI_OPCODE_ATOMUMAX:
+   case TGSI_OPCODE_ATOMIMIN:
+   case TGSI_OPCODE_ATOMIMAX:
+      exec_atomop(mach, inst);
+      break;
+
+   case TGSI_OPCODE_RESQ:
+      exec_resq(mach, inst);
+      break;
+   case TGSI_OPCODE_BARRIER:
+   case TGSI_OPCODE_MEMBAR:
+      break;
    default:
       assert( 0 );
    }
@@ -4430,6 +5670,8 @@ tgsi_exec_machine_run( struct tgsi_exec_machine *mach )
       default_mask = 0x1;
    }
 
+   if (mach->NonHelperMask == 0)
+      mach->NonHelperMask = default_mask;
    mach->CondMask = default_mask;
    mach->LoopMask = default_mask;
    mach->ContMask = default_mask;