mesa: s/mesaFormat/attFormat/
[mesa.git] / src / mesa / program / ir_to_mesa.cpp
index 17ebdbb220bae9b4f4d7a699614eba949c5b4da2..33c262f8ca4cea2833bc838a80f9551020ed0a8d 100644 (file)
@@ -26,8 +26,7 @@
 /**
  * \file ir_to_mesa.cpp
  *
- * Translates the IR to ARB_fragment_program text if possible,
- * printing the result
+ * Translate GLSL IR to Mesa's gl_program representation.
  */
 
 #include <stdio.h>
@@ -54,6 +53,7 @@ extern "C" {
 #include "program/program.h"
 #include "program/prog_uniform.h"
 #include "program/prog_parameter.h"
+#include "program/sampler.h"
 }
 
 static int swizzle_for_size(int size);
@@ -65,7 +65,7 @@ static int swizzle_for_size(int size);
 typedef struct ir_to_mesa_src_reg {
    ir_to_mesa_src_reg(int file, int index, const glsl_type *type)
    {
-      this->file = file;
+      this->file = (gl_register_file) file;
       this->index = index;
       if (type && (type->is_scalar() || type->is_vector() || type->is_matrix()))
         this->swizzle = swizzle_for_size(type->vector_elements);
@@ -84,7 +84,7 @@ typedef struct ir_to_mesa_src_reg {
       this->reladdr = NULL;
    }
 
-   int file; /**< PROGRAM_* from Mesa */
+   gl_register_file file; /**< PROGRAM_* from Mesa */
    int index; /**< temporary index, VERT_ATTRIB_*, FRAG_ATTRIB_*, etc. */
    GLuint swizzle; /**< SWIZZLE_XYZWONEZERO swizzles from Mesa. */
    int negate; /**< NEGATE_XYZW mask from mesa */
@@ -105,12 +105,25 @@ extern ir_to_mesa_src_reg ir_to_mesa_undef;
 
 class ir_to_mesa_instruction : public exec_node {
 public:
+   /* Callers of this ralloc-based new need not call delete. It's
+    * easier to just ralloc_free 'ctx' (or any of its ancestors). */
+   static void* operator new(size_t size, void *ctx)
+   {
+      void *node;
+
+      node = rzalloc_size(ctx, size);
+      assert(node != NULL);
+
+      return node;
+   }
+
    enum prog_opcode op;
    ir_to_mesa_dst_reg dst_reg;
    ir_to_mesa_src_reg src_reg[3];
    /** Pointer to the ir source this tree came from for debugging */
    ir_instruction *ir;
    GLboolean cond_update;
+   bool saturate;
    int sampler; /**< sampler index */
    int tex_target; /**< One of TEXTURE_*_INDEX */
    GLboolean tex_shadow;
@@ -120,13 +133,13 @@ public:
 
 class variable_storage : public exec_node {
 public:
-   variable_storage(ir_variable *var, int file, int index)
+   variable_storage(ir_variable *var, gl_register_file file, int index)
       : file(file), index(index), var(var)
    {
       /* empty */
    }
 
-   int file;
+   gl_register_file file;
    int index;
    ir_variable *var; /* variable that maps to this, if any */
 };
@@ -171,8 +184,10 @@ public:
 
    function_entry *current_function;
 
-   GLcontext *ctx;
+   struct gl_context *ctx;
    struct gl_program *prog;
+   struct gl_shader_program *shader_program;
+   struct gl_shader_compiler_options *options;
 
    int next_temp;
 
@@ -246,6 +261,17 @@ public:
                                               ir_to_mesa_src_reg src1,
                                               ir_to_mesa_src_reg src2);
 
+   /**
+    * Emit the correct dot-product instruction for the type of arguments
+    *
+    * \sa ir_to_mesa_emit_op2
+    */
+   void ir_to_mesa_emit_dp(ir_instruction *ir,
+                          ir_to_mesa_dst_reg dst,
+                          ir_to_mesa_src_reg src0,
+                          ir_to_mesa_src_reg src1,
+                          unsigned elements);
+
    void ir_to_mesa_emit_scalar_op1(ir_instruction *ir,
                                   enum prog_opcode op,
                                   ir_to_mesa_dst_reg dst,
@@ -257,22 +283,19 @@ public:
                                   ir_to_mesa_src_reg src0,
                                   ir_to_mesa_src_reg src1);
 
+   void emit_scs(ir_instruction *ir, enum prog_opcode op,
+                ir_to_mesa_dst_reg dst,
+                const ir_to_mesa_src_reg &src);
+
    GLboolean try_emit_mad(ir_expression *ir,
                          int mul_operand);
+   GLboolean try_emit_sat(ir_expression *ir);
 
-   int add_uniform(const char *name,
-                  const glsl_type *type,
-                  ir_constant *constant);
-   void add_aggregate_uniform(ir_instruction *ir,
-                             const char *name,
-                             const struct glsl_type *type,
-                             ir_constant *constant,
-                             struct ir_to_mesa_dst_reg temp);
+   void emit_swz(ir_expression *ir);
 
-   struct hash_table *sampler_map;
+   bool process_move_condition(ir_rvalue *ir);
 
-   void set_sampler_location(ir_variable *sampler, int location);
-   int get_sampler_location(ir_variable *sampler);
+   void copy_propagate(void);
 
    void *mem_ctx;
 };
@@ -287,7 +310,22 @@ ir_to_mesa_dst_reg ir_to_mesa_address_reg = {
    PROGRAM_ADDRESS, 0, WRITEMASK_X, COND_TR, NULL
 };
 
-static int swizzle_for_size(int size)
+static void
+fail_link(struct gl_shader_program *prog, const char *fmt, ...) PRINTFLIKE(2, 3);
+
+static void
+fail_link(struct gl_shader_program *prog, const char *fmt, ...)
+{
+   va_list args;
+   va_start(args, fmt);
+   ralloc_vasprintf_append(&prog->InfoLog, fmt, args);
+   va_end(args);
+
+   prog->LinkStatus = GL_FALSE;
+}
+
+static int
+swizzle_for_size(int size)
 {
    int size_swizzles[4] = {
       MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X),
@@ -296,6 +334,7 @@ static int swizzle_for_size(int size)
       MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W),
    };
 
+   assert((size >= 1) && (size <= 4));
    return size_swizzles[size - 1];
 }
 
@@ -378,22 +417,18 @@ ir_to_mesa_visitor::ir_to_mesa_emit_op0(ir_instruction *ir,
 }
 
 void
-ir_to_mesa_visitor::set_sampler_location(ir_variable *sampler, int location)
-{
-   if (this->sampler_map == NULL) {
-      this->sampler_map = hash_table_ctor(0, hash_table_pointer_hash,
-                                         hash_table_pointer_compare);
-   }
-
-   hash_table_insert(this->sampler_map, (void *)(uintptr_t)location, sampler);
-}
-
-int
-ir_to_mesa_visitor::get_sampler_location(ir_variable *sampler)
+ir_to_mesa_visitor::ir_to_mesa_emit_dp(ir_instruction *ir,
+                                      ir_to_mesa_dst_reg dst,
+                                      ir_to_mesa_src_reg src0,
+                                      ir_to_mesa_src_reg src1,
+                                      unsigned elements)
 {
-   void *result = hash_table_find(this->sampler_map, sampler);
+   static const gl_inst_opcode dot_opcodes[] = {
+      OPCODE_DP2, OPCODE_DP3, OPCODE_DP4
+   };
 
-   return (int)(uintptr_t)result;
+   ir_to_mesa_emit_op3(ir, dot_opcodes[elements - 2],
+                      dst, src0, src1, ir_to_mesa_undef);
 }
 
 inline ir_to_mesa_dst_reg
@@ -450,6 +485,10 @@ ir_to_mesa_visitor::ir_to_mesa_emit_scalar_op2(ir_instruction *ir,
       GLuint src0_swiz = GET_SWZ(src0.swizzle, i);
       GLuint src1_swiz = GET_SWZ(src1.swizzle, i);
       for (j = i + 1; j < 4; j++) {
+        /* If there is another enabled component in the destination that is
+         * derived from the same inputs, generate its value on this pass as
+         * well.
+         */
         if (!(done_mask & (1 << j)) &&
             GET_SWZ(src0.swizzle, j) == src0_swiz &&
             GET_SWZ(src1.swizzle, j) == src1_swiz) {
@@ -483,6 +522,102 @@ ir_to_mesa_visitor::ir_to_mesa_emit_scalar_op1(ir_instruction *ir,
    ir_to_mesa_emit_scalar_op2(ir, op, dst, src0, undef);
 }
 
+/**
+ * Emit an OPCODE_SCS instruction
+ *
+ * The \c SCS opcode functions a bit differently than the other Mesa (or
+ * ARB_fragment_program) opcodes.  Instead of splatting its result across all
+ * four components of the destination, it writes one value to the \c x
+ * component and another value to the \c y component.
+ *
+ * \param ir        IR instruction being processed
+ * \param op        Either \c OPCODE_SIN or \c OPCODE_COS depending on which
+ *                  value is desired.
+ * \param dst       Destination register
+ * \param src       Source register
+ */
+void
+ir_to_mesa_visitor::emit_scs(ir_instruction *ir, enum prog_opcode op,
+                            ir_to_mesa_dst_reg dst,
+                            const ir_to_mesa_src_reg &src)
+{
+   /* Vertex programs cannot use the SCS opcode.
+    */
+   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB) {
+      ir_to_mesa_emit_scalar_op1(ir, op, dst, src);
+      return;
+   }
+
+   const unsigned component = (op == OPCODE_SIN) ? 0 : 1;
+   const unsigned scs_mask = (1U << component);
+   int done_mask = ~dst.writemask;
+   ir_to_mesa_src_reg tmp;
+
+   assert(op == OPCODE_SIN || op == OPCODE_COS);
+
+   /* If there are compnents in the destination that differ from the component
+    * that will be written by the SCS instrution, we'll need a temporary.
+    */
+   if (scs_mask != unsigned(dst.writemask)) {
+      tmp = get_temp(glsl_type::vec4_type);
+   }
+
+   for (unsigned i = 0; i < 4; i++) {
+      unsigned this_mask = (1U << i);
+      ir_to_mesa_src_reg src0 = src;
+
+      if ((done_mask & this_mask) != 0)
+        continue;
+
+      /* The source swizzle specified which component of the source generates
+       * sine / cosine for the current component in the destination.  The SCS
+       * instruction requires that this value be swizzle to the X component.
+       * Replace the current swizzle with a swizzle that puts the source in
+       * the X component.
+       */
+      unsigned src0_swiz = GET_SWZ(src.swizzle, i);
+
+      src0.swizzle = MAKE_SWIZZLE4(src0_swiz, src0_swiz,
+                                  src0_swiz, src0_swiz);
+      for (unsigned j = i + 1; j < 4; j++) {
+        /* If there is another enabled component in the destination that is
+         * derived from the same inputs, generate its value on this pass as
+         * well.
+         */
+        if (!(done_mask & (1 << j)) &&
+            GET_SWZ(src0.swizzle, j) == src0_swiz) {
+           this_mask |= (1 << j);
+        }
+      }
+
+      if (this_mask != scs_mask) {
+        ir_to_mesa_instruction *inst;
+        ir_to_mesa_dst_reg tmp_dst = ir_to_mesa_dst_reg_from_src(tmp);
+
+        /* Emit the SCS instruction.
+         */
+        inst = ir_to_mesa_emit_op1(ir, OPCODE_SCS, tmp_dst, src0);
+        inst->dst_reg.writemask = scs_mask;
+
+        /* Move the result of the SCS instruction to the desired location in
+         * the destination.
+         */
+        tmp.swizzle = MAKE_SWIZZLE4(component, component,
+                                    component, component);
+        inst = ir_to_mesa_emit_op1(ir, OPCODE_SCS, dst, tmp);
+        inst->dst_reg.writemask = this_mask;
+      } else {
+        /* Emit the SCS instruction to write directly to the destination.
+         */
+        ir_to_mesa_instruction *inst =
+           ir_to_mesa_emit_op1(ir, OPCODE_SCS, dst, src0);
+        inst->dst_reg.writemask = scs_mask;
+      }
+
+      done_mask |= this_mask;
+   }
+}
+
 struct ir_to_mesa_src_reg
 ir_to_mesa_visitor::src_reg_for_float(float val)
 {
@@ -516,6 +651,7 @@ type_size(const struct glsl_type *type)
         return 1;
       }
    case GLSL_TYPE_ARRAY:
+      assert(type->length > 0);
       return type_size(type->fields.array) * type->length;
    case GLSL_TYPE_STRUCT:
       size = 0;
@@ -524,10 +660,10 @@ type_size(const struct glsl_type *type)
       }
       return size;
    case GLSL_TYPE_SAMPLER:
-      /* Samplers take up no register space, since they're baked in at
-       * link time.
+      /* Samplers take up one slot in UNIFORMS[], but they're baked in
+       * at link time.
        */
-      return 0;
+      return 1;
    default:
       assert(0);
       return 0;
@@ -591,19 +727,176 @@ ir_to_mesa_visitor::visit(ir_variable *ir)
 
       fp->OriginUpperLeft = ir->origin_upper_left;
       fp->PixelCenterInteger = ir->pixel_center_integer;
+
+   } else if (strcmp(ir->name, "gl_FragDepth") == 0) {
+      struct gl_fragment_program *fp = (struct gl_fragment_program *)this->prog;
+      switch (ir->depth_layout) {
+      case ir_depth_layout_none:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_NONE;
+        break;
+      case ir_depth_layout_any:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_ANY;
+        break;
+      case ir_depth_layout_greater:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_GREATER;
+        break;
+      case ir_depth_layout_less:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_LESS;
+        break;
+      case ir_depth_layout_unchanged:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_UNCHANGED;
+        break;
+      default:
+        assert(0);
+        break;
+      }
+   }
+
+   if (ir->mode == ir_var_uniform && strncmp(ir->name, "gl_", 3) == 0) {
+      unsigned int i;
+      const struct gl_builtin_uniform_desc *statevar;
+
+      for (i = 0; _mesa_builtin_uniform_desc[i].name; i++) {
+        if (strcmp(ir->name, _mesa_builtin_uniform_desc[i].name) == 0)
+           break;
+      }
+
+      if (!_mesa_builtin_uniform_desc[i].name) {
+        fail_link(this->shader_program,
+                  "Failed to find builtin uniform `%s'\n", ir->name);
+        return;
+      }
+
+      statevar = &_mesa_builtin_uniform_desc[i];
+
+      int array_count;
+      if (ir->type->is_array()) {
+        array_count = ir->type->length;
+      } else {
+        array_count = 1;
+      }
+
+      /* Check if this statevar's setup in the STATE file exactly
+       * matches how we'll want to reference it as a
+       * struct/array/whatever.  If not, then we need to move it into
+       * temporary storage and hope that it'll get copy-propagated
+       * out.
+       */
+      for (i = 0; i < statevar->num_elements; i++) {
+        if (statevar->elements[i].swizzle != SWIZZLE_XYZW) {
+           break;
+        }
+      }
+
+      struct variable_storage *storage;
+      ir_to_mesa_dst_reg dst;
+      if (i == statevar->num_elements) {
+        /* We'll set the index later. */
+        storage = new(mem_ctx) variable_storage(ir, PROGRAM_STATE_VAR, -1);
+        this->variables.push_tail(storage);
+
+        dst = ir_to_mesa_undef_dst;
+      } else {
+        storage = new(mem_ctx) variable_storage(ir, PROGRAM_TEMPORARY,
+                                                this->next_temp);
+        this->variables.push_tail(storage);
+        this->next_temp += type_size(ir->type);
+
+        dst = ir_to_mesa_dst_reg_from_src(ir_to_mesa_src_reg(PROGRAM_TEMPORARY,
+                                                             storage->index,
+                                                             NULL));
+      }
+
+
+      for (int a = 0; a < array_count; a++) {
+        for (unsigned int i = 0; i < statevar->num_elements; i++) {
+           struct gl_builtin_uniform_element *element = &statevar->elements[i];
+           int tokens[STATE_LENGTH];
+
+           memcpy(tokens, element->tokens, sizeof(element->tokens));
+           if (ir->type->is_array()) {
+              tokens[1] = a;
+           }
+
+           int index = _mesa_add_state_reference(this->prog->Parameters,
+                                                 (gl_state_index *)tokens);
+
+           if (storage->file == PROGRAM_STATE_VAR) {
+              if (storage->index == -1) {
+                 storage->index = index;
+              } else {
+                 assert(index ==
+                         (int)(storage->index + a * statevar->num_elements + i));
+              }
+           } else {
+              ir_to_mesa_src_reg src(PROGRAM_STATE_VAR, index, NULL);
+              src.swizzle = element->swizzle;
+              ir_to_mesa_emit_op1(ir, OPCODE_MOV, dst, src);
+              /* even a float takes up a whole vec4 reg in a struct/array. */
+              dst.index++;
+           }
+        }
+      }
+      if (storage->file == PROGRAM_TEMPORARY &&
+         dst.index != storage->index + type_size(ir->type)) {
+        fail_link(this->shader_program,
+                  "failed to load builtin uniform `%s'  (%d/%d regs loaded)\n",
+                  ir->name, dst.index - storage->index,
+                  type_size(ir->type));
+      }
    }
 }
 
 void
 ir_to_mesa_visitor::visit(ir_loop *ir)
 {
-   assert(!ir->from);
-   assert(!ir->to);
-   assert(!ir->increment);
-   assert(!ir->counter);
+   ir_dereference_variable *counter = NULL;
+
+   if (ir->counter != NULL)
+      counter = new(ir) ir_dereference_variable(ir->counter);
+
+   if (ir->from != NULL) {
+      assert(ir->counter != NULL);
+
+      ir_assignment *a = new(ir) ir_assignment(counter, ir->from, NULL);
+
+      a->accept(this);
+      delete a;
+   }
 
    ir_to_mesa_emit_op0(NULL, OPCODE_BGNLOOP);
+
+   if (ir->to) {
+      ir_expression *e =
+        new(ir) ir_expression(ir->cmp, glsl_type::bool_type,
+                              counter, ir->to);
+      ir_if *if_stmt =  new(ir) ir_if(e);
+
+      ir_loop_jump *brk = new(ir) ir_loop_jump(ir_loop_jump::jump_break);
+
+      if_stmt->then_instructions.push_tail(brk);
+
+      if_stmt->accept(this);
+
+      delete if_stmt;
+      delete e;
+      delete brk;
+   }
+
    visit_exec_list(&ir->body_instructions, this);
+
+   if (ir->increment) {
+      ir_expression *e =
+        new(ir) ir_expression(ir_binop_add, counter->type,
+                              counter, ir->increment);
+
+      ir_assignment *a = new(ir) ir_assignment(counter, e, NULL);
+
+      a->accept(this);
+      delete a;
+      delete e;
+   }
+
    ir_to_mesa_emit_op0(NULL, OPCODE_ENDLOOP);
 }
 
@@ -674,6 +967,32 @@ ir_to_mesa_visitor::try_emit_mad(ir_expression *ir, int mul_operand)
    return true;
 }
 
+GLboolean
+ir_to_mesa_visitor::try_emit_sat(ir_expression *ir)
+{
+   /* Saturates were only introduced to vertex programs in
+    * NV_vertex_program3, so don't give them to drivers in the VP.
+    */
+   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB)
+      return false;
+
+   ir_rvalue *sat_src = ir->as_rvalue_to_saturate();
+   if (!sat_src)
+      return false;
+
+   sat_src->accept(this);
+   ir_to_mesa_src_reg src = this->result;
+
+   this->result = get_temp(ir->type);
+   ir_to_mesa_instruction *inst;
+   inst = ir_to_mesa_emit_op1(ir, OPCODE_MOV,
+                             ir_to_mesa_dst_reg_from_src(this->result),
+                             src);
+   inst->saturate = true;
+
+   return true;
+}
+
 void
 ir_to_mesa_visitor::reladdr_to_temp(ir_instruction *ir,
                                    ir_to_mesa_src_reg *reg, int *num_reladdr)
@@ -694,16 +1013,130 @@ ir_to_mesa_visitor::reladdr_to_temp(ir_instruction *ir,
    (*num_reladdr)--;
 }
 
+void
+ir_to_mesa_visitor::emit_swz(ir_expression *ir)
+{
+   /* Assume that the vector operator is in a form compatible with OPCODE_SWZ.
+    * This means that each of the operands is either an immediate value of -1,
+    * 0, or 1, or is a component from one source register (possibly with
+    * negation).
+    */
+   uint8_t components[4] = { 0 };
+   bool negate[4] = { false };
+   ir_variable *var = NULL;
+
+   for (unsigned i = 0; i < ir->type->vector_elements; i++) {
+      ir_rvalue *op = ir->operands[i];
+
+      assert(op->type->is_scalar());
+
+      while (op != NULL) {
+        switch (op->ir_type) {
+        case ir_type_constant: {
+
+           assert(op->type->is_scalar());
+
+           const ir_constant *const c = op->as_constant();
+           if (c->is_one()) {
+              components[i] = SWIZZLE_ONE;
+           } else if (c->is_zero()) {
+              components[i] = SWIZZLE_ZERO;
+           } else if (c->is_negative_one()) {
+              components[i] = SWIZZLE_ONE;
+              negate[i] = true;
+           } else {
+              assert(!"SWZ constant must be 0.0 or 1.0.");
+           }
+
+           op = NULL;
+           break;
+        }
+
+        case ir_type_dereference_variable: {
+           ir_dereference_variable *const deref =
+              (ir_dereference_variable *) op;
+
+           assert((var == NULL) || (deref->var == var));
+           components[i] = SWIZZLE_X;
+           var = deref->var;
+           op = NULL;
+           break;
+        }
+
+        case ir_type_expression: {
+           ir_expression *const expr = (ir_expression *) op;
+
+           assert(expr->operation == ir_unop_neg);
+           negate[i] = true;
+
+           op = expr->operands[0];
+           break;
+        }
+
+        case ir_type_swizzle: {
+           ir_swizzle *const swiz = (ir_swizzle *) op;
+
+           components[i] = swiz->mask.x;
+           op = swiz->val;
+           break;
+        }
+
+        default:
+           assert(!"Should not get here.");
+           return;
+        }
+      }
+   }
+
+   assert(var != NULL);
+
+   ir_dereference_variable *const deref =
+      new(mem_ctx) ir_dereference_variable(var);
+
+   this->result.file = PROGRAM_UNDEFINED;
+   deref->accept(this);
+   if (this->result.file == PROGRAM_UNDEFINED) {
+      ir_print_visitor v;
+      printf("Failed to get tree for expression operand:\n");
+      deref->accept(&v);
+      exit(1);
+   }
+
+   ir_to_mesa_src_reg src;
+
+   src = this->result;
+   src.swizzle = MAKE_SWIZZLE4(components[0],
+                              components[1],
+                              components[2],
+                              components[3]);
+   src.negate = ((unsigned(negate[0]) << 0)
+                | (unsigned(negate[1]) << 1)
+                | (unsigned(negate[2]) << 2)
+                | (unsigned(negate[3]) << 3));
+
+   /* Storage for our result.  Ideally for an assignment we'd be using the
+    * actual storage for the result here, instead.
+    */
+   const ir_to_mesa_src_reg result_src = get_temp(ir->type);
+   ir_to_mesa_dst_reg result_dst = ir_to_mesa_dst_reg_from_src(result_src);
+
+   /* Limit writes to the channels that will be used by result_src later.
+    * This does limit this temp's use as a temporary for multi-instruction
+    * sequences.
+    */
+   result_dst.writemask = (1 << ir->type->vector_elements) - 1;
+
+   ir_to_mesa_emit_op1(ir, OPCODE_SWZ, result_dst, src);
+   this->result = result_src;
+}
+
 void
 ir_to_mesa_visitor::visit(ir_expression *ir)
 {
    unsigned int operand;
-   struct ir_to_mesa_src_reg op[2];
+   struct ir_to_mesa_src_reg op[Elements(ir->operands)];
    struct ir_to_mesa_src_reg result_src;
    struct ir_to_mesa_dst_reg result_dst;
-   const glsl_type *vec4_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 4, 1);
-   const glsl_type *vec3_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 3, 1);
-   const glsl_type *vec2_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 2, 1);
 
    /* Quick peephole: Emit OPCODE_MAD(a, b, c) instead of ADD(MUL(a, b), c)
     */
@@ -713,6 +1146,13 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       if (try_emit_mad(ir, 0))
         return;
    }
+   if (try_emit_sat(ir))
+      return;
+
+   if (ir->operation == ir_quadop_vector) {
+      this->emit_swz(ir);
+      return;
+   }
 
    for (operand = 0; operand < ir->get_num_operands(); operand++) {
       this->result.file = PROGRAM_UNDEFINED;
@@ -731,6 +1171,12 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       assert(!ir->operands[operand]->type->is_matrix());
    }
 
+   int vector_elements = ir->operands[0]->type->vector_elements;
+   if (ir->operands[1]) {
+      vector_elements = MAX2(vector_elements,
+                            ir->operands[1]->type->vector_elements);
+   }
+
    this->result.file = PROGRAM_UNDEFINED;
 
    /* Storage for our result.  Ideally for an assignment we'd be using
@@ -780,6 +1226,12 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
    case ir_unop_cos:
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_COS, result_dst, op[0]);
       break;
+   case ir_unop_sin_reduced:
+      emit_scs(ir, OPCODE_SIN, result_dst, op[0]);
+      break;
+   case ir_unop_cos_reduced:
+      emit_scs(ir, OPCODE_COS, result_dst, op[0]);
+      break;
 
    case ir_unop_dFdx:
       ir_to_mesa_emit_op1(ir, OPCODE_DDX, result_dst, op[0]);
@@ -788,6 +1240,16 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       ir_to_mesa_emit_op1(ir, OPCODE_DDY, result_dst, op[0]);
       break;
 
+   case ir_unop_noise: {
+      const enum prog_opcode opcode =
+        prog_opcode(OPCODE_NOISE1
+                    + (ir->operands[0]->type->vector_elements) - 1);
+      assert((opcode >= OPCODE_NOISE1) && (opcode <= OPCODE_NOISE4));
+
+      ir_to_mesa_emit_op1(ir, opcode, result_dst, op[0]);
+      break;
+   }
+
    case ir_binop_add:
       ir_to_mesa_emit_op2(ir, OPCODE_ADD, result_dst, op[0], op[1]);
       break;
@@ -817,33 +1279,48 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       ir_to_mesa_emit_op2(ir, OPCODE_SGE, result_dst, op[0], op[1]);
       break;
    case ir_binop_equal:
+      ir_to_mesa_emit_op2(ir, OPCODE_SEQ, result_dst, op[0], op[1]);
+      break;
+   case ir_binop_nequal:
+      ir_to_mesa_emit_op2(ir, OPCODE_SNE, result_dst, op[0], op[1]);
+      break;
+   case ir_binop_all_equal:
       /* "==" operator producing a scalar boolean. */
       if (ir->operands[0]->type->is_vector() ||
          ir->operands[1]->type->is_vector()) {
         ir_to_mesa_src_reg temp = get_temp(glsl_type::vec4_type);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             ir_to_mesa_dst_reg_from_src(temp), op[0], op[1]);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4, result_dst, temp, temp);
+        ir_to_mesa_emit_dp(ir, result_dst, temp, temp, vector_elements);
         ir_to_mesa_emit_op2(ir, OPCODE_SEQ,
                             result_dst, result_src, src_reg_for_float(0.0));
       } else {
         ir_to_mesa_emit_op2(ir, OPCODE_SEQ, result_dst, op[0], op[1]);
       }
       break;
-   case ir_binop_nequal:
+   case ir_binop_any_nequal:
       /* "!=" operator producing a scalar boolean. */
       if (ir->operands[0]->type->is_vector() ||
          ir->operands[1]->type->is_vector()) {
         ir_to_mesa_src_reg temp = get_temp(glsl_type::vec4_type);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             ir_to_mesa_dst_reg_from_src(temp), op[0], op[1]);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4, result_dst, temp, temp);
+        ir_to_mesa_emit_dp(ir, result_dst, temp, temp, vector_elements);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             result_dst, result_src, src_reg_for_float(0.0));
       } else {
         ir_to_mesa_emit_op2(ir, OPCODE_SNE, result_dst, op[0], op[1]);
       }
       break;
+
+   case ir_unop_any:
+      assert(ir->operands[0]->type->is_vector());
+      ir_to_mesa_emit_dp(ir, result_dst, op[0], op[0],
+                        ir->operands[0]->type->vector_elements);
+      ir_to_mesa_emit_op2(ir, OPCODE_SNE,
+                         result_dst, result_src, src_reg_for_float(0.0));
+      break;
+
    case ir_binop_logic_xor:
       ir_to_mesa_emit_op2(ir, OPCODE_SNE, result_dst, op[0], op[1]);
       break;
@@ -867,34 +1344,20 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       break;
 
    case ir_binop_dot:
-      if (ir->operands[0]->type == vec4_type) {
-        assert(ir->operands[1]->type == vec4_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4,
-                            result_dst,
-                            op[0], op[1]);
-      } else if (ir->operands[0]->type == vec3_type) {
-        assert(ir->operands[1]->type == vec3_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP3,
-                            result_dst,
-                            op[0], op[1]);
-      } else if (ir->operands[0]->type == vec2_type) {
-        assert(ir->operands[1]->type == vec2_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP2,
-                            result_dst,
-                            op[0], op[1]);
-      }
-      break;
-
-   case ir_binop_cross:
-      ir_to_mesa_emit_op2(ir, OPCODE_XPD, result_dst, op[0], op[1]);
+      assert(ir->operands[0]->type->is_vector());
+      assert(ir->operands[0]->type == ir->operands[1]->type);
+      ir_to_mesa_emit_dp(ir, result_dst, op[0], op[1],
+                        ir->operands[0]->type->vector_elements);
       break;
 
    case ir_unop_sqrt:
+      /* sqrt(x) = x * rsq(x). */
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_RSQ, result_dst, op[0]);
-      ir_to_mesa_emit_scalar_op1(ir, OPCODE_RCP, result_dst, result_src);
-      /* For incoming channels < 0, set the result to 0. */
+      ir_to_mesa_emit_op2(ir, OPCODE_MUL, result_dst, result_src, op[0]);
+      /* For incoming channels <= 0, set the result to 0. */
+      op[0].negate = ~op[0].negate;
       ir_to_mesa_emit_op3(ir, OPCODE_CMP, result_dst,
-                         op[0], src_reg_for_float(0.0), result_src);
+                         op[0], result_src, src_reg_for_float(0.0));
       break;
    case ir_unop_rsq:
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_RSQ, result_dst, op[0]);
@@ -911,7 +1374,7 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
    case ir_unop_f2b:
    case ir_unop_i2b:
       ir_to_mesa_emit_op2(ir, OPCODE_SNE, result_dst,
-                         result_src, src_reg_for_float(0.0));
+                         op[0], src_reg_for_float(0.0));
       break;
    case ir_unop_trunc:
       ir_to_mesa_emit_op1(ir, OPCODE_TRUNC, result_dst, op[0]);
@@ -945,8 +1408,15 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
    case ir_binop_bit_and:
    case ir_binop_bit_xor:
    case ir_binop_bit_or:
+   case ir_unop_round_even:
       assert(!"GLSL 1.30 features unsupported");
       break;
+
+   case ir_quadop_vector:
+      /* This operation should have already been handled.
+       */
+      assert(!"Should not get here.");
+      break;
    }
 
    this->result = result_src;
@@ -1001,489 +1471,20 @@ ir_to_mesa_visitor::visit(ir_swizzle *ir)
    this->result = src_reg;
 }
 
-static const struct {
-   const char *name;
-   const char *field;
-   int tokens[STATE_LENGTH];
-   int swizzle;
-   bool array_indexed;
-} statevars[] = {
-   {"gl_DepthRange", "near",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_XXXX, false},
-   {"gl_DepthRange", "far",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_YYYY, false},
-   {"gl_DepthRange", "diff",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_ZZZZ, false},
-
-   {"gl_ClipPlane", NULL,
-    {STATE_CLIPPLANE, 0, 0}, SWIZZLE_XYZW, true}
-,
-   {"gl_Point", "size",
-    {STATE_POINT_SIZE}, SWIZZLE_XXXX, false},
-   {"gl_Point", "sizeMin",
-    {STATE_POINT_SIZE}, SWIZZLE_YYYY, false},
-   {"gl_Point", "sizeMax",
-    {STATE_POINT_SIZE}, SWIZZLE_ZZZZ, false},
-   {"gl_Point", "fadeThresholdSize",
-    {STATE_POINT_SIZE}, SWIZZLE_WWWW, false},
-   {"gl_Point", "distanceConstantAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_XXXX, false},
-   {"gl_Point", "distanceLinearAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_YYYY, false},
-   {"gl_Point", "distanceQuadraticAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_ZZZZ, false},
-
-   {"gl_FrontMaterial", "emission",
-    {STATE_MATERIAL, 0, STATE_EMISSION}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "ambient",
-    {STATE_MATERIAL, 0, STATE_AMBIENT}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "diffuse",
-    {STATE_MATERIAL, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "specular",
-    {STATE_MATERIAL, 0, STATE_SPECULAR}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "shininess",
-    {STATE_MATERIAL, 0, STATE_SHININESS}, SWIZZLE_XXXX, false},
-
-   {"gl_BackMaterial", "emission",
-    {STATE_MATERIAL, 1, STATE_EMISSION}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "ambient",
-    {STATE_MATERIAL, 1, STATE_AMBIENT}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "diffuse",
-    {STATE_MATERIAL, 1, STATE_DIFFUSE}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "specular",
-    {STATE_MATERIAL, 1, STATE_SPECULAR}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "shininess",
-    {STATE_MATERIAL, 1, STATE_SHININESS}, SWIZZLE_XXXX, false},
-
-   {"gl_LightSource", "ambient",
-    {STATE_LIGHT, 0, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "diffuse",
-    {STATE_LIGHT, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "specular",
-    {STATE_LIGHT, 0, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "position",
-    {STATE_LIGHT, 0, STATE_POSITION}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "halfVector",
-    {STATE_LIGHT, 0, STATE_HALF_VECTOR}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "spotDirection",
-    {STATE_LIGHT, 0, STATE_SPOT_DIRECTION}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "spotCosCutoff",
-    {STATE_LIGHT, 0, STATE_SPOT_DIRECTION}, SWIZZLE_WWWW, true},
-   {"gl_LightSource", "spotCutoff",
-    {STATE_LIGHT, 0, STATE_SPOT_CUTOFF}, SWIZZLE_XXXX, true},
-   {"gl_LightSource", "spotExponent",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_WWWW, true},
-   {"gl_LightSource", "constantAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_XXXX, true},
-   {"gl_LightSource", "linearAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_YYYY, true},
-   {"gl_LightSource", "quadraticAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_ZZZZ, true},
-
-   {"gl_LightModel", "ambient",
-    {STATE_LIGHTMODEL_AMBIENT, 0}, SWIZZLE_XYZW, false},
-
-   {"gl_FrontLightModelProduct", "sceneColor",
-    {STATE_LIGHTMODEL_SCENECOLOR, 0}, SWIZZLE_XYZW, false},
-   {"gl_BackLightModelProduct", "sceneColor",
-    {STATE_LIGHTMODEL_SCENECOLOR, 1}, SWIZZLE_XYZW, false},
-
-   {"gl_FrontLightProduct", "ambient",
-    {STATE_LIGHTPROD, 0, 0, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_FrontLightProduct", "diffuse",
-    {STATE_LIGHTPROD, 0, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_FrontLightProduct", "specular",
-    {STATE_LIGHTPROD, 0, 0, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-
-   {"gl_BackLightProduct", "ambient",
-    {STATE_LIGHTPROD, 0, 1, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_BackLightProduct", "diffuse",
-    {STATE_LIGHTPROD, 0, 1, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_BackLightProduct", "specular",
-    {STATE_LIGHTPROD, 0, 1, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-
-   {"gl_TextureEnvColor", NULL,
-    {STATE_TEXENV_COLOR, 0}, SWIZZLE_XYZW, true},
-
-   {"gl_EyePlaneS", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_S}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneT", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_T}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneR", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_R}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneQ", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_Q}, SWIZZLE_XYZW, true},
-
-   {"gl_ObjectPlaneS", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_S}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneT", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_T}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneR", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_R}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneQ", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_Q}, SWIZZLE_XYZW, true},
-
-   {"gl_Fog", "color",
-    {STATE_FOG_COLOR}, SWIZZLE_XYZW, false},
-   {"gl_Fog", "density",
-    {STATE_FOG_PARAMS}, SWIZZLE_XXXX, false},
-   {"gl_Fog", "start",
-    {STATE_FOG_PARAMS}, SWIZZLE_YYYY, false},
-   {"gl_Fog", "end",
-    {STATE_FOG_PARAMS}, SWIZZLE_ZZZZ, false},
-   {"gl_Fog", "scale",
-    {STATE_FOG_PARAMS}, SWIZZLE_WWWW, false},
-};
-
-static ir_to_mesa_src_reg
-get_builtin_uniform_reg(struct gl_program *prog,
-                       const char *name, int array_index, const char *field)
-{
-   unsigned int i;
-   ir_to_mesa_src_reg src_reg;
-   int tokens[STATE_LENGTH];
-
-   for (i = 0; i < Elements(statevars); i++) {
-      if (strcmp(statevars[i].name, name) != 0)
-        continue;
-      if (!field && statevars[i].field) {
-        assert(!"FINISHME: whole-structure state var dereference");
-      }
-      if (field && (!statevars[i].field || strcmp(statevars[i].field, field) != 0))
-        continue;
-      break;
-   }
-
-   if (i ==  Elements(statevars)) {
-      printf("builtin uniform %s%s%s not found\n",
-            name,
-            field ? "." : "",
-            field ? field : "");
-      abort();
-   }
-
-   memcpy(&tokens, statevars[i].tokens, sizeof(tokens));
-   if (statevars[i].array_indexed)
-      tokens[1] = array_index;
-
-   src_reg.file = PROGRAM_STATE_VAR;
-   src_reg.index = _mesa_add_state_reference(prog->Parameters,
-                                            (gl_state_index *)tokens);
-   src_reg.swizzle = statevars[i].swizzle;
-   src_reg.negate = 0;
-   src_reg.reladdr = false;
-
-   return src_reg;
-}
-
-static int
-add_matrix_ref(struct gl_program *prog, int *tokens)
-{
-   int base_pos = -1;
-   int i;
-
-   /* Add a ref for each column.  It looks like the reason we do
-    * it this way is that _mesa_add_state_reference doesn't work
-    * for things that aren't vec4s, so the tokens[2]/tokens[3]
-    * range has to be equal.
-    */
-   for (i = 0; i < 4; i++) {
-      tokens[2] = i;
-      tokens[3] = i;
-      int pos = _mesa_add_state_reference(prog->Parameters,
-                                         (gl_state_index *)tokens);
-      if (base_pos == -1)
-        base_pos = pos;
-      else
-        assert(base_pos + i == pos);
-   }
-
-   return base_pos;
-}
-
-static variable_storage *
-get_builtin_matrix_ref(void *mem_ctx, struct gl_program *prog, ir_variable *var,
-                      ir_rvalue *array_index)
-{
-   /*
-    * NOTE: The ARB_vertex_program extension specified that matrices get
-    * loaded in registers in row-major order.  With GLSL, we want column-
-    * major order.  So, we need to transpose all matrices here...
-    */
-   static const struct {
-      const char *name;
-      int matrix;
-      int modifier;
-   } matrices[] = {
-      { "gl_ModelViewMatrix", STATE_MODELVIEW_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ModelViewMatrixInverse", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ModelViewMatrixTranspose", STATE_MODELVIEW_MATRIX, 0 },
-      { "gl_ModelViewMatrixInverseTranspose", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_ProjectionMatrix", STATE_PROJECTION_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ProjectionMatrixInverse", STATE_PROJECTION_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ProjectionMatrixTranspose", STATE_PROJECTION_MATRIX, 0 },
-      { "gl_ProjectionMatrixInverseTranspose", STATE_PROJECTION_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_ModelViewProjectionMatrix", STATE_MVP_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ModelViewProjectionMatrixInverse", STATE_MVP_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ModelViewProjectionMatrixTranspose", STATE_MVP_MATRIX, 0 },
-      { "gl_ModelViewProjectionMatrixInverseTranspose", STATE_MVP_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_TextureMatrix", STATE_TEXTURE_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_TextureMatrixInverse", STATE_TEXTURE_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_TextureMatrixTranspose", STATE_TEXTURE_MATRIX, 0 },
-      { "gl_TextureMatrixInverseTranspose", STATE_TEXTURE_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_NormalMatrix", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVERSE },
-
-   };
-   unsigned int i;
-   variable_storage *entry;
-
-   /* C++ gets angry when we try to use an int as a gl_state_index, so we use
-    * ints for gl_state_index.  Make sure they're compatible.
-    */
-   assert(sizeof(gl_state_index) == sizeof(int));
-
-   for (i = 0; i < Elements(matrices); i++) {
-      if (strcmp(var->name, matrices[i].name) == 0) {
-        int tokens[STATE_LENGTH];
-        int base_pos = -1;
-
-        tokens[0] = matrices[i].matrix;
-        tokens[4] = matrices[i].modifier;
-        if (matrices[i].matrix == STATE_TEXTURE_MATRIX) {
-           ir_constant *index = array_index->constant_expression_value();
-           if (index) {
-              tokens[1] = index->value.i[0];
-              base_pos = add_matrix_ref(prog, tokens);
-           } else {
-              for (i = 0; i < var->type->length; i++) {
-                 tokens[1] = i;
-                 int pos = add_matrix_ref(prog, tokens);
-                 if (base_pos == -1)
-                    base_pos = pos;
-                 else
-                    assert(base_pos + (int)i * 4 == pos);
-              }
-           }
-        } else {
-           tokens[1] = 0; /* unused array index */
-           base_pos = add_matrix_ref(prog, tokens);
-        }
-        tokens[4] = matrices[i].modifier;
-
-        entry = new(mem_ctx) variable_storage(var,
-                                              PROGRAM_STATE_VAR,
-                                              base_pos);
-
-        return entry;
-      }
-   }
-
-   return NULL;
-}
-
-int
-ir_to_mesa_visitor::add_uniform(const char *name,
-                               const glsl_type *type,
-                               ir_constant *constant)
-{
-   int len;
-
-   if (type->is_vector() ||
-       type->is_scalar()) {
-      len = type->vector_elements;
-   } else {
-      len = type_size(type) * 4;
-   }
-
-   float *values = NULL;
-   if (constant && type->is_array()) {
-      values = (float *)malloc(type->length * 4 * sizeof(float));
-
-      assert(type->fields.array->is_scalar() ||
-            type->fields.array->is_vector() ||
-            !"FINISHME: uniform array initializers for non-vector");
-
-      for (unsigned int i = 0; i < type->length; i++) {
-        ir_constant *element = constant->array_elements[i];
-        unsigned int c;
-
-        for (c = 0; c < type->fields.array->vector_elements; c++) {
-           switch (type->fields.array->base_type) {
-           case GLSL_TYPE_FLOAT:
-              values[4 * i + c] = element->value.f[c];
-              break;
-           case GLSL_TYPE_INT:
-              values[4 * i + c] = element->value.i[c];
-              break;
-           case GLSL_TYPE_UINT:
-              values[4 * i + c] = element->value.u[c];
-              break;
-           case GLSL_TYPE_BOOL:
-              values[4 * i + c] = element->value.b[c];
-              break;
-           default:
-              assert(!"not reached");
-           }
-        }
-      }
-   } else if (constant) {
-      values = (float *)malloc(16 * sizeof(float));
-      for (unsigned int i = 0; i < type->components(); i++) {
-        switch (type->base_type) {
-        case GLSL_TYPE_FLOAT:
-           values[i] = constant->value.f[i];
-           break;
-        case GLSL_TYPE_INT:
-           values[i] = constant->value.i[i];
-           break;
-        case GLSL_TYPE_UINT:
-           values[i] = constant->value.u[i];
-           break;
-        case GLSL_TYPE_BOOL:
-           values[i] = constant->value.b[i];
-           break;
-        default:
-           assert(!"not reached");
-        }
-      }
-   }
-
-   int loc = _mesa_add_uniform(this->prog->Parameters,
-                              name,
-                              len,
-                              type->gl_type,
-                              values);
-   free(values);
-
-   return loc;
-}
-
-/* Recursively add all the members of the aggregate uniform as uniform names
- * to Mesa, moving those uniforms to our structured temporary.
- */
-void
-ir_to_mesa_visitor::add_aggregate_uniform(ir_instruction *ir,
-                                         const char *name,
-                                         const struct glsl_type *type,
-                                         ir_constant *constant,
-                                         struct ir_to_mesa_dst_reg temp)
-{
-   int loc;
-
-   if (type->is_record()) {
-      void *mem_ctx = talloc_new(NULL);
-      ir_constant *field_constant = NULL;
-
-      if (constant)
-        field_constant = (ir_constant *)constant->components.get_head();
-
-      for (unsigned int i = 0; i < type->length; i++) {
-        const glsl_type *field_type = type->fields.structure[i].type;
-
-        add_aggregate_uniform(ir,
-                              talloc_asprintf(mem_ctx, "%s.%s", name,
-                                              type->fields.structure[i].name),
-                              field_type, field_constant, temp);
-        temp.index += type_size(field_type);
-
-        if (constant)
-           field_constant = (ir_constant *)field_constant->next;
-      }
-
-      talloc_free(mem_ctx);
-
-      return;
-   }
-
-   assert(type->is_vector() || type->is_scalar() || !"FINISHME: other types");
-
-   loc = add_uniform(name, type, constant);
-
-   ir_to_mesa_src_reg uniform(PROGRAM_UNIFORM, loc, type);
-
-   for (int i = 0; i < type_size(type); i++) {
-      ir_to_mesa_emit_op1(ir, OPCODE_MOV, temp, uniform);
-      temp.index++;
-      uniform.index++;
-   }
-}
-
-
 void
 ir_to_mesa_visitor::visit(ir_dereference_variable *ir)
 {
    variable_storage *entry = find_variable_storage(ir->var);
-   unsigned int loc;
+   ir_variable *var = ir->var;
 
    if (!entry) {
-      switch (ir->var->mode) {
+      switch (var->mode) {
       case ir_var_uniform:
-        entry = get_builtin_matrix_ref(this->mem_ctx, this->prog, ir->var,
-                                       NULL);
-        if (entry)
-           break;
-
-        /* FINISHME: Fix up uniform name for arrays and things */
-        if (ir->var->type->base_type == GLSL_TYPE_SAMPLER ||
-            (ir->var->type->base_type == GLSL_TYPE_ARRAY &&
-             ir->var->type->fields.array->base_type == GLSL_TYPE_SAMPLER)) {
-           int array_length;
-
-           if (ir->var->type->base_type == GLSL_TYPE_ARRAY)
-              array_length = ir->var->type->length;
-           else
-              array_length = 1;
-           int sampler = _mesa_add_sampler(this->prog->Parameters,
-                                           ir->var->name,
-                                           ir->var->type->gl_type,
-                                           array_length);
-           set_sampler_location(ir->var, sampler);
-
-           entry = new(mem_ctx) variable_storage(ir->var, PROGRAM_SAMPLER,
-                                                 sampler);
-           this->variables.push_tail(entry);
-           break;
-        }
-
-        assert(ir->var->type->gl_type != 0 &&
-               ir->var->type->gl_type != GL_INVALID_ENUM);
-
-        /* Oh, the joy of aggregate types in Mesa.  Like constants,
-         * we can only really do vec4s.  So, make a temp, chop the
-         * aggregate up into vec4s, and move those vec4s to the temp.
-         */
-        if (ir->var->type->is_record()) {
-           ir_to_mesa_src_reg temp = get_temp(ir->var->type);
-
-           entry = new(mem_ctx) variable_storage(ir->var,
-                                                 temp.file,
-                                                 temp.index);
-           this->variables.push_tail(entry);
-
-           add_aggregate_uniform(ir->var, ir->var->name, ir->var->type,
-                                 ir->var->constant_value,
-                                 ir_to_mesa_dst_reg_from_src(temp));
-           break;
-        }
-
-        loc = add_uniform(ir->var->name,
-                          ir->var->type,
-                          ir->var->constant_value);
-
-        /* Always mark the uniform used at this point.  If it isn't
-         * used, dead code elimination should have nuked the decl already.
-         */
-        this->prog->Parameters->Parameters[loc].Used = GL_TRUE;
-
-        entry = new(mem_ctx) variable_storage(ir->var, PROGRAM_UNIFORM, loc);
+        entry = new(mem_ctx) variable_storage(var, PROGRAM_UNIFORM,
+                                              var->location);
         this->variables.push_tail(entry);
         break;
       case ir_var_in:
-      case ir_var_out:
       case ir_var_inout:
         /* The linker assigns locations for varyings and attributes,
          * including deprecated builtins (like gl_Color), user-assign
@@ -1492,101 +1493,58 @@ ir_to_mesa_visitor::visit(ir_dereference_variable *ir)
          *
          * FINISHME: We would hit this path for function arguments.  Fix!
          */
-        assert(ir->var->location != -1);
-        if (ir->var->mode == ir_var_in ||
-            ir->var->mode == ir_var_inout) {
-           entry = new(mem_ctx) variable_storage(ir->var,
-                                                 PROGRAM_INPUT,
-                                                 ir->var->location);
-
-           if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
-               ir->var->location >= VERT_ATTRIB_GENERIC0) {
-              _mesa_add_attribute(prog->Attributes,
-                                  ir->var->name,
-                                  _mesa_sizeof_glsl_type(ir->var->type->gl_type),
-                                  ir->var->type->gl_type,
-                                  ir->var->location - VERT_ATTRIB_GENERIC0);
-           }
-        } else {
-           entry = new(mem_ctx) variable_storage(ir->var,
-                                                 PROGRAM_OUTPUT,
-                                                 ir->var->location);
-        }
-
+        assert(var->location != -1);
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_INPUT,
+                                               var->location);
+         if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
+             var->location >= VERT_ATTRIB_GENERIC0) {
+            _mesa_add_attribute(this->prog->Attributes,
+                                var->name,
+                                _mesa_sizeof_glsl_type(var->type->gl_type),
+                                var->type->gl_type,
+                                var->location - VERT_ATTRIB_GENERIC0);
+         }
+         break;
+      case ir_var_out:
+        assert(var->location != -1);
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_OUTPUT,
+                                               var->location);
         break;
+      case ir_var_system_value:
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_SYSTEM_VALUE,
+                                               var->location);
+         break;
       case ir_var_auto:
       case ir_var_temporary:
-        entry = new(mem_ctx) variable_storage(ir->var, PROGRAM_TEMPORARY,
+        entry = new(mem_ctx) variable_storage(var, PROGRAM_TEMPORARY,
                                               this->next_temp);
         this->variables.push_tail(entry);
 
-        next_temp += type_size(ir->var->type);
+        next_temp += type_size(var->type);
         break;
       }
 
       if (!entry) {
-        printf("Failed to make storage for %s\n", ir->var->name);
+        printf("Failed to make storage for %s\n", var->name);
         exit(1);
       }
    }
 
-   this->result = ir_to_mesa_src_reg(entry->file, entry->index, ir->var->type);
+   this->result = ir_to_mesa_src_reg(entry->file, entry->index, var->type);
 }
 
 void
 ir_to_mesa_visitor::visit(ir_dereference_array *ir)
 {
-   ir_variable *var = ir->variable_referenced();
    ir_constant *index;
    ir_to_mesa_src_reg src_reg;
-   ir_dereference_variable *deref_var = ir->array->as_dereference_variable();
    int element_size = type_size(ir->type);
 
    index = ir->array_index->constant_expression_value();
 
-   if (deref_var && strncmp(deref_var->var->name,
-                           "gl_TextureMatrix",
-                           strlen("gl_TextureMatrix")) == 0) {
-      struct variable_storage *entry;
-
-      entry = get_builtin_matrix_ref(this->mem_ctx, this->prog, deref_var->var,
-                                    ir->array_index);
-      assert(entry);
-
-      ir_to_mesa_src_reg src_reg(entry->file, entry->index, ir->type);
-
-      if (index) {
-        src_reg.reladdr = NULL;
-      } else {
-        ir_to_mesa_src_reg index_reg = get_temp(glsl_type::float_type);
-
-        ir->array_index->accept(this);
-        ir_to_mesa_emit_op2(ir, OPCODE_MUL,
-                            ir_to_mesa_dst_reg_from_src(index_reg),
-                            this->result, src_reg_for_float(element_size));
-
-        src_reg.reladdr = talloc(mem_ctx, ir_to_mesa_src_reg);
-        memcpy(src_reg.reladdr, &index_reg, sizeof(index_reg));
-      }
-
-      this->result = src_reg;
-      return;
-   }
-
-   if (strncmp(var->name, "gl_", 3) == 0 && var->mode == ir_var_uniform &&
-       !var->type->is_matrix()) {
-      ir_dereference_record *record = NULL;
-      if (ir->array->ir_type == ir_type_dereference_record)
-        record = (ir_dereference_record *)ir->array;
-
-      assert(index || !"FINISHME: variable-indexed builtin uniform access");
-
-      this->result = get_builtin_uniform_reg(prog,
-                                            var->name,
-                                            index->value.i[0],
-                                            record ? record->field : NULL);
-   }
-
    ir->array->accept(this);
    src_reg = this->result;
 
@@ -1612,7 +1570,7 @@ ir_to_mesa_visitor::visit(ir_dereference_array *ir)
                             this->result, src_reg_for_float(element_size));
       }
 
-      src_reg.reladdr = talloc(mem_ctx, ir_to_mesa_src_reg);
+      src_reg.reladdr = ralloc(mem_ctx, ir_to_mesa_src_reg);
       memcpy(src_reg.reladdr, &index_reg, sizeof(index_reg));
    }
 
@@ -1631,17 +1589,6 @@ ir_to_mesa_visitor::visit(ir_dereference_record *ir)
    unsigned int i;
    const glsl_type *struct_type = ir->record->type;
    int offset = 0;
-   ir_variable *var = ir->record->variable_referenced();
-
-   if (strncmp(var->name, "gl_", 3) == 0 && var->mode == ir_var_uniform) {
-      assert(var);
-
-      this->result = get_builtin_uniform_reg(prog,
-                                            var->name,
-                                            0,
-                                            ir->field);
-      return;
-   }
 
    ir->record->accept(this);
 
@@ -1650,7 +1597,13 @@ ir_to_mesa_visitor::visit(ir_dereference_record *ir)
         break;
       offset += type_size(struct_type->fields.structure[i].type);
    }
-   this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
+
+   /* If the type is smaller than a vec4, replicate the last channel out. */
+   if (ir->type->is_scalar() || ir->type->is_vector())
+      this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
+   else
+      this->result.swizzle = SWIZZLE_NOOP;
+
    this->result.index += offset;
 }
 
@@ -1679,6 +1632,93 @@ get_assignment_lhs(ir_dereference *ir, ir_to_mesa_visitor *v)
    return ir_to_mesa_dst_reg_from_src(v->result);
 }
 
+/**
+ * Process the condition of a conditional assignment
+ *
+ * Examines the condition of a conditional assignment to generate the optimal
+ * first operand of a \c CMP instruction.  If the condition is a relational
+ * operator with 0 (e.g., \c ir_binop_less), the value being compared will be
+ * used as the source for the \c CMP instruction.  Otherwise the comparison
+ * is processed to a boolean result, and the boolean result is used as the
+ * operand to the CMP instruction.
+ */
+bool
+ir_to_mesa_visitor::process_move_condition(ir_rvalue *ir)
+{
+   ir_rvalue *src_ir = ir;
+   bool negate = true;
+   bool switch_order = false;
+
+   ir_expression *const expr = ir->as_expression();
+   if ((expr != NULL) && (expr->get_num_operands() == 2)) {
+      bool zero_on_left = false;
+
+      if (expr->operands[0]->is_zero()) {
+        src_ir = expr->operands[1];
+        zero_on_left = true;
+      } else if (expr->operands[1]->is_zero()) {
+        src_ir = expr->operands[0];
+        zero_on_left = false;
+      }
+
+      /*      a is -  0  +            -  0  +
+       * (a <  0)  T  F  F  ( a < 0)  T  F  F
+       * (0 <  a)  F  F  T  (-a < 0)  F  F  T
+       * (a <= 0)  T  T  F  (-a < 0)  F  F  T  (swap order of other operands)
+       * (0 <= a)  F  T  T  ( a < 0)  T  F  F  (swap order of other operands)
+       * (a >  0)  F  F  T  (-a < 0)  F  F  T
+       * (0 >  a)  T  F  F  ( a < 0)  T  F  F
+       * (a >= 0)  F  T  T  ( a < 0)  T  F  F  (swap order of other operands)
+       * (0 >= a)  T  T  F  (-a < 0)  F  F  T  (swap order of other operands)
+       *
+       * Note that exchanging the order of 0 and 'a' in the comparison simply
+       * means that the value of 'a' should be negated.
+       */
+      if (src_ir != ir) {
+        switch (expr->operation) {
+        case ir_binop_less:
+           switch_order = false;
+           negate = zero_on_left;
+           break;
+
+        case ir_binop_greater:
+           switch_order = false;
+           negate = !zero_on_left;
+           break;
+
+        case ir_binop_lequal:
+           switch_order = true;
+           negate = !zero_on_left;
+           break;
+
+        case ir_binop_gequal:
+           switch_order = true;
+           negate = zero_on_left;
+           break;
+
+        default:
+           /* This isn't the right kind of comparison afterall, so make sure
+            * the whole condition is visited.
+            */
+           src_ir = ir;
+           break;
+        }
+      }
+   }
+
+   src_ir->accept(this);
+
+   /* We use the OPCODE_CMP (a < 0 ? b : c) for conditional moves, and the
+    * condition we produced is 0.0 or 1.0.  By flipping the sign, we can
+    * choose which value OPCODE_CMP produces without an extra instruction
+    * computing the condition.
+    */
+   if (negate)
+      this->result.negate = ~this->result.negate;
+
+   return switch_order;
+}
+
 void
 ir_to_mesa_visitor::visit(ir_assignment *ir)
 {
@@ -1699,33 +1739,57 @@ ir_to_mesa_visitor::visit(ir_assignment *ir)
       assert(!ir->lhs->type->is_scalar() && !ir->lhs->type->is_vector());
       l.writemask = WRITEMASK_XYZW;
    } else if (ir->lhs->type->is_scalar()) {
-      /* FINISHME: This hack makes writing to gl_FragData, which lives in the
+      /* FINISHME: This hack makes writing to gl_FragDepth, which lives in the
        * FINISHME: W component of fragment shader output zero, work correctly.
        */
       l.writemask = WRITEMASK_XYZW;
    } else {
+      int swizzles[4];
+      int first_enabled_chan = 0;
+      int rhs_chan = 0;
+
       assert(ir->lhs->type->is_vector());
       l.writemask = ir->write_mask;
+
+      for (int i = 0; i < 4; i++) {
+        if (l.writemask & (1 << i)) {
+           first_enabled_chan = GET_SWZ(r.swizzle, i);
+           break;
+        }
+      }
+
+      /* Swizzle a small RHS vector into the channels being written.
+       *
+       * glsl ir treats write_mask as dictating how many channels are
+       * present on the RHS while Mesa IR treats write_mask as just
+       * showing which channels of the vec4 RHS get written.
+       */
+      for (int i = 0; i < 4; i++) {
+        if (l.writemask & (1 << i))
+           swizzles[i] = GET_SWZ(r.swizzle, rhs_chan++);
+        else
+           swizzles[i] = first_enabled_chan;
+      }
+      r.swizzle = MAKE_SWIZZLE4(swizzles[0], swizzles[1],
+                               swizzles[2], swizzles[3]);
    }
 
    assert(l.file != PROGRAM_UNDEFINED);
    assert(r.file != PROGRAM_UNDEFINED);
 
    if (ir->condition) {
-      ir_to_mesa_src_reg condition;
-
-      ir->condition->accept(this);
-      condition = this->result;
+      const bool switch_order = this->process_move_condition(ir->condition);
+      ir_to_mesa_src_reg condition = this->result;
 
-      /* We use the OPCODE_CMP (a < 0 ? b : c) for conditional moves,
-       * and the condition we produced is 0.0 or 1.0.  By flipping the
-       * sign, we can choose which value OPCODE_CMP produces without
-       * an extra computing the condition.
-       */
-      condition.negate = ~condition.negate;
       for (i = 0; i < type_size(ir->lhs->type); i++) {
-        ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
-                            condition, r, ir_to_mesa_src_reg_from_dst(l));
+        if (switch_order) {
+           ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
+                               condition, ir_to_mesa_src_reg_from_dst(l), r);
+        } else {
+           ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
+                               condition, r, ir_to_mesa_src_reg_from_dst(l));
+        }
+
         l.index++;
         r.index++;
       }
@@ -1743,7 +1807,7 @@ void
 ir_to_mesa_visitor::visit(ir_constant *ir)
 {
    ir_to_mesa_src_reg src_reg;
-   GLfloat stack_vals[4];
+   GLfloat stack_vals[4] = { 0 };
    GLfloat *values = stack_vals;
    unsigned int i;
 
@@ -1863,7 +1927,7 @@ ir_to_mesa_visitor::get_function_signature(ir_function_signature *sig)
         return entry;
    }
 
-   entry = talloc(mem_ctx, function_entry);
+   entry = ralloc(mem_ctx, function_entry);
    entry->sig = sig;
    entry->sig_id = this->next_signature_id++;
    entry->bgn_inst = NULL;
@@ -1974,7 +2038,6 @@ ir_to_mesa_visitor::visit(ir_call *ir)
    this->result = entry->return_reg;
 }
 
-
 void
 ir_to_mesa_visitor::visit(ir_texture *ir)
 {
@@ -2074,34 +2137,11 @@ ir_to_mesa_visitor::visit(ir_texture *ir)
    if (ir->shadow_comparitor)
       inst->tex_shadow = GL_TRUE;
 
-   ir_variable *sampler = ir->sampler->variable_referenced();
-
-   /* generate the mapping, remove when we generate storage at
-    * declaration time
-    */
-   ir->sampler->accept(this);
-
-   inst->sampler = get_sampler_location(sampler);
-
-   ir_dereference_array *sampler_array = ir->sampler->as_dereference_array();
-   if (sampler_array) {
-      ir_constant *array_index =
-        sampler_array->array_index->constant_expression_value();
-
-      /* GLSL 1.10 and 1.20 allowed variable sampler array indices,
-       * while GLSL 1.30 requires that the array indices be constant
-       * integer expressions.  We don't expect any driver to actually
-       * work with a really variable array index, and in 1.20 all that
-       * would work would be an unrolled loop counter, so assert that
-       * we ended up with a constant at least..
-       */
-      assert(array_index);
-      inst->sampler += array_index->value.i[0];
-   }
+   inst->sampler = _mesa_get_sampler_uniform_value(ir->sampler,
+                                                  this->shader_program,
+                                                  this->prog);
 
-   const glsl_type *sampler_type = sampler->type;
-   while (sampler_type->base_type == GLSL_TYPE_ARRAY)
-      sampler_type = sampler_type->fields.array;
+   const glsl_type *sampler_type = ir->sampler->type;
 
    switch (sampler_type->sampler_dimensionality) {
    case GLSL_SAMPLER_DIM_1D:
@@ -2158,9 +2198,17 @@ ir_to_mesa_visitor::visit(ir_return *ir)
 void
 ir_to_mesa_visitor::visit(ir_discard *ir)
 {
-   assert(ir->condition == NULL); /* FINISHME */
+   struct gl_fragment_program *fp = (struct gl_fragment_program *)this->prog;
+
+   if (ir->condition) {
+      ir->condition->accept(this);
+      this->result.negate = ~this->result.negate;
+      ir_to_mesa_emit_op1(ir, OPCODE_KIL, ir_to_mesa_undef_dst, this->result);
+   } else {
+      ir_to_mesa_emit_op0(ir, OPCODE_KIL_NV);
+   }
 
-   ir_to_mesa_emit_op0(ir, OPCODE_KIL_NV);
+   fp->UsesKill = GL_TRUE;
 }
 
 void
@@ -2174,7 +2222,7 @@ ir_to_mesa_visitor::visit(ir_if *ir)
    ir->condition->accept(this);
    assert(this->result.file != PROGRAM_UNDEFINED);
 
-   if (ctx->Shader.EmitCondCodes) {
+   if (this->options->EmitCondCodes) {
       cond_inst = (ir_to_mesa_instruction *)this->instructions.get_tail();
 
       /* See if we actually generated any instruction for generating
@@ -2215,16 +2263,13 @@ ir_to_mesa_visitor::ir_to_mesa_visitor()
    result.file = PROGRAM_UNDEFINED;
    next_temp = 1;
    next_signature_id = 1;
-   sampler_map = NULL;
    current_function = NULL;
-   mem_ctx = talloc_new(NULL);
+   mem_ctx = ralloc_context(NULL);
 }
 
 ir_to_mesa_visitor::~ir_to_mesa_visitor()
 {
-   talloc_free(mem_ctx);
-   if (this->sampler_map)
-      hash_table_dtor(this->sampler_map);
+   ralloc_free(mem_ctx);
 }
 
 static struct prog_src_register
@@ -2233,7 +2278,7 @@ mesa_src_reg_from_ir_src_reg(ir_to_mesa_src_reg reg)
    struct prog_src_register mesa_reg;
 
    mesa_reg.File = reg.file;
-   assert(reg.index < (1 << INST_INDEX_BITS) - 1);
+   assert(reg.index < (1 << INST_INDEX_BITS));
    mesa_reg.Index = reg.index;
    mesa_reg.Swizzle = reg.swizzle;
    mesa_reg.RelAddr = reg.reladdr != NULL;
@@ -2273,8 +2318,8 @@ set_branchtargets(ir_to_mesa_visitor *v,
       }
    }
 
-   if_stack = talloc_zero_array(v->mem_ctx, int, if_count);
-   loop_stack = talloc_zero_array(v->mem_ctx, int, loop_count);
+   if_stack = rzalloc_array(v->mem_ctx, int, if_count);
+   loop_stack = rzalloc_array(v->mem_ctx, int, loop_count);
 
    for (i = 0; i < num_instructions; i++) {
       switch (mesa_instructions[i].Opcode) {
@@ -2371,12 +2416,6 @@ count_resources(struct gl_program *prog)
    for (i = 0; i < prog->NumInstructions; i++) {
       struct prog_instruction *inst = &prog->Instructions[i];
 
-      /* Instead of just using the uniform's value to map to a
-       * sampler, Mesa first allocates a separate number for the
-       * sampler (_mesa_add_sampler), then we reindex it down to a
-       * small integer (sampler_map[], SamplersUsed), then that gets
-       * mapped to the uniform's value, and we get an actual sampler.
-       */
       if (_mesa_is_tex_instruction(inst->Opcode)) {
         prog->SamplerTargets[inst->TexSrcUnit] =
            (gl_texture_index)inst->TexSrcTarget;
@@ -2390,31 +2429,420 @@ count_resources(struct gl_program *prog)
    _mesa_update_shader_textures_used(prog);
 }
 
-/* Each stage has some uniforms in its Parameters list.  The Uniforms
- * list for the linked shader program has a pointer to these uniforms
- * in each of the stage's Parameters list, so that their values can be
- * updated when a uniform is set.
+struct uniform_sort {
+   struct gl_uniform *u;
+   int pos;
+};
+
+/* The shader_program->Uniforms list is almost sorted in increasing
+ * uniform->{Frag,Vert}Pos locations, but not quite when there are
+ * uniforms shared between targets.  We need to add parameters in
+ * increasing order for the targets.
+ */
+static int
+sort_uniforms(const void *a, const void *b)
+{
+   struct uniform_sort *u1 = (struct uniform_sort *)a;
+   struct uniform_sort *u2 = (struct uniform_sort *)b;
+
+   return u1->pos - u2->pos;
+}
+
+/* Add the uniforms to the parameters.  The linker chose locations
+ * in our parameters lists (which weren't created yet), which the
+ * uniforms code will use to poke values into our parameters list
+ * when uniforms are updated.
  */
 static void
-link_uniforms_to_shared_uniform_list(struct gl_uniform_list *uniforms,
-                                    struct gl_program *prog)
+add_uniforms_to_parameters_list(struct gl_shader_program *shader_program,
+                               struct gl_shader *shader,
+                               struct gl_program *prog)
 {
    unsigned int i;
+   unsigned int next_sampler = 0, num_uniforms = 0;
+   struct uniform_sort *sorted_uniforms;
+
+   sorted_uniforms = ralloc_array(NULL, struct uniform_sort,
+                                 shader_program->Uniforms->NumUniforms);
+
+   for (i = 0; i < shader_program->Uniforms->NumUniforms; i++) {
+      struct gl_uniform *uniform = shader_program->Uniforms->Uniforms + i;
+      int parameter_index = -1;
+
+      switch (shader->Type) {
+      case GL_VERTEX_SHADER:
+        parameter_index = uniform->VertPos;
+        break;
+      case GL_FRAGMENT_SHADER:
+        parameter_index = uniform->FragPos;
+        break;
+      case GL_GEOMETRY_SHADER:
+        parameter_index = uniform->GeomPos;
+        break;
+      }
+
+      /* Only add uniforms used in our target. */
+      if (parameter_index != -1) {
+        sorted_uniforms[num_uniforms].pos = parameter_index;
+        sorted_uniforms[num_uniforms].u = uniform;
+        num_uniforms++;
+      }
+   }
+
+   qsort(sorted_uniforms, num_uniforms, sizeof(struct uniform_sort),
+        sort_uniforms);
+
+   for (i = 0; i < num_uniforms; i++) {
+      struct gl_uniform *uniform = sorted_uniforms[i].u;
+      int parameter_index = sorted_uniforms[i].pos;
+      const glsl_type *type = uniform->Type;
+      unsigned int size;
+
+      if (type->is_vector() ||
+         type->is_scalar()) {
+        size = type->vector_elements;
+      } else {
+        size = type_size(type) * 4;
+      }
+
+      gl_register_file file;
+      if (type->is_sampler() ||
+         (type->is_array() && type->fields.array->is_sampler())) {
+        file = PROGRAM_SAMPLER;
+      } else {
+        file = PROGRAM_UNIFORM;
+      }
+
+      GLint index = _mesa_lookup_parameter_index(prog->Parameters, -1,
+                                                uniform->Name);
+
+      if (index < 0) {
+        index = _mesa_add_parameter(prog->Parameters, file,
+                                    uniform->Name, size, type->gl_type,
+                                    NULL, NULL, 0x0);
+
+        /* Sampler uniform values are stored in prog->SamplerUnits,
+         * and the entry in that array is selected by this index we
+         * store in ParameterValues[].
+         */
+        if (file == PROGRAM_SAMPLER) {
+           for (unsigned int j = 0; j < size / 4; j++)
+              prog->Parameters->ParameterValues[index + j][0] = next_sampler++;
+        }
+
+        /* The location chosen in the Parameters list here (returned
+         * from _mesa_add_uniform) has to match what the linker chose.
+         */
+        if (index != parameter_index) {
+           fail_link(shader_program, "Allocation of uniform `%s' to target "
+                     "failed (%d vs %d)\n",
+                     uniform->Name, index, parameter_index);
+        }
+      }
+   }
+
+   ralloc_free(sorted_uniforms);
+}
+
+static void
+set_uniform_initializer(struct gl_context *ctx, void *mem_ctx,
+                       struct gl_shader_program *shader_program,
+                       const char *name, const glsl_type *type,
+                       ir_constant *val)
+{
+   if (type->is_record()) {
+      ir_constant *field_constant;
+
+      field_constant = (ir_constant *)val->components.get_head();
+
+      for (unsigned int i = 0; i < type->length; i++) {
+        const glsl_type *field_type = type->fields.structure[i].type;
+        const char *field_name = ralloc_asprintf(mem_ctx, "%s.%s", name,
+                                           type->fields.structure[i].name);
+        set_uniform_initializer(ctx, mem_ctx, shader_program, field_name,
+                                field_type, field_constant);
+        field_constant = (ir_constant *)field_constant->next;
+      }
+      return;
+   }
+
+   int loc = _mesa_get_uniform_location(ctx, shader_program, name);
+
+   if (loc == -1) {
+      fail_link(shader_program,
+               "Couldn't find uniform for initializer %s\n", name);
+      return;
+   }
+
+   for (unsigned int i = 0; i < (type->is_array() ? type->length : 1); i++) {
+      ir_constant *element;
+      const glsl_type *element_type;
+      if (type->is_array()) {
+        element = val->array_elements[i];
+        element_type = type->fields.array;
+      } else {
+        element = val;
+        element_type = type;
+      }
+
+      void *values;
+
+      if (element_type->base_type == GLSL_TYPE_BOOL) {
+        int *conv = ralloc_array(mem_ctx, int, element_type->components());
+        for (unsigned int j = 0; j < element_type->components(); j++) {
+           conv[j] = element->value.b[j];
+        }
+        values = (void *)conv;
+        element_type = glsl_type::get_instance(GLSL_TYPE_INT,
+                                               element_type->vector_elements,
+                                               1);
+      } else {
+        values = &element->value;
+      }
+
+      if (element_type->is_matrix()) {
+        _mesa_uniform_matrix(ctx, shader_program,
+                             element_type->matrix_columns,
+                             element_type->vector_elements,
+                             loc, 1, GL_FALSE, (GLfloat *)values);
+        loc += element_type->matrix_columns;
+      } else {
+        _mesa_uniform(ctx, shader_program, loc, element_type->matrix_columns,
+                      values, element_type->gl_type);
+        loc += type_size(element_type);
+      }
+   }
+}
+
+static void
+set_uniform_initializers(struct gl_context *ctx,
+                        struct gl_shader_program *shader_program)
+{
+   void *mem_ctx = NULL;
+
+   for (unsigned int i = 0; i < MESA_SHADER_TYPES; i++) {
+      struct gl_shader *shader = shader_program->_LinkedShaders[i];
+
+      if (shader == NULL)
+        continue;
+
+      foreach_iter(exec_list_iterator, iter, *shader->ir) {
+        ir_instruction *ir = (ir_instruction *)iter.get();
+        ir_variable *var = ir->as_variable();
+
+        if (!var || var->mode != ir_var_uniform || !var->constant_value)
+           continue;
+
+        if (!mem_ctx)
+           mem_ctx = ralloc_context(NULL);
+
+        set_uniform_initializer(ctx, mem_ctx, shader_program, var->name,
+                                var->type, var->constant_value);
+      }
+   }
+
+   ralloc_free(mem_ctx);
+}
+
+/*
+ * On a basic block basis, tracks available PROGRAM_TEMPORARY register
+ * channels for copy propagation and updates following instructions to
+ * use the original versions.
+ *
+ * The ir_to_mesa_visitor lazily produces code assuming that this pass
+ * will occur.  As an example, a TXP production before this pass:
+ *
+ * 0: MOV TEMP[1], INPUT[4].xyyy;
+ * 1: MOV TEMP[1].w, INPUT[4].wwww;
+ * 2: TXP TEMP[2], TEMP[1], texture[0], 2D;
+ *
+ * and after:
+ *
+ * 0: MOV TEMP[1], INPUT[4].xyyy;
+ * 1: MOV TEMP[1].w, INPUT[4].wwww;
+ * 2: TXP TEMP[2], INPUT[4].xyyw, texture[0], 2D;
+ *
+ * which allows for dead code elimination on TEMP[1]'s writes.
+ */
+void
+ir_to_mesa_visitor::copy_propagate(void)
+{
+   ir_to_mesa_instruction **acp = rzalloc_array(mem_ctx,
+                                                   ir_to_mesa_instruction *,
+                                                   this->next_temp * 4);
+   int *acp_level = rzalloc_array(mem_ctx, int, this->next_temp * 4);
+   int level = 0;
+
+   foreach_iter(exec_list_iterator, iter, this->instructions) {
+      ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
+
+      assert(inst->dst_reg.file != PROGRAM_TEMPORARY
+            || inst->dst_reg.index < this->next_temp);
+
+      /* First, do any copy propagation possible into the src regs. */
+      for (int r = 0; r < 3; r++) {
+        ir_to_mesa_instruction *first = NULL;
+        bool good = true;
+        int acp_base = inst->src_reg[r].index * 4;
+
+        if (inst->src_reg[r].file != PROGRAM_TEMPORARY ||
+            inst->src_reg[r].reladdr)
+           continue;
+
+        /* See if we can find entries in the ACP consisting of MOVs
+         * from the same src register for all the swizzled channels
+         * of this src register reference.
+         */
+        for (int i = 0; i < 4; i++) {
+           int src_chan = GET_SWZ(inst->src_reg[r].swizzle, i);
+           ir_to_mesa_instruction *copy_chan = acp[acp_base + src_chan];
+
+           if (!copy_chan) {
+              good = false;
+              break;
+           }
+
+           assert(acp_level[acp_base + src_chan] <= level);
+
+           if (!first) {
+              first = copy_chan;
+           } else {
+              if (first->src_reg[0].file != copy_chan->src_reg[0].file ||
+                  first->src_reg[0].index != copy_chan->src_reg[0].index) {
+                 good = false;
+                 break;
+              }
+           }
+        }
+
+        if (good) {
+           /* We've now validated that we can copy-propagate to
+            * replace this src register reference.  Do it.
+            */
+           inst->src_reg[r].file = first->src_reg[0].file;
+           inst->src_reg[r].index = first->src_reg[0].index;
+
+           int swizzle = 0;
+           for (int i = 0; i < 4; i++) {
+              int src_chan = GET_SWZ(inst->src_reg[r].swizzle, i);
+              ir_to_mesa_instruction *copy_inst = acp[acp_base + src_chan];
+              swizzle |= (GET_SWZ(copy_inst->src_reg[0].swizzle, src_chan) <<
+                          (3 * i));
+           }
+           inst->src_reg[r].swizzle = swizzle;
+        }
+      }
+
+      switch (inst->op) {
+      case OPCODE_BGNLOOP:
+      case OPCODE_ENDLOOP:
+        /* End of a basic block, clear the ACP entirely. */
+        memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
+        break;
+
+      case OPCODE_IF:
+        ++level;
+        break;
+
+      case OPCODE_ENDIF:
+      case OPCODE_ELSE:
+        /* Clear all channels written inside the block from the ACP, but
+         * leaving those that were not touched.
+         */
+        for (int r = 0; r < this->next_temp; r++) {
+           for (int c = 0; c < 4; c++) {
+              if (!acp[4 * r + c])
+                 continue;
+
+              if (acp_level[4 * r + c] >= level)
+                 acp[4 * r + c] = NULL;
+           }
+        }
+        if (inst->op == OPCODE_ENDIF)
+           --level;
+        break;
+
+      default:
+        /* Continuing the block, clear any written channels from
+         * the ACP.
+         */
+        if (inst->dst_reg.file == PROGRAM_TEMPORARY && inst->dst_reg.reladdr) {
+           /* Any temporary might be written, so no copy propagation
+            * across this instruction.
+            */
+           memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
+        } else if (inst->dst_reg.file == PROGRAM_OUTPUT &&
+                   inst->dst_reg.reladdr) {
+           /* Any output might be written, so no copy propagation
+            * from outputs across this instruction.
+            */
+           for (int r = 0; r < this->next_temp; r++) {
+              for (int c = 0; c < 4; c++) {
+                 if (!acp[4 * r + c])
+                    continue;
+
+                 if (acp[4 * r + c]->src_reg[0].file == PROGRAM_OUTPUT)
+                    acp[4 * r + c] = NULL;
+              }
+           }
+        } else if (inst->dst_reg.file == PROGRAM_TEMPORARY ||
+                   inst->dst_reg.file == PROGRAM_OUTPUT) {
+           /* Clear where it's used as dst. */
+           if (inst->dst_reg.file == PROGRAM_TEMPORARY) {
+              for (int c = 0; c < 4; c++) {
+                 if (inst->dst_reg.writemask & (1 << c)) {
+                    acp[4 * inst->dst_reg.index + c] = NULL;
+                 }
+              }
+           }
+
+           /* Clear where it's used as src. */
+           for (int r = 0; r < this->next_temp; r++) {
+              for (int c = 0; c < 4; c++) {
+                 if (!acp[4 * r + c])
+                    continue;
+
+                 int src_chan = GET_SWZ(acp[4 * r + c]->src_reg[0].swizzle, c);
 
-   for (i = 0; i < prog->Parameters->NumParameters; i++) {
-      const struct gl_program_parameter *p = prog->Parameters->Parameters + i;
+                 if (acp[4 * r + c]->src_reg[0].file == inst->dst_reg.file &&
+                     acp[4 * r + c]->src_reg[0].index == inst->dst_reg.index &&
+                     inst->dst_reg.writemask & (1 << src_chan))
+                 {
+                    acp[4 * r + c] = NULL;
+                 }
+              }
+           }
+        }
+        break;
+      }
 
-      if (p->Type == PROGRAM_UNIFORM || p->Type == PROGRAM_SAMPLER) {
-        struct gl_uniform *uniform =
-           _mesa_append_uniform(uniforms, p->Name, prog->Target, i);
-        if (uniform)
-           uniform->Initialized = p->Initialized;
+      /* If this is a copy, add it to the ACP. */
+      if (inst->op == OPCODE_MOV &&
+         inst->dst_reg.file == PROGRAM_TEMPORARY &&
+         !inst->dst_reg.reladdr &&
+         !inst->saturate &&
+         !inst->src_reg[0].reladdr &&
+         !inst->src_reg[0].negate) {
+        for (int i = 0; i < 4; i++) {
+           if (inst->dst_reg.writemask & (1 << i)) {
+              acp[4 * inst->dst_reg.index + i] = inst;
+              acp_level[4 * inst->dst_reg.index + i] = level;
+           }
+        }
       }
    }
+
+   ralloc_free(acp_level);
+   ralloc_free(acp);
 }
 
-struct gl_program *
-get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
+
+/**
+ * Convert a shader's GLSL IR into a Mesa gl_program.
+ */
+static struct gl_program *
+get_mesa_program(struct gl_context *ctx,
+                 struct gl_shader_program *shader_program,
                 struct gl_shader *shader)
 {
    ir_to_mesa_visitor v;
@@ -2425,6 +2853,8 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    GLenum target;
    const char *target_string;
    GLboolean progress;
+   struct gl_shader_compiler_options *options =
+         &ctx->ShaderCompilerOptions[_mesa_shader_type_to_index(shader->Type)];
 
    switch (shader->Type) {
    case GL_VERTEX_SHADER:
@@ -2435,9 +2865,13 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
       target = GL_FRAGMENT_PROGRAM_ARB;
       target_string = "fragment";
       break;
+   case GL_GEOMETRY_SHADER:
+      target = GL_GEOMETRY_PROGRAM_NV;
+      target_string = "geometry";
+      break;
    default:
       assert(!"should not be reached");
-      break;
+      return NULL;
    }
 
    validate_ir_tree(shader->ir);
@@ -2450,6 +2884,10 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    prog->Attributes = _mesa_new_parameter_list();
    v.ctx = ctx;
    v.prog = prog;
+   v.shader_program = shader_program;
+   v.options = options;
+
+   add_uniforms_to_parameters_list(shader_program, shader, prog);
 
    /* Emit Mesa IR for main(). */
    visit_exec_list(shader->ir, &v);
@@ -2494,16 +2932,22 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    mesa_instructions =
       (struct prog_instruction *)calloc(num_instructions,
                                        sizeof(*mesa_instructions));
-   mesa_instruction_annotation = talloc_array(v.mem_ctx, ir_instruction *,
+   mesa_instruction_annotation = ralloc_array(v.mem_ctx, ir_instruction *,
                                              num_instructions);
 
+   v.copy_propagate();
+
+   /* Convert ir_mesa_instructions into prog_instructions.
+    */
    mesa_inst = mesa_instructions;
    i = 0;
    foreach_iter(exec_list_iterator, iter, v.instructions) {
-      ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
+      const ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
 
       mesa_inst->Opcode = inst->op;
       mesa_inst->CondUpdate = inst->cond_update;
+      if (inst->saturate)
+        mesa_inst->SaturateMode = SATURATE_ZERO_ONE;
       mesa_inst->DstReg.File = inst->dst_reg.file;
       mesa_inst->DstReg.Index = inst->dst_reg.index;
       mesa_inst->DstReg.CondMask = inst->dst_reg.cond_mask;
@@ -2517,11 +2961,17 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
       mesa_inst->TexShadow = inst->tex_shadow;
       mesa_instruction_annotation[i] = inst->ir;
 
-      if (ctx->Shader.EmitNoIfs && mesa_inst->Opcode == OPCODE_IF) {
-        shader_program->InfoLog =
-           talloc_asprintf_append(shader_program->InfoLog,
-                                  "Couldn't flatten if statement\n");
-        shader_program->LinkStatus = false;
+      /* Set IndirectRegisterFiles. */
+      if (mesa_inst->DstReg.RelAddr)
+         prog->IndirectRegisterFiles |= 1 << mesa_inst->DstReg.File;
+
+      /* Update program's bitmask of indirectly accessed register files */
+      for (unsigned src = 0; src < 3; src++)
+         if (mesa_inst->SrcReg[src].RelAddr)
+            prog->IndirectRegisterFiles |= 1 << mesa_inst->SrcReg[src].File;
+
+      if (options->EmitNoIfs && mesa_inst->Opcode == OPCODE_IF) {
+        fail_link(shader_program, "Couldn't flatten if statement\n");
       }
 
       switch (mesa_inst->Opcode) {
@@ -2544,6 +2994,15 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
 
       mesa_inst++;
       i++;
+
+      if (!shader_program->LinkStatus)
+         break;
+   }
+
+   if (!shader_program->LinkStatus) {
+      free(mesa_instructions);
+      _mesa_reference_program(ctx, &shader->Program, NULL);
+      return NULL;
    }
 
    set_branchtargets(&v, mesa_instructions, num_instructions);
@@ -2577,8 +3036,14 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
 }
 
 extern "C" {
+
+/**
+ * Called via ctx->Driver.CompilerShader().
+ * This is a no-op.
+ * XXX can we remove the ctx->Driver.CompileShader() hook?
+ */
 GLboolean
-_mesa_ir_compile_shader(GLcontext *ctx, struct gl_shader *shader)
+_mesa_ir_compile_shader(struct gl_context *ctx, struct gl_shader *shader)
 {
    assert(shader->CompileStatus);
    (void) ctx;
@@ -2586,28 +3051,62 @@ _mesa_ir_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    return GL_TRUE;
 }
 
+
+/**
+ * Link a shader.
+ * Called via ctx->Driver.LinkShader()
+ * This actually involves converting GLSL IR into Mesa gl_programs with
+ * code lowering and other optimizations.
+ */
 GLboolean
-_mesa_ir_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
+_mesa_ir_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 {
    assert(prog->LinkStatus);
 
-   for (unsigned i = 0; i < prog->_NumLinkedShaders; i++) {
+   for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
+      if (prog->_LinkedShaders[i] == NULL)
+        continue;
+
       bool progress;
       exec_list *ir = prog->_LinkedShaders[i]->ir;
+      const struct gl_shader_compiler_options *options =
+            &ctx->ShaderCompilerOptions[_mesa_shader_type_to_index(prog->_LinkedShaders[i]->Type)];
 
       do {
         progress = false;
 
         /* Lowering */
         do_mat_op_to_vec(ir);
-        do_mod_to_fract(ir);
-        do_div_to_mul_rcp(ir);
-        do_explog_to_explog2(ir);
+        lower_instructions(ir, (MOD_TO_FRACT | DIV_TO_MUL_RCP | EXP_TO_EXP2
+                                | LOG_TO_LOG2
+                                | ((options->EmitNoPow) ? POW_TO_EXP2 : 0)));
+
+        progress = do_lower_jumps(ir, true, true, options->EmitNoMainReturn, options->EmitNoCont, options->EmitNoLoops) || progress;
+
+        progress = do_common_optimization(ir, true, options->MaxUnrollIterations) || progress;
+
+        progress = lower_quadop_vector(ir, true) || progress;
+
+        if (options->EmitNoIfs) {
+           progress = lower_discard(ir) || progress;
+           progress = lower_if_to_cond_assign(ir) || progress;
+        }
 
-        progress = do_common_optimization(ir, true) || progress;
+        if (options->EmitNoNoise)
+           progress = lower_noise(ir) || progress;
 
-        if (ctx->Shader.EmitNoIfs)
-           progress = do_if_to_cond_assign(ir) || progress;
+        /* If there are forms of indirect addressing that the driver
+         * cannot handle, perform the lowering pass.
+         */
+        if (options->EmitNoIndirectInput || options->EmitNoIndirectOutput
+            || options->EmitNoIndirectTemp || options->EmitNoIndirectUniform)
+          progress =
+            lower_variable_index_to_cond_assign(ir,
+                                                options->EmitNoIndirectInput,
+                                                options->EmitNoIndirectOutput,
+                                                options->EmitNoIndirectTemp,
+                                                options->EmitNoIndirectUniform)
+            || progress;
 
         progress = do_vec_index_to_cond_assign(ir) || progress;
       } while (progress);
@@ -2615,46 +3114,69 @@ _mesa_ir_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
       validate_ir_tree(ir);
    }
 
-   for (unsigned i = 0; i < prog->_NumLinkedShaders; i++) {
+   for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
       struct gl_program *linked_prog;
-      bool ok = true;
 
-      linked_prog = get_mesa_program(ctx, prog, prog->_LinkedShaders[i]);
+      if (prog->_LinkedShaders[i] == NULL)
+        continue;
 
-      link_uniforms_to_shared_uniform_list(prog->Uniforms, linked_prog);
+      linked_prog = get_mesa_program(ctx, prog, prog->_LinkedShaders[i]);
 
-      switch (prog->_LinkedShaders[i]->Type) {
-      case GL_VERTEX_SHADER:
-        _mesa_reference_vertprog(ctx, &prog->VertexProgram,
-                                 (struct gl_vertex_program *)linked_prog);
-        ok = ctx->Driver.ProgramStringNotify(ctx, GL_VERTEX_PROGRAM_ARB,
-                                             linked_prog);
-        break;
-      case GL_FRAGMENT_SHADER:
-        _mesa_reference_fragprog(ctx, &prog->FragmentProgram,
-                                 (struct gl_fragment_program *)linked_prog);
-        ok = ctx->Driver.ProgramStringNotify(ctx, GL_FRAGMENT_PROGRAM_ARB,
-                                             linked_prog);
-        break;
-      }
-      if (!ok) {
-        return GL_FALSE;
+      if (linked_prog) {
+         bool ok = true;
+
+         switch (prog->_LinkedShaders[i]->Type) {
+         case GL_VERTEX_SHADER:
+            _mesa_reference_vertprog(ctx, &prog->VertexProgram,
+                                     (struct gl_vertex_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_VERTEX_PROGRAM_ARB,
+                                                 linked_prog);
+            break;
+         case GL_FRAGMENT_SHADER:
+            _mesa_reference_fragprog(ctx, &prog->FragmentProgram,
+                                     (struct gl_fragment_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_FRAGMENT_PROGRAM_ARB,
+                                                 linked_prog);
+            break;
+         case GL_GEOMETRY_SHADER:
+            _mesa_reference_geomprog(ctx, &prog->GeometryProgram,
+                                     (struct gl_geometry_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_GEOMETRY_PROGRAM_NV,
+                                                 linked_prog);
+            break;
+         }
+         if (!ok) {
+            return GL_FALSE;
+         }
       }
+
       _mesa_reference_program(ctx, &linked_prog, NULL);
    }
 
    return GL_TRUE;
 }
 
+
+/**
+ * Compile a GLSL shader.  Called via glCompileShader().
+ */
 void
-_mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
+_mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader)
 {
    struct _mesa_glsl_parse_state *state =
       new(shader) _mesa_glsl_parse_state(ctx, shader->Type, shader);
 
    const char *source = shader->Source;
+   /* Check if the user called glCompileShader without first calling
+    * glShaderSource.  This should fail to compile, but not raise a GL_ERROR.
+    */
+   if (source == NULL) {
+      shader->CompileStatus = GL_FALSE;
+      return;
+   }
+
    state->error = preprocess(state, &source, &state->info_log,
-                            &ctx->Extensions);
+                            &ctx->Extensions, ctx->API);
 
    if (ctx->Shader.Flags & GLSL_DUMP) {
       printf("GLSL source for shader %d:\n", shader->Name);
@@ -2667,7 +3189,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
      _mesa_glsl_lexer_dtor(state);
    }
 
-   talloc_free(shader->ir);
+   ralloc_free(shader->ir);
    shader->ir = new(shader) exec_list;
    if (!state->error && !state->translation_unit.is_empty())
       _mesa_ast_to_hir(shader->ir, state);
@@ -2678,7 +3200,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
       /* Do some optimization at compile time to reduce shader IR size
        * and reduce later work if the same shader is linked multiple times
        */
-      while (do_common_optimization(shader->ir, false))
+      while (do_common_optimization(shader->ir, false, 32))
         ;
 
       validate_ir_tree(shader->ir);
@@ -2714,7 +3236,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    /* Retain any live IR, but trash the rest. */
    reparent_ir(shader->ir, shader->ir);
 
-   talloc_free(state);
+   ralloc_free(state);
 
    if (shader->CompileStatus) {
       if (!ctx->Driver.CompileShader(ctx, shader))
@@ -2722,8 +3244,12 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    }
 }
 
+
+/**
+ * Link a GLSL shader program.  Called via glLinkProgram().
+ */
 void
-_mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
+_mesa_glsl_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 {
    unsigned int i;
 
@@ -2733,9 +3259,7 @@ _mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
 
    for (i = 0; i < prog->NumShaders; i++) {
       if (!prog->Shaders[i]->CompileStatus) {
-        prog->InfoLog =
-           talloc_asprintf_append(prog->InfoLog,
-                                  "linking with uncompiled shader");
+        fail_link(prog, "linking with uncompiled shader");
         prog->LinkStatus = GL_FALSE;
       }
    }
@@ -2743,15 +3267,10 @@ _mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
    prog->Varying = _mesa_new_parameter_list();
    _mesa_reference_vertprog(ctx, &prog->VertexProgram, NULL);
    _mesa_reference_fragprog(ctx, &prog->FragmentProgram, NULL);
+   _mesa_reference_geomprog(ctx, &prog->GeometryProgram, NULL);
 
    if (prog->LinkStatus) {
       link_shaders(ctx, prog);
-
-      /* We don't use the linker's uniforms list, and cook up our own at
-       * generate time.
-       */
-      _mesa_free_uniform_list(prog->Uniforms);
-      prog->Uniforms = _mesa_new_uniform_list();
    }
 
    if (prog->LinkStatus) {
@@ -2760,6 +3279,8 @@ _mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
       }
    }
 
+   set_uniform_initializers(ctx, prog);
+
    if (ctx->Shader.Flags & GLSL_DUMP) {
       if (!prog->LinkStatus) {
         printf("GLSL shader program %d failed to link\n", prog->Name);