Convert everything from the talloc API to the ralloc API.
[mesa.git] / src / mesa / program / ir_to_mesa.cpp
index af6d7345a54a4d142b35053d2ebdac68f9a952de..3794c0de02af3746b2d77d52639bf349b49ae1cc 100644 (file)
@@ -26,8 +26,7 @@
 /**
  * \file ir_to_mesa.cpp
  *
- * Translates the IR to ARB_fragment_program text if possible,
- * printing the result
+ * Translate GLSL IR to Mesa's gl_program representation.
  */
 
 #include <stdio.h>
@@ -54,6 +53,7 @@ extern "C" {
 #include "program/program.h"
 #include "program/prog_uniform.h"
 #include "program/prog_parameter.h"
+#include "program/sampler.h"
 }
 
 static int swizzle_for_size(int size);
@@ -65,7 +65,7 @@ static int swizzle_for_size(int size);
 typedef struct ir_to_mesa_src_reg {
    ir_to_mesa_src_reg(int file, int index, const glsl_type *type)
    {
-      this->file = file;
+      this->file = (gl_register_file) file;
       this->index = index;
       if (type && (type->is_scalar() || type->is_vector() || type->is_matrix()))
         this->swizzle = swizzle_for_size(type->vector_elements);
@@ -84,7 +84,7 @@ typedef struct ir_to_mesa_src_reg {
       this->reladdr = NULL;
    }
 
-   int file; /**< PROGRAM_* from Mesa */
+   gl_register_file file; /**< PROGRAM_* from Mesa */
    int index; /**< temporary index, VERT_ATTRIB_*, FRAG_ATTRIB_*, etc. */
    GLuint swizzle; /**< SWIZZLE_XYZWONEZERO swizzles from Mesa. */
    int negate; /**< NEGATE_XYZW mask from mesa */
@@ -105,12 +105,25 @@ extern ir_to_mesa_src_reg ir_to_mesa_undef;
 
 class ir_to_mesa_instruction : public exec_node {
 public:
+   /* Callers of this ralloc-based new need not call delete. It's
+    * easier to just ralloc_free 'ctx' (or any of its ancestors). */
+   static void* operator new(size_t size, void *ctx)
+   {
+      void *node;
+
+      node = rzalloc_size(ctx, size);
+      assert(node != NULL);
+
+      return node;
+   }
+
    enum prog_opcode op;
    ir_to_mesa_dst_reg dst_reg;
    ir_to_mesa_src_reg src_reg[3];
    /** Pointer to the ir source this tree came from for debugging */
    ir_instruction *ir;
    GLboolean cond_update;
+   bool saturate;
    int sampler; /**< sampler index */
    int tex_target; /**< One of TEXTURE_*_INDEX */
    GLboolean tex_shadow;
@@ -120,13 +133,13 @@ public:
 
 class variable_storage : public exec_node {
 public:
-   variable_storage(ir_variable *var, int file, int index)
+   variable_storage(ir_variable *var, gl_register_file file, int index)
       : file(file), index(index), var(var)
    {
       /* empty */
    }
 
-   int file;
+   gl_register_file file;
    int index;
    ir_variable *var; /* variable that maps to this, if any */
 };
@@ -171,9 +184,10 @@ public:
 
    function_entry *current_function;
 
-   GLcontext *ctx;
+   struct gl_context *ctx;
    struct gl_program *prog;
    struct gl_shader_program *shader_program;
+   struct gl_shader_compiler_options *options;
 
    int next_temp;
 
@@ -247,6 +261,17 @@ public:
                                               ir_to_mesa_src_reg src1,
                                               ir_to_mesa_src_reg src2);
 
+   /**
+    * Emit the correct dot-product instruction for the type of arguments
+    *
+    * \sa ir_to_mesa_emit_op2
+    */
+   void ir_to_mesa_emit_dp(ir_instruction *ir,
+                          ir_to_mesa_dst_reg dst,
+                          ir_to_mesa_src_reg src0,
+                          ir_to_mesa_src_reg src1,
+                          unsigned elements);
+
    void ir_to_mesa_emit_scalar_op1(ir_instruction *ir,
                                   enum prog_opcode op,
                                   ir_to_mesa_dst_reg dst,
@@ -258,10 +283,19 @@ public:
                                   ir_to_mesa_src_reg src0,
                                   ir_to_mesa_src_reg src1);
 
+   void emit_scs(ir_instruction *ir, enum prog_opcode op,
+                ir_to_mesa_dst_reg dst,
+                const ir_to_mesa_src_reg &src);
+
    GLboolean try_emit_mad(ir_expression *ir,
                          int mul_operand);
+   GLboolean try_emit_sat(ir_expression *ir);
+
+   void emit_swz(ir_expression *ir);
 
-   int get_sampler_uniform_value(ir_dereference *deref);
+   bool process_move_condition(ir_rvalue *ir);
+
+   void copy_propagate(void);
 
    void *mem_ctx;
 };
@@ -276,7 +310,22 @@ ir_to_mesa_dst_reg ir_to_mesa_address_reg = {
    PROGRAM_ADDRESS, 0, WRITEMASK_X, COND_TR, NULL
 };
 
-static int swizzle_for_size(int size)
+static void
+fail_link(struct gl_shader_program *prog, const char *fmt, ...) PRINTFLIKE(2, 3);
+
+static void
+fail_link(struct gl_shader_program *prog, const char *fmt, ...)
+{
+   va_list args;
+   va_start(args, fmt);
+   ralloc_vasprintf_append(&prog->InfoLog, fmt, args);
+   va_end(args);
+
+   prog->LinkStatus = GL_FALSE;
+}
+
+static int
+swizzle_for_size(int size)
 {
    int size_swizzles[4] = {
       MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_X, SWIZZLE_X, SWIZZLE_X),
@@ -285,6 +334,7 @@ static int swizzle_for_size(int size)
       MAKE_SWIZZLE4(SWIZZLE_X, SWIZZLE_Y, SWIZZLE_Z, SWIZZLE_W),
    };
 
+   assert((size >= 1) && (size <= 4));
    return size_swizzles[size - 1];
 }
 
@@ -366,6 +416,21 @@ ir_to_mesa_visitor::ir_to_mesa_emit_op0(ir_instruction *ir,
                              ir_to_mesa_undef);
 }
 
+void
+ir_to_mesa_visitor::ir_to_mesa_emit_dp(ir_instruction *ir,
+                                      ir_to_mesa_dst_reg dst,
+                                      ir_to_mesa_src_reg src0,
+                                      ir_to_mesa_src_reg src1,
+                                      unsigned elements)
+{
+   static const gl_inst_opcode dot_opcodes[] = {
+      OPCODE_DP2, OPCODE_DP3, OPCODE_DP4
+   };
+
+   ir_to_mesa_emit_op3(ir, dot_opcodes[elements - 2],
+                      dst, src0, src1, ir_to_mesa_undef);
+}
+
 inline ir_to_mesa_dst_reg
 ir_to_mesa_dst_reg_from_src(ir_to_mesa_src_reg reg)
 {
@@ -420,6 +485,10 @@ ir_to_mesa_visitor::ir_to_mesa_emit_scalar_op2(ir_instruction *ir,
       GLuint src0_swiz = GET_SWZ(src0.swizzle, i);
       GLuint src1_swiz = GET_SWZ(src1.swizzle, i);
       for (j = i + 1; j < 4; j++) {
+        /* If there is another enabled component in the destination that is
+         * derived from the same inputs, generate its value on this pass as
+         * well.
+         */
         if (!(done_mask & (1 << j)) &&
             GET_SWZ(src0.swizzle, j) == src0_swiz &&
             GET_SWZ(src1.swizzle, j) == src1_swiz) {
@@ -453,6 +522,102 @@ ir_to_mesa_visitor::ir_to_mesa_emit_scalar_op1(ir_instruction *ir,
    ir_to_mesa_emit_scalar_op2(ir, op, dst, src0, undef);
 }
 
+/**
+ * Emit an OPCODE_SCS instruction
+ *
+ * The \c SCS opcode functions a bit differently than the other Mesa (or
+ * ARB_fragment_program) opcodes.  Instead of splatting its result across all
+ * four components of the destination, it writes one value to the \c x
+ * component and another value to the \c y component.
+ *
+ * \param ir        IR instruction being processed
+ * \param op        Either \c OPCODE_SIN or \c OPCODE_COS depending on which
+ *                  value is desired.
+ * \param dst       Destination register
+ * \param src       Source register
+ */
+void
+ir_to_mesa_visitor::emit_scs(ir_instruction *ir, enum prog_opcode op,
+                            ir_to_mesa_dst_reg dst,
+                            const ir_to_mesa_src_reg &src)
+{
+   /* Vertex programs cannot use the SCS opcode.
+    */
+   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB) {
+      ir_to_mesa_emit_scalar_op1(ir, op, dst, src);
+      return;
+   }
+
+   const unsigned component = (op == OPCODE_SIN) ? 0 : 1;
+   const unsigned scs_mask = (1U << component);
+   int done_mask = ~dst.writemask;
+   ir_to_mesa_src_reg tmp;
+
+   assert(op == OPCODE_SIN || op == OPCODE_COS);
+
+   /* If there are compnents in the destination that differ from the component
+    * that will be written by the SCS instrution, we'll need a temporary.
+    */
+   if (scs_mask != unsigned(dst.writemask)) {
+      tmp = get_temp(glsl_type::vec4_type);
+   }
+
+   for (unsigned i = 0; i < 4; i++) {
+      unsigned this_mask = (1U << i);
+      ir_to_mesa_src_reg src0 = src;
+
+      if ((done_mask & this_mask) != 0)
+        continue;
+
+      /* The source swizzle specified which component of the source generates
+       * sine / cosine for the current component in the destination.  The SCS
+       * instruction requires that this value be swizzle to the X component.
+       * Replace the current swizzle with a swizzle that puts the source in
+       * the X component.
+       */
+      unsigned src0_swiz = GET_SWZ(src.swizzle, i);
+
+      src0.swizzle = MAKE_SWIZZLE4(src0_swiz, src0_swiz,
+                                  src0_swiz, src0_swiz);
+      for (unsigned j = i + 1; j < 4; j++) {
+        /* If there is another enabled component in the destination that is
+         * derived from the same inputs, generate its value on this pass as
+         * well.
+         */
+        if (!(done_mask & (1 << j)) &&
+            GET_SWZ(src0.swizzle, j) == src0_swiz) {
+           this_mask |= (1 << j);
+        }
+      }
+
+      if (this_mask != scs_mask) {
+        ir_to_mesa_instruction *inst;
+        ir_to_mesa_dst_reg tmp_dst = ir_to_mesa_dst_reg_from_src(tmp);
+
+        /* Emit the SCS instruction.
+         */
+        inst = ir_to_mesa_emit_op1(ir, OPCODE_SCS, tmp_dst, src0);
+        inst->dst_reg.writemask = scs_mask;
+
+        /* Move the result of the SCS instruction to the desired location in
+         * the destination.
+         */
+        tmp.swizzle = MAKE_SWIZZLE4(component, component,
+                                    component, component);
+        inst = ir_to_mesa_emit_op1(ir, OPCODE_SCS, dst, tmp);
+        inst->dst_reg.writemask = this_mask;
+      } else {
+        /* Emit the SCS instruction to write directly to the destination.
+         */
+        ir_to_mesa_instruction *inst =
+           ir_to_mesa_emit_op1(ir, OPCODE_SCS, dst, src0);
+        inst->dst_reg.writemask = scs_mask;
+      }
+
+      done_mask |= this_mask;
+   }
+}
+
 struct ir_to_mesa_src_reg
 ir_to_mesa_visitor::src_reg_for_float(float val)
 {
@@ -486,6 +651,7 @@ type_size(const struct glsl_type *type)
         return 1;
       }
    case GLSL_TYPE_ARRAY:
+      assert(type->length > 0);
       return type_size(type->fields.array) * type->length;
    case GLSL_TYPE_STRUCT:
       size = 0;
@@ -561,19 +727,176 @@ ir_to_mesa_visitor::visit(ir_variable *ir)
 
       fp->OriginUpperLeft = ir->origin_upper_left;
       fp->PixelCenterInteger = ir->pixel_center_integer;
+
+   } else if (strcmp(ir->name, "gl_FragDepth") == 0) {
+      struct gl_fragment_program *fp = (struct gl_fragment_program *)this->prog;
+      switch (ir->depth_layout) {
+      case ir_depth_layout_none:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_NONE;
+        break;
+      case ir_depth_layout_any:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_ANY;
+        break;
+      case ir_depth_layout_greater:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_GREATER;
+        break;
+      case ir_depth_layout_less:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_LESS;
+        break;
+      case ir_depth_layout_unchanged:
+        fp->FragDepthLayout = FRAG_DEPTH_LAYOUT_UNCHANGED;
+        break;
+      default:
+        assert(0);
+        break;
+      }
+   }
+
+   if (ir->mode == ir_var_uniform && strncmp(ir->name, "gl_", 3) == 0) {
+      unsigned int i;
+      const struct gl_builtin_uniform_desc *statevar;
+
+      for (i = 0; _mesa_builtin_uniform_desc[i].name; i++) {
+        if (strcmp(ir->name, _mesa_builtin_uniform_desc[i].name) == 0)
+           break;
+      }
+
+      if (!_mesa_builtin_uniform_desc[i].name) {
+        fail_link(this->shader_program,
+                  "Failed to find builtin uniform `%s'\n", ir->name);
+        return;
+      }
+
+      statevar = &_mesa_builtin_uniform_desc[i];
+
+      int array_count;
+      if (ir->type->is_array()) {
+        array_count = ir->type->length;
+      } else {
+        array_count = 1;
+      }
+
+      /* Check if this statevar's setup in the STATE file exactly
+       * matches how we'll want to reference it as a
+       * struct/array/whatever.  If not, then we need to move it into
+       * temporary storage and hope that it'll get copy-propagated
+       * out.
+       */
+      for (i = 0; i < statevar->num_elements; i++) {
+        if (statevar->elements[i].swizzle != SWIZZLE_XYZW) {
+           break;
+        }
+      }
+
+      struct variable_storage *storage;
+      ir_to_mesa_dst_reg dst;
+      if (i == statevar->num_elements) {
+        /* We'll set the index later. */
+        storage = new(mem_ctx) variable_storage(ir, PROGRAM_STATE_VAR, -1);
+        this->variables.push_tail(storage);
+
+        dst = ir_to_mesa_undef_dst;
+      } else {
+        storage = new(mem_ctx) variable_storage(ir, PROGRAM_TEMPORARY,
+                                                this->next_temp);
+        this->variables.push_tail(storage);
+        this->next_temp += type_size(ir->type);
+
+        dst = ir_to_mesa_dst_reg_from_src(ir_to_mesa_src_reg(PROGRAM_TEMPORARY,
+                                                             storage->index,
+                                                             NULL));
+      }
+
+
+      for (int a = 0; a < array_count; a++) {
+        for (unsigned int i = 0; i < statevar->num_elements; i++) {
+           struct gl_builtin_uniform_element *element = &statevar->elements[i];
+           int tokens[STATE_LENGTH];
+
+           memcpy(tokens, element->tokens, sizeof(element->tokens));
+           if (ir->type->is_array()) {
+              tokens[1] = a;
+           }
+
+           int index = _mesa_add_state_reference(this->prog->Parameters,
+                                                 (gl_state_index *)tokens);
+
+           if (storage->file == PROGRAM_STATE_VAR) {
+              if (storage->index == -1) {
+                 storage->index = index;
+              } else {
+                 assert(index ==
+                         (int)(storage->index + a * statevar->num_elements + i));
+              }
+           } else {
+              ir_to_mesa_src_reg src(PROGRAM_STATE_VAR, index, NULL);
+              src.swizzle = element->swizzle;
+              ir_to_mesa_emit_op1(ir, OPCODE_MOV, dst, src);
+              /* even a float takes up a whole vec4 reg in a struct/array. */
+              dst.index++;
+           }
+        }
+      }
+      if (storage->file == PROGRAM_TEMPORARY &&
+         dst.index != storage->index + type_size(ir->type)) {
+        fail_link(this->shader_program,
+                  "failed to load builtin uniform `%s'  (%d/%d regs loaded)\n",
+                  ir->name, dst.index - storage->index,
+                  type_size(ir->type));
+      }
    }
 }
 
 void
 ir_to_mesa_visitor::visit(ir_loop *ir)
 {
-   assert(!ir->from);
-   assert(!ir->to);
-   assert(!ir->increment);
-   assert(!ir->counter);
+   ir_dereference_variable *counter = NULL;
+
+   if (ir->counter != NULL)
+      counter = new(ir) ir_dereference_variable(ir->counter);
+
+   if (ir->from != NULL) {
+      assert(ir->counter != NULL);
+
+      ir_assignment *a = new(ir) ir_assignment(counter, ir->from, NULL);
+
+      a->accept(this);
+      delete a;
+   }
 
    ir_to_mesa_emit_op0(NULL, OPCODE_BGNLOOP);
+
+   if (ir->to) {
+      ir_expression *e =
+        new(ir) ir_expression(ir->cmp, glsl_type::bool_type,
+                              counter, ir->to);
+      ir_if *if_stmt =  new(ir) ir_if(e);
+
+      ir_loop_jump *brk = new(ir) ir_loop_jump(ir_loop_jump::jump_break);
+
+      if_stmt->then_instructions.push_tail(brk);
+
+      if_stmt->accept(this);
+
+      delete if_stmt;
+      delete e;
+      delete brk;
+   }
+
    visit_exec_list(&ir->body_instructions, this);
+
+   if (ir->increment) {
+      ir_expression *e =
+        new(ir) ir_expression(ir_binop_add, counter->type,
+                              counter, ir->increment);
+
+      ir_assignment *a = new(ir) ir_assignment(counter, e, NULL);
+
+      a->accept(this);
+      delete a;
+      delete e;
+   }
+
    ir_to_mesa_emit_op0(NULL, OPCODE_ENDLOOP);
 }
 
@@ -644,6 +967,32 @@ ir_to_mesa_visitor::try_emit_mad(ir_expression *ir, int mul_operand)
    return true;
 }
 
+GLboolean
+ir_to_mesa_visitor::try_emit_sat(ir_expression *ir)
+{
+   /* Saturates were only introduced to vertex programs in
+    * NV_vertex_program3, so don't give them to drivers in the VP.
+    */
+   if (this->prog->Target == GL_VERTEX_PROGRAM_ARB)
+      return false;
+
+   ir_rvalue *sat_src = ir->as_rvalue_to_saturate();
+   if (!sat_src)
+      return false;
+
+   sat_src->accept(this);
+   ir_to_mesa_src_reg src = this->result;
+
+   this->result = get_temp(ir->type);
+   ir_to_mesa_instruction *inst;
+   inst = ir_to_mesa_emit_op1(ir, OPCODE_MOV,
+                             ir_to_mesa_dst_reg_from_src(this->result),
+                             src);
+   inst->saturate = true;
+
+   return true;
+}
+
 void
 ir_to_mesa_visitor::reladdr_to_temp(ir_instruction *ir,
                                    ir_to_mesa_src_reg *reg, int *num_reladdr)
@@ -664,16 +1013,130 @@ ir_to_mesa_visitor::reladdr_to_temp(ir_instruction *ir,
    (*num_reladdr)--;
 }
 
+void
+ir_to_mesa_visitor::emit_swz(ir_expression *ir)
+{
+   /* Assume that the vector operator is in a form compatible with OPCODE_SWZ.
+    * This means that each of the operands is either an immediate value of -1,
+    * 0, or 1, or is a component from one source register (possibly with
+    * negation).
+    */
+   uint8_t components[4] = { 0 };
+   bool negate[4] = { false };
+   ir_variable *var = NULL;
+
+   for (unsigned i = 0; i < ir->type->vector_elements; i++) {
+      ir_rvalue *op = ir->operands[i];
+
+      assert(op->type->is_scalar());
+
+      while (op != NULL) {
+        switch (op->ir_type) {
+        case ir_type_constant: {
+
+           assert(op->type->is_scalar());
+
+           const ir_constant *const c = op->as_constant();
+           if (c->is_one()) {
+              components[i] = SWIZZLE_ONE;
+           } else if (c->is_zero()) {
+              components[i] = SWIZZLE_ZERO;
+           } else if (c->is_negative_one()) {
+              components[i] = SWIZZLE_ONE;
+              negate[i] = true;
+           } else {
+              assert(!"SWZ constant must be 0.0 or 1.0.");
+           }
+
+           op = NULL;
+           break;
+        }
+
+        case ir_type_dereference_variable: {
+           ir_dereference_variable *const deref =
+              (ir_dereference_variable *) op;
+
+           assert((var == NULL) || (deref->var == var));
+           components[i] = SWIZZLE_X;
+           var = deref->var;
+           op = NULL;
+           break;
+        }
+
+        case ir_type_expression: {
+           ir_expression *const expr = (ir_expression *) op;
+
+           assert(expr->operation == ir_unop_neg);
+           negate[i] = true;
+
+           op = expr->operands[0];
+           break;
+        }
+
+        case ir_type_swizzle: {
+           ir_swizzle *const swiz = (ir_swizzle *) op;
+
+           components[i] = swiz->mask.x;
+           op = swiz->val;
+           break;
+        }
+
+        default:
+           assert(!"Should not get here.");
+           return;
+        }
+      }
+   }
+
+   assert(var != NULL);
+
+   ir_dereference_variable *const deref =
+      new(mem_ctx) ir_dereference_variable(var);
+
+   this->result.file = PROGRAM_UNDEFINED;
+   deref->accept(this);
+   if (this->result.file == PROGRAM_UNDEFINED) {
+      ir_print_visitor v;
+      printf("Failed to get tree for expression operand:\n");
+      deref->accept(&v);
+      exit(1);
+   }
+
+   ir_to_mesa_src_reg src;
+
+   src = this->result;
+   src.swizzle = MAKE_SWIZZLE4(components[0],
+                              components[1],
+                              components[2],
+                              components[3]);
+   src.negate = ((unsigned(negate[0]) << 0)
+                | (unsigned(negate[1]) << 1)
+                | (unsigned(negate[2]) << 2)
+                | (unsigned(negate[3]) << 3));
+
+   /* Storage for our result.  Ideally for an assignment we'd be using the
+    * actual storage for the result here, instead.
+    */
+   const ir_to_mesa_src_reg result_src = get_temp(ir->type);
+   ir_to_mesa_dst_reg result_dst = ir_to_mesa_dst_reg_from_src(result_src);
+
+   /* Limit writes to the channels that will be used by result_src later.
+    * This does limit this temp's use as a temporary for multi-instruction
+    * sequences.
+    */
+   result_dst.writemask = (1 << ir->type->vector_elements) - 1;
+
+   ir_to_mesa_emit_op1(ir, OPCODE_SWZ, result_dst, src);
+   this->result = result_src;
+}
+
 void
 ir_to_mesa_visitor::visit(ir_expression *ir)
 {
    unsigned int operand;
-   struct ir_to_mesa_src_reg op[2];
+   struct ir_to_mesa_src_reg op[Elements(ir->operands)];
    struct ir_to_mesa_src_reg result_src;
    struct ir_to_mesa_dst_reg result_dst;
-   const glsl_type *vec4_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 4, 1);
-   const glsl_type *vec3_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 3, 1);
-   const glsl_type *vec2_type = glsl_type::get_instance(GLSL_TYPE_FLOAT, 2, 1);
 
    /* Quick peephole: Emit OPCODE_MAD(a, b, c) instead of ADD(MUL(a, b), c)
     */
@@ -683,6 +1146,13 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       if (try_emit_mad(ir, 0))
         return;
    }
+   if (try_emit_sat(ir))
+      return;
+
+   if (ir->operation == ir_quadop_vector) {
+      this->emit_swz(ir);
+      return;
+   }
 
    for (operand = 0; operand < ir->get_num_operands(); operand++) {
       this->result.file = PROGRAM_UNDEFINED;
@@ -701,6 +1171,12 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       assert(!ir->operands[operand]->type->is_matrix());
    }
 
+   int vector_elements = ir->operands[0]->type->vector_elements;
+   if (ir->operands[1]) {
+      vector_elements = MAX2(vector_elements,
+                            ir->operands[1]->type->vector_elements);
+   }
+
    this->result.file = PROGRAM_UNDEFINED;
 
    /* Storage for our result.  Ideally for an assignment we'd be using
@@ -750,6 +1226,12 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
    case ir_unop_cos:
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_COS, result_dst, op[0]);
       break;
+   case ir_unop_sin_reduced:
+      emit_scs(ir, OPCODE_SIN, result_dst, op[0]);
+      break;
+   case ir_unop_cos_reduced:
+      emit_scs(ir, OPCODE_COS, result_dst, op[0]);
+      break;
 
    case ir_unop_dFdx:
       ir_to_mesa_emit_op1(ir, OPCODE_DDX, result_dst, op[0]);
@@ -758,6 +1240,16 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       ir_to_mesa_emit_op1(ir, OPCODE_DDY, result_dst, op[0]);
       break;
 
+   case ir_unop_noise: {
+      const enum prog_opcode opcode =
+        prog_opcode(OPCODE_NOISE1
+                    + (ir->operands[0]->type->vector_elements) - 1);
+      assert((opcode >= OPCODE_NOISE1) && (opcode <= OPCODE_NOISE4));
+
+      ir_to_mesa_emit_op1(ir, opcode, result_dst, op[0]);
+      break;
+   }
+
    case ir_binop_add:
       ir_to_mesa_emit_op2(ir, OPCODE_ADD, result_dst, op[0], op[1]);
       break;
@@ -787,27 +1279,33 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       ir_to_mesa_emit_op2(ir, OPCODE_SGE, result_dst, op[0], op[1]);
       break;
    case ir_binop_equal:
+      ir_to_mesa_emit_op2(ir, OPCODE_SEQ, result_dst, op[0], op[1]);
+      break;
+   case ir_binop_nequal:
+      ir_to_mesa_emit_op2(ir, OPCODE_SNE, result_dst, op[0], op[1]);
+      break;
+   case ir_binop_all_equal:
       /* "==" operator producing a scalar boolean. */
       if (ir->operands[0]->type->is_vector() ||
          ir->operands[1]->type->is_vector()) {
         ir_to_mesa_src_reg temp = get_temp(glsl_type::vec4_type);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             ir_to_mesa_dst_reg_from_src(temp), op[0], op[1]);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4, result_dst, temp, temp);
+        ir_to_mesa_emit_dp(ir, result_dst, temp, temp, vector_elements);
         ir_to_mesa_emit_op2(ir, OPCODE_SEQ,
                             result_dst, result_src, src_reg_for_float(0.0));
       } else {
         ir_to_mesa_emit_op2(ir, OPCODE_SEQ, result_dst, op[0], op[1]);
       }
       break;
-   case ir_binop_nequal:
+   case ir_binop_any_nequal:
       /* "!=" operator producing a scalar boolean. */
       if (ir->operands[0]->type->is_vector() ||
          ir->operands[1]->type->is_vector()) {
         ir_to_mesa_src_reg temp = get_temp(glsl_type::vec4_type);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             ir_to_mesa_dst_reg_from_src(temp), op[0], op[1]);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4, result_dst, temp, temp);
+        ir_to_mesa_emit_dp(ir, result_dst, temp, temp, vector_elements);
         ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                             result_dst, result_src, src_reg_for_float(0.0));
       } else {
@@ -816,20 +1314,9 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       break;
 
    case ir_unop_any:
-      switch (ir->operands[0]->type->vector_elements) {
-      case 4:
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4, result_dst, op[0], op[0]);
-        break;
-      case 3:
-        ir_to_mesa_emit_op2(ir, OPCODE_DP3, result_dst, op[0], op[0]);
-        break;
-      case 2:
-        ir_to_mesa_emit_op2(ir, OPCODE_DP2, result_dst, op[0], op[0]);
-        break;
-      default:
-        assert(!"unreached: ir_unop_any of non-bvec");
-        break;
-      }
+      assert(ir->operands[0]->type->is_vector());
+      ir_to_mesa_emit_dp(ir, result_dst, op[0], op[0],
+                        ir->operands[0]->type->vector_elements);
       ir_to_mesa_emit_op2(ir, OPCODE_SNE,
                          result_dst, result_src, src_reg_for_float(0.0));
       break;
@@ -857,34 +1344,20 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
       break;
 
    case ir_binop_dot:
-      if (ir->operands[0]->type == vec4_type) {
-        assert(ir->operands[1]->type == vec4_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP4,
-                            result_dst,
-                            op[0], op[1]);
-      } else if (ir->operands[0]->type == vec3_type) {
-        assert(ir->operands[1]->type == vec3_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP3,
-                            result_dst,
-                            op[0], op[1]);
-      } else if (ir->operands[0]->type == vec2_type) {
-        assert(ir->operands[1]->type == vec2_type);
-        ir_to_mesa_emit_op2(ir, OPCODE_DP2,
-                            result_dst,
-                            op[0], op[1]);
-      }
-      break;
-
-   case ir_binop_cross:
-      ir_to_mesa_emit_op2(ir, OPCODE_XPD, result_dst, op[0], op[1]);
+      assert(ir->operands[0]->type->is_vector());
+      assert(ir->operands[0]->type == ir->operands[1]->type);
+      ir_to_mesa_emit_dp(ir, result_dst, op[0], op[1],
+                        ir->operands[0]->type->vector_elements);
       break;
 
    case ir_unop_sqrt:
+      /* sqrt(x) = x * rsq(x). */
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_RSQ, result_dst, op[0]);
-      ir_to_mesa_emit_scalar_op1(ir, OPCODE_RCP, result_dst, result_src);
-      /* For incoming channels < 0, set the result to 0. */
+      ir_to_mesa_emit_op2(ir, OPCODE_MUL, result_dst, result_src, op[0]);
+      /* For incoming channels <= 0, set the result to 0. */
+      op[0].negate = ~op[0].negate;
       ir_to_mesa_emit_op3(ir, OPCODE_CMP, result_dst,
-                         op[0], src_reg_for_float(0.0), result_src);
+                         op[0], result_src, src_reg_for_float(0.0));
       break;
    case ir_unop_rsq:
       ir_to_mesa_emit_scalar_op1(ir, OPCODE_RSQ, result_dst, op[0]);
@@ -935,8 +1408,15 @@ ir_to_mesa_visitor::visit(ir_expression *ir)
    case ir_binop_bit_and:
    case ir_binop_bit_xor:
    case ir_binop_bit_or:
+   case ir_unop_round_even:
       assert(!"GLSL 1.30 features unsupported");
       break;
+
+   case ir_quadop_vector:
+      /* This operation should have already been handled.
+       */
+      assert(!"Should not get here.");
+      break;
    }
 
    this->result = result_src;
@@ -991,308 +1471,20 @@ ir_to_mesa_visitor::visit(ir_swizzle *ir)
    this->result = src_reg;
 }
 
-static const struct {
-   const char *name;
-   const char *field;
-   int tokens[STATE_LENGTH];
-   int swizzle;
-   bool array_indexed;
-} statevars[] = {
-   {"gl_DepthRange", "near",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_XXXX, false},
-   {"gl_DepthRange", "far",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_YYYY, false},
-   {"gl_DepthRange", "diff",
-    {STATE_DEPTH_RANGE, 0, 0}, SWIZZLE_ZZZZ, false},
-
-   {"gl_ClipPlane", NULL,
-    {STATE_CLIPPLANE, 0, 0}, SWIZZLE_XYZW, true}
-,
-   {"gl_Point", "size",
-    {STATE_POINT_SIZE}, SWIZZLE_XXXX, false},
-   {"gl_Point", "sizeMin",
-    {STATE_POINT_SIZE}, SWIZZLE_YYYY, false},
-   {"gl_Point", "sizeMax",
-    {STATE_POINT_SIZE}, SWIZZLE_ZZZZ, false},
-   {"gl_Point", "fadeThresholdSize",
-    {STATE_POINT_SIZE}, SWIZZLE_WWWW, false},
-   {"gl_Point", "distanceConstantAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_XXXX, false},
-   {"gl_Point", "distanceLinearAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_YYYY, false},
-   {"gl_Point", "distanceQuadraticAttenuation",
-    {STATE_POINT_ATTENUATION}, SWIZZLE_ZZZZ, false},
-
-   {"gl_FrontMaterial", "emission",
-    {STATE_MATERIAL, 0, STATE_EMISSION}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "ambient",
-    {STATE_MATERIAL, 0, STATE_AMBIENT}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "diffuse",
-    {STATE_MATERIAL, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "specular",
-    {STATE_MATERIAL, 0, STATE_SPECULAR}, SWIZZLE_XYZW, false},
-   {"gl_FrontMaterial", "shininess",
-    {STATE_MATERIAL, 0, STATE_SHININESS}, SWIZZLE_XXXX, false},
-
-   {"gl_BackMaterial", "emission",
-    {STATE_MATERIAL, 1, STATE_EMISSION}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "ambient",
-    {STATE_MATERIAL, 1, STATE_AMBIENT}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "diffuse",
-    {STATE_MATERIAL, 1, STATE_DIFFUSE}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "specular",
-    {STATE_MATERIAL, 1, STATE_SPECULAR}, SWIZZLE_XYZW, false},
-   {"gl_BackMaterial", "shininess",
-    {STATE_MATERIAL, 1, STATE_SHININESS}, SWIZZLE_XXXX, false},
-
-   {"gl_LightSource", "ambient",
-    {STATE_LIGHT, 0, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "diffuse",
-    {STATE_LIGHT, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "specular",
-    {STATE_LIGHT, 0, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "position",
-    {STATE_LIGHT, 0, STATE_POSITION}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "halfVector",
-    {STATE_LIGHT, 0, STATE_HALF_VECTOR}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "spotDirection",
-    {STATE_LIGHT, 0, STATE_SPOT_DIRECTION}, SWIZZLE_XYZW, true},
-   {"gl_LightSource", "spotCosCutoff",
-    {STATE_LIGHT, 0, STATE_SPOT_DIRECTION}, SWIZZLE_WWWW, true},
-   {"gl_LightSource", "spotCutoff",
-    {STATE_LIGHT, 0, STATE_SPOT_CUTOFF}, SWIZZLE_XXXX, true},
-   {"gl_LightSource", "spotExponent",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_WWWW, true},
-   {"gl_LightSource", "constantAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_XXXX, true},
-   {"gl_LightSource", "linearAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_YYYY, true},
-   {"gl_LightSource", "quadraticAttenuation",
-    {STATE_LIGHT, 0, STATE_ATTENUATION}, SWIZZLE_ZZZZ, true},
-
-   {"gl_LightModel", "ambient",
-    {STATE_LIGHTMODEL_AMBIENT, 0}, SWIZZLE_XYZW, false},
-
-   {"gl_FrontLightModelProduct", "sceneColor",
-    {STATE_LIGHTMODEL_SCENECOLOR, 0}, SWIZZLE_XYZW, false},
-   {"gl_BackLightModelProduct", "sceneColor",
-    {STATE_LIGHTMODEL_SCENECOLOR, 1}, SWIZZLE_XYZW, false},
-
-   {"gl_FrontLightProduct", "ambient",
-    {STATE_LIGHTPROD, 0, 0, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_FrontLightProduct", "diffuse",
-    {STATE_LIGHTPROD, 0, 0, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_FrontLightProduct", "specular",
-    {STATE_LIGHTPROD, 0, 0, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-
-   {"gl_BackLightProduct", "ambient",
-    {STATE_LIGHTPROD, 0, 1, STATE_AMBIENT}, SWIZZLE_XYZW, true},
-   {"gl_BackLightProduct", "diffuse",
-    {STATE_LIGHTPROD, 0, 1, STATE_DIFFUSE}, SWIZZLE_XYZW, true},
-   {"gl_BackLightProduct", "specular",
-    {STATE_LIGHTPROD, 0, 1, STATE_SPECULAR}, SWIZZLE_XYZW, true},
-
-   {"gl_TextureEnvColor", NULL,
-    {STATE_TEXENV_COLOR, 0}, SWIZZLE_XYZW, true},
-
-   {"gl_EyePlaneS", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_S}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneT", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_T}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneR", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_R}, SWIZZLE_XYZW, true},
-   {"gl_EyePlaneQ", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_EYE_Q}, SWIZZLE_XYZW, true},
-
-   {"gl_ObjectPlaneS", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_S}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneT", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_T}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneR", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_R}, SWIZZLE_XYZW, true},
-   {"gl_ObjectPlaneQ", NULL,
-    {STATE_TEXGEN, 0, STATE_TEXGEN_OBJECT_Q}, SWIZZLE_XYZW, true},
-
-   {"gl_Fog", "color",
-    {STATE_FOG_COLOR}, SWIZZLE_XYZW, false},
-   {"gl_Fog", "density",
-    {STATE_FOG_PARAMS}, SWIZZLE_XXXX, false},
-   {"gl_Fog", "start",
-    {STATE_FOG_PARAMS}, SWIZZLE_YYYY, false},
-   {"gl_Fog", "end",
-    {STATE_FOG_PARAMS}, SWIZZLE_ZZZZ, false},
-   {"gl_Fog", "scale",
-    {STATE_FOG_PARAMS}, SWIZZLE_WWWW, false},
-};
-
-static ir_to_mesa_src_reg
-get_builtin_uniform_reg(struct gl_program *prog,
-                       const char *name, int array_index, const char *field)
-{
-   unsigned int i;
-   ir_to_mesa_src_reg src_reg;
-   int tokens[STATE_LENGTH];
-
-   for (i = 0; i < Elements(statevars); i++) {
-      if (strcmp(statevars[i].name, name) != 0)
-        continue;
-      if (!field && statevars[i].field) {
-        assert(!"FINISHME: whole-structure state var dereference");
-      }
-      if (field && (!statevars[i].field || strcmp(statevars[i].field, field) != 0))
-        continue;
-      break;
-   }
-
-   if (i ==  Elements(statevars)) {
-      printf("builtin uniform %s%s%s not found\n",
-            name,
-            field ? "." : "",
-            field ? field : "");
-      abort();
-   }
-
-   memcpy(&tokens, statevars[i].tokens, sizeof(tokens));
-   if (statevars[i].array_indexed)
-      tokens[1] = array_index;
-
-   src_reg.file = PROGRAM_STATE_VAR;
-   src_reg.index = _mesa_add_state_reference(prog->Parameters,
-                                            (gl_state_index *)tokens);
-   src_reg.swizzle = statevars[i].swizzle;
-   src_reg.negate = 0;
-   src_reg.reladdr = false;
-
-   return src_reg;
-}
-
-static int
-add_matrix_ref(struct gl_program *prog, int *tokens)
-{
-   int base_pos = -1;
-   int i;
-
-   /* Add a ref for each column.  It looks like the reason we do
-    * it this way is that _mesa_add_state_reference doesn't work
-    * for things that aren't vec4s, so the tokens[2]/tokens[3]
-    * range has to be equal.
-    */
-   for (i = 0; i < 4; i++) {
-      tokens[2] = i;
-      tokens[3] = i;
-      int pos = _mesa_add_state_reference(prog->Parameters,
-                                         (gl_state_index *)tokens);
-      if (base_pos == -1)
-        base_pos = pos;
-      else
-        assert(base_pos + i == pos);
-   }
-
-   return base_pos;
-}
-
-static variable_storage *
-get_builtin_matrix_ref(void *mem_ctx, struct gl_program *prog, ir_variable *var,
-                      ir_rvalue *array_index)
-{
-   /*
-    * NOTE: The ARB_vertex_program extension specified that matrices get
-    * loaded in registers in row-major order.  With GLSL, we want column-
-    * major order.  So, we need to transpose all matrices here...
-    */
-   static const struct {
-      const char *name;
-      int matrix;
-      int modifier;
-   } matrices[] = {
-      { "gl_ModelViewMatrix", STATE_MODELVIEW_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ModelViewMatrixInverse", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ModelViewMatrixTranspose", STATE_MODELVIEW_MATRIX, 0 },
-      { "gl_ModelViewMatrixInverseTranspose", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_ProjectionMatrix", STATE_PROJECTION_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ProjectionMatrixInverse", STATE_PROJECTION_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ProjectionMatrixTranspose", STATE_PROJECTION_MATRIX, 0 },
-      { "gl_ProjectionMatrixInverseTranspose", STATE_PROJECTION_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_ModelViewProjectionMatrix", STATE_MVP_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_ModelViewProjectionMatrixInverse", STATE_MVP_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_ModelViewProjectionMatrixTranspose", STATE_MVP_MATRIX, 0 },
-      { "gl_ModelViewProjectionMatrixInverseTranspose", STATE_MVP_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_TextureMatrix", STATE_TEXTURE_MATRIX, STATE_MATRIX_TRANSPOSE },
-      { "gl_TextureMatrixInverse", STATE_TEXTURE_MATRIX, STATE_MATRIX_INVTRANS },
-      { "gl_TextureMatrixTranspose", STATE_TEXTURE_MATRIX, 0 },
-      { "gl_TextureMatrixInverseTranspose", STATE_TEXTURE_MATRIX, STATE_MATRIX_INVERSE },
-
-      { "gl_NormalMatrix", STATE_MODELVIEW_MATRIX, STATE_MATRIX_INVERSE },
-
-   };
-   unsigned int i;
-   variable_storage *entry;
-
-   /* C++ gets angry when we try to use an int as a gl_state_index, so we use
-    * ints for gl_state_index.  Make sure they're compatible.
-    */
-   assert(sizeof(gl_state_index) == sizeof(int));
-
-   for (i = 0; i < Elements(matrices); i++) {
-      if (strcmp(var->name, matrices[i].name) == 0) {
-        int tokens[STATE_LENGTH];
-        int base_pos = -1;
-
-        tokens[0] = matrices[i].matrix;
-        tokens[4] = matrices[i].modifier;
-        if (matrices[i].matrix == STATE_TEXTURE_MATRIX) {
-           ir_constant *index = array_index->constant_expression_value();
-           if (index) {
-              tokens[1] = index->value.i[0];
-              base_pos = add_matrix_ref(prog, tokens);
-           } else {
-              for (i = 0; i < var->type->length; i++) {
-                 tokens[1] = i;
-                 int pos = add_matrix_ref(prog, tokens);
-                 if (base_pos == -1)
-                    base_pos = pos;
-                 else
-                    assert(base_pos + (int)i * 4 == pos);
-              }
-           }
-        } else {
-           tokens[1] = 0; /* unused array index */
-           base_pos = add_matrix_ref(prog, tokens);
-        }
-
-        entry = new(mem_ctx) variable_storage(var,
-                                              PROGRAM_STATE_VAR,
-                                              base_pos);
-
-        return entry;
-      }
-   }
-
-   return NULL;
-}
-
 void
 ir_to_mesa_visitor::visit(ir_dereference_variable *ir)
 {
    variable_storage *entry = find_variable_storage(ir->var);
+   ir_variable *var = ir->var;
 
    if (!entry) {
-      switch (ir->var->mode) {
+      switch (var->mode) {
       case ir_var_uniform:
-        entry = get_builtin_matrix_ref(this->mem_ctx, this->prog, ir->var,
-                                       NULL);
-        if (entry)
-           break;
-
-        entry = new(mem_ctx) variable_storage(ir->var, PROGRAM_UNIFORM,
-                                              ir->var->location);
+        entry = new(mem_ctx) variable_storage(var, PROGRAM_UNIFORM,
+                                              var->location);
         this->variables.push_tail(entry);
         break;
       case ir_var_in:
-      case ir_var_out:
       case ir_var_inout:
         /* The linker assigns locations for varyings and attributes,
          * including deprecated builtins (like gl_Color), user-assign
@@ -1301,102 +1493,58 @@ ir_to_mesa_visitor::visit(ir_dereference_variable *ir)
          *
          * FINISHME: We would hit this path for function arguments.  Fix!
          */
-        assert(ir->var->location != -1);
-        if (ir->var->mode == ir_var_in ||
-            ir->var->mode == ir_var_inout) {
-           entry = new(mem_ctx) variable_storage(ir->var,
-                                                 PROGRAM_INPUT,
-                                                 ir->var->location);
-
-           if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
-               ir->var->location >= VERT_ATTRIB_GENERIC0) {
-              _mesa_add_attribute(prog->Attributes,
-                                  ir->var->name,
-                                  _mesa_sizeof_glsl_type(ir->var->type->gl_type),
-                                  ir->var->type->gl_type,
-                                  ir->var->location - VERT_ATTRIB_GENERIC0);
-           }
-        } else {
-           entry = new(mem_ctx) variable_storage(ir->var,
-                                                 PROGRAM_OUTPUT,
-                                                 ir->var->location);
-        }
-
+        assert(var->location != -1);
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_INPUT,
+                                               var->location);
+         if (this->prog->Target == GL_VERTEX_PROGRAM_ARB &&
+             var->location >= VERT_ATTRIB_GENERIC0) {
+            _mesa_add_attribute(this->prog->Attributes,
+                                var->name,
+                                _mesa_sizeof_glsl_type(var->type->gl_type),
+                                var->type->gl_type,
+                                var->location - VERT_ATTRIB_GENERIC0);
+         }
+         break;
+      case ir_var_out:
+        assert(var->location != -1);
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_OUTPUT,
+                                               var->location);
         break;
+      case ir_var_system_value:
+         entry = new(mem_ctx) variable_storage(var,
+                                               PROGRAM_SYSTEM_VALUE,
+                                               var->location);
+         break;
       case ir_var_auto:
       case ir_var_temporary:
-        entry = new(mem_ctx) variable_storage(ir->var, PROGRAM_TEMPORARY,
+        entry = new(mem_ctx) variable_storage(var, PROGRAM_TEMPORARY,
                                               this->next_temp);
         this->variables.push_tail(entry);
 
-        next_temp += type_size(ir->var->type);
+        next_temp += type_size(var->type);
         break;
       }
 
       if (!entry) {
-        printf("Failed to make storage for %s\n", ir->var->name);
+        printf("Failed to make storage for %s\n", var->name);
         exit(1);
       }
    }
 
-   this->result = ir_to_mesa_src_reg(entry->file, entry->index, ir->var->type);
+   this->result = ir_to_mesa_src_reg(entry->file, entry->index, var->type);
 }
 
 void
 ir_to_mesa_visitor::visit(ir_dereference_array *ir)
 {
-   ir_variable *var = ir->variable_referenced();
    ir_constant *index;
    ir_to_mesa_src_reg src_reg;
-   ir_dereference_variable *deref_var = ir->array->as_dereference_variable();
    int element_size = type_size(ir->type);
 
    index = ir->array_index->constant_expression_value();
 
-   if (deref_var && strncmp(deref_var->var->name,
-                           "gl_TextureMatrix",
-                           strlen("gl_TextureMatrix")) == 0) {
-      struct variable_storage *entry;
-
-      entry = get_builtin_matrix_ref(this->mem_ctx, this->prog, deref_var->var,
-                                    ir->array_index);
-      assert(entry);
-
-      ir_to_mesa_src_reg src_reg(entry->file, entry->index, ir->type);
-
-      if (index) {
-        src_reg.reladdr = NULL;
-      } else {
-        ir_to_mesa_src_reg index_reg = get_temp(glsl_type::float_type);
-
-        ir->array_index->accept(this);
-        ir_to_mesa_emit_op2(ir, OPCODE_MUL,
-                            ir_to_mesa_dst_reg_from_src(index_reg),
-                            this->result, src_reg_for_float(element_size));
-
-        src_reg.reladdr = talloc(mem_ctx, ir_to_mesa_src_reg);
-        memcpy(src_reg.reladdr, &index_reg, sizeof(index_reg));
-      }
-
-      this->result = src_reg;
-      return;
-   }
-
-   if (var &&
-       strncmp(var->name, "gl_", 3) == 0 && var->mode == ir_var_uniform &&
-       !var->type->is_matrix()) {
-      ir_dereference_record *record = NULL;
-      if (ir->array->ir_type == ir_type_dereference_record)
-        record = (ir_dereference_record *)ir->array;
-
-      assert(index || !"FINISHME: variable-indexed builtin uniform access");
-
-      this->result = get_builtin_uniform_reg(prog,
-                                            var->name,
-                                            index->value.i[0],
-                                            record ? record->field : NULL);
-   }
-
    ir->array->accept(this);
    src_reg = this->result;
 
@@ -1422,7 +1570,7 @@ ir_to_mesa_visitor::visit(ir_dereference_array *ir)
                             this->result, src_reg_for_float(element_size));
       }
 
-      src_reg.reladdr = talloc(mem_ctx, ir_to_mesa_src_reg);
+      src_reg.reladdr = ralloc(mem_ctx, ir_to_mesa_src_reg);
       memcpy(src_reg.reladdr, &index_reg, sizeof(index_reg));
    }
 
@@ -1441,17 +1589,6 @@ ir_to_mesa_visitor::visit(ir_dereference_record *ir)
    unsigned int i;
    const glsl_type *struct_type = ir->record->type;
    int offset = 0;
-   ir_variable *var = ir->record->variable_referenced();
-
-   if (strncmp(var->name, "gl_", 3) == 0 && var->mode == ir_var_uniform) {
-      assert(var);
-
-      this->result = get_builtin_uniform_reg(prog,
-                                            var->name,
-                                            0,
-                                            ir->field);
-      return;
-   }
 
    ir->record->accept(this);
 
@@ -1460,7 +1597,13 @@ ir_to_mesa_visitor::visit(ir_dereference_record *ir)
         break;
       offset += type_size(struct_type->fields.structure[i].type);
    }
-   this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
+
+   /* If the type is smaller than a vec4, replicate the last channel out. */
+   if (ir->type->is_scalar() || ir->type->is_vector())
+      this->result.swizzle = swizzle_for_size(ir->type->vector_elements);
+   else
+      this->result.swizzle = SWIZZLE_NOOP;
+
    this->result.index += offset;
 }
 
@@ -1489,6 +1632,93 @@ get_assignment_lhs(ir_dereference *ir, ir_to_mesa_visitor *v)
    return ir_to_mesa_dst_reg_from_src(v->result);
 }
 
+/**
+ * Process the condition of a conditional assignment
+ *
+ * Examines the condition of a conditional assignment to generate the optimal
+ * first operand of a \c CMP instruction.  If the condition is a relational
+ * operator with 0 (e.g., \c ir_binop_less), the value being compared will be
+ * used as the source for the \c CMP instruction.  Otherwise the comparison
+ * is processed to a boolean result, and the boolean result is used as the
+ * operand to the CMP instruction.
+ */
+bool
+ir_to_mesa_visitor::process_move_condition(ir_rvalue *ir)
+{
+   ir_rvalue *src_ir = ir;
+   bool negate = true;
+   bool switch_order = false;
+
+   ir_expression *const expr = ir->as_expression();
+   if ((expr != NULL) && (expr->get_num_operands() == 2)) {
+      bool zero_on_left = false;
+
+      if (expr->operands[0]->is_zero()) {
+        src_ir = expr->operands[1];
+        zero_on_left = true;
+      } else if (expr->operands[1]->is_zero()) {
+        src_ir = expr->operands[0];
+        zero_on_left = false;
+      }
+
+      /*      a is -  0  +            -  0  +
+       * (a <  0)  T  F  F  ( a < 0)  T  F  F
+       * (0 <  a)  F  F  T  (-a < 0)  F  F  T
+       * (a <= 0)  T  T  F  (-a < 0)  F  F  T  (swap order of other operands)
+       * (0 <= a)  F  T  T  ( a < 0)  T  F  F  (swap order of other operands)
+       * (a >  0)  F  F  T  (-a < 0)  F  F  T
+       * (0 >  a)  T  F  F  ( a < 0)  T  F  F
+       * (a >= 0)  F  T  T  ( a < 0)  T  F  F  (swap order of other operands)
+       * (0 >= a)  T  T  F  (-a < 0)  F  F  T  (swap order of other operands)
+       *
+       * Note that exchanging the order of 0 and 'a' in the comparison simply
+       * means that the value of 'a' should be negated.
+       */
+      if (src_ir != ir) {
+        switch (expr->operation) {
+        case ir_binop_less:
+           switch_order = false;
+           negate = zero_on_left;
+           break;
+
+        case ir_binop_greater:
+           switch_order = false;
+           negate = !zero_on_left;
+           break;
+
+        case ir_binop_lequal:
+           switch_order = true;
+           negate = !zero_on_left;
+           break;
+
+        case ir_binop_gequal:
+           switch_order = true;
+           negate = zero_on_left;
+           break;
+
+        default:
+           /* This isn't the right kind of comparison afterall, so make sure
+            * the whole condition is visited.
+            */
+           src_ir = ir;
+           break;
+        }
+      }
+   }
+
+   src_ir->accept(this);
+
+   /* We use the OPCODE_CMP (a < 0 ? b : c) for conditional moves, and the
+    * condition we produced is 0.0 or 1.0.  By flipping the sign, we can
+    * choose which value OPCODE_CMP produces without an extra instruction
+    * computing the condition.
+    */
+   if (negate)
+      this->result.negate = ~this->result.negate;
+
+   return switch_order;
+}
+
 void
 ir_to_mesa_visitor::visit(ir_assignment *ir)
 {
@@ -1509,33 +1739,57 @@ ir_to_mesa_visitor::visit(ir_assignment *ir)
       assert(!ir->lhs->type->is_scalar() && !ir->lhs->type->is_vector());
       l.writemask = WRITEMASK_XYZW;
    } else if (ir->lhs->type->is_scalar()) {
-      /* FINISHME: This hack makes writing to gl_FragData, which lives in the
+      /* FINISHME: This hack makes writing to gl_FragDepth, which lives in the
        * FINISHME: W component of fragment shader output zero, work correctly.
        */
       l.writemask = WRITEMASK_XYZW;
    } else {
+      int swizzles[4];
+      int first_enabled_chan = 0;
+      int rhs_chan = 0;
+
       assert(ir->lhs->type->is_vector());
       l.writemask = ir->write_mask;
+
+      for (int i = 0; i < 4; i++) {
+        if (l.writemask & (1 << i)) {
+           first_enabled_chan = GET_SWZ(r.swizzle, i);
+           break;
+        }
+      }
+
+      /* Swizzle a small RHS vector into the channels being written.
+       *
+       * glsl ir treats write_mask as dictating how many channels are
+       * present on the RHS while Mesa IR treats write_mask as just
+       * showing which channels of the vec4 RHS get written.
+       */
+      for (int i = 0; i < 4; i++) {
+        if (l.writemask & (1 << i))
+           swizzles[i] = GET_SWZ(r.swizzle, rhs_chan++);
+        else
+           swizzles[i] = first_enabled_chan;
+      }
+      r.swizzle = MAKE_SWIZZLE4(swizzles[0], swizzles[1],
+                               swizzles[2], swizzles[3]);
    }
 
    assert(l.file != PROGRAM_UNDEFINED);
    assert(r.file != PROGRAM_UNDEFINED);
 
    if (ir->condition) {
-      ir_to_mesa_src_reg condition;
-
-      ir->condition->accept(this);
-      condition = this->result;
+      const bool switch_order = this->process_move_condition(ir->condition);
+      ir_to_mesa_src_reg condition = this->result;
 
-      /* We use the OPCODE_CMP (a < 0 ? b : c) for conditional moves,
-       * and the condition we produced is 0.0 or 1.0.  By flipping the
-       * sign, we can choose which value OPCODE_CMP produces without
-       * an extra computing the condition.
-       */
-      condition.negate = ~condition.negate;
       for (i = 0; i < type_size(ir->lhs->type); i++) {
-        ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
-                            condition, r, ir_to_mesa_src_reg_from_dst(l));
+        if (switch_order) {
+           ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
+                               condition, ir_to_mesa_src_reg_from_dst(l), r);
+        } else {
+           ir_to_mesa_emit_op3(ir, OPCODE_CMP, l,
+                               condition, r, ir_to_mesa_src_reg_from_dst(l));
+        }
+
         l.index++;
         r.index++;
       }
@@ -1673,7 +1927,7 @@ ir_to_mesa_visitor::get_function_signature(ir_function_signature *sig)
         return entry;
    }
 
-   entry = talloc(mem_ctx, function_entry);
+   entry = ralloc(mem_ctx, function_entry);
    entry->sig = sig;
    entry->sig_id = this->next_signature_id++;
    entry->bgn_inst = NULL;
@@ -1784,92 +2038,6 @@ ir_to_mesa_visitor::visit(ir_call *ir)
    this->result = entry->return_reg;
 }
 
-class get_sampler_name : public ir_hierarchical_visitor
-{
-public:
-   get_sampler_name(ir_to_mesa_visitor *mesa, ir_dereference *last)
-   {
-      this->mem_ctx = mesa->mem_ctx;
-      this->mesa = mesa;
-      this->name = NULL;
-      this->offset = 0;
-      this->last = last;
-   }
-
-   virtual ir_visitor_status visit(ir_dereference_variable *ir)
-   {
-      this->name = ir->var->name;
-      return visit_continue;
-   }
-
-   virtual ir_visitor_status visit_leave(ir_dereference_record *ir)
-   {
-      this->name = talloc_asprintf(mem_ctx, "%s.%s", name, ir->field);
-      return visit_continue;
-   }
-
-   virtual ir_visitor_status visit_leave(ir_dereference_array *ir)
-   {
-      ir_constant *index = ir->array_index->as_constant();
-      int i;
-
-      if (index) {
-        i = index->value.i[0];
-      } else {
-        /* GLSL 1.10 and 1.20 allowed variable sampler array indices,
-         * while GLSL 1.30 requires that the array indices be
-         * constant integer expressions.  We don't expect any driver
-         * to actually work with a really variable array index, so
-         * all that would work would be an unrolled loop counter that ends
-         * up being constant above.
-         */
-        mesa->shader_program->InfoLog =
-           talloc_asprintf_append(mesa->shader_program->InfoLog,
-                                  "warning: Variable sampler array index "
-                                  "unsupported.\nThis feature of the language "
-                                  "was removed in GLSL 1.20 and is unlikely "
-                                  "to be supported for 1.10 in Mesa.\n");
-        i = 0;
-      }
-      if (ir != last) {
-        this->name = talloc_asprintf(mem_ctx, "%s[%d]", name, i);
-      } else {
-        offset = i;
-      }
-      return visit_continue;
-   }
-
-   ir_to_mesa_visitor *mesa;
-   const char *name;
-   void *mem_ctx;
-   int offset;
-   ir_dereference *last;
-};
-
-int
-ir_to_mesa_visitor::get_sampler_uniform_value(ir_dereference *sampler)
-{
-   get_sampler_name getname(this, sampler);
-
-   sampler->accept(&getname);
-
-   GLint index = _mesa_lookup_parameter_index(prog->Parameters, -1,
-                                             getname.name);
-
-   if (index < 0) {
-      this->shader_program->InfoLog =
-        talloc_asprintf_append(this->shader_program->InfoLog,
-                               "failed to find sampler named %s.\n",
-                               getname.name);
-      this->shader_program->LinkStatus = GL_FALSE;
-      return 0;
-   }
-
-   index += getname.offset;
-
-   return this->prog->Parameters->ParameterValues[index][0];
-}
-
 void
 ir_to_mesa_visitor::visit(ir_texture *ir)
 {
@@ -1969,7 +2137,9 @@ ir_to_mesa_visitor::visit(ir_texture *ir)
    if (ir->shadow_comparitor)
       inst->tex_shadow = GL_TRUE;
 
-   inst->sampler = get_sampler_uniform_value(ir->sampler);
+   inst->sampler = _mesa_get_sampler_uniform_value(ir->sampler,
+                                                  this->shader_program,
+                                                  this->prog);
 
    const glsl_type *sampler_type = ir->sampler->type;
 
@@ -2028,9 +2198,17 @@ ir_to_mesa_visitor::visit(ir_return *ir)
 void
 ir_to_mesa_visitor::visit(ir_discard *ir)
 {
-   assert(ir->condition == NULL); /* FINISHME */
+   struct gl_fragment_program *fp = (struct gl_fragment_program *)this->prog;
+
+   if (ir->condition) {
+      ir->condition->accept(this);
+      this->result.negate = ~this->result.negate;
+      ir_to_mesa_emit_op1(ir, OPCODE_KIL, ir_to_mesa_undef_dst, this->result);
+   } else {
+      ir_to_mesa_emit_op0(ir, OPCODE_KIL_NV);
+   }
 
-   ir_to_mesa_emit_op0(ir, OPCODE_KIL_NV);
+   fp->UsesKill = GL_TRUE;
 }
 
 void
@@ -2044,7 +2222,7 @@ ir_to_mesa_visitor::visit(ir_if *ir)
    ir->condition->accept(this);
    assert(this->result.file != PROGRAM_UNDEFINED);
 
-   if (ctx->Shader.EmitCondCodes) {
+   if (this->options->EmitCondCodes) {
       cond_inst = (ir_to_mesa_instruction *)this->instructions.get_tail();
 
       /* See if we actually generated any instruction for generating
@@ -2086,12 +2264,12 @@ ir_to_mesa_visitor::ir_to_mesa_visitor()
    next_temp = 1;
    next_signature_id = 1;
    current_function = NULL;
-   mem_ctx = talloc_new(NULL);
+   mem_ctx = ralloc_context(NULL);
 }
 
 ir_to_mesa_visitor::~ir_to_mesa_visitor()
 {
-   talloc_free(mem_ctx);
+   ralloc_free(mem_ctx);
 }
 
 static struct prog_src_register
@@ -2100,7 +2278,7 @@ mesa_src_reg_from_ir_src_reg(ir_to_mesa_src_reg reg)
    struct prog_src_register mesa_reg;
 
    mesa_reg.File = reg.file;
-   assert(reg.index < (1 << INST_INDEX_BITS) - 1);
+   assert(reg.index < (1 << INST_INDEX_BITS));
    mesa_reg.Index = reg.index;
    mesa_reg.Swizzle = reg.swizzle;
    mesa_reg.RelAddr = reg.reladdr != NULL;
@@ -2140,8 +2318,8 @@ set_branchtargets(ir_to_mesa_visitor *v,
       }
    }
 
-   if_stack = talloc_zero_array(v->mem_ctx, int, if_count);
-   loop_stack = talloc_zero_array(v->mem_ctx, int, loop_count);
+   if_stack = rzalloc_array(v->mem_ctx, int, if_count);
+   loop_stack = rzalloc_array(v->mem_ctx, int, loop_count);
 
    for (i = 0; i < num_instructions; i++) {
       switch (mesa_instructions[i].Opcode) {
@@ -2251,6 +2429,25 @@ count_resources(struct gl_program *prog)
    _mesa_update_shader_textures_used(prog);
 }
 
+struct uniform_sort {
+   struct gl_uniform *u;
+   int pos;
+};
+
+/* The shader_program->Uniforms list is almost sorted in increasing
+ * uniform->{Frag,Vert}Pos locations, but not quite when there are
+ * uniforms shared between targets.  We need to add parameters in
+ * increasing order for the targets.
+ */
+static int
+sort_uniforms(const void *a, const void *b)
+{
+   struct uniform_sort *u1 = (struct uniform_sort *)a;
+   struct uniform_sort *u2 = (struct uniform_sort *)b;
+
+   return u1->pos - u2->pos;
+}
+
 /* Add the uniforms to the parameters.  The linker chose locations
  * in our parameters lists (which weren't created yet), which the
  * uniforms code will use to poke values into our parameters list
@@ -2262,12 +2459,14 @@ add_uniforms_to_parameters_list(struct gl_shader_program *shader_program,
                                struct gl_program *prog)
 {
    unsigned int i;
-   unsigned int next_sampler = 0;
+   unsigned int next_sampler = 0, num_uniforms = 0;
+   struct uniform_sort *sorted_uniforms;
+
+   sorted_uniforms = ralloc_array(NULL, struct uniform_sort,
+                                 shader_program->Uniforms->NumUniforms);
 
    for (i = 0; i < shader_program->Uniforms->NumUniforms; i++) {
       struct gl_uniform *uniform = shader_program->Uniforms->Uniforms + i;
-      const glsl_type *type = uniform->Type;
-      unsigned int size;
       int parameter_index = -1;
 
       switch (shader->Type) {
@@ -2283,8 +2482,21 @@ add_uniforms_to_parameters_list(struct gl_shader_program *shader_program,
       }
 
       /* Only add uniforms used in our target. */
-      if (parameter_index == -1)
-        continue;
+      if (parameter_index != -1) {
+        sorted_uniforms[num_uniforms].pos = parameter_index;
+        sorted_uniforms[num_uniforms].u = uniform;
+        num_uniforms++;
+      }
+   }
+
+   qsort(sorted_uniforms, num_uniforms, sizeof(struct uniform_sort),
+        sort_uniforms);
+
+   for (i = 0; i < num_uniforms; i++) {
+      struct gl_uniform *uniform = sorted_uniforms[i].u;
+      int parameter_index = sorted_uniforms[i].pos;
+      const glsl_type *type = uniform->Type;
+      unsigned int size;
 
       if (type->is_vector() ||
          type->is_scalar()) {
@@ -2322,19 +2534,18 @@ add_uniforms_to_parameters_list(struct gl_shader_program *shader_program,
          * from _mesa_add_uniform) has to match what the linker chose.
          */
         if (index != parameter_index) {
-           shader_program->InfoLog =
-              talloc_asprintf_append(shader_program->InfoLog,
-                                     "Allocation of uniform `%s' to target "
-                                     "failed (%d vs %d)\n", uniform->Name,
-                                     index, parameter_index);
-           shader_program->LinkStatus = false;
+           fail_link(shader_program, "Allocation of uniform `%s' to target "
+                     "failed (%d vs %d)\n",
+                     uniform->Name, index, parameter_index);
         }
       }
    }
+
+   ralloc_free(sorted_uniforms);
 }
 
 static void
-set_uniform_initializer(GLcontext *ctx, void *mem_ctx,
+set_uniform_initializer(struct gl_context *ctx, void *mem_ctx,
                        struct gl_shader_program *shader_program,
                        const char *name, const glsl_type *type,
                        ir_constant *val)
@@ -2346,7 +2557,7 @@ set_uniform_initializer(GLcontext *ctx, void *mem_ctx,
 
       for (unsigned int i = 0; i < type->length; i++) {
         const glsl_type *field_type = type->fields.structure[i].type;
-        const char *field_name = talloc_asprintf(mem_ctx, "%s.%s", name,
+        const char *field_name = ralloc_asprintf(mem_ctx, "%s.%s", name,
                                            type->fields.structure[i].name);
         set_uniform_initializer(ctx, mem_ctx, shader_program, field_name,
                                 field_type, field_constant);
@@ -2358,12 +2569,9 @@ set_uniform_initializer(GLcontext *ctx, void *mem_ctx,
    int loc = _mesa_get_uniform_location(ctx, shader_program, name);
 
    if (loc == -1) {
-      shader_program->InfoLog =
-        talloc_asprintf_append(shader_program->InfoLog,
-                               "Couldn't find uniform for "
-                               "initializer %s\n", name);
-      shader_program->LinkStatus = false;
-      abort();
+      fail_link(shader_program,
+               "Couldn't find uniform for initializer %s\n", name);
+      return;
    }
 
    for (unsigned int i = 0; i < (type->is_array() ? type->length : 1); i++) {
@@ -2380,7 +2588,7 @@ set_uniform_initializer(GLcontext *ctx, void *mem_ctx,
       void *values;
 
       if (element_type->base_type == GLSL_TYPE_BOOL) {
-        int *conv = talloc_array(mem_ctx, int, element_type->components());
+        int *conv = ralloc_array(mem_ctx, int, element_type->components());
         for (unsigned int j = 0; j < element_type->components(); j++) {
            conv[j] = element->value.b[j];
         }
@@ -2407,13 +2615,17 @@ set_uniform_initializer(GLcontext *ctx, void *mem_ctx,
 }
 
 static void
-set_uniform_initializers(GLcontext *ctx,
+set_uniform_initializers(struct gl_context *ctx,
                         struct gl_shader_program *shader_program)
 {
    void *mem_ctx = NULL;
 
-   for (unsigned int i = 0; i < shader_program->_NumLinkedShaders; i++) {
+   for (unsigned int i = 0; i < MESA_SHADER_TYPES; i++) {
       struct gl_shader *shader = shader_program->_LinkedShaders[i];
+
+      if (shader == NULL)
+        continue;
+
       foreach_iter(exec_list_iterator, iter, *shader->ir) {
         ir_instruction *ir = (ir_instruction *)iter.get();
         ir_variable *var = ir->as_variable();
@@ -2422,18 +2634,153 @@ set_uniform_initializers(GLcontext *ctx,
            continue;
 
         if (!mem_ctx)
-           mem_ctx = talloc_new(NULL);
+           mem_ctx = ralloc_context(NULL);
 
         set_uniform_initializer(ctx, mem_ctx, shader_program, var->name,
                                 var->type, var->constant_value);
       }
    }
 
-   talloc_free(mem_ctx);
+   ralloc_free(mem_ctx);
 }
 
-struct gl_program *
-get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
+/*
+ * On a basic block basis, tracks available PROGRAM_TEMPORARY register
+ * channels for copy propagation and updates following instructions to
+ * use the original versions.
+ *
+ * The ir_to_mesa_visitor lazily produces code assuming that this pass
+ * will occur.  As an example, a TXP production before this pass:
+ *
+ * 0: MOV TEMP[1], INPUT[4].xyyy;
+ * 1: MOV TEMP[1].w, INPUT[4].wwww;
+ * 2: TXP TEMP[2], TEMP[1], texture[0], 2D;
+ *
+ * and after:
+ *
+ * 0: MOV TEMP[1], INPUT[4].xyyy;
+ * 1: MOV TEMP[1].w, INPUT[4].wwww;
+ * 2: TXP TEMP[2], INPUT[4].xyyw, texture[0], 2D;
+ *
+ * which allows for dead code elimination on TEMP[1]'s writes.
+ */
+void
+ir_to_mesa_visitor::copy_propagate(void)
+{
+   ir_to_mesa_instruction **acp = rzalloc_array(mem_ctx,
+                                                   ir_to_mesa_instruction *,
+                                                   this->next_temp * 4);
+
+   foreach_iter(exec_list_iterator, iter, this->instructions) {
+      ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
+
+      assert(inst->dst_reg.file != PROGRAM_TEMPORARY
+            || inst->dst_reg.index < this->next_temp);
+
+      /* First, do any copy propagation possible into the src regs. */
+      for (int r = 0; r < 3; r++) {
+        ir_to_mesa_instruction *first = NULL;
+        bool good = true;
+        int acp_base = inst->src_reg[r].index * 4;
+
+        if (inst->src_reg[r].file != PROGRAM_TEMPORARY ||
+            inst->src_reg[r].reladdr)
+           continue;
+
+        /* See if we can find entries in the ACP consisting of MOVs
+         * from the same src register for all the swizzled channels
+         * of this src register reference.
+         */
+        for (int i = 0; i < 4; i++) {
+           int src_chan = GET_SWZ(inst->src_reg[r].swizzle, i);
+           ir_to_mesa_instruction *copy_chan = acp[acp_base + src_chan];
+
+           if (!copy_chan) {
+              good = false;
+              break;
+           }
+
+           if (!first) {
+              first = copy_chan;
+           } else {
+              if (first->src_reg[0].file != copy_chan->src_reg[0].file ||
+                  first->src_reg[0].index != copy_chan->src_reg[0].index) {
+                 good = false;
+                 break;
+              }
+           }
+        }
+
+        if (good) {
+           /* We've now validated that we can copy-propagate to
+            * replace this src register reference.  Do it.
+            */
+           inst->src_reg[r].file = first->src_reg[0].file;
+           inst->src_reg[r].index = first->src_reg[0].index;
+
+           int swizzle = 0;
+           for (int i = 0; i < 4; i++) {
+              int src_chan = GET_SWZ(inst->src_reg[r].swizzle, i);
+              ir_to_mesa_instruction *copy_inst = acp[acp_base + src_chan];
+              swizzle |= (GET_SWZ(copy_inst->src_reg[0].swizzle, src_chan) <<
+                          (3 * i));
+           }
+           inst->src_reg[r].swizzle = swizzle;
+        }
+      }
+
+      switch (inst->op) {
+      case OPCODE_BGNLOOP:
+      case OPCODE_ENDLOOP:
+      case OPCODE_ELSE:
+      case OPCODE_ENDIF:
+        /* End of a basic block, clear the ACP entirely. */
+        memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
+        break;
+
+      default:
+        /* Continuing the block, clear any written channels from
+         * the ACP.
+         */
+        if (inst->dst_reg.file == PROGRAM_TEMPORARY) {
+           if (inst->dst_reg.reladdr) {
+              memset(acp, 0, sizeof(*acp) * this->next_temp * 4);
+           } else {
+              for (int i = 0; i < 4; i++) {
+                 if (inst->dst_reg.writemask & (1 << i)) {
+                    acp[4 * inst->dst_reg.index + i] = NULL;
+                 }
+              }
+           }
+        }
+        break;
+      }
+
+      /* If this is a copy, add it to the ACP. */
+      if (inst->op == OPCODE_MOV &&
+         inst->dst_reg.file == PROGRAM_TEMPORARY &&
+         !inst->dst_reg.reladdr &&
+         !inst->saturate &&
+         !inst->src_reg[0].reladdr &&
+         !inst->src_reg[0].negate) {
+        for (int i = 0; i < 4; i++) {
+           if (inst->dst_reg.writemask & (1 << i)) {
+              acp[4 * inst->dst_reg.index + i] = inst;
+           }
+        }
+      }
+   }
+
+   ralloc_free(acp);
+}
+
+
+/**
+ * Convert a shader's GLSL IR into a Mesa gl_program.
+ */
+static struct gl_program *
+get_mesa_program(struct gl_context *ctx,
+                 struct gl_shader_program *shader_program,
                 struct gl_shader *shader)
 {
    ir_to_mesa_visitor v;
@@ -2444,6 +2791,8 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    GLenum target;
    const char *target_string;
    GLboolean progress;
+   struct gl_shader_compiler_options *options =
+         &ctx->ShaderCompilerOptions[_mesa_shader_type_to_index(shader->Type)];
 
    switch (shader->Type) {
    case GL_VERTEX_SHADER:
@@ -2454,6 +2803,10 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
       target = GL_FRAGMENT_PROGRAM_ARB;
       target_string = "fragment";
       break;
+   case GL_GEOMETRY_SHADER:
+      target = GL_GEOMETRY_PROGRAM_NV;
+      target_string = "geometry";
+      break;
    default:
       assert(!"should not be reached");
       return NULL;
@@ -2470,6 +2823,7 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    v.ctx = ctx;
    v.prog = prog;
    v.shader_program = shader_program;
+   v.options = options;
 
    add_uniforms_to_parameters_list(shader_program, shader, prog);
 
@@ -2516,16 +2870,22 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
    mesa_instructions =
       (struct prog_instruction *)calloc(num_instructions,
                                        sizeof(*mesa_instructions));
-   mesa_instruction_annotation = talloc_array(v.mem_ctx, ir_instruction *,
+   mesa_instruction_annotation = ralloc_array(v.mem_ctx, ir_instruction *,
                                              num_instructions);
 
+   v.copy_propagate();
+
+   /* Convert ir_mesa_instructions into prog_instructions.
+    */
    mesa_inst = mesa_instructions;
    i = 0;
    foreach_iter(exec_list_iterator, iter, v.instructions) {
-      ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
+      const ir_to_mesa_instruction *inst = (ir_to_mesa_instruction *)iter.get();
 
       mesa_inst->Opcode = inst->op;
       mesa_inst->CondUpdate = inst->cond_update;
+      if (inst->saturate)
+        mesa_inst->SaturateMode = SATURATE_ZERO_ONE;
       mesa_inst->DstReg.File = inst->dst_reg.file;
       mesa_inst->DstReg.Index = inst->dst_reg.index;
       mesa_inst->DstReg.CondMask = inst->dst_reg.cond_mask;
@@ -2543,15 +2903,13 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
       if (mesa_inst->DstReg.RelAddr)
          prog->IndirectRegisterFiles |= 1 << mesa_inst->DstReg.File;
 
+      /* Update program's bitmask of indirectly accessed register files */
       for (unsigned src = 0; src < 3; src++)
          if (mesa_inst->SrcReg[src].RelAddr)
             prog->IndirectRegisterFiles |= 1 << mesa_inst->SrcReg[src].File;
 
-      if (ctx->Shader.EmitNoIfs && mesa_inst->Opcode == OPCODE_IF) {
-        shader_program->InfoLog =
-           talloc_asprintf_append(shader_program->InfoLog,
-                                  "Couldn't flatten if statement\n");
-        shader_program->LinkStatus = false;
+      if (options->EmitNoIfs && mesa_inst->Opcode == OPCODE_IF) {
+        fail_link(shader_program, "Couldn't flatten if statement\n");
       }
 
       switch (mesa_inst->Opcode) {
@@ -2574,6 +2932,15 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
 
       mesa_inst++;
       i++;
+
+      if (!shader_program->LinkStatus)
+         break;
+   }
+
+   if (!shader_program->LinkStatus) {
+      free(mesa_instructions);
+      _mesa_reference_program(ctx, &shader->Program, NULL);
+      return NULL;
    }
 
    set_branchtargets(&v, mesa_instructions, num_instructions);
@@ -2607,8 +2974,14 @@ get_mesa_program(GLcontext *ctx, struct gl_shader_program *shader_program,
 }
 
 extern "C" {
+
+/**
+ * Called via ctx->Driver.CompilerShader().
+ * This is a no-op.
+ * XXX can we remove the ctx->Driver.CompileShader() hook?
+ */
 GLboolean
-_mesa_ir_compile_shader(GLcontext *ctx, struct gl_shader *shader)
+_mesa_ir_compile_shader(struct gl_context *ctx, struct gl_shader *shader)
 {
    assert(shader->CompileStatus);
    (void) ctx;
@@ -2616,28 +2989,62 @@ _mesa_ir_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    return GL_TRUE;
 }
 
+
+/**
+ * Link a shader.
+ * Called via ctx->Driver.LinkShader()
+ * This actually involves converting GLSL IR into Mesa gl_programs with
+ * code lowering and other optimizations.
+ */
 GLboolean
-_mesa_ir_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
+_mesa_ir_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 {
    assert(prog->LinkStatus);
 
-   for (unsigned i = 0; i < prog->_NumLinkedShaders; i++) {
+   for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
+      if (prog->_LinkedShaders[i] == NULL)
+        continue;
+
       bool progress;
       exec_list *ir = prog->_LinkedShaders[i]->ir;
+      const struct gl_shader_compiler_options *options =
+            &ctx->ShaderCompilerOptions[_mesa_shader_type_to_index(prog->_LinkedShaders[i]->Type)];
 
       do {
         progress = false;
 
         /* Lowering */
         do_mat_op_to_vec(ir);
-        do_mod_to_fract(ir);
-        do_div_to_mul_rcp(ir);
-        do_explog_to_explog2(ir);
+        lower_instructions(ir, (MOD_TO_FRACT | DIV_TO_MUL_RCP | EXP_TO_EXP2
+                                | LOG_TO_LOG2
+                                | ((options->EmitNoPow) ? POW_TO_EXP2 : 0)));
+
+        progress = do_lower_jumps(ir, true, true, options->EmitNoMainReturn, options->EmitNoCont, options->EmitNoLoops) || progress;
 
-        progress = do_common_optimization(ir, true) || progress;
+        progress = do_common_optimization(ir, true, options->MaxUnrollIterations) || progress;
 
-        if (ctx->Shader.EmitNoIfs)
-           progress = do_if_to_cond_assign(ir) || progress;
+        progress = lower_quadop_vector(ir, true) || progress;
+
+        if (options->EmitNoIfs) {
+           progress = lower_discard(ir) || progress;
+           progress = lower_if_to_cond_assign(ir) || progress;
+        }
+
+        if (options->EmitNoNoise)
+           progress = lower_noise(ir) || progress;
+
+        /* If there are forms of indirect addressing that the driver
+         * cannot handle, perform the lowering pass.
+         */
+        if (options->EmitNoIndirectInput || options->EmitNoIndirectOutput
+            || options->EmitNoIndirectTemp || options->EmitNoIndirectUniform)
+          progress =
+            lower_variable_index_to_cond_assign(ir,
+                                                options->EmitNoIndirectInput,
+                                                options->EmitNoIndirectOutput,
+                                                options->EmitNoIndirectTemp,
+                                                options->EmitNoIndirectUniform)
+            || progress;
 
         progress = do_vec_index_to_cond_assign(ir) || progress;
       } while (progress);
@@ -2645,44 +3052,69 @@ _mesa_ir_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
       validate_ir_tree(ir);
    }
 
-   for (unsigned i = 0; i < prog->_NumLinkedShaders; i++) {
+   for (unsigned i = 0; i < MESA_SHADER_TYPES; i++) {
       struct gl_program *linked_prog;
-      bool ok = true;
+
+      if (prog->_LinkedShaders[i] == NULL)
+        continue;
 
       linked_prog = get_mesa_program(ctx, prog, prog->_LinkedShaders[i]);
 
-      switch (prog->_LinkedShaders[i]->Type) {
-      case GL_VERTEX_SHADER:
-        _mesa_reference_vertprog(ctx, &prog->VertexProgram,
-                                 (struct gl_vertex_program *)linked_prog);
-        ok = ctx->Driver.ProgramStringNotify(ctx, GL_VERTEX_PROGRAM_ARB,
-                                             linked_prog);
-        break;
-      case GL_FRAGMENT_SHADER:
-        _mesa_reference_fragprog(ctx, &prog->FragmentProgram,
-                                 (struct gl_fragment_program *)linked_prog);
-        ok = ctx->Driver.ProgramStringNotify(ctx, GL_FRAGMENT_PROGRAM_ARB,
-                                             linked_prog);
-        break;
-      }
-      if (!ok) {
-        return GL_FALSE;
+      if (linked_prog) {
+         bool ok = true;
+
+         switch (prog->_LinkedShaders[i]->Type) {
+         case GL_VERTEX_SHADER:
+            _mesa_reference_vertprog(ctx, &prog->VertexProgram,
+                                     (struct gl_vertex_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_VERTEX_PROGRAM_ARB,
+                                                 linked_prog);
+            break;
+         case GL_FRAGMENT_SHADER:
+            _mesa_reference_fragprog(ctx, &prog->FragmentProgram,
+                                     (struct gl_fragment_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_FRAGMENT_PROGRAM_ARB,
+                                                 linked_prog);
+            break;
+         case GL_GEOMETRY_SHADER:
+            _mesa_reference_geomprog(ctx, &prog->GeometryProgram,
+                                     (struct gl_geometry_program *)linked_prog);
+            ok = ctx->Driver.ProgramStringNotify(ctx, GL_GEOMETRY_PROGRAM_NV,
+                                                 linked_prog);
+            break;
+         }
+         if (!ok) {
+            return GL_FALSE;
+         }
       }
+
       _mesa_reference_program(ctx, &linked_prog, NULL);
    }
 
    return GL_TRUE;
 }
 
+
+/**
+ * Compile a GLSL shader.  Called via glCompileShader().
+ */
 void
-_mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
+_mesa_glsl_compile_shader(struct gl_context *ctx, struct gl_shader *shader)
 {
    struct _mesa_glsl_parse_state *state =
       new(shader) _mesa_glsl_parse_state(ctx, shader->Type, shader);
 
    const char *source = shader->Source;
+   /* Check if the user called glCompileShader without first calling
+    * glShaderSource.  This should fail to compile, but not raise a GL_ERROR.
+    */
+   if (source == NULL) {
+      shader->CompileStatus = GL_FALSE;
+      return;
+   }
+
    state->error = preprocess(state, &source, &state->info_log,
-                            &ctx->Extensions);
+                            &ctx->Extensions, ctx->API);
 
    if (ctx->Shader.Flags & GLSL_DUMP) {
       printf("GLSL source for shader %d:\n", shader->Name);
@@ -2695,7 +3127,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
      _mesa_glsl_lexer_dtor(state);
    }
 
-   talloc_free(shader->ir);
+   ralloc_free(shader->ir);
    shader->ir = new(shader) exec_list;
    if (!state->error && !state->translation_unit.is_empty())
       _mesa_ast_to_hir(shader->ir, state);
@@ -2706,7 +3138,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
       /* Do some optimization at compile time to reduce shader IR size
        * and reduce later work if the same shader is linked multiple times
        */
-      while (do_common_optimization(shader->ir, false))
+      while (do_common_optimization(shader->ir, false, 32))
         ;
 
       validate_ir_tree(shader->ir);
@@ -2742,7 +3174,7 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    /* Retain any live IR, but trash the rest. */
    reparent_ir(shader->ir, shader->ir);
 
-   talloc_free(state);
+   ralloc_free(state);
 
    if (shader->CompileStatus) {
       if (!ctx->Driver.CompileShader(ctx, shader))
@@ -2750,8 +3182,12 @@ _mesa_glsl_compile_shader(GLcontext *ctx, struct gl_shader *shader)
    }
 }
 
+
+/**
+ * Link a GLSL shader program.  Called via glLinkProgram().
+ */
 void
-_mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
+_mesa_glsl_link_shader(struct gl_context *ctx, struct gl_shader_program *prog)
 {
    unsigned int i;
 
@@ -2761,9 +3197,7 @@ _mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
 
    for (i = 0; i < prog->NumShaders; i++) {
       if (!prog->Shaders[i]->CompileStatus) {
-        prog->InfoLog =
-           talloc_asprintf_append(prog->InfoLog,
-                                  "linking with uncompiled shader");
+        fail_link(prog, "linking with uncompiled shader");
         prog->LinkStatus = GL_FALSE;
       }
    }
@@ -2771,6 +3205,7 @@ _mesa_glsl_link_shader(GLcontext *ctx, struct gl_shader_program *prog)
    prog->Varying = _mesa_new_parameter_list();
    _mesa_reference_vertprog(ctx, &prog->VertexProgram, NULL);
    _mesa_reference_fragprog(ctx, &prog->FragmentProgram, NULL);
+   _mesa_reference_geomprog(ctx, &prog->GeometryProgram, NULL);
 
    if (prog->LinkStatus) {
       link_shaders(ctx, prog);