Merge remote-tracking branch 'mesa-public/master' into vulkan
[mesa.git] / src / glsl / ir.cpp
index 2c54525938044b113c94040f858d9fe92a55aa57..724861b1e9f971a33d04d183bfcfdc9567bb860e 100644 (file)
 #include <string.h>
 #include "main/core.h" /* for MAX2 */
 #include "ir.h"
-#include "ir_visitor.h"
 #include "glsl_types.h"
 
-ir_rvalue::ir_rvalue()
+ir_rvalue::ir_rvalue(enum ir_node_type t)
+   : ir_instruction(t)
 {
    this->type = glsl_type::error_type;
 }
@@ -46,11 +46,6 @@ bool ir_rvalue::is_negative_one() const
    return false;
 }
 
-bool ir_rvalue::is_basis() const
-{
-   return false;
-}
-
 /**
  * Modify the swizzle make to move one component to another
  *
@@ -68,8 +63,6 @@ update_rhs_swizzle(ir_swizzle_mask &m, unsigned from, unsigned to)
    case 3: m.w = from; break;
    default: assert(!"Should not get here.");
    }
-
-   m.num_components = MAX2(m.num_components, (to + 1));
 }
 
 void
@@ -100,6 +93,7 @@ ir_assignment::set_lhs(ir_rvalue *lhs)
 
         write_mask |= (((this->write_mask >> i) & 1) << c);
         update_rhs_swizzle(rhs_swiz, i, c);
+         rhs_swiz.num_components = swiz->val->type->vector_elements;
       }
 
       this->write_mask = write_mask;
@@ -119,6 +113,7 @@ ir_assignment::set_lhs(ir_rvalue *lhs)
         if (write_mask & (1 << i))
            update_rhs_swizzle(rhs_swiz, i, rhs_chan++);
       }
+      rhs_swiz.num_components = rhs_chan;
       this->rhs = new(mem_ctx) ir_swizzle(this->rhs, rhs_swiz);
    }
 
@@ -153,8 +148,8 @@ ir_assignment::whole_variable_written()
 
 ir_assignment::ir_assignment(ir_dereference *lhs, ir_rvalue *rhs,
                             ir_rvalue *condition, unsigned write_mask)
+   : ir_instruction(ir_type_assignment)
 {
-   this->ir_type = ir_type_assignment;
    this->condition = condition;
    this->rhs = rhs;
    this->lhs = lhs;
@@ -173,8 +168,8 @@ ir_assignment::ir_assignment(ir_dereference *lhs, ir_rvalue *rhs,
 
 ir_assignment::ir_assignment(ir_rvalue *lhs, ir_rvalue *rhs,
                             ir_rvalue *condition)
+   : ir_instruction(ir_type_assignment)
 {
-   this->ir_type = ir_type_assignment;
    this->condition = condition;
    this->rhs = rhs;
 
@@ -198,8 +193,8 @@ ir_assignment::ir_assignment(ir_rvalue *lhs, ir_rvalue *rhs,
 ir_expression::ir_expression(int op, const struct glsl_type *type,
                             ir_rvalue *op0, ir_rvalue *op1,
                             ir_rvalue *op2, ir_rvalue *op3)
+   : ir_rvalue(ir_type_expression)
 {
-   this->ir_type = ir_type_expression;
    this->type = type;
    this->operation = ir_expression_operation(op);
    this->operands[0] = op0;
@@ -215,9 +210,8 @@ ir_expression::ir_expression(int op, const struct glsl_type *type,
 }
 
 ir_expression::ir_expression(int op, ir_rvalue *op0)
+   : ir_rvalue(ir_type_expression)
 {
-   this->ir_type = ir_type_expression;
-
    this->operation = ir_expression_operation(op);
    this->operands[0] = op0;
    this->operands[1] = NULL;
@@ -246,17 +240,27 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
    case ir_unop_round_even:
    case ir_unop_sin:
    case ir_unop_cos:
-   case ir_unop_sin_reduced:
-   case ir_unop_cos_reduced:
    case ir_unop_dFdx:
+   case ir_unop_dFdx_coarse:
+   case ir_unop_dFdx_fine:
    case ir_unop_dFdy:
+   case ir_unop_dFdy_coarse:
+   case ir_unop_dFdy_fine:
+   case ir_unop_bitfield_reverse:
+   case ir_unop_interpolate_at_centroid:
+   case ir_unop_saturate:
       this->type = op0->type;
       break;
 
    case ir_unop_f2i:
    case ir_unop_b2i:
    case ir_unop_u2i:
+   case ir_unop_d2i:
    case ir_unop_bitcast_f2i:
+   case ir_unop_bit_count:
+   case ir_unop_find_msb:
+   case ir_unop_find_lsb:
+   case ir_unop_subroutine_to_int:
       this->type = glsl_type::get_instance(GLSL_TYPE_INT,
                                           op0->type->vector_elements, 1);
       break;
@@ -264,6 +268,7 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
    case ir_unop_b2f:
    case ir_unop_i2f:
    case ir_unop_u2f:
+   case ir_unop_d2f:
    case ir_unop_bitcast_i2f:
    case ir_unop_bitcast_u2f:
       this->type = glsl_type::get_instance(GLSL_TYPE_FLOAT,
@@ -272,12 +277,21 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
 
    case ir_unop_f2b:
    case ir_unop_i2b:
+   case ir_unop_d2b:
       this->type = glsl_type::get_instance(GLSL_TYPE_BOOL,
                                           op0->type->vector_elements, 1);
       break;
 
+   case ir_unop_f2d:
+   case ir_unop_i2d:
+   case ir_unop_u2d:
+      this->type = glsl_type::get_instance(GLSL_TYPE_DOUBLE,
+                                          op0->type->vector_elements, 1);
+      break;
+
    case ir_unop_i2u:
    case ir_unop_f2u:
+   case ir_unop_d2u:
    case ir_unop_bitcast_f2u:
       this->type = glsl_type::get_instance(GLSL_TYPE_UINT,
                                           op0->type->vector_elements, 1);
@@ -289,6 +303,10 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
       this->type = glsl_type::float_type;
       break;
 
+   case ir_unop_unpack_double_2x32:
+      this->type = glsl_type::uvec2_type;
+      break;
+
    case ir_unop_any:
       this->type = glsl_type::bool_type;
       break;
@@ -301,6 +319,10 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
       this->type = glsl_type::uint_type;
       break;
 
+   case ir_unop_pack_double_2x32:
+      this->type = glsl_type::double_type;
+      break;
+
    case ir_unop_unpack_snorm_2x16:
    case ir_unop_unpack_unorm_2x16:
    case ir_unop_unpack_half_2x16:
@@ -312,6 +334,14 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
       this->type = glsl_type::vec4_type;
       break;
 
+   case ir_unop_frexp_sig:
+      this->type = op0->type;
+      break;
+   case ir_unop_frexp_exp:
+      this->type = glsl_type::get_instance(GLSL_TYPE_INT,
+                                          op0->type->vector_elements, 1);
+      break;
+
    default:
       assert(!"not reached: missing automatic type setup for ir_expression");
       this->type = op0->type;
@@ -320,9 +350,8 @@ ir_expression::ir_expression(int op, ir_rvalue *op0)
 }
 
 ir_expression::ir_expression(int op, ir_rvalue *op0, ir_rvalue *op1)
+   : ir_rvalue(ir_type_expression)
 {
-   this->ir_type = ir_type_expression;
-
    this->operation = ir_expression_operation(op);
    this->operands[0] = op0;
    this->operands[1] = op1;
@@ -350,10 +379,12 @@ ir_expression::ir_expression(int op, ir_rvalue *op0, ir_rvalue *op1)
       } else if (op1->type->is_scalar()) {
         this->type = op0->type;
       } else {
-        /* FINISHME: matrix types */
-        assert(!op0->type->is_matrix() && !op1->type->is_matrix());
-        assert(op0->type == op1->type);
-        this->type = op0->type;
+         if (this->operation == ir_binop_mul) {
+            this->type = glsl_type::get_mul_type(op0->type, op1->type);
+         } else {
+            assert(op0->type == op1->type);
+            this->type = op0->type;
+         }
       }
       break;
 
@@ -387,18 +418,60 @@ ir_expression::ir_expression(int op, ir_rvalue *op0, ir_rvalue *op1)
       break;
 
    case ir_binop_dot:
-      this->type = glsl_type::float_type;
+      this->type = op0->type->get_base_type();
       break;
 
    case ir_binop_pack_half_2x16_split:
       this->type = glsl_type::uint_type;
       break;
 
+   case ir_binop_imul_high:
+   case ir_binop_carry:
+   case ir_binop_borrow:
    case ir_binop_lshift:
    case ir_binop_rshift:
+   case ir_binop_bfm:
+   case ir_binop_ldexp:
+   case ir_binop_interpolate_at_offset:
+   case ir_binop_interpolate_at_sample:
+      this->type = op0->type;
+      break;
+
+   case ir_binop_vector_extract:
+      this->type = op0->type->get_scalar_type();
+      break;
+
+   default:
+      assert(!"not reached: missing automatic type setup for ir_expression");
+      this->type = glsl_type::float_type;
+   }
+}
+
+ir_expression::ir_expression(int op, ir_rvalue *op0, ir_rvalue *op1,
+                             ir_rvalue *op2)
+   : ir_rvalue(ir_type_expression)
+{
+   this->operation = ir_expression_operation(op);
+   this->operands[0] = op0;
+   this->operands[1] = op1;
+   this->operands[2] = op2;
+   this->operands[3] = NULL;
+
+   assert(op > ir_last_binop && op <= ir_last_triop);
+
+   switch (this->operation) {
+   case ir_triop_fma:
+   case ir_triop_lrp:
+   case ir_triop_bitfield_extract:
+   case ir_triop_vector_insert:
       this->type = op0->type;
       break;
 
+   case ir_triop_bfi:
+   case ir_triop_csel:
+      this->type = op1->type;
+      break;
+
    default:
       assert(!"not reached: missing automatic type setup for ir_expression");
       this->type = glsl_type::float_type;
@@ -449,6 +522,13 @@ static const char *const operator_strs[] = {
    "u2f",
    "i2u",
    "u2i",
+   "d2f",
+   "f2d",
+   "d2i",
+   "i2d",
+   "d2u",
+   "u2d",
+   "d2b",
    "bitcast_i2f",
    "bitcast_f2i",
    "bitcast_u2f",
@@ -461,10 +541,12 @@ static const char *const operator_strs[] = {
    "round_even",
    "sin",
    "cos",
-   "sin_reduced",
-   "cos_reduced",
    "dFdx",
+   "dFdxCoarse",
+   "dFdxFine",
    "dFdy",
+   "dFdyCoarse",
+   "dFdyFine",
    "packSnorm2x16",
    "packSnorm4x8",
    "packUnorm2x16",
@@ -481,11 +563,21 @@ static const char *const operator_strs[] = {
    "bit_count",
    "find_msb",
    "find_lsb",
+   "sat",
+   "packDouble2x32",
+   "unpackDouble2x32",
+   "frexp_sig",
+   "frexp_exp",
    "noise",
+   "subroutine_to_int",
+   "interpolate_at_centroid",
    "+",
    "-",
    "*",
+   "imul_high",
    "/",
+   "carry",
+   "borrow",
    "%",
    "<",
    ">",
@@ -510,17 +602,24 @@ static const char *const operator_strs[] = {
    "packHalf2x16_split",
    "bfm",
    "ubo_load",
+   "ldexp",
+   "vector_extract",
+   "interpolate_at_offset",
+   "interpolate_at_sample",
+   "fma",
    "lrp",
+   "csel",
    "bfi",
    "bitfield_extract",
+   "vector_insert",
    "bitfield_insert",
    "vector",
 };
 
 const char *ir_expression::operator_string(ir_expression_operation op)
 {
-   assert((unsigned int) op < Elements(operator_strs));
-   assert(Elements(operator_strs) == (ir_quadop_vector + 1));
+   assert((unsigned int) op < ARRAY_SIZE(operator_strs));
+   assert(ARRAY_SIZE(operator_strs) == (ir_quadop_vector + 1));
    return operator_strs[op];
 }
 
@@ -557,64 +656,89 @@ ir_expression::get_operator(const char *str)
 }
 
 ir_constant::ir_constant()
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
 }
 
 ir_constant::ir_constant(const struct glsl_type *type,
                         const ir_constant_data *data)
+   : ir_rvalue(ir_type_constant)
 {
    assert((type->base_type >= GLSL_TYPE_UINT)
          && (type->base_type <= GLSL_TYPE_BOOL));
 
-   this->ir_type = ir_type_constant;
    this->type = type;
    memcpy(& this->value, data, sizeof(this->value));
 }
 
-ir_constant::ir_constant(float f)
+ir_constant::ir_constant(float f, unsigned vector_elements)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
-   this->type = glsl_type::float_type;
-   this->value.f[0] = f;
-   for (int i = 1; i < 16; i++)  {
+   assert(vector_elements <= 4);
+   this->type = glsl_type::get_instance(GLSL_TYPE_FLOAT, vector_elements, 1);
+   for (unsigned i = 0; i < vector_elements; i++) {
+      this->value.f[i] = f;
+   }
+   for (unsigned i = vector_elements; i < 16; i++)  {
       this->value.f[i] = 0;
    }
 }
 
-ir_constant::ir_constant(unsigned int u)
+ir_constant::ir_constant(double d, unsigned vector_elements)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
-   this->type = glsl_type::uint_type;
-   this->value.u[0] = u;
-   for (int i = 1; i < 16; i++) {
+   assert(vector_elements <= 4);
+   this->type = glsl_type::get_instance(GLSL_TYPE_DOUBLE, vector_elements, 1);
+   for (unsigned i = 0; i < vector_elements; i++) {
+      this->value.d[i] = d;
+   }
+   for (unsigned i = vector_elements; i < 16; i++)  {
+      this->value.d[i] = 0.0;
+   }
+}
+
+ir_constant::ir_constant(unsigned int u, unsigned vector_elements)
+   : ir_rvalue(ir_type_constant)
+{
+   assert(vector_elements <= 4);
+   this->type = glsl_type::get_instance(GLSL_TYPE_UINT, vector_elements, 1);
+   for (unsigned i = 0; i < vector_elements; i++) {
+      this->value.u[i] = u;
+   }
+   for (unsigned i = vector_elements; i < 16; i++) {
       this->value.u[i] = 0;
    }
 }
 
-ir_constant::ir_constant(int i)
+ir_constant::ir_constant(int integer, unsigned vector_elements)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
-   this->type = glsl_type::int_type;
-   this->value.i[0] = i;
-   for (int i = 1; i < 16; i++) {
+   assert(vector_elements <= 4);
+   this->type = glsl_type::get_instance(GLSL_TYPE_INT, vector_elements, 1);
+   for (unsigned i = 0; i < vector_elements; i++) {
+      this->value.i[i] = integer;
+   }
+   for (unsigned i = vector_elements; i < 16; i++) {
       this->value.i[i] = 0;
    }
 }
 
-ir_constant::ir_constant(bool b)
+ir_constant::ir_constant(bool b, unsigned vector_elements)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
-   this->type = glsl_type::bool_type;
-   this->value.b[0] = b;
-   for (int i = 1; i < 16; i++) {
+   assert(vector_elements <= 4);
+   this->type = glsl_type::get_instance(GLSL_TYPE_BOOL, vector_elements, 1);
+   for (unsigned i = 0; i < vector_elements; i++) {
+      this->value.b[i] = b;
+   }
+   for (unsigned i = vector_elements; i < 16; i++) {
       this->value.b[i] = false;
    }
 }
 
 ir_constant::ir_constant(const ir_constant *c, unsigned i)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
    this->type = c->type->get_base_type();
 
    switch (this->type->base_type) {
@@ -622,13 +746,14 @@ ir_constant::ir_constant(const ir_constant *c, unsigned i)
    case GLSL_TYPE_INT:   this->value.i[0] = c->value.i[i]; break;
    case GLSL_TYPE_FLOAT: this->value.f[0] = c->value.f[i]; break;
    case GLSL_TYPE_BOOL:  this->value.b[0] = c->value.b[i]; break;
+   case GLSL_TYPE_DOUBLE: this->value.d[0] = c->value.d[i]; break;
    default:              assert(!"Should not get here."); break;
    }
 }
 
 ir_constant::ir_constant(const struct glsl_type *type, exec_list *value_list)
+   : ir_rvalue(ir_type_constant)
 {
-   this->ir_type = ir_type_constant;
    this->type = type;
 
    assert(type->is_scalar() || type->is_vector() || type->is_matrix()
@@ -637,8 +762,7 @@ ir_constant::ir_constant(const struct glsl_type *type, exec_list *value_list)
    if (type->is_array()) {
       this->array_elements = ralloc_array(this, ir_constant *, type->length);
       unsigned i = 0;
-      foreach_list(node, value_list) {
-        ir_constant *value = (ir_constant *) node;
+      foreach_in_list(ir_constant, value, value_list) {
         assert(value->as_constant() != NULL);
 
         this->array_elements[i++] = value;
@@ -674,9 +798,16 @@ ir_constant::ir_constant(const struct glsl_type *type, exec_list *value_list)
    if (value->type->is_scalar() && value->next->is_tail_sentinel()) {
       if (type->is_matrix()) {
         /* Matrix - fill diagonal (rest is already set to 0) */
-        assert(type->base_type == GLSL_TYPE_FLOAT);
-        for (unsigned i = 0; i < type->matrix_columns; i++)
-           this->value.f[i * type->vector_elements + i] = value->value.f[0];
+         assert(type->base_type == GLSL_TYPE_FLOAT ||
+                type->base_type == GLSL_TYPE_DOUBLE);
+         for (unsigned i = 0; i < type->matrix_columns; i++) {
+            if (type->base_type == GLSL_TYPE_FLOAT)
+               this->value.f[i * type->vector_elements + i] =
+                  value->value.f[0];
+            else
+               this->value.d[i * type->vector_elements + i] =
+                  value->value.d[0];
+         }
       } else {
         /* Vector or scalar - fill all components */
         switch (type->base_type) {
@@ -689,6 +820,10 @@ ir_constant::ir_constant(const struct glsl_type *type, exec_list *value_list)
            for (unsigned i = 0; i < type->components(); i++)
               this->value.f[i] = value->value.f[0];
            break;
+        case GLSL_TYPE_DOUBLE:
+           for (unsigned i = 0; i < type->components(); i++)
+              this->value.d[i] = value->value.d[0];
+           break;
         case GLSL_TYPE_BOOL:
            for (unsigned i = 0; i < type->components(); i++)
               this->value.b[i] = value->value.b[0];
@@ -747,6 +882,9 @@ ir_constant::ir_constant(const struct glsl_type *type, exec_list *value_list)
         case GLSL_TYPE_BOOL:
            this->value.b[i] = value->get_bool_component(j);
            break;
+        case GLSL_TYPE_DOUBLE:
+           this->value.d[i] = value->get_double_component(j);
+           break;
         default:
            /* FINISHME: What to do?  Exceptions are not the answer.
             */
@@ -776,7 +914,7 @@ ir_constant::zero(void *mem_ctx, const glsl_type *type)
       c->array_elements = ralloc_array(c, ir_constant *, type->length);
 
       for (unsigned i = 0; i < type->length; i++)
-        c->array_elements[i] = ir_constant::zero(c, type->element_type());
+        c->array_elements[i] = ir_constant::zero(c, type->fields.array);
    }
 
    if (type->is_record()) {
@@ -797,6 +935,7 @@ ir_constant::get_bool_component(unsigned i) const
    case GLSL_TYPE_INT:   return this->value.i[i] != 0;
    case GLSL_TYPE_FLOAT: return ((int)this->value.f[i]) != 0;
    case GLSL_TYPE_BOOL:  return this->value.b[i];
+   case GLSL_TYPE_DOUBLE: return this->value.d[i] != 0.0;
    default:              assert(!"Should not get here."); break;
    }
 
@@ -814,6 +953,25 @@ ir_constant::get_float_component(unsigned i) const
    case GLSL_TYPE_INT:   return (float) this->value.i[i];
    case GLSL_TYPE_FLOAT: return this->value.f[i];
    case GLSL_TYPE_BOOL:  return this->value.b[i] ? 1.0f : 0.0f;
+   case GLSL_TYPE_DOUBLE: return (float) this->value.d[i];
+   default:              assert(!"Should not get here."); break;
+   }
+
+   /* Must return something to make the compiler happy.  This is clearly an
+    * error case.
+    */
+   return 0.0;
+}
+
+double
+ir_constant::get_double_component(unsigned i) const
+{
+   switch (this->type->base_type) {
+   case GLSL_TYPE_UINT:  return (double) this->value.u[i];
+   case GLSL_TYPE_INT:   return (double) this->value.i[i];
+   case GLSL_TYPE_FLOAT: return (double) this->value.f[i];
+   case GLSL_TYPE_BOOL:  return this->value.b[i] ? 1.0 : 0.0;
+   case GLSL_TYPE_DOUBLE: return this->value.d[i];
    default:              assert(!"Should not get here."); break;
    }
 
@@ -831,6 +989,7 @@ ir_constant::get_int_component(unsigned i) const
    case GLSL_TYPE_INT:   return this->value.i[i];
    case GLSL_TYPE_FLOAT: return (int) this->value.f[i];
    case GLSL_TYPE_BOOL:  return this->value.b[i] ? 1 : 0;
+   case GLSL_TYPE_DOUBLE: return (int) this->value.d[i];
    default:              assert(!"Should not get here."); break;
    }
 
@@ -848,6 +1007,7 @@ ir_constant::get_uint_component(unsigned i) const
    case GLSL_TYPE_INT:   return this->value.i[i];
    case GLSL_TYPE_FLOAT: return (unsigned) this->value.f[i];
    case GLSL_TYPE_BOOL:  return this->value.b[i] ? 1 : 0;
+   case GLSL_TYPE_DOUBLE: return (unsigned) this->value.d[i];
    default:              assert(!"Should not get here."); break;
    }
 
@@ -912,6 +1072,7 @@ ir_constant::copy_offset(ir_constant *src, int offset)
    case GLSL_TYPE_UINT:
    case GLSL_TYPE_INT:
    case GLSL_TYPE_FLOAT:
+   case GLSL_TYPE_DOUBLE:
    case GLSL_TYPE_BOOL: {
       unsigned int size = src->type->components();
       assert (size <= this->type->components() - offset);
@@ -929,6 +1090,9 @@ ir_constant::copy_offset(ir_constant *src, int offset)
         case GLSL_TYPE_BOOL:
            value.b[i+offset] = src->get_bool_component(i);
            break;
+        case GLSL_TYPE_DOUBLE:
+           value.d[i+offset] = src->get_double_component(i);
+           break;
         default: // Shut up the compiler
            break;
         }
@@ -939,9 +1103,7 @@ ir_constant::copy_offset(ir_constant *src, int offset)
    case GLSL_TYPE_STRUCT: {
       assert (src->type == this->type);
       this->components.make_empty();
-      foreach_list(node, &src->components) {
-        ir_constant *const orig = (ir_constant *) node;
-
+      foreach_in_list(ir_constant, orig, &src->components) {
         this->components.push_tail(orig->clone(this, NULL));
       }
       break;
@@ -987,6 +1149,9 @@ ir_constant::copy_masked_offset(ir_constant *src, int offset, unsigned int mask)
         case GLSL_TYPE_BOOL:
            value.b[i+offset] = src->get_bool_component(id++);
            break;
+        case GLSL_TYPE_DOUBLE:
+           value.d[i+offset] = src->get_double_component(id++);
+           break;
         default:
            assert(!"Should not get here.");
            return;
@@ -1047,6 +1212,10 @@ ir_constant::has_value(const ir_constant *c) const
         if (this->value.b[i] != c->value.b[i])
            return false;
         break;
+      case GLSL_TYPE_DOUBLE:
+        if (this->value.d[i] != c->value.d[i])
+           return false;
+        break;
       default:
         assert(!"Should not get here.");
         return false;
@@ -1057,27 +1226,35 @@ ir_constant::has_value(const ir_constant *c) const
 }
 
 bool
-ir_constant::is_zero() const
+ir_constant::is_value(float f, int i) const
 {
    if (!this->type->is_scalar() && !this->type->is_vector())
       return false;
 
+   /* Only accept boolean values for 0/1. */
+   if (int(bool(i)) != i && this->type->is_boolean())
+      return false;
+
    for (unsigned c = 0; c < this->type->vector_elements; c++) {
       switch (this->type->base_type) {
       case GLSL_TYPE_FLOAT:
-        if (this->value.f[c] != 0.0)
+        if (this->value.f[c] != f)
            return false;
         break;
       case GLSL_TYPE_INT:
-        if (this->value.i[c] != 0)
+        if (this->value.i[c] != i)
            return false;
         break;
       case GLSL_TYPE_UINT:
-        if (this->value.u[c] != 0)
+        if (this->value.u[c] != unsigned(i))
            return false;
         break;
       case GLSL_TYPE_BOOL:
-        if (this->value.b[c] != false)
+        if (this->value.b[c] != bool(i))
+           return false;
+        break;
+      case GLSL_TYPE_DOUBLE:
+        if (this->value.d[c] != double(f))
            return false;
         break;
       default:
@@ -1094,137 +1271,43 @@ ir_constant::is_zero() const
 }
 
 bool
-ir_constant::is_one() const
+ir_constant::is_zero() const
 {
-   if (!this->type->is_scalar() && !this->type->is_vector())
-      return false;
-
-   for (unsigned c = 0; c < this->type->vector_elements; c++) {
-      switch (this->type->base_type) {
-      case GLSL_TYPE_FLOAT:
-        if (this->value.f[c] != 1.0)
-           return false;
-        break;
-      case GLSL_TYPE_INT:
-        if (this->value.i[c] != 1)
-           return false;
-        break;
-      case GLSL_TYPE_UINT:
-        if (this->value.u[c] != 1)
-           return false;
-        break;
-      case GLSL_TYPE_BOOL:
-        if (this->value.b[c] != true)
-           return false;
-        break;
-      default:
-        /* The only other base types are structures, arrays, and samplers.
-         * Samplers cannot be constants, and the others should have been
-         * filtered out above.
-         */
-        assert(!"Should not get here.");
-        return false;
-      }
-   }
+   return is_value(0.0, 0);
+}
 
-   return true;
+bool
+ir_constant::is_one() const
+{
+   return is_value(1.0, 1);
 }
 
 bool
 ir_constant::is_negative_one() const
 {
-   if (!this->type->is_scalar() && !this->type->is_vector())
-      return false;
-
-   if (this->type->is_boolean())
-      return false;
-
-   for (unsigned c = 0; c < this->type->vector_elements; c++) {
-      switch (this->type->base_type) {
-      case GLSL_TYPE_FLOAT:
-        if (this->value.f[c] != -1.0)
-           return false;
-        break;
-      case GLSL_TYPE_INT:
-        if (this->value.i[c] != -1)
-           return false;
-        break;
-      case GLSL_TYPE_UINT:
-        if (int(this->value.u[c]) != -1)
-           return false;
-        break;
-      default:
-        /* The only other base types are structures, arrays, samplers, and
-         * booleans.  Samplers cannot be constants, and the others should
-         * have been filtered out above.
-         */
-        assert(!"Should not get here.");
-        return false;
-      }
-   }
-
-   return true;
+   return is_value(-1.0, -1);
 }
 
 bool
-ir_constant::is_basis() const
+ir_constant::is_uint16_constant() const
 {
-   if (!this->type->is_scalar() && !this->type->is_vector())
+   if (!type->is_integer())
       return false;
 
-   if (this->type->is_boolean())
-      return false;
-
-   unsigned ones = 0;
-   for (unsigned c = 0; c < this->type->vector_elements; c++) {
-      switch (this->type->base_type) {
-      case GLSL_TYPE_FLOAT:
-        if (this->value.f[c] == 1.0)
-           ones++;
-        else if (this->value.f[c] != 0.0)
-           return false;
-        break;
-      case GLSL_TYPE_INT:
-        if (this->value.i[c] == 1)
-           ones++;
-        else if (this->value.i[c] != 0)
-           return false;
-        break;
-      case GLSL_TYPE_UINT:
-        if (int(this->value.u[c]) == 1)
-           ones++;
-        else if (int(this->value.u[c]) != 0)
-           return false;
-        break;
-      default:
-        /* The only other base types are structures, arrays, samplers, and
-         * booleans.  Samplers cannot be constants, and the others should
-         * have been filtered out above.
-         */
-        assert(!"Should not get here.");
-        return false;
-      }
-   }
-
-   return ones == 1;
+   return value.u[0] < (1 << 16);
 }
 
 ir_loop::ir_loop()
+   : ir_instruction(ir_type_loop)
 {
-   this->ir_type = ir_type_loop;
-   this->cmp = ir_unop_neg;
-   this->from = NULL;
-   this->to = NULL;
-   this->increment = NULL;
-   this->counter = NULL;
 }
 
 
 ir_dereference_variable::ir_dereference_variable(ir_variable *var)
+   : ir_dereference(ir_type_dereference_variable)
 {
    assert(var != NULL);
 
-   this->ir_type = ir_type_dereference_variable;
    this->var = var;
    this->type = var->type;
 }
@@ -1232,8 +1315,8 @@ ir_dereference_variable::ir_dereference_variable(ir_variable *var)
 
 ir_dereference_array::ir_dereference_array(ir_rvalue *value,
                                           ir_rvalue *array_index)
+   : ir_dereference(ir_type_dereference_array)
 {
-   this->ir_type = ir_type_dereference_array;
    this->array_index = array_index;
    this->set_array(value);
 }
@@ -1241,10 +1324,10 @@ ir_dereference_array::ir_dereference_array(ir_rvalue *value,
 
 ir_dereference_array::ir_dereference_array(ir_variable *var,
                                           ir_rvalue *array_index)
+   : ir_dereference(ir_type_dereference_array)
 {
    void *ctx = ralloc_parent(var);
 
-   this->ir_type = ir_type_dereference_array;
    this->array_index = array_index;
    this->set_array(new(ctx) ir_dereference_variable(var));
 }
@@ -1260,7 +1343,7 @@ ir_dereference_array::set_array(ir_rvalue *value)
    const glsl_type *const vt = this->array->type;
 
    if (vt->is_array()) {
-      type = vt->element_type();
+      type = vt->fields.array;
    } else if (vt->is_matrix()) {
       type = vt->column_type();
    } else if (vt->is_vector()) {
@@ -1271,10 +1354,10 @@ ir_dereference_array::set_array(ir_rvalue *value)
 
 ir_dereference_record::ir_dereference_record(ir_rvalue *value,
                                             const char *field)
+   : ir_dereference(ir_type_dereference_record)
 {
    assert(value != NULL);
 
-   this->ir_type = ir_type_dereference_record;
    this->record = value;
    this->field = ralloc_strdup(this, field);
    this->type = this->record->type->field_type(field);
@@ -1283,10 +1366,10 @@ ir_dereference_record::ir_dereference_record(ir_rvalue *value,
 
 ir_dereference_record::ir_dereference_record(ir_variable *var,
                                             const char *field)
+   : ir_dereference(ir_type_dereference_record)
 {
    void *ctx = ralloc_parent(var);
 
-   this->ir_type = ir_type_dereference_record;
    this->record = new(ctx) ir_dereference_variable(var);
    this->field = ralloc_strdup(this, field);
    this->type = this->record->type->field_type(field);
@@ -1299,23 +1382,23 @@ ir_dereference::is_lvalue() const
 
    /* Every l-value derference chain eventually ends in a variable.
     */
-   if ((var == NULL) || var->read_only)
+   if ((var == NULL) || var->data.read_only)
       return false;
 
-   /* From page 17 (page 23 of the PDF) of the GLSL 1.20 spec:
+   /* From section 4.1.7 of the GLSL 4.40 spec:
     *
-    *    "Samplers cannot be treated as l-values; hence cannot be used
-    *     as out or inout function parameters, nor can they be
-    *     assigned into."
+    *   "Opaque variables cannot be treated as l-values; hence cannot
+    *    be used as out or inout function parameters, nor can they be
+    *    assigned into."
     */
-   if (this->type->contains_sampler())
+   if (this->type->contains_opaque())
       return false;
 
    return true;
 }
 
 
-static const char *tex_opcode_strs[] = { "tex", "txb", "txl", "txd", "txf", "txf_ms", "txs", "lod" };
+static const char * const tex_opcode_strs[] = { "tex", "txb", "txl", "txd", "txf", "txf_ms", "txs", "lod", "tg4", "query_levels" };
 
 const char *ir_texture::opcode_string()
 {
@@ -1344,7 +1427,7 @@ ir_texture::set_sampler(ir_dereference *sampler, const glsl_type *type)
    this->sampler = sampler;
    this->type = type;
 
-   if (this->op == ir_txs) {
+   if (this->op == ir_txs || this->op == ir_query_levels) {
       assert(type->base_type == GLSL_TYPE_INT);
    } else if (this->op == ir_lod) {
       assert(type->vector_elements == 2);
@@ -1403,24 +1486,22 @@ ir_swizzle::init_mask(const unsigned *comp, unsigned count)
 
 ir_swizzle::ir_swizzle(ir_rvalue *val, unsigned x, unsigned y, unsigned z,
                       unsigned w, unsigned count)
-   : val(val)
+   : ir_rvalue(ir_type_swizzle), val(val)
 {
    const unsigned components[4] = { x, y, z, w };
-   this->ir_type = ir_type_swizzle;
    this->init_mask(components, count);
 }
 
 ir_swizzle::ir_swizzle(ir_rvalue *val, const unsigned *comp,
                       unsigned count)
-   : val(val)
+   : ir_rvalue(ir_type_swizzle), val(val)
 {
-   this->ir_type = ir_type_swizzle;
    this->init_mask(comp, count);
 }
 
 ir_swizzle::ir_swizzle(ir_rvalue *val, ir_swizzle_mask mask)
+   : ir_rvalue(ir_type_swizzle)
 {
-   this->ir_type = ir_type_swizzle;
    this->val = val;
    this->mask = mask;
    this->type = glsl_type::get_instance(val->type->base_type,
@@ -1517,35 +1598,82 @@ ir_swizzle::variable_referenced() const
 }
 
 
+bool ir_variable::temporaries_allocate_names = false;
+
+const char ir_variable::tmp_name[] = "compiler_temp";
+
 ir_variable::ir_variable(const struct glsl_type *type, const char *name,
                         ir_variable_mode mode)
-   : max_array_access(0), read_only(false), centroid(false), invariant(false),
-     mode(mode), interpolation(INTERP_QUALIFIER_NONE)
+   : ir_instruction(ir_type_variable)
 {
-   this->ir_type = ir_type_variable;
    this->type = type;
-   this->name = ralloc_strdup(this, name);
-   this->explicit_location = false;
-   this->has_initializer = false;
-   this->location = -1;
-   this->location_frac = 0;
-   this->warn_extension = NULL;
+
+   if (mode == ir_var_temporary && !ir_variable::temporaries_allocate_names)
+      name = NULL;
+
+   /* The ir_variable clone method may call this constructor with name set to
+    * tmp_name.
+    */
+   assert(name != NULL
+          || mode == ir_var_temporary
+          || mode == ir_var_function_in
+          || mode == ir_var_function_out
+          || mode == ir_var_function_inout);
+   assert(name != ir_variable::tmp_name
+          || mode == ir_var_temporary);
+   if (mode == ir_var_temporary
+       && (name == NULL || name == ir_variable::tmp_name)) {
+      this->name = ir_variable::tmp_name;
+   } else {
+      this->name = ralloc_strdup(this, name);
+   }
+
+   this->u.max_ifc_array_access = NULL;
+
+   this->data.explicit_location = false;
+   this->data.has_initializer = false;
+   this->data.location = -1;
+   this->data.location_frac = 0;
+   this->data.binding = 0;
+   this->data.warn_extension_index = 0;
    this->constant_value = NULL;
    this->constant_initializer = NULL;
-   this->origin_upper_left = false;
-   this->pixel_center_integer = false;
-   this->depth_layout = ir_depth_layout_none;
-   this->used = false;
-
-   if (type && type->base_type == GLSL_TYPE_SAMPLER)
-      this->read_only = true;
+   this->data.origin_upper_left = false;
+   this->data.pixel_center_integer = false;
+   this->data.depth_layout = ir_depth_layout_none;
+   this->data.used = false;
+   this->data.read_only = false;
+   this->data.centroid = false;
+   this->data.sample = false;
+   this->data.patch = false;
+   this->data.invariant = false;
+   this->data.how_declared = ir_var_declared_normally;
+   this->data.mode = mode;
+   this->data.interpolation = INTERP_QUALIFIER_NONE;
+   this->data.max_array_access = 0;
+   this->data.atomic.offset = 0;
+   this->data.image_read_only = false;
+   this->data.image_write_only = false;
+   this->data.image_coherent = false;
+   this->data.image_volatile = false;
+   this->data.image_restrict = false;
+
+   if (type != NULL) {
+      if (type->base_type == GLSL_TYPE_SAMPLER)
+         this->data.read_only = true;
+
+      if (type->is_interface())
+         this->init_interface_type(type);
+      else if (type->is_array() && type->fields.array->is_interface())
+         this->init_interface_type(type->fields.array);
+   }
 }
 
 
 const char *
-ir_variable::interpolation_string() const
+interpolation_string(unsigned interpolation)
 {
-   switch (this->interpolation) {
+   switch (interpolation) {
    case INTERP_QUALIFIER_NONE:          return "no";
    case INTERP_QUALIFIER_SMOOTH:        return "smooth";
    case INTERP_QUALIFIER_FLAT:          return "flat";
@@ -1560,9 +1688,9 @@ ir_variable::interpolation_string() const
 glsl_interp_qualifier
 ir_variable::determine_interpolation_mode(bool flat_shade)
 {
-   if (this->interpolation != INTERP_QUALIFIER_NONE)
-      return (glsl_interp_qualifier) this->interpolation;
-   int location = this->location;
+   if (this->data.interpolation != INTERP_QUALIFIER_NONE)
+      return (glsl_interp_qualifier) this->data.interpolation;
+   int location = this->data.location;
    bool is_gl_Color =
       location == VARYING_SLOT_COL0 || location == VARYING_SLOT_COL1;
    if (flat_shade && is_gl_Color)
@@ -1571,16 +1699,67 @@ ir_variable::determine_interpolation_mode(bool flat_shade)
       return INTERP_QUALIFIER_SMOOTH;
 }
 
+const char *const ir_variable::warn_extension_table[] = {
+   "",
+   "GL_ARB_shader_stencil_export",
+   "GL_AMD_shader_stencil_export",
+};
+
+void
+ir_variable::enable_extension_warning(const char *extension)
+{
+   for (unsigned i = 0; i < ARRAY_SIZE(warn_extension_table); i++) {
+      if (strcmp(warn_extension_table[i], extension) == 0) {
+         this->data.warn_extension_index = i;
+         return;
+      }
+   }
+
+   assert(!"Should not get here.");
+   this->data.warn_extension_index = 0;
+}
+
+const char *
+ir_variable::get_extension_warning() const
+{
+   return this->data.warn_extension_index == 0
+      ? NULL : warn_extension_table[this->data.warn_extension_index];
+}
 
-ir_function_signature::ir_function_signature(const glsl_type *return_type)
-   : return_type(return_type), is_defined(false), _function(NULL)
+ir_function_signature::ir_function_signature(const glsl_type *return_type,
+                                             builtin_available_predicate b)
+   : ir_instruction(ir_type_function_signature),
+     return_type(return_type), is_defined(false), is_intrinsic(false),
+     builtin_avail(b), _function(NULL)
 {
-   this->ir_type = ir_type_function_signature;
-   this->is_builtin = false;
    this->origin = NULL;
 }
 
 
+bool
+ir_function_signature::is_builtin() const
+{
+   return builtin_avail != NULL;
+}
+
+
+bool
+ir_function_signature::is_builtin_available(const _mesa_glsl_parse_state *state) const
+{
+   /* We can't call the predicate without a state pointer, so just say that
+    * the signature is available.  At compile time, we need the filtering,
+    * but also receive a valid state pointer.  At link time, we're resolving
+    * imported built-in prototypes to their definitions, which will always
+    * be an exact match.  So we can skip the filtering.
+    */
+   if (state == NULL)
+      return true;
+
+   assert(builtin_avail != NULL);
+   return builtin_avail(state);
+}
+
+
 static bool
 modes_match(unsigned a, unsigned b)
 {
@@ -1599,25 +1778,26 @@ modes_match(unsigned a, unsigned b)
 const char *
 ir_function_signature::qualifiers_match(exec_list *params)
 {
-   exec_list_iterator iter_a = parameters.iterator();
-   exec_list_iterator iter_b = params->iterator();
-
    /* check that the qualifiers match. */
-   while (iter_a.has_next()) {
-      ir_variable *a = (ir_variable *)iter_a.get();
-      ir_variable *b = (ir_variable *)iter_b.get();
-
-      if (a->read_only != b->read_only ||
-         !modes_match(a->mode, b->mode) ||
-         a->interpolation != b->interpolation ||
-         a->centroid != b->centroid) {
+   foreach_two_lists(a_node, &this->parameters, b_node, params) {
+      ir_variable *a = (ir_variable *) a_node;
+      ir_variable *b = (ir_variable *) b_node;
+
+      if (a->data.read_only != b->data.read_only ||
+         !modes_match(a->data.mode, b->data.mode) ||
+         a->data.interpolation != b->data.interpolation ||
+         a->data.centroid != b->data.centroid ||
+          a->data.sample != b->data.sample ||
+          a->data.patch != b->data.patch ||
+          a->data.image_read_only != b->data.image_read_only ||
+          a->data.image_write_only != b->data.image_write_only ||
+          a->data.image_coherent != b->data.image_coherent ||
+          a->data.image_volatile != b->data.image_volatile ||
+          a->data.image_restrict != b->data.image_restrict) {
 
         /* parameter a's qualifiers don't match */
         return a->name;
       }
-
-      iter_a.next();
-      iter_b.next();
    }
    return NULL;
 }
@@ -1630,19 +1810,13 @@ ir_function_signature::replace_parameters(exec_list *new_params)
     * parameter information comes from the function prototype, it may either
     * specify incorrect parameter names or not have names at all.
     */
-   foreach_iter(exec_list_iterator, iter, parameters) {
-      assert(((ir_instruction *) iter.get())->as_variable() != NULL);
-
-      iter.remove();
-   }
-
    new_params->move_nodes_to(&parameters);
 }
 
 
 ir_function::ir_function(const char *name)
+   : ir_instruction(ir_type_function)
 {
-   this->ir_type = ir_type_function;
    this->name = ralloc_strdup(this, name);
 }
 
@@ -1650,9 +1824,8 @@ ir_function::ir_function(const char *name)
 bool
 ir_function::has_user_signature()
 {
-   foreach_list(n, &this->signatures) {
-      ir_function_signature *const sig = (ir_function_signature *) n;
-      if (!sig->is_builtin)
+   foreach_in_list(ir_function_signature, sig, &this->signatures) {
+      if (!sig->is_builtin())
         return true;
    }
    return false;
@@ -1662,7 +1835,7 @@ ir_function::has_user_signature()
 ir_rvalue *
 ir_rvalue::error_value(void *mem_ctx)
 {
-   ir_rvalue *v = new(mem_ctx) ir_rvalue;
+   ir_rvalue *v = new(mem_ctx) ir_rvalue(ir_type_unset);
 
    v->type = glsl_type::error_type;
    return v;
@@ -1672,8 +1845,8 @@ ir_rvalue::error_value(void *mem_ctx)
 void
 visit_exec_list(exec_list *list, ir_visitor *visitor)
 {
-   foreach_iter(exec_list_iterator, iter, *list) {
-      ((ir_instruction *)iter.get())->accept(visitor);
+   foreach_in_list_safe(ir_instruction, node, list) {
+      node->accept(visitor);
    }
 }
 
@@ -1682,6 +1855,7 @@ static void
 steal_memory(ir_instruction *ir, void *new_ctx)
 {
    ir_variable *var = ir->as_variable();
+   ir_function *fn = ir->as_function();
    ir_constant *constant = ir->as_constant();
    if (var != NULL && var->constant_value != NULL)
       steal_memory(var->constant_value, ir);
@@ -1689,13 +1863,15 @@ steal_memory(ir_instruction *ir, void *new_ctx)
    if (var != NULL && var->constant_initializer != NULL)
       steal_memory(var->constant_initializer, ir);
 
+   if (fn != NULL && fn->subroutine_types)
+      ralloc_steal(new_ctx, fn->subroutine_types);
+
    /* The components of aggregate constants are not visited by the normal
     * visitor, so steal their values by hand.
     */
    if (constant != NULL) {
       if (constant->type->is_record()) {
-        foreach_iter(exec_list_iterator, iter, constant->components) {
-           ir_constant *field = (ir_constant *)iter.get();
+        foreach_in_list(ir_constant, field, &constant->components) {
            steal_memory(field, ir);
         }
       } else if (constant->type->is_array()) {
@@ -1712,8 +1888,8 @@ steal_memory(ir_instruction *ir, void *new_ctx)
 void
 reparent_ir(exec_list *list, void *mem_ctx)
 {
-   foreach_list(node, list) {
-      visit_tree((ir_instruction *) node, steal_memory, mem_ctx);
+   foreach_in_list(ir_instruction, node, list) {
+      visit_tree(node, steal_memory, mem_ctx);
    }
 }
 
@@ -1772,3 +1948,70 @@ ir_rvalue::as_rvalue_to_saturate()
 
    return NULL;
 }
+
+
+unsigned
+vertices_per_prim(GLenum prim)
+{
+   switch (prim) {
+   case GL_POINTS:
+      return 1;
+   case GL_LINES:
+      return 2;
+   case GL_TRIANGLES:
+      return 3;
+   case GL_LINES_ADJACENCY:
+      return 4;
+   case GL_TRIANGLES_ADJACENCY:
+      return 6;
+   default:
+      assert(!"Bad primitive");
+      return 3;
+   }
+}
+
+/**
+ * Generate a string describing the mode of a variable
+ */
+const char *
+mode_string(const ir_variable *var)
+{
+   switch (var->data.mode) {
+   case ir_var_auto:
+      return (var->data.read_only) ? "global constant" : "global variable";
+
+   case ir_var_uniform:
+      return "uniform";
+
+   case ir_var_shader_storage:
+      return "buffer";
+
+   case ir_var_shader_in:
+      return "shader input";
+
+   case ir_var_shader_out:
+      return "shader output";
+
+   case ir_var_function_in:
+   case ir_var_const_in:
+      return "function input";
+
+   case ir_var_function_out:
+      return "function output";
+
+   case ir_var_function_inout:
+      return "function inout";
+
+   case ir_var_system_value:
+      return "shader input";
+
+   case ir_var_temporary:
+      return "compiler temporary";
+
+   case ir_var_mode_count:
+      break;
+   }
+
+   assert(!"Should not get here.");
+   return "invalid variable";
+}