X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir_lower_vars_to_ssa.c;h=30870e6e35c41b239687d63fbdc70ee9774bba98;hb=025bdbac3e09ae9bac9eefb831e9446b9574d120;hp=a3f3fcfd9b481264efc3065e2191cc2d61c6ef6a;hpb=59f57289959702e528b68bdd0d06488089517a00;p=mesa.git

diff --git a/src/compiler/nir/nir_lower_vars_to_ssa.c b/src/compiler/nir/nir_lower_vars_to_ssa.c
index a3f3fcfd9b4..30870e6e35c 100644
--- a/src/compiler/nir/nir_lower_vars_to_ssa.c
+++ b/src/compiler/nir/nir_lower_vars_to_ssa.c
@@ -27,6 +27,7 @@
 
 #include "nir.h"
 #include "nir_builder.h"
+#include "nir_deref.h"
 #include "nir_phi_builder.h"
 #include "nir_vla.h"
 
@@ -38,10 +39,10 @@ struct deref_node {
    bool lower_to_ssa;
 
    /* Only valid for things that end up in the direct list.
-    * Note that multiple nir_deref_vars may correspond to this node, but they
-    * will all be equivalent, so any is as good as the other.
+    * Note that multiple nir_deref_instrs may correspond to this node, but
+    * they will all be equivalent, so any is as good as the other.
     */
-   nir_deref_var *deref;
+   nir_deref_path path;
    struct exec_node direct_derefs_link;
 
    struct set *loads;
@@ -50,11 +51,24 @@ struct deref_node {
 
    struct nir_phi_builder_value *pb_value;
 
+   /* True if this node is fully direct.  If set, it must be in the children
+    * array of its parent.
+    */
+   bool is_direct;
+
+   /* Set on a root node for a variable to indicate that variable is used by a
+    * cast or passed through some other sequence of instructions that are not
+    * derefs.
+    */
+   bool has_complex_use;
+
    struct deref_node *wildcard;
    struct deref_node *indirect;
    struct deref_node *children[0];
 };
 
+#define UNDEF_NODE ((struct deref_node *)(uintptr_t)1)
+
 struct lower_variables_state {
    nir_shader *shader;
    void *dead_ctx;
@@ -92,16 +106,17 @@ struct lower_variables_state {
 
 static struct deref_node *
 deref_node_create(struct deref_node *parent,
-                  const struct glsl_type *type, nir_shader *shader)
+                  const struct glsl_type *type,
+                  bool is_direct, void *mem_ctx)
 {
    size_t size = sizeof(struct deref_node) +
                  glsl_get_length(type) * sizeof(struct deref_node *);
 
-   struct deref_node *node = rzalloc_size(shader, size);
+   struct deref_node *node = rzalloc_size(mem_ctx, size);
    node->type = type;
    node->parent = parent;
-   node->deref = NULL;
    exec_node_init(&node->direct_derefs_link);
+   node->is_direct = is_direct;
 
    return node;
 }
@@ -120,7 +135,7 @@ get_deref_node_for_var(nir_variable *var, struct lower_variables_state *state)
    if (var_entry) {
       return var_entry->data;
    } else {
-      node = deref_node_create(NULL, var->type, state->dead_ctx);
+      node = deref_node_create(NULL, var->type, true, state->dead_ctx);
       _mesa_hash_table_insert(state->deref_var_nodes, var, node);
       return node;
    }
@@ -132,82 +147,98 @@ get_deref_node_for_var(nir_variable *var, struct lower_variables_state *state)
  * table of of fully-qualified direct derefs.
  */
 static struct deref_node *
-get_deref_node(nir_deref_var *deref, struct lower_variables_state *state)
+get_deref_node_recur(nir_deref_instr *deref,
+                     struct lower_variables_state *state)
 {
-   bool is_direct = true;
+   if (deref->deref_type == nir_deref_type_var)
+      return get_deref_node_for_var(deref->var, state);
 
-   /* Start at the base of the chain. */
-   struct deref_node *node = get_deref_node_for_var(deref->var, state);
-   assert(deref->deref.type == node->type);
+   if (deref->deref_type == nir_deref_type_cast)
+      return NULL;
 
-   for (nir_deref *tail = deref->deref.child; tail; tail = tail->child) {
-      switch (tail->deref_type) {
-      case nir_deref_type_struct: {
-         nir_deref_struct *deref_struct = nir_deref_as_struct(tail);
+   struct deref_node *parent =
+      get_deref_node_recur(nir_deref_instr_parent(deref), state);
+   if (parent == NULL)
+      return NULL;
 
-         assert(deref_struct->index < glsl_get_length(node->type));
+   if (parent == UNDEF_NODE)
+      return UNDEF_NODE;
 
-         if (node->children[deref_struct->index] == NULL)
-            node->children[deref_struct->index] =
-               deref_node_create(node, tail->type, state->dead_ctx);
+   switch (deref->deref_type) {
+   case nir_deref_type_struct:
+      assert(glsl_type_is_struct_or_ifc(parent->type));
+      assert(deref->strct.index < glsl_get_length(parent->type));
 
-         node = node->children[deref_struct->index];
-         break;
+      if (parent->children[deref->strct.index] == NULL) {
+         parent->children[deref->strct.index] =
+            deref_node_create(parent, deref->type, parent->is_direct,
+                              state->dead_ctx);
       }
 
-      case nir_deref_type_array: {
-         nir_deref_array *arr = nir_deref_as_array(tail);
-
-         switch (arr->deref_array_type) {
-         case nir_deref_array_type_direct:
-            /* This is possible if a loop unrolls and generates an
-             * out-of-bounds offset.  We need to handle this at least
-             * somewhat gracefully.
-             */
-            if (arr->base_offset >= glsl_get_length(node->type))
-               return NULL;
-
-            if (node->children[arr->base_offset] == NULL)
-               node->children[arr->base_offset] =
-                  deref_node_create(node, tail->type, state->dead_ctx);
-
-            node = node->children[arr->base_offset];
-            break;
+      return parent->children[deref->strct.index];
+
+   case nir_deref_type_array: {
+      if (nir_src_is_const(deref->arr.index)) {
+         uint32_t index = nir_src_as_uint(deref->arr.index);
+         /* This is possible if a loop unrolls and generates an
+          * out-of-bounds offset.  We need to handle this at least
+          * somewhat gracefully.
+          */
+         if (index >= glsl_get_length(parent->type))
+            return UNDEF_NODE;
+
+         if (parent->children[index] == NULL) {
+            parent->children[index] =
+               deref_node_create(parent, deref->type, parent->is_direct,
+                                 state->dead_ctx);
+         }
 
-         case nir_deref_array_type_indirect:
-            if (node->indirect == NULL)
-               node->indirect = deref_node_create(node, tail->type,
-                                                  state->dead_ctx);
+         return parent->children[index];
+      } else {
+         if (parent->indirect == NULL) {
+            parent->indirect =
+               deref_node_create(parent, deref->type, false, state->dead_ctx);
+         }
 
-            node = node->indirect;
-            is_direct = false;
-            break;
+         return parent->indirect;
+      }
+      break;
+   }
 
-         case nir_deref_array_type_wildcard:
-            if (node->wildcard == NULL)
-               node->wildcard = deref_node_create(node, tail->type,
-                                                  state->dead_ctx);
+   case nir_deref_type_array_wildcard:
+      if (parent->wildcard == NULL) {
+         parent->wildcard =
+            deref_node_create(parent, deref->type, false, state->dead_ctx);
+      }
 
-            node = node->wildcard;
-            is_direct = false;
-            break;
+      return parent->wildcard;
 
-         default:
-            unreachable("Invalid array deref type");
-         }
-         break;
-      }
-      default:
-         unreachable("Invalid deref type");
-      }
+   default:
+      unreachable("Invalid deref type");
    }
+}
+
+static struct deref_node *
+get_deref_node(nir_deref_instr *deref, struct lower_variables_state *state)
+{
+   /* This pass only works on local variables.  Just ignore any derefs with
+    * a non-local mode.
+    */
+   if (deref->mode != nir_var_function_temp)
+      return NULL;
 
-   assert(node);
+   struct deref_node *node = get_deref_node_recur(deref, state);
+   if (!node)
+      return NULL;
 
-   /* Only insert if it isn't already in the list. */
-   if (is_direct && state->add_to_direct_deref_nodes &&
+   /* Insert the node in the direct derefs list.  We only do this if it's not
+    * already in the list and we only bother for deref nodes which are used
+    * directly in a load or store.
+    */
+   if (node != UNDEF_NODE && node->is_direct &&
+       state->add_to_direct_deref_nodes &&
        node->direct_derefs_link.next == NULL) {
-      node->deref = deref;
+      nir_deref_path_init(&node->path, deref, state->dead_ctx);
       assert(deref->var != NULL);
       exec_list_push_tail(&state->direct_deref_nodes,
                           &node->direct_derefs_link);
@@ -217,41 +248,42 @@ get_deref_node(nir_deref_var *deref, struct lower_variables_state *state)
 }
 
 /* \sa foreach_deref_node_match */
-static bool
-foreach_deref_node_worker(struct deref_node *node, nir_deref *deref,
-                          bool (* cb)(struct deref_node *node,
+static void
+foreach_deref_node_worker(struct deref_node *node, nir_deref_instr **path,
+                          void (* cb)(struct deref_node *node,
                                       struct lower_variables_state *state),
                           struct lower_variables_state *state)
 {
-   if (deref->child == NULL) {
-      return cb(node, state);
-   } else {
-      switch (deref->child->deref_type) {
-      case nir_deref_type_array: {
-         nir_deref_array *arr = nir_deref_as_array(deref->child);
-         assert(arr->deref_array_type == nir_deref_array_type_direct);
-         if (node->children[arr->base_offset] &&
-             !foreach_deref_node_worker(node->children[arr->base_offset],
-                                        deref->child, cb, state))
-            return false;
-
-         if (node->wildcard &&
-             !foreach_deref_node_worker(node->wildcard,
-                                        deref->child, cb, state))
-            return false;
+   if (*path == NULL) {
+      cb(node, state);
+      return;
+   }
 
-         return true;
+   switch ((*path)->deref_type) {
+   case nir_deref_type_struct:
+      if (node->children[(*path)->strct.index]) {
+         foreach_deref_node_worker(node->children[(*path)->strct.index],
+                                   path + 1, cb, state);
       }
+      return;
+
+   case nir_deref_type_array: {
+      uint32_t index = nir_src_as_uint((*path)->arr.index);
 
-      case nir_deref_type_struct: {
-         nir_deref_struct *str = nir_deref_as_struct(deref->child);
-         return foreach_deref_node_worker(node->children[str->index],
-                                          deref->child, cb, state);
+      if (node->children[index]) {
+         foreach_deref_node_worker(node->children[index],
+                                   path + 1, cb, state);
       }
 
-      default:
-         unreachable("Invalid deref child type");
+      if (node->wildcard) {
+         foreach_deref_node_worker(node->wildcard,
+                                   path + 1, cb, state);
       }
+      return;
+   }
+
+   default:
+      unreachable("Unsupported deref type");
    }
 }
 
@@ -267,67 +299,62 @@ foreach_deref_node_worker(struct deref_node *node, nir_deref *deref,
  * The given deref must be a full-length and fully qualified (no wildcards
  * or indirects) deref chain.
  */
-static bool
-foreach_deref_node_match(nir_deref_var *deref,
-                         bool (* cb)(struct deref_node *node,
+static void
+foreach_deref_node_match(nir_deref_path *path,
+                         void (* cb)(struct deref_node *node,
                                      struct lower_variables_state *state),
                          struct lower_variables_state *state)
 {
-   nir_deref_var var_deref = *deref;
-   var_deref.deref.child = NULL;
-   struct deref_node *node = get_deref_node(&var_deref, state);
+   assert(path->path[0]->deref_type == nir_deref_type_var);
+   struct deref_node *node = get_deref_node_for_var(path->path[0]->var, state);
 
    if (node == NULL)
-      return false;
+      return;
 
-   return foreach_deref_node_worker(node, &deref->deref, cb, state);
+   foreach_deref_node_worker(node, &path->path[1], cb, state);
 }
 
 /* \sa deref_may_be_aliased */
 static bool
-deref_may_be_aliased_node(struct deref_node *node, nir_deref *deref,
-                          struct lower_variables_state *state)
+path_may_be_aliased_node(struct deref_node *node, nir_deref_instr **path,
+                         struct lower_variables_state *state)
 {
-   if (deref->child == NULL) {
+   if (*path == NULL)
       return false;
-   } else {
-      switch (deref->child->deref_type) {
-      case nir_deref_type_array: {
-         nir_deref_array *arr = nir_deref_as_array(deref->child);
-         if (arr->deref_array_type == nir_deref_array_type_indirect)
-            return true;
 
-         /* If there is an indirect at this level, we're aliased. */
-         if (node->indirect)
-            return true;
+   switch ((*path)->deref_type) {
+   case nir_deref_type_struct:
+      if (node->children[(*path)->strct.index]) {
+         return path_may_be_aliased_node(node->children[(*path)->strct.index],
+                                         path + 1, state);
+      } else {
+         return false;
+      }
 
-         assert(arr->deref_array_type == nir_deref_array_type_direct);
+   case nir_deref_type_array: {
+      if (!nir_src_is_const((*path)->arr.index))
+         return true;
 
-         if (node->children[arr->base_offset] &&
-             deref_may_be_aliased_node(node->children[arr->base_offset],
-                                       deref->child, state))
-            return true;
+      uint32_t index = nir_src_as_uint((*path)->arr.index);
 
-         if (node->wildcard &&
-             deref_may_be_aliased_node(node->wildcard, deref->child, state))
-            return true;
+      /* If there is an indirect at this level, we're aliased. */
+      if (node->indirect)
+         return true;
 
-         return false;
-      }
+      if (node->children[index] &&
+          path_may_be_aliased_node(node->children[index],
+                                   path + 1, state))
+         return true;
 
-      case nir_deref_type_struct: {
-         nir_deref_struct *str = nir_deref_as_struct(deref->child);
-         if (node->children[str->index]) {
-             return deref_may_be_aliased_node(node->children[str->index],
-                                              deref->child, state);
-         } else {
-            return false;
-         }
-      }
+      if (node->wildcard &&
+          path_may_be_aliased_node(node->wildcard, path + 1, state))
+         return true;
 
-      default:
-         unreachable("Invalid nir_deref child type");
-      }
+      return false;
+   }
+
+   default:
+      unreachable("Unsupported deref type");
    }
 }
 
@@ -346,24 +373,46 @@ deref_may_be_aliased_node(struct deref_node *node, nir_deref *deref,
  * references.
  */
 static bool
-deref_may_be_aliased(nir_deref_var *deref,
+path_may_be_aliased(nir_deref_path *path,
+                    struct lower_variables_state *state)
+{
+   assert(path->path[0]->deref_type == nir_deref_type_var);
+   nir_variable *var = path->path[0]->var;
+   struct deref_node *var_node = get_deref_node_for_var(var, state);
+
+   /* First see if this variable is ever used by anything other than a
+    * load/store.  If there's even so much as a cast in the way, we have to
+    * assume aliasing and bail.
+    */
+   if (var_node->has_complex_use)
+      return true;
+
+   return path_may_be_aliased_node(var_node, &path->path[1], state);
+}
+
+static void
+register_complex_use(nir_deref_instr *deref,
                      struct lower_variables_state *state)
 {
-   return deref_may_be_aliased_node(get_deref_node_for_var(deref->var, state),
-                                    &deref->deref, state);
+   assert(deref->deref_type == nir_deref_type_var);
+   struct deref_node *node = get_deref_node_for_var(deref->var, state);
+   if (node == NULL)
+      return;
+
+   node->has_complex_use = true;
 }
 
 static void
 register_load_instr(nir_intrinsic_instr *load_instr,
                     struct lower_variables_state *state)
 {
-   struct deref_node *node = get_deref_node(load_instr->variables[0], state);
-   if (node == NULL)
+   nir_deref_instr *deref = nir_src_as_deref(load_instr->src[0]);
+   struct deref_node *node = get_deref_node(deref, state);
+   if (node == NULL || node == UNDEF_NODE)
       return;
 
    if (node->loads == NULL)
-      node->loads = _mesa_set_create(state->dead_ctx, _mesa_hash_pointer,
-                                     _mesa_key_pointer_equal);
+      node->loads = _mesa_pointer_set_create(state->dead_ctx);
 
    _mesa_set_add(node->loads, load_instr);
 }
@@ -372,13 +421,13 @@ static void
 register_store_instr(nir_intrinsic_instr *store_instr,
                      struct lower_variables_state *state)
 {
-   struct deref_node *node = get_deref_node(store_instr->variables[0], state);
-   if (node == NULL)
+   nir_deref_instr *deref = nir_src_as_deref(store_instr->src[0]);
+   struct deref_node *node = get_deref_node(deref, state);
+   if (node == NULL || node == UNDEF_NODE)
       return;
 
    if (node->stores == NULL)
-      node->stores = _mesa_set_create(state->dead_ctx, _mesa_hash_pointer,
-                                     _mesa_key_pointer_equal);
+      node->stores = _mesa_pointer_set_create(state->dead_ctx);
 
    _mesa_set_add(node->stores, store_instr);
 }
@@ -388,72 +437,85 @@ register_copy_instr(nir_intrinsic_instr *copy_instr,
                     struct lower_variables_state *state)
 {
    for (unsigned idx = 0; idx < 2; idx++) {
-      struct deref_node *node =
-         get_deref_node(copy_instr->variables[idx], state);
-
-      if (node == NULL)
+      nir_deref_instr *deref = nir_src_as_deref(copy_instr->src[idx]);
+      struct deref_node *node = get_deref_node(deref, state);
+      if (node == NULL || node == UNDEF_NODE)
          continue;
 
       if (node->copies == NULL)
-         node->copies = _mesa_set_create(state->dead_ctx, _mesa_hash_pointer,
-                                         _mesa_key_pointer_equal);
+         node->copies = _mesa_pointer_set_create(state->dead_ctx);
 
       _mesa_set_add(node->copies, copy_instr);
    }
 }
 
-/* Registers all variable uses in the given block. */
-static bool
-register_variable_uses_block(nir_block *block, void *void_state)
+static void
+register_variable_uses(nir_function_impl *impl,
+                       struct lower_variables_state *state)
 {
-   struct lower_variables_state *state = void_state;
+   nir_foreach_block(block, impl) {
+      nir_foreach_instr_safe(instr, block) {
+         switch (instr->type) {
+         case nir_instr_type_deref: {
+            nir_deref_instr *deref = nir_instr_as_deref(instr);
 
-   nir_foreach_instr_safe(block, instr) {
-      if (instr->type != nir_instr_type_intrinsic)
-         continue;
+            if (deref->deref_type == nir_deref_type_var &&
+                nir_deref_instr_has_complex_use(deref))
+               register_complex_use(deref, state);
+
+            break;
+         }
 
-      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+         case nir_instr_type_intrinsic: {
+            nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
 
-      switch (intrin->intrinsic) {
-      case nir_intrinsic_load_var:
-         register_load_instr(intrin, state);
-         break;
+            switch (intrin->intrinsic) {
+            case nir_intrinsic_load_deref:
+               register_load_instr(intrin, state);
+               break;
 
-      case nir_intrinsic_store_var:
-         register_store_instr(intrin, state);
-         break;
+            case nir_intrinsic_store_deref:
+               register_store_instr(intrin, state);
+               break;
 
-      case nir_intrinsic_copy_var:
-         register_copy_instr(intrin, state);
-         break;
+            case nir_intrinsic_copy_deref:
+               register_copy_instr(intrin, state);
+               break;
 
-      default:
-         continue;
+            default:
+               continue;
+            }
+            break;
+         }
+
+         default:
+            break;
+         }
       }
    }
-
-   return true;
 }
 
 /* Walks over all of the copy instructions to or from the given deref_node
  * and lowers them to load/store intrinsics.
  */
-static bool
+static void
 lower_copies_to_load_store(struct deref_node *node,
                            struct lower_variables_state *state)
 {
    if (!node->copies)
-      return true;
+      return;
+
+   nir_builder b;
+   nir_builder_init(&b, state->impl);
 
-   struct set_entry *copy_entry;
    set_foreach(node->copies, copy_entry) {
       nir_intrinsic_instr *copy = (void *)copy_entry->key;
 
-      nir_lower_var_copy_instr(copy, state->shader);
+      nir_lower_deref_copy_instr(&b, copy);
 
       for (unsigned i = 0; i < 2; ++i) {
-         struct deref_node *arg_node =
-            get_deref_node(copy->variables[i], state);
+         nir_deref_instr *arg_deref = nir_src_as_deref(copy->src[i]);
+         struct deref_node *arg_node = get_deref_node(arg_deref, state);
 
          /* Only bother removing copy entries for other nodes */
          if (arg_node == NULL || arg_node == node)
@@ -461,150 +523,160 @@ lower_copies_to_load_store(struct deref_node *node,
 
          struct set_entry *arg_entry = _mesa_set_search(arg_node->copies, copy);
          assert(arg_entry);
-         _mesa_set_remove(node->copies, arg_entry);
+         _mesa_set_remove(arg_node->copies, arg_entry);
       }
 
       nir_instr_remove(&copy->instr);
    }
 
    node->copies = NULL;
-
-   return true;
 }
 
-/* Performs variable renaming by doing a DFS of the dominance tree
+/* Performs variable renaming
  *
  * This algorithm is very similar to the one outlined in "Efficiently
  * Computing Static Single Assignment Form and the Control Dependence
- * Graph" by Cytron et. al.  The primary difference is that we only put one
+ * Graph" by Cytron et al.  The primary difference is that we only put one
  * SSA def on the stack per block.
  */
 static bool
-rename_variables_block(nir_block *block, struct lower_variables_state *state)
+rename_variables(struct lower_variables_state *state)
 {
    nir_builder b;
    nir_builder_init(&b, state->impl);
 
-   nir_foreach_instr_safe(block, instr) {
-      if (instr->type != nir_instr_type_intrinsic)
-         continue;
+   nir_foreach_block(block, state->impl) {
+      nir_foreach_instr_safe(instr, block) {
+         if (instr->type != nir_instr_type_intrinsic)
+            continue;
+
+         nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+
+         switch (intrin->intrinsic) {
+         case nir_intrinsic_load_deref: {
+            nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+            if (deref->mode != nir_var_function_temp)
+               continue;
+
+            struct deref_node *node = get_deref_node(deref, state);
+            if (node == NULL)
+               continue;
+
+            if (node == UNDEF_NODE) {
+               /* If we hit this path then we are referencing an invalid
+                * value.  Most likely, we unrolled something and are
+                * reading past the end of some array.  In any case, this
+                * should result in an undefined value.
+                */
+               nir_ssa_undef_instr *undef =
+                  nir_ssa_undef_instr_create(state->shader,
+                                             intrin->num_components,
+                                             intrin->dest.ssa.bit_size);
+
+               nir_instr_insert_before(&intrin->instr, &undef->instr);
+               nir_instr_remove(&intrin->instr);
+
+               nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
+                                        nir_src_for_ssa(&undef->def));
+               continue;
+            }
+
+            if (!node->lower_to_ssa)
+               continue;
 
-      nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
+            nir_alu_instr *mov = nir_alu_instr_create(state->shader,
+                                                      nir_op_mov);
+            mov->src[0].src = nir_src_for_ssa(
+               nir_phi_builder_value_get_block_def(node->pb_value, block));
+            for (unsigned i = intrin->num_components; i < NIR_MAX_VEC_COMPONENTS; i++)
+               mov->src[0].swizzle[i] = 0;
 
-      switch (intrin->intrinsic) {
-      case nir_intrinsic_load_var: {
-         struct deref_node *node =
-            get_deref_node(intrin->variables[0], state);
+            assert(intrin->dest.is_ssa);
 
-         if (node == NULL) {
-            /* If we hit this path then we are referencing an invalid
-             * value.  Most likely, we unrolled something and are
-             * reading past the end of some array.  In any case, this
-             * should result in an undefined value.
-             */
-            nir_ssa_undef_instr *undef =
-               nir_ssa_undef_instr_create(state->shader,
-                                          intrin->num_components);
+            mov->dest.write_mask = (1 << intrin->num_components) - 1;
+            nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
+                              intrin->num_components,
+                              intrin->dest.ssa.bit_size, NULL);
 
-            nir_instr_insert_before(&intrin->instr, &undef->instr);
+            nir_instr_insert_before(&intrin->instr, &mov->instr);
             nir_instr_remove(&intrin->instr);
 
             nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
-                                     nir_src_for_ssa(&undef->def));
-            continue;
+                                     nir_src_for_ssa(&mov->dest.dest.ssa));
+            break;
          }
 
-         if (!node->lower_to_ssa)
-            continue;
+         case nir_intrinsic_store_deref: {
+            nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
+            if (deref->mode != nir_var_function_temp)
+               continue;
 
-         nir_alu_instr *mov = nir_alu_instr_create(state->shader,
-                                                   nir_op_imov);
-         mov->src[0].src = nir_src_for_ssa(
-            nir_phi_builder_value_get_block_def(node->pb_value, block));
-         for (unsigned i = intrin->num_components; i < 4; i++)
-            mov->src[0].swizzle[i] = 0;
+            struct deref_node *node = get_deref_node(deref, state);
+            if (node == NULL)
+               continue;
 
-         assert(intrin->dest.is_ssa);
+            assert(intrin->src[1].is_ssa);
+            nir_ssa_def *value = intrin->src[1].ssa;
 
-         mov->dest.write_mask = (1 << intrin->num_components) - 1;
-         nir_ssa_dest_init(&mov->instr, &mov->dest.dest,
-                           intrin->num_components, NULL);
-
-         nir_instr_insert_before(&intrin->instr, &mov->instr);
-         nir_instr_remove(&intrin->instr);
+            if (node == UNDEF_NODE) {
+               /* Probably an out-of-bounds array store.  That should be a
+                * no-op. */
+               nir_instr_remove(&intrin->instr);
+               continue;
+            }
 
-         nir_ssa_def_rewrite_uses(&intrin->dest.ssa,
-                                  nir_src_for_ssa(&mov->dest.dest.ssa));
-         break;
-      }
+            if (!node->lower_to_ssa)
+               continue;
+
+            assert(intrin->num_components ==
+                   glsl_get_vector_elements(node->type));
+
+            nir_ssa_def *new_def;
+            b.cursor = nir_before_instr(&intrin->instr);
+
+            unsigned wrmask = nir_intrinsic_write_mask(intrin);
+            if (wrmask == (1 << intrin->num_components) - 1) {
+               /* Whole variable store - just copy the source.  Note that
+                * intrin->num_components and value->num_components
+                * may differ.
+                */
+               unsigned swiz[NIR_MAX_VEC_COMPONENTS];
+               for (unsigned i = 0; i < NIR_MAX_VEC_COMPONENTS; i++)
+                  swiz[i] = i < intrin->num_components ? i : 0;
+
+               new_def = nir_swizzle(&b, value, swiz,
+                                     intrin->num_components);
+            } else {
+               nir_ssa_def *old_def =
+                  nir_phi_builder_value_get_block_def(node->pb_value, block);
+               /* For writemasked store_var intrinsics, we combine the newly
+                * written values with the existing contents of unwritten
+                * channels, creating a new SSA value for the whole vector.
+                */
+               nir_ssa_def *srcs[NIR_MAX_VEC_COMPONENTS];
+               for (unsigned i = 0; i < intrin->num_components; i++) {
+                  if (wrmask & (1 << i)) {
+                     srcs[i] = nir_channel(&b, value, i);
+                  } else {
+                     srcs[i] = nir_channel(&b, old_def, i);
+                  }
+               }
+               new_def = nir_vec(&b, srcs, intrin->num_components);
+            }
 
-      case nir_intrinsic_store_var: {
-         struct deref_node *node =
-            get_deref_node(intrin->variables[0], state);
+            assert(new_def->num_components == intrin->num_components);
 
-         if (node == NULL) {
-            /* Probably an out-of-bounds array store.  That should be a
-             * no-op. */
+            nir_phi_builder_value_set_block_def(node->pb_value, block, new_def);
             nir_instr_remove(&intrin->instr);
-            continue;
+            break;
          }
 
-         if (!node->lower_to_ssa)
-            continue;
-
-         assert(intrin->num_components ==
-                glsl_get_vector_elements(node->type));
-
-         assert(intrin->src[0].is_ssa);
-
-         nir_ssa_def *new_def;
-         b.cursor = nir_before_instr(&intrin->instr);
-
-         unsigned wrmask = nir_intrinsic_write_mask(intrin);
-         if (wrmask == (1 << intrin->num_components) - 1) {
-            /* Whole variable store - just copy the source.  Note that
-             * intrin->num_components and intrin->src[0].ssa->num_components
-             * may differ.
-             */
-            unsigned swiz[4];
-            for (unsigned i = 0; i < 4; i++)
-               swiz[i] = i < intrin->num_components ? i : 0;
-
-            new_def = nir_swizzle(&b, intrin->src[0].ssa, swiz,
-                                  intrin->num_components, false);
-         } else {
-            nir_ssa_def *old_def =
-               nir_phi_builder_value_get_block_def(node->pb_value, block);
-            /* For writemasked store_var intrinsics, we combine the newly
-             * written values with the existing contents of unwritten
-             * channels, creating a new SSA value for the whole vector.
-             */
-            nir_ssa_def *srcs[4];
-            for (unsigned i = 0; i < intrin->num_components; i++) {
-               if (wrmask & (1 << i)) {
-                  srcs[i] = nir_channel(&b, intrin->src[0].ssa, i);
-               } else {
-                  srcs[i] = nir_channel(&b, old_def, i);
-               }
-            }
-            new_def = nir_vec(&b, srcs, intrin->num_components);
+         default:
+            break;
          }
-
-         assert(new_def->num_components == intrin->num_components);
-
-         nir_phi_builder_value_set_block_def(node->pb_value, block, new_def);
-         nir_instr_remove(&intrin->instr);
-         break;
-      }
-
-      default:
-         break;
       }
    }
 
-   for (unsigned i = 0; i < block->num_dom_children; ++i)
-      rename_variables_block(block->dom_children[i], state);
-
    return true;
 }
 
@@ -620,14 +692,12 @@ rename_variables_block(nir_block *block, struct lower_variables_state *state)
  *     fully-direct references we see and store them in the
  *     direct_deref_nodes hash table.
  *
- *  2) Walk over the the list of fully-qualified direct derefs generated in
+ *  2) Walk over the list of fully-qualified direct derefs generated in
  *     the previous pass.  For each deref, we determine if it can ever be
  *     aliased, i.e. if there is an indirect reference anywhere that may
  *     refer to it.  If it cannot be aliased, we mark it for lowering to an
  *     SSA value.  At this point, we lower any var_copy instructions that
- *     use the given deref to load/store operations and, if the deref has a
- *     constant initializer, we go ahead and add a load_const value at the
- *     beginning of the function with the initialized value.
+ *     use the given deref to load/store operations.
  *
  *  3) Walk over the list of derefs we plan to lower to SSA values and
  *     insert phi nodes as needed.
@@ -644,14 +714,13 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
    state.dead_ctx = ralloc_context(state.shader);
    state.impl = impl;
 
-   state.deref_var_nodes = _mesa_hash_table_create(state.dead_ctx,
-                                                   _mesa_hash_pointer,
-                                                   _mesa_key_pointer_equal);
+   state.deref_var_nodes = _mesa_pointer_hash_table_create(state.dead_ctx);
    exec_list_make_empty(&state.direct_deref_nodes);
 
    /* Build the initial deref structures and direct_deref_nodes table */
    state.add_to_direct_deref_nodes = true;
-   nir_foreach_block(impl, register_variable_uses_block, &state);
+
+   register_variable_uses(impl, &state);
 
    bool progress = false;
 
@@ -662,14 +731,14 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
 
    foreach_list_typed_safe(struct deref_node, node, direct_derefs_link,
                            &state.direct_deref_nodes) {
-      nir_deref_var *deref = node->deref;
+      nir_deref_path *path = &node->path;
 
-      if (deref->var->data.mode != nir_var_local) {
-         exec_node_remove(&node->direct_derefs_link);
-         continue;
-      }
+      assert(path->path[0]->deref_type == nir_deref_type_var);
 
-      if (deref_may_be_aliased(deref, &state)) {
+      /* We don't build deref nodes for non-local variables */
+      assert(path->path[0]->var->data.mode == nir_var_function_temp);
+
+      if (path_may_be_aliased(path, &state)) {
          exec_node_remove(&node->direct_derefs_link);
          continue;
       }
@@ -677,11 +746,13 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
       node->lower_to_ssa = true;
       progress = true;
 
-      foreach_deref_node_match(deref, lower_copies_to_load_store, &state);
+      foreach_deref_node_match(path, lower_copies_to_load_store, &state);
    }
 
-   if (!progress)
+   if (!progress) {
+      nir_metadata_preserve(impl, nir_metadata_all);
       return false;
+   }
 
    nir_metadata_require(impl, nir_metadata_dominance);
 
@@ -691,11 +762,13 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
     * added load/store instructions are registered.  We need this
     * information for phi node insertion below.
     */
-   nir_foreach_block(impl, register_variable_uses_block, &state);
+   register_variable_uses(impl, &state);
 
    state.phi_builder = nir_phi_builder_create(state.impl);
 
-   NIR_VLA(BITSET_WORD, store_blocks, BITSET_WORDS(state.impl->num_blocks));
+   BITSET_WORD *store_blocks =
+      ralloc_array(state.dead_ctx, BITSET_WORD,
+                   BITSET_WORDS(state.impl->num_blocks));
    foreach_list_typed(struct deref_node, node, direct_derefs_link,
                       &state.direct_deref_nodes) {
       if (!node->lower_to_ssa)
@@ -704,8 +777,10 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
       memset(store_blocks, 0,
              BITSET_WORDS(state.impl->num_blocks) * sizeof(*store_blocks));
 
+      assert(node->path.path[0]->var->constant_initializer == NULL &&
+             node->path.path[0]->var->pointer_initializer == NULL);
+
       if (node->stores) {
-         struct set_entry *store_entry;
          set_foreach(node->stores, store_entry) {
             nir_intrinsic_instr *store =
                (nir_intrinsic_instr *)store_entry->key;
@@ -713,24 +788,14 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
          }
       }
 
-      if (node->deref->var->constant_initializer)
-         BITSET_SET(store_blocks, 0);
-
       node->pb_value =
          nir_phi_builder_add_value(state.phi_builder,
                                    glsl_get_vector_elements(node->type),
+                                   glsl_get_bit_size(node->type),
                                    store_blocks);
-
-      if (node->deref->var->constant_initializer) {
-         nir_load_const_instr *load =
-            nir_deref_get_const_initializer_load(state.shader, node->deref);
-         nir_instr_insert_before_cf_list(&impl->body, &load->instr);
-         nir_phi_builder_value_set_block_def(node->pb_value,
-                                             nir_start_block(impl), &load->def);
-      }
    }
 
-   rename_variables_block(nir_start_block(impl), &state);
+   rename_variables(&state);
 
    nir_phi_builder_finish(state.phi_builder);
 
@@ -742,11 +807,15 @@ nir_lower_vars_to_ssa_impl(nir_function_impl *impl)
    return progress;
 }
 
-void
+bool
 nir_lower_vars_to_ssa(nir_shader *shader)
 {
-   nir_foreach_function(shader, function) {
+   bool progress = false;
+
+   nir_foreach_function(function, shader) {
       if (function->impl)
-         nir_lower_vars_to_ssa_impl(function->impl);
+         progress |= nir_lower_vars_to_ssa_impl(function->impl);
    }
+
+   return progress;
 }