X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=src%2Fcompiler%2Fnir%2Fnir_clone.c;h=f57fce599c1c7af127bb5bc24b7920520667b106;hb=d70fff99c5bc3a721e20869e7f0be8024ffe5ecd;hp=7444dfe6e590348920d4f0e1fe45731959b15f86;hpb=3124ce699bb3844e793f00e00bfbea5c91744f90;p=mesa.git

diff --git a/src/compiler/nir/nir_clone.c b/src/compiler/nir/nir_clone.c
index 7444dfe6e59..f57fce599c1 100644
--- a/src/compiler/nir/nir_clone.c
+++ b/src/compiler/nir/nir_clone.c
@@ -22,7 +22,7 @@
  */
 
 #include "nir.h"
-#include "nir_control_flow_private.h"
+#include "nir_control_flow.h"
 
 /* Secret Decoder Ring:
  *   clone_foo():
@@ -35,6 +35,13 @@ typedef struct {
    /* True if we are cloning an entire shader. */
    bool global_clone;
 
+   /* If true allows the clone operation to fall back to the original pointer
+    * if no clone pointer is found in the remap table.  This allows us to
+    * clone a loop body without having to add srcs from outside the loop to
+    * the remap table. This is useful for loop unrolling.
+    */
+   bool allow_remap_fallback;
+
    /* maps orig ptr -> cloned ptr: */
    struct hash_table *remap_table;
 
@@ -46,11 +53,18 @@ typedef struct {
 } clone_state;
 
 static void
-init_clone_state(clone_state *state, bool global)
+init_clone_state(clone_state *state, struct hash_table *remap_table,
+                 bool global, bool allow_remap_fallback)
 {
    state->global_clone = global;
-   state->remap_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
-                                                _mesa_key_pointer_equal);
+   state->allow_remap_fallback = allow_remap_fallback;
+
+   if (remap_table) {
+      state->remap_table = remap_table;
+   } else {
+      state->remap_table = _mesa_pointer_hash_table_create(NULL);
+   }
+
    list_inithead(&state->phi_srcs);
 }
 
@@ -71,10 +85,16 @@ _lookup_ptr(clone_state *state, const void *ptr, bool global)
    if (!state->global_clone && global)
       return (void *)ptr;
 
+   if (unlikely(!state->remap_table)) {
+      assert(state->allow_remap_fallback);
+      return (void *)ptr;
+   }
+
    entry = _mesa_hash_table_search(state->remap_table, ptr);
-   assert(entry && "Failed to find pointer!");
-   if (!entry)
-      return NULL;
+   if (!entry) {
+      assert(state->allow_remap_fallback);
+      return (void *)ptr;
+   }
 
    return entry->data;
 }
@@ -100,7 +120,7 @@ remap_global(clone_state *state, const void *ptr)
 static nir_register *
 remap_reg(clone_state *state, const nir_register *reg)
 {
-   return _lookup_ptr(state, reg, reg->is_global);
+   return _lookup_ptr(state, reg, false);
 }
 
 static nir_variable *
@@ -109,43 +129,61 @@ remap_var(clone_state *state, const nir_variable *var)
    return _lookup_ptr(state, var, nir_variable_is_global(var));
 }
 
-static nir_constant *
-clone_constant(clone_state *state, const nir_constant *c, nir_variable *nvar)
+nir_constant *
+nir_constant_clone(const nir_constant *c, nir_variable *nvar)
 {
    nir_constant *nc = ralloc(nvar, nir_constant);
 
-   nc->value = c->value;
+   memcpy(nc->values, c->values, sizeof(nc->values));
    nc->num_elements = c->num_elements;
    nc->elements = ralloc_array(nvar, nir_constant *, c->num_elements);
    for (unsigned i = 0; i < c->num_elements; i++) {
-      nc->elements[i] = clone_constant(state, c->elements[i], nvar);
+      nc->elements[i] = nir_constant_clone(c->elements[i], nvar);
    }
 
    return nc;
 }
 
-/* NOTE: for cloning nir_variable's, bypass nir_variable_create to avoid
+/* NOTE: for cloning nir_variables, bypass nir_variable_create to avoid
  * having to deal with locals and globals separately:
  */
-static nir_variable *
-clone_variable(clone_state *state, const nir_variable *var)
+nir_variable *
+nir_variable_clone(const nir_variable *var, nir_shader *shader)
 {
-   nir_variable *nvar = rzalloc(state->ns, nir_variable);
-   add_remap(state, nvar, var);
+   nir_variable *nvar = rzalloc(shader, nir_variable);
 
    nvar->type = var->type;
    nvar->name = ralloc_strdup(nvar, var->name);
    nvar->data = var->data;
    nvar->num_state_slots = var->num_state_slots;
-   nvar->state_slots = ralloc_array(nvar, nir_state_slot, var->num_state_slots);
-   memcpy(nvar->state_slots, var->state_slots,
-          var->num_state_slots * sizeof(nir_state_slot));
+   if (var->num_state_slots) {
+      nvar->state_slots = ralloc_array(nvar, nir_state_slot, var->num_state_slots);
+      memcpy(nvar->state_slots, var->state_slots,
+             var->num_state_slots * sizeof(nir_state_slot));
+   }
    if (var->constant_initializer) {
       nvar->constant_initializer =
-         clone_constant(state, var->constant_initializer, nvar);
+         nir_constant_clone(var->constant_initializer, nvar);
    }
    nvar->interface_type = var->interface_type;
 
+   nvar->num_members = var->num_members;
+   if (var->num_members) {
+      nvar->members = ralloc_array(nvar, struct nir_variable_data,
+                                   var->num_members);
+      memcpy(nvar->members, var->members,
+             var->num_members * sizeof(*var->members));
+   }
+
+   return nvar;
+}
+
+static nir_variable *
+clone_variable(clone_state *state, const nir_variable *var)
+{
+   nir_variable *nvar = nir_variable_clone(var, state->ns);
+   add_remap(state, nvar, var);
+
    return nvar;
 }
 
@@ -161,7 +199,7 @@ clone_var_list(clone_state *state, struct exec_list *dst,
    }
 }
 
-/* NOTE: for cloning nir_register's, bypass nir_global/local_reg_create()
+/* NOTE: for cloning nir_registers, bypass nir_global/local_reg_create()
  * to avoid having to deal with locals and globals separately:
  */
 static nir_register *
@@ -171,11 +209,10 @@ clone_register(clone_state *state, const nir_register *reg)
    add_remap(state, nreg, reg);
 
    nreg->num_components = reg->num_components;
+   nreg->bit_size = reg->bit_size;
    nreg->num_array_elems = reg->num_array_elems;
    nreg->index = reg->index;
    nreg->name = ralloc_strdup(nreg, reg->name);
-   nreg->is_global = reg->is_global;
-   nreg->is_packed = reg->is_packed;
 
    /* reconstructing uses/defs/if_uses handled by nir_instr_insert() */
    list_inithead(&nreg->uses);
@@ -222,7 +259,8 @@ __clone_dst(clone_state *state, nir_instr *ninstr,
    if (dst->is_ssa) {
       nir_ssa_dest_init(ninstr, ndst, dst->ssa.num_components,
                         dst->ssa.bit_size, dst->ssa.name);
-      add_remap(state, &ndst->ssa, &dst->ssa);
+      if (likely(state->remap_table))
+         add_remap(state, &ndst->ssa, &dst->ssa);
    } else {
       ndst->reg.reg = remap_reg(state, dst->reg.reg);
       if (dst->reg.indirect) {
@@ -233,91 +271,81 @@ __clone_dst(clone_state *state, nir_instr *ninstr,
    }
 }
 
-static nir_deref *clone_deref(clone_state *state, const nir_deref *deref,
-                              nir_instr *ninstr, nir_deref *parent);
-
-static nir_deref_var *
-clone_deref_var(clone_state *state, const nir_deref_var *dvar,
-                nir_instr *ninstr)
+static nir_alu_instr *
+clone_alu(clone_state *state, const nir_alu_instr *alu)
 {
-   nir_variable *nvar = remap_var(state, dvar->var);
-   nir_deref_var *ndvar = nir_deref_var_create(ninstr, nvar);
+   nir_alu_instr *nalu = nir_alu_instr_create(state->ns, alu->op);
+   nalu->exact = alu->exact;
+   nalu->no_signed_wrap = alu->no_signed_wrap;
+   nalu->no_unsigned_wrap = alu->no_unsigned_wrap;
 
-   if (dvar->deref.child)
-      ndvar->deref.child = clone_deref(state, dvar->deref.child,
-                                       ninstr, &ndvar->deref);
+   __clone_dst(state, &nalu->instr, &nalu->dest.dest, &alu->dest.dest);
+   nalu->dest.saturate = alu->dest.saturate;
+   nalu->dest.write_mask = alu->dest.write_mask;
+
+   for (unsigned i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
+      __clone_src(state, &nalu->instr, &nalu->src[i].src, &alu->src[i].src);
+      nalu->src[i].negate = alu->src[i].negate;
+      nalu->src[i].abs = alu->src[i].abs;
+      memcpy(nalu->src[i].swizzle, alu->src[i].swizzle,
+             sizeof(nalu->src[i].swizzle));
+   }
 
-   return ndvar;
+   return nalu;
 }
 
-static nir_deref_array *
-clone_deref_array(clone_state *state, const nir_deref_array *darr,
-                  nir_instr *ninstr, nir_deref *parent)
+nir_alu_instr *
+nir_alu_instr_clone(nir_shader *shader, const nir_alu_instr *orig)
 {
-   nir_deref_array *ndarr = nir_deref_array_create(parent);
+   clone_state state = {
+      .allow_remap_fallback = true,
+      .ns = shader,
+   };
+   return clone_alu(&state, orig);
+}
 
-   ndarr->deref.type = darr->deref.type;
-   if (darr->deref.child)
-      ndarr->deref.child = clone_deref(state, darr->deref.child,
-                                       ninstr, &ndarr->deref);
+static nir_deref_instr *
+clone_deref_instr(clone_state *state, const nir_deref_instr *deref)
+{
+   nir_deref_instr *nderef =
+      nir_deref_instr_create(state->ns, deref->deref_type);
 
-   ndarr->deref_array_type = darr->deref_array_type;
-   ndarr->base_offset = darr->base_offset;
-   if (ndarr->deref_array_type == nir_deref_array_type_indirect)
-      __clone_src(state, ninstr, &ndarr->indirect, &darr->indirect);
+   __clone_dst(state, &nderef->instr, &nderef->dest, &deref->dest);
 
-   return ndarr;
-}
+   nderef->mode = deref->mode;
+   nderef->type = deref->type;
 
-static nir_deref_struct *
-clone_deref_struct(clone_state *state, const nir_deref_struct *dstr,
-                   nir_instr *ninstr, nir_deref *parent)
-{
-   nir_deref_struct *ndstr = nir_deref_struct_create(parent, dstr->index);
+   if (deref->deref_type == nir_deref_type_var) {
+      nderef->var = remap_var(state, deref->var);
+      return nderef;
+   }
 
-   ndstr->deref.type = dstr->deref.type;
-   if (dstr->deref.child)
-      ndstr->deref.child = clone_deref(state, dstr->deref.child,
-                                       ninstr, &ndstr->deref);
+   __clone_src(state, &nderef->instr, &nderef->parent, &deref->parent);
 
-   return ndstr;
-}
+   switch (deref->deref_type) {
+   case nir_deref_type_struct:
+      nderef->strct.index = deref->strct.index;
+      break;
 
-static nir_deref *
-clone_deref(clone_state *state, const nir_deref *dref,
-            nir_instr *ninstr, nir_deref *parent)
-{
-   switch (dref->deref_type) {
    case nir_deref_type_array:
-      return &clone_deref_array(state, nir_deref_as_array(dref),
-                                ninstr, parent)->deref;
-   case nir_deref_type_struct:
-      return &clone_deref_struct(state, nir_deref_as_struct(dref),
-                                 ninstr, parent)->deref;
-   default:
-      unreachable("bad deref type");
-      return NULL;
-   }
-}
+   case nir_deref_type_ptr_as_array:
+      __clone_src(state, &nderef->instr,
+                  &nderef->arr.index, &deref->arr.index);
+      break;
 
-static nir_alu_instr *
-clone_alu(clone_state *state, const nir_alu_instr *alu)
-{
-   nir_alu_instr *nalu = nir_alu_instr_create(state->ns, alu->op);
+   case nir_deref_type_array_wildcard:
+      /* Nothing to do */
+      break;
 
-   __clone_dst(state, &nalu->instr, &nalu->dest.dest, &alu->dest.dest);
-   nalu->dest.saturate = alu->dest.saturate;
-   nalu->dest.write_mask = alu->dest.write_mask;
+   case nir_deref_type_cast:
+      nderef->cast.ptr_stride = deref->cast.ptr_stride;
+      break;
 
-   for (unsigned i = 0; i < nir_op_infos[alu->op].num_inputs; i++) {
-      __clone_src(state, &nalu->instr, &nalu->src[i].src, &alu->src[i].src);
-      nalu->src[i].negate = alu->src[i].negate;
-      nalu->src[i].abs = alu->src[i].abs;
-      memcpy(nalu->src[i].swizzle, alu->src[i].swizzle,
-             sizeof(nalu->src[i].swizzle));
+   default:
+      unreachable("Invalid instruction deref type");
    }
 
-   return nalu;
+   return nderef;
 }
 
 static nir_intrinsic_instr *
@@ -326,7 +354,6 @@ clone_intrinsic(clone_state *state, const nir_intrinsic_instr *itr)
    nir_intrinsic_instr *nitr =
       nir_intrinsic_instr_create(state->ns, itr->intrinsic);
 
-   unsigned num_variables = nir_intrinsic_infos[itr->intrinsic].num_variables;
    unsigned num_srcs = nir_intrinsic_infos[itr->intrinsic].num_srcs;
 
    if (nir_intrinsic_infos[itr->intrinsic].has_dest)
@@ -335,11 +362,6 @@ clone_intrinsic(clone_state *state, const nir_intrinsic_instr *itr)
    nitr->num_components = itr->num_components;
    memcpy(nitr->const_index, itr->const_index, sizeof(nitr->const_index));
 
-   for (unsigned i = 0; i < num_variables; i++) {
-      nitr->variables[i] = clone_deref_var(state, itr->variables[i],
-                                           &nitr->instr);
-   }
-
    for (unsigned i = 0; i < num_srcs; i++)
       __clone_src(state, &nitr->instr, &nitr->src[i], &itr->src[i]);
 
@@ -350,9 +372,10 @@ static nir_load_const_instr *
 clone_load_const(clone_state *state, const nir_load_const_instr *lc)
 {
    nir_load_const_instr *nlc =
-      nir_load_const_instr_create(state->ns, lc->def.num_components);
+      nir_load_const_instr_create(state->ns, lc->def.num_components,
+                                  lc->def.bit_size);
 
-   memcpy(&nlc->value, &lc->value, sizeof(nlc->value));
+   memcpy(&nlc->value, &lc->value, sizeof(*nlc->value) * lc->def.num_components);
 
    add_remap(state, &nlc->def, &lc->def);
 
@@ -363,7 +386,8 @@ static nir_ssa_undef_instr *
 clone_ssa_undef(clone_state *state, const nir_ssa_undef_instr *sa)
 {
    nir_ssa_undef_instr *nsa =
-      nir_ssa_undef_instr_create(state->ns, sa->def.num_components);
+      nir_ssa_undef_instr_create(state->ns, sa->def.num_components,
+                                 sa->def.bit_size);
 
    add_remap(state, &nsa->def, &sa->def);
 
@@ -388,15 +412,13 @@ clone_tex(clone_state *state, const nir_tex_instr *tex)
    ntex->is_shadow = tex->is_shadow;
    ntex->is_new_style_shadow = tex->is_new_style_shadow;
    ntex->component = tex->component;
+   memcpy(ntex->tg4_offsets, tex->tg4_offsets, sizeof(tex->tg4_offsets));
 
    ntex->texture_index = tex->texture_index;
-   if (tex->texture)
-      ntex->texture = clone_deref_var(state, tex->texture, &ntex->instr);
-   ntex->texture_array_size = tex->texture_array_size;
-
    ntex->sampler_index = tex->sampler_index;
-   if (tex->sampler)
-      ntex->sampler = clone_deref_var(state, tex->sampler, &ntex->instr);
+
+   ntex->texture_non_uniform = tex->texture_non_uniform;
+   ntex->sampler_non_uniform = tex->sampler_non_uniform;
 
    return ntex;
 }
@@ -460,10 +482,7 @@ clone_call(clone_state *state, const nir_call_instr *call)
    nir_call_instr *ncall = nir_call_instr_create(state->ns, ncallee);
 
    for (unsigned i = 0; i < ncall->num_params; i++)
-      ncall->params[i] = clone_deref_var(state, call->params[i], &ncall->instr);
-
-   ncall->return_deref = clone_deref_var(state, call->return_deref,
-                                         &ncall->instr);
+      __clone_src(state, ncall, &ncall->params[i], &call->params[i]);
 
    return ncall;
 }
@@ -474,6 +493,8 @@ clone_instr(clone_state *state, const nir_instr *instr)
    switch (instr->type) {
    case nir_instr_type_alu:
       return &clone_alu(state, nir_instr_as_alu(instr))->instr;
+   case nir_instr_type_deref:
+      return &clone_deref_instr(state, nir_instr_as_deref(instr))->instr;
    case nir_instr_type_intrinsic:
       return &clone_intrinsic(state, nir_instr_as_intrinsic(instr))->instr;
    case nir_instr_type_load_const:
@@ -511,7 +532,7 @@ clone_block(clone_state *state, struct exec_list *cf_list, const nir_block *blk)
    /* We need this for phi sources */
    add_remap(state, nblk, blk);
 
-   nir_foreach_instr(blk, instr) {
+   nir_foreach_instr(instr, blk) {
       if (instr->type == nir_instr_type_phi) {
          /* Phi instructions are a bit of a special case when cloning because
           * we don't want inserting the instruction to automatically handle
@@ -536,6 +557,7 @@ static nir_if *
 clone_if(clone_state *state, struct exec_list *cf_list, const nir_if *i)
 {
    nir_if *ni = nir_if_create(state->ns);
+   ni->control = i->control;
 
    __clone_src(state, ni, &ni->condition, &i->condition);
 
@@ -551,6 +573,8 @@ static nir_loop *
 clone_loop(clone_state *state, struct exec_list *cf_list, const nir_loop *loop)
 {
    nir_loop *nloop = nir_loop_create(state->ns);
+   nloop->control = loop->control;
+   nloop->partially_unrolled = loop->partially_unrolled;
 
    nir_cf_node_insert_end(cf_list, &nloop->cf_node);
 
@@ -581,6 +605,59 @@ clone_cf_list(clone_state *state, struct exec_list *dst,
    }
 }
 
+/* After we've cloned almost everything, we have to walk the list of phi
+ * sources and fix them up.  Thanks to loops, the block and SSA value for a
+ * phi source may not be defined when we first encounter it.  Instead, we
+ * add it to the phi_srcs list and we fix it up here.
+ */
+static void
+fixup_phi_srcs(clone_state *state)
+{
+   list_for_each_entry_safe(nir_phi_src, src, &state->phi_srcs, src.use_link) {
+      src->pred = remap_local(state, src->pred);
+
+      /* Remove from this list */
+      list_del(&src->src.use_link);
+
+      if (src->src.is_ssa) {
+         src->src.ssa = remap_local(state, src->src.ssa);
+         list_addtail(&src->src.use_link, &src->src.ssa->uses);
+      } else {
+         src->src.reg.reg = remap_reg(state, src->src.reg.reg);
+         list_addtail(&src->src.use_link, &src->src.reg.reg->uses);
+      }
+   }
+   assert(list_is_empty(&state->phi_srcs));
+}
+
+void
+nir_cf_list_clone(nir_cf_list *dst, nir_cf_list *src, nir_cf_node *parent,
+                  struct hash_table *remap_table)
+{
+   exec_list_make_empty(&dst->list);
+   dst->impl = src->impl;
+
+   if (exec_list_is_empty(&src->list))
+      return;
+
+   clone_state state;
+   init_clone_state(&state, remap_table, false, true);
+
+   /* We use the same shader */
+   state.ns = src->impl->function->shader;
+
+   /* The control-flow code assumes that the list of cf_nodes always starts
+    * and ends with a block.  We start by adding an empty block.
+    */
+   nir_block *nblk = nir_block_create(state.ns);
+   nblk->cf_node.parent = parent;
+   exec_list_push_tail(&dst->list, &nblk->cf_node.node);
+
+   clone_cf_list(&state, &dst->list, &src->list);
+
+   fixup_phi_srcs(&state);
+}
+
 static nir_function_impl *
 clone_function_impl(clone_state *state, const nir_function_impl *fi)
 {
@@ -590,33 +667,11 @@ clone_function_impl(clone_state *state, const nir_function_impl *fi)
    clone_reg_list(state, &nfi->registers, &fi->registers);
    nfi->reg_alloc = fi->reg_alloc;
 
-   nfi->num_params = fi->num_params;
-   nfi->params = ralloc_array(state->ns, nir_variable *, fi->num_params);
-   for (unsigned i = 0; i < fi->num_params; i++) {
-      nfi->params[i] = clone_variable(state, fi->params[i]);
-   }
-   if (fi->return_var)
-      nfi->return_var = clone_variable(state, fi->return_var);
-
-   assert(list_empty(&state->phi_srcs));
+   assert(list_is_empty(&state->phi_srcs));
 
    clone_cf_list(state, &nfi->body, &fi->body);
 
-   /* After we've cloned almost everything, we have to walk the list of phi
-    * sources and fix them up.  Thanks to loops, the block and SSA value for a
-    * phi source may not be defined when we first encounter it.  Instead, we
-    * add it to the phi_srcs list and we fix it up here.
-    */
-   list_for_each_entry_safe(nir_phi_src, src, &state->phi_srcs, src.use_link) {
-      src->pred = remap_local(state, src->pred);
-      assert(src->src.is_ssa);
-      src->src.ssa = remap_local(state, src->src.ssa);
-
-      /* Remove from this list and place in the uses of the SSA def */
-      list_del(&src->src.use_link);
-      list_addtail(&src->src.use_link, &src->src.ssa->uses);
-   }
-   assert(list_empty(&state->phi_srcs));
+   fixup_phi_srcs(state);
 
    /* All metadata is invalidated in the cloning process */
    nfi->valid_metadata = 0;
@@ -625,13 +680,12 @@ clone_function_impl(clone_state *state, const nir_function_impl *fi)
 }
 
 nir_function_impl *
-nir_function_impl_clone(const nir_function_impl *fi)
+nir_function_impl_clone(nir_shader *shader, const nir_function_impl *fi)
 {
    clone_state state;
-   init_clone_state(&state, false);
+   init_clone_state(&state, NULL, false, false);
 
-   /* We use the same shader */
-   state.ns = fi->function->shader;
+   state.ns = shader;
 
    nir_function_impl *nfi = clone_function_impl(&state, fi);
 
@@ -650,14 +704,15 @@ clone_function(clone_state *state, const nir_function *fxn, nir_shader *ns)
    add_remap(state, nfxn, fxn);
 
    nfxn->num_params = fxn->num_params;
-   nfxn->params = ralloc_array(state->ns, nir_parameter, fxn->num_params);
-   memcpy(nfxn->params, fxn->params, sizeof(nir_parameter) * fxn->num_params);
-
-   nfxn->return_type = fxn->return_type;
+   if (fxn->num_params) {
+           nfxn->params = ralloc_array(state->ns, nir_parameter, fxn->num_params);
+           memcpy(nfxn->params, fxn->params, sizeof(nir_parameter) * fxn->num_params);
+   }
+   nfxn->is_entrypoint = fxn->is_entrypoint;
 
    /* At first glance, it looks like we should clone the function_impl here.
     * However, call instructions need to be able to reference at least the
-    * function and those will get processed as we clone the function_impl's.
+    * function and those will get processed as we clone the function_impls.
     * We stop here and do function_impls as a second pass.
     */
 
@@ -668,36 +723,28 @@ nir_shader *
 nir_shader_clone(void *mem_ctx, const nir_shader *s)
 {
    clone_state state;
-   init_clone_state(&state, true);
+   init_clone_state(&state, NULL, true, false);
 
-   nir_shader *ns = nir_shader_create(mem_ctx, s->stage, s->options);
+   nir_shader *ns = nir_shader_create(mem_ctx, s->info.stage, s->options, NULL);
    state.ns = ns;
 
-   clone_var_list(&state, &ns->uniforms, &s->uniforms);
-   clone_var_list(&state, &ns->inputs,   &s->inputs);
-   clone_var_list(&state, &ns->outputs,  &s->outputs);
-   clone_var_list(&state, &ns->shared,   &s->shared);
-   clone_var_list(&state, &ns->globals,  &s->globals);
-   clone_var_list(&state, &ns->system_values, &s->system_values);
+   clone_var_list(&state, &ns->variables, &s->variables);
 
    /* Go through and clone functions */
    foreach_list_typed(nir_function, fxn, node, &s->functions)
       clone_function(&state, fxn, ns);
 
    /* Only after all functions are cloned can we clone the actual function
-    * implementations.  This is because nir_call_instr's need to reference the
+    * implementations.  This is because nir_call_instrs need to reference the
     * functions of other functions and we don't know what order the functions
     * will have in the list.
     */
-   nir_foreach_function(s, fxn) {
+   nir_foreach_function(fxn, s) {
       nir_function *nfxn = remap_global(&state, fxn);
       nfxn->impl = clone_function_impl(&state, fxn->impl);
       nfxn->impl->function = nfxn;
    }
 
-   clone_reg_list(&state, &ns->registers, &s->registers);
-   ns->reg_alloc = s->reg_alloc;
-
    ns->info = s->info;
    ns->info.name = ralloc_strdup(ns, ns->info.name);
    if (ns->info.label)
@@ -707,8 +754,49 @@ nir_shader_clone(void *mem_ctx, const nir_shader *s)
    ns->num_uniforms = s->num_uniforms;
    ns->num_outputs = s->num_outputs;
    ns->num_shared = s->num_shared;
+   ns->scratch_size = s->scratch_size;
+
+   ns->constant_data_size = s->constant_data_size;
+   if (s->constant_data_size > 0) {
+      ns->constant_data = ralloc_size(ns, s->constant_data_size);
+      memcpy(ns->constant_data, s->constant_data, s->constant_data_size);
+   }
 
    free_clone_state(&state);
 
    return ns;
 }
+
+/** Overwrites dst and replaces its contents with src
+ *
+ * Everything ralloc parented to dst and src itself (but not its children)
+ * will be freed.
+ *
+ * This should only be used by test code which needs to swap out shaders with
+ * a cloned or deserialized version.
+ */
+void
+nir_shader_replace(nir_shader *dst, nir_shader *src)
+{
+   /* Delete all of dest's ralloc children */
+   void *dead_ctx = ralloc_context(NULL);
+   ralloc_adopt(dead_ctx, dst);
+   ralloc_free(dead_ctx);
+
+   /* Re-parent all of src's ralloc children to dst */
+   ralloc_adopt(dst, src);
+
+   memcpy(dst, src, sizeof(*dst));
+
+   /* We have to move all the linked lists over separately because we need the
+    * pointers in the list elements to point to the lists in dst and not src.
+    */
+   exec_list_move_nodes_to(&src->variables, &dst->variables);
+
+   /* Now move the functions over.  This takes a tiny bit more work */
+   exec_list_move_nodes_to(&src->functions, &dst->functions);
+   nir_foreach_function(function, dst)
+      function->shader = dst;
+
+   ralloc_free(src);
+}