nir/nir: Use a linked list instead of a hash set for use/def sets

This commit switches us from the current setup of using hash sets for
use/def sets to using linked lists.  Doing so should save us quite a bit of
memory because we aren't carrying around 3 hash sets per register and 2 per
SSA value.  It should also save us CPU time because adding/removing things
from use/def sets is 4 pointer manipulations instead of a hash lookup.

Running shader-db 50 times with USE_NIR=0, NIR, and NIR + use/def lists:

   GLSL IR Only:        586.4 +/- 1.653833
   NIR with hash sets:  675.4 +/- 2.502108
   NIR + use/def lists: 641.2 +/- 1.557043

I also ran a memory usage experiment with Ken's patch to delete GLSL IR and
keep NIR.  This patch cuts an aditional 42.9 MiB of ralloc'd memory over
and above what we gained by deleting the GLSL IR on the same dota trace.

On the code complexity side of things, some things are now much easier and
others are a bit harder.  One of the operations we perform constantly in
optimization passes is to replace one source with another.  Due to the fact
that an instruction can use the same SSA value multiple times, we had to
iterate through the sources of the instruction and determine if the use we
were replacing was the only one before removing it from the set of uses.
With this patch, uses are per-source not per-instruction so we can just
remove it safely.  On the other hand, trying to iterate over all of the
instructions that use a given value is more difficult.  Fortunately, the
two places we do that are the ffma peephole where it doesn't matter and GCM
where we already gracefully handle duplicates visits to an instruction.

Another aspect here is that using linked lists in this way can be tricky to
get right.  With sets, things were quite forgiving and the worst that
happened if you didn't properly remove a use was that it would get caught
in the validator.  With linked lists, it can lead to linked list corruption
which can be harder to track.  However, we do just as much validation of
the linked lists as we did of the sets so the validator should still catch
these problems.  While working on this series, the vast majority of the
bugs I had to fix were caught by assertions.  I don't think the lists are
going to be that much worse than the sets.

Reviewed-by: Connor Abbott <cwabbott0@gmail.com>

diff --git a/src/glsl/nir/nir.c b/src/glsl/nir/nir.c
index b8f5dd491a4eaed3071645a7154f594aa57a7c98..f03e80a4e0ea3e098cddd0723dd9fb19c8a59408 100644 (file)
--- a/src/glsl/nir/nir.c
+++ b/src/glsl/nir/nir.c
@@ -58,12 +58,9 @@ reg_create(void *mem_ctx, struct exec_list *list)
    nir_register *reg = ralloc(mem_ctx, nir_register);
 
    reg->parent_instr = NULL;
-   reg->uses = _mesa_set_create(reg, _mesa_hash_pointer,
-                                _mesa_key_pointer_equal);
-   reg->defs = _mesa_set_create(reg, _mesa_hash_pointer,
-                                _mesa_key_pointer_equal);
-   reg->if_uses = _mesa_set_create(reg, _mesa_hash_pointer,
-                                   _mesa_key_pointer_equal);
+   list_inithead(&reg->uses);
+   list_inithead(&reg->defs);
+   list_inithead(&reg->if_uses);
 
    reg->num_components = 0;
    reg->num_array_elems = 0;
@@ -1070,11 +1067,14 @@ update_if_uses(nir_cf_node *node)
 
    nir_if *if_stmt = nir_cf_node_as_if(node);
 
-   struct set *if_uses_set = if_stmt->condition.is_ssa ?
-                             if_stmt->condition.ssa->if_uses :
-                             if_stmt->condition.reg.reg->uses;
-
-   _mesa_set_add(if_uses_set, if_stmt);
+   if_stmt->condition.parent_if = if_stmt;
+   if (if_stmt->condition.is_ssa) {
+      list_addtail(&if_stmt->condition.use_link,
+                   &if_stmt->condition.ssa->if_uses);
+   } else {
+      list_addtail(&if_stmt->condition.use_link,
+                   &if_stmt->condition.reg.reg->if_uses);
+   }
 }
 
 void
@@ -1227,16 +1227,7 @@ cleanup_cf_node(nir_cf_node *node)
       foreach_list_typed(nir_cf_node, child, node, &if_stmt->else_list)
          cleanup_cf_node(child);
 
-      struct set *if_uses;
-      if (if_stmt->condition.is_ssa) {
-         if_uses = if_stmt->condition.ssa->if_uses;
-      } else {
-         if_uses = if_stmt->condition.reg.reg->if_uses;
-      }
-
-      struct set_entry *entry = _mesa_set_search(if_uses, if_stmt);
-      assert(entry);
-      _mesa_set_remove(if_uses, entry);
+      list_del(&if_stmt->condition.use_link);
       break;
    }
 
@@ -1293,9 +1284,9 @@ add_use_cb(nir_src *src, void *state)
 {
    nir_instr *instr = state;
 
-   struct set *uses_set = src->is_ssa ? src->ssa->uses : src->reg.reg->uses;
-
-   _mesa_set_add(uses_set, instr);
+   src->parent_instr = instr;
+   list_addtail(&src->use_link,
+                src->is_ssa ? &src->ssa->uses : &src->reg.reg->uses);
 
    return true;
 }
@@ -1320,8 +1311,10 @@ add_reg_def_cb(nir_dest *dest, void *state)
 {
    nir_instr *instr = state;
 
-   if (!dest->is_ssa)
-      _mesa_set_add(dest->reg.reg->defs, instr);
+   if (!dest->is_ssa) {
+      dest->reg.parent_instr = instr;
+      list_addtail(&dest->reg.def_link, &dest->reg.reg->defs);
+   }
 
    return true;
 }
@@ -1436,13 +1429,7 @@ nir_instr_insert_after_cf_list(struct exec_list *list, nir_instr *after)
 static bool
 remove_use_cb(nir_src *src, void *state)
 {
-   nir_instr *instr = state;
-
-   struct set *uses_set = src->is_ssa ? src->ssa->uses : src->reg.reg->uses;
-
-   struct set_entry *entry = _mesa_set_search(uses_set, instr);
-   if (entry)
-      _mesa_set_remove(uses_set, entry);
+   list_del(&src->use_link);
 
    return true;
 }
@@ -1450,16 +1437,8 @@ remove_use_cb(nir_src *src, void *state)
 static bool
 remove_def_cb(nir_dest *dest, void *state)
 {
-   nir_instr *instr = state;
-
-   if (dest->is_ssa)
-      return true;
-
-   nir_register *reg = dest->reg.reg;
-
-   struct set_entry *entry = _mesa_set_search(reg->defs, instr);
-   if (entry)
-      _mesa_set_remove(reg->defs, entry);
+   if (!dest->is_ssa)
+      list_del(&dest->reg.def_link);
 
    return true;
 }
@@ -1834,86 +1813,77 @@ nir_srcs_equal(nir_src src1, nir_src src2)
 }
 
 static bool
-src_does_not_use_def(nir_src *src, void *void_def)
+src_is_valid(const nir_src *src)
 {
-   nir_ssa_def *def = void_def;
-
-   if (src->is_ssa) {
-      return src->ssa != def;
-   } else {
-      return true;
-   }
+   return src->is_ssa ? (src->ssa != NULL) : (src->reg.reg != NULL);
 }
 
-static bool
-src_does_not_use_reg(nir_src *src, void *void_reg)
+static void
+src_remove_all_uses(nir_src *src)
 {
-   nir_register *reg = void_reg;
+   for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
+      if (!src_is_valid(src))
+         continue;
 
-   if (src->is_ssa) {
-      return true;
-   } else {
-      return src->reg.reg != reg;
+      list_del(&src->use_link);
+   }
+}
+
+static void
+src_add_all_uses(nir_src *src, nir_instr *parent_instr, nir_if *parent_if)
+{
+   for (; src; src = src->is_ssa ? NULL : src->reg.indirect) {
+      if (!src_is_valid(src))
+         continue;
+
+      if (parent_instr) {
+         src->parent_instr = parent_instr;
+         if (src->is_ssa)
+            list_addtail(&src->use_link, &src->ssa->uses);
+         else
+            list_addtail(&src->use_link, &src->reg.reg->uses);
+      } else {
+         assert(parent_if);
+         src->parent_if = parent_if;
+         if (src->is_ssa)
+            list_addtail(&src->use_link, &src->ssa->if_uses);
+         else
+            list_addtail(&src->use_link, &src->reg.reg->if_uses);
+      }
    }
 }
 
 void
 nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src)
 {
-   nir_src old_src = *src;
+   assert(!src_is_valid(src) || src->parent_instr == instr);
+
+   src_remove_all_uses(src);
    *src = new_src;
+   src_add_all_uses(src, instr, NULL);
+}
 
-   for (nir_src *iter_src = &old_src; iter_src;
-        iter_src = iter_src->is_ssa ? NULL : iter_src->reg.indirect) {
-      if (iter_src->is_ssa) {
-         nir_ssa_def *ssa = iter_src->ssa;
-         if (ssa && nir_foreach_src(instr, src_does_not_use_def, ssa)) {
-            struct set_entry *entry = _mesa_set_search(ssa->uses, instr);
-            assert(entry);
-            _mesa_set_remove(ssa->uses, entry);
-         }
-      } else {
-         nir_register *reg = iter_src->reg.reg;
-         if (reg && nir_foreach_src(instr, src_does_not_use_reg, reg)) {
-            struct set_entry *entry = _mesa_set_search(reg->uses, instr);
-            assert(entry);
-            _mesa_set_remove(reg->uses, entry);
-         }
-      }
-   }
+void
+nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src)
+{
+   assert(!src_is_valid(dest) || dest->parent_instr == dest_instr);
 
-   for (nir_src *iter_src = &new_src; iter_src;
-        iter_src = iter_src->is_ssa ? NULL : iter_src->reg.indirect) {
-      if (iter_src->is_ssa) {
-         if (iter_src->ssa)
-            _mesa_set_add(iter_src->ssa->uses, instr);
-      } else {
-         if (iter_src->reg.reg)
-            _mesa_set_add(iter_src->reg.reg->uses, instr);
-      }
-   }
+   src_remove_all_uses(dest);
+   src_remove_all_uses(src);
+   *dest = *src;
+   *src = NIR_SRC_INIT;
+   src_add_all_uses(dest, dest_instr, NULL);
 }
 
 void
 nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src)
 {
-   for (nir_src *src = &if_stmt->condition; src;
-        src = src->is_ssa ? NULL : src->reg.indirect) {
-      struct set *uses = src->is_ssa ? src->ssa->if_uses
-                                     : src->reg.reg->if_uses;
-      struct set_entry *entry = _mesa_set_search(uses, if_stmt);
-      assert(entry);
-      _mesa_set_remove(uses, entry);
-   }
-
-   if_stmt->condition = new_src;
+   nir_src *src = &if_stmt->condition;
+   assert(!src_is_valid(src) || src->parent_if == if_stmt);
 
-   for (nir_src *src = &if_stmt->condition; src;
-        src = src->is_ssa ? NULL : src->reg.indirect) {
-      struct set *uses = src->is_ssa ? src->ssa->if_uses
-                                     : src->reg.reg->if_uses;
-      _mesa_set_add(uses, if_stmt);
-   }
+   src_remove_all_uses(src);
+   *src = new_src;
+   src_add_all_uses(src, NULL, if_stmt);
 }
 
 void
@@ -1922,10 +1892,8 @@ nir_ssa_def_init(nir_instr *instr, nir_ssa_def *def,
 {
    def->name = name;
    def->parent_instr = instr;
-   def->uses = _mesa_set_create(instr, _mesa_hash_pointer,
-                                _mesa_key_pointer_equal);
-   def->if_uses = _mesa_set_create(instr, _mesa_hash_pointer,
-                                   _mesa_key_pointer_equal);
+   list_inithead(&def->uses);
+   list_inithead(&def->if_uses);
    def->num_components = num_components;
 
    if (instr->block) {
@@ -1946,57 +1914,23 @@ nir_ssa_dest_init(nir_instr *instr, nir_dest *dest,
    nir_ssa_def_init(instr, &dest->ssa, num_components, name);
 }
 
-struct ssa_def_rewrite_state {
-   void *mem_ctx;
-   nir_ssa_def *old;
-   nir_src new_src;
-};
-
-static bool
-ssa_def_rewrite_uses_src(nir_src *src, void *void_state)
-{
-   struct ssa_def_rewrite_state *state = void_state;
-
-   if (src->is_ssa && src->ssa == state->old)
-      nir_src_copy(src, &state->new_src, state->mem_ctx);
-
-   return true;
-}
-
 void
 nir_ssa_def_rewrite_uses(nir_ssa_def *def, nir_src new_src, void *mem_ctx)
 {
-   struct ssa_def_rewrite_state state;
-   state.mem_ctx = mem_ctx;
-   state.old = def;
-   state.new_src = new_src;
-
    assert(!new_src.is_ssa || def != new_src.ssa);
 
-   struct set *new_uses, *new_if_uses;
-   if (new_src.is_ssa) {
-      new_uses = new_src.ssa->uses;
-      new_if_uses = new_src.ssa->if_uses;
-   } else {
-      new_uses = new_src.reg.reg->uses;
-      new_if_uses = new_src.reg.reg->if_uses;
-   }
-
-   struct set_entry *entry;
-   set_foreach(def->uses, entry) {
-      nir_instr *instr = (nir_instr *)entry->key;
-
-      _mesa_set_remove(def->uses, entry);
-      nir_foreach_src(instr, ssa_def_rewrite_uses_src, &state);
-      _mesa_set_add(new_uses, instr);
+   nir_foreach_use_safe(def, use_src) {
+      nir_instr *src_parent_instr = use_src->parent_instr;
+      list_del(&use_src->use_link);
+      nir_src_copy(use_src, &new_src, mem_ctx);
+      src_add_all_uses(use_src, src_parent_instr, NULL);
    }
 
-   set_foreach(def->if_uses, entry) {
-      nir_if *if_use = (nir_if *)entry->key;
-
-      _mesa_set_remove(def->if_uses, entry);
-      nir_src_copy(&if_use->condition, &new_src, mem_ctx);
-      _mesa_set_add(new_if_uses, if_use);
+   nir_foreach_if_use_safe(def, use_src) {
+      nir_if *src_parent_if = use_src->parent_if;
+      list_del(&use_src->use_link);
+      nir_src_copy(use_src, &new_src, mem_ctx);
+      src_add_all_uses(use_src, NULL, src_parent_if);
    }
 }
 

diff --git a/src/glsl/nir/nir.h b/src/glsl/nir/nir.h
index aaf1c572ebadf7218edcf858912e21e89d50baf9..697d37e95acd14131abe9fea87c636aa593aed72 100644 (file)
--- a/src/glsl/nir/nir.h
+++ b/src/glsl/nir/nir.h
@@ -30,6 +30,7 @@
 #include "util/hash_table.h"
 #include "../list.h"
 #include "GL/gl.h" /* GLenum */
+#include "util/list.h"
 #include "util/ralloc.h"
 #include "util/set.h"
 #include "util/bitset.h"
@@ -397,13 +398,13 @@ typedef struct {
    struct nir_instr *parent_instr;
 
    /** set of nir_instr's where this register is used (read from) */
-   struct set *uses;
+   struct list_head uses;
 
    /** set of nir_instr's where this register is defined (written to) */
-   struct set *defs;
+   struct list_head defs;
 
    /** set of nir_if's where this register is used as a condition */
-   struct set *if_uses;
+   struct list_head if_uses;
 } nir_register;
 
 typedef enum {
@@ -462,10 +463,10 @@ typedef struct {
    nir_instr *parent_instr;
 
    /** set of nir_instr's where this register is used (read from) */
-   struct set *uses;
+   struct list_head uses;
 
    /** set of nir_if's where this register is used as a condition */
-   struct set *if_uses;
+   struct list_head if_uses;
 
    uint8_t num_components;
 } nir_ssa_def;
@@ -481,6 +482,9 @@ typedef struct {
 } nir_reg_src;
 
 typedef struct {
+   nir_instr *parent_instr;
+   struct list_head def_link;
+
    nir_register *reg;
    struct nir_src *indirect; /** < NULL for no indirect offset */
    unsigned base_offset;
@@ -488,7 +492,16 @@ typedef struct {
    /* TODO def-use chain goes here */
 } nir_reg_dest;
 
+struct nir_if;
+
 typedef struct nir_src {
+   union {
+      nir_instr *parent_instr;
+      struct nir_if *parent_if;
+   };
+
+   struct list_head use_link;
+
    union {
       nir_reg_src reg;
       nir_ssa_def *ssa;
@@ -497,7 +510,19 @@ typedef struct nir_src {
    bool is_ssa;
 } nir_src;
 
-#define NIR_SRC_INIT (nir_src) { { { NULL } } }
+#define NIR_SRC_INIT (nir_src) { { NULL } }
+
+#define nir_foreach_use(reg_or_ssa_def, src) \
+   list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->uses, use_link)
+
+#define nir_foreach_use_safe(reg_or_ssa_def, src) \
+   list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->uses, use_link)
+
+#define nir_foreach_if_use(reg_or_ssa_def, src) \
+   list_for_each_entry(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)
+
+#define nir_foreach_if_use_safe(reg_or_ssa_def, src) \
+   list_for_each_entry_safe(nir_src, src, &(reg_or_ssa_def)->if_uses, use_link)
 
 typedef struct {
    union {
@@ -510,6 +535,12 @@ typedef struct {
 
 #define NIR_DEST_INIT (nir_dest) { { { NULL } } }
 
+#define nir_foreach_def(reg, dest) \
+   list_for_each_entry(nir_dest, dest, &(reg)->defs, reg.def_link)
+
+#define nir_foreach_def_safe(reg, dest) \
+   list_for_each_entry_safe(nir_dest, dest, &(reg)->defs, reg.def_link)
+
 static inline nir_src
 nir_src_for_ssa(nir_ssa_def *def)
 {
@@ -1208,7 +1239,7 @@ nir_block_last_instr(nir_block *block)
 #define nir_foreach_instr_safe(block, instr) \
    foreach_list_typed_safe(nir_instr, instr, node, &(block)->instr_list)
 
-typedef struct {
+typedef struct nir_if {
    nir_cf_node cf_node;
    nir_src condition;
 
@@ -1549,6 +1580,7 @@ bool nir_foreach_src(nir_instr *instr, nir_foreach_src_cb cb, void *state);
 nir_const_value *nir_src_as_const_value(nir_src src);
 bool nir_srcs_equal(nir_src src1, nir_src src2);
 void nir_instr_rewrite_src(nir_instr *instr, nir_src *src, nir_src new_src);
+void nir_instr_move_src(nir_instr *dest_instr, nir_src *dest, nir_src *src);
 void nir_if_rewrite_condition(nir_if *if_stmt, nir_src new_src);
 
 void nir_ssa_dest_init(nir_instr *instr, nir_dest *dest,

diff --git a/src/glsl/nir/nir_from_ssa.c b/src/glsl/nir/nir_from_ssa.c
index 512fdacc53a5145f550c73bec3fe2d6182155fe3..d1bc109182139edef85ae7923e27ad8d523e7649 100644 (file)
--- a/src/glsl/nir/nir_from_ssa.c
+++ b/src/glsl/nir/nir_from_ssa.c
@@ -503,7 +503,7 @@ rewrite_ssa_def(nir_ssa_def *def, void *void_state)
    }
 
    nir_ssa_def_rewrite_uses(def, nir_src_for_reg(reg), state->mem_ctx);
-   assert(def->uses->entries == 0 && def->if_uses->entries == 0);
+   assert(list_empty(&def->uses) && list_empty(&def->if_uses));
 
    if (def->parent_instr->type == nir_instr_type_ssa_undef)
       return true;
@@ -515,12 +515,9 @@ rewrite_ssa_def(nir_ssa_def *def, void *void_state)
     */
    nir_dest *dest = exec_node_data(nir_dest, def, ssa);
 
-   _mesa_set_destroy(dest->ssa.uses, NULL);
-   _mesa_set_destroy(dest->ssa.if_uses, NULL);
-
    *dest = nir_dest_for_reg(reg);
-
-   _mesa_set_add(reg->defs, state->instr);
+   dest->reg.parent_instr = state->instr;
+   list_addtail(&dest->reg.def_link, &reg->defs);
 
    return true;
 }

diff --git a/src/glsl/nir/nir_lower_locals_to_regs.c b/src/glsl/nir/nir_lower_locals_to_regs.c
index bc6a3d320c4a95727e9bcfac41213e9169cf28a6..28fdec50e048b063b86088bf508dd0ee60e9ede5 100644 (file)
--- a/src/glsl/nir/nir_lower_locals_to_regs.c
+++ b/src/glsl/nir/nir_lower_locals_to_regs.c
@@ -269,18 +269,16 @@ lower_locals_to_regs_block(nir_block *block, void *void_state)
 static nir_block *
 compute_reg_usedef_lca(nir_register *reg)
 {
-   struct set_entry *entry;
    nir_block *lca = NULL;
 
-   set_foreach(reg->defs, entry)
-      lca = nir_dominance_lca(lca, ((nir_instr *)entry->key)->block);
+   list_for_each_entry(nir_dest, def_dest, &reg->defs, reg.def_link)
+      lca = nir_dominance_lca(lca, def_dest->reg.parent_instr->block);
 
-   set_foreach(reg->uses, entry)
-      lca = nir_dominance_lca(lca, ((nir_instr *)entry->key)->block);
+   list_for_each_entry(nir_src, use_src, &reg->uses, use_link)
+      lca = nir_dominance_lca(lca, use_src->parent_instr->block);
 
-   set_foreach(reg->if_uses, entry) {
-      nir_if *if_stmt = (nir_if *)entry->key;
-      nir_cf_node *prev_node = nir_cf_node_prev(&if_stmt->cf_node);
+   list_for_each_entry(nir_src, use_src, &reg->if_uses, use_link) {
+      nir_cf_node *prev_node = nir_cf_node_prev(&use_src->parent_if->cf_node);
       assert(prev_node->type == nir_cf_node_block);
       lca = nir_dominance_lca(lca, nir_cf_node_as_block(prev_node));
    }

diff --git a/src/glsl/nir/nir_lower_samplers.cpp b/src/glsl/nir/nir_lower_samplers.cpp
index e9e152f6cbf82878a8cfa41589c65b507789c478..8fc5909b711f0872eaa4703e79d6216552604735 100644 (file)
--- a/src/glsl/nir/nir_lower_samplers.cpp
+++ b/src/glsl/nir/nir_lower_samplers.cpp
@@ -83,21 +83,30 @@ lower_sampler(nir_tex_instr *instr, const struct gl_shader_program *shader_progr
                ralloc_asprintf_append(&name, "[%u]", deref_array->base_offset);
             break;
          case nir_deref_array_type_indirect: {
-            instr->src = reralloc(instr, instr->src, nir_tex_src,
-                                  instr->num_srcs + 1);
-            memset(&instr->src[instr->num_srcs], 0, sizeof *instr->src);
+            /* First, we have to resize the array of texture sources */
+            nir_tex_src *new_srcs = rzalloc_array(instr, nir_tex_src,
+                                                  instr->num_srcs + 1);
+
+            for (unsigned i = 0; i < instr->num_srcs; i++) {
+               new_srcs[i].src_type = instr->src[i].src_type;
+               nir_instr_move_src(&instr->instr, &new_srcs[i].src,
+                                  &instr->src[i].src);
+            }
+
+            ralloc_free(instr->src);
+            instr->src = new_srcs;
+
+            /* Now we can go ahead and move the source over to being a
+             * first-class texture source.
+             */
             instr->src[instr->num_srcs].src_type = nir_tex_src_sampler_offset;
             instr->num_srcs++;
-
-            nir_instr_rewrite_src(&instr->instr,
-                                  &instr->src[instr->num_srcs - 1].src,
-                                  deref_array->indirect);
+            nir_instr_move_src(&instr->instr,
+                               &instr->src[instr->num_srcs - 1].src,
+                               &deref_array->indirect);
 
             instr->sampler_array_size = glsl_get_length(deref->type);
 
-            nir_instr_rewrite_src(&instr->instr, &deref_array->indirect,
-                                  NIR_SRC_INIT);
-
             if (deref_array->deref.child)
                ralloc_strcat(&name, "[0]");
             break;

diff --git a/src/glsl/nir/nir_lower_tex_projector.c b/src/glsl/nir/nir_lower_tex_projector.c
index 6e2cc803fb8a0f46585439615211a939e3be1674..357131cd7280c609229a3a4280b40aeeb187966f 100644 (file)
--- a/src/glsl/nir/nir_lower_tex_projector.c
+++ b/src/glsl/nir/nir_lower_tex_projector.c
@@ -111,9 +111,10 @@ nir_lower_tex_projector_block(nir_block *block, void *void_state)
        */
       nir_instr_rewrite_src(&tex->instr, &tex->src[proj_index].src,
                             NIR_SRC_INIT);
-      memmove(&tex->src[proj_index],
-              &tex->src[proj_index + 1],
-              (tex->num_srcs - proj_index) * sizeof(*tex->src));
+      for (int i = proj_index + 1; i < tex->num_srcs; i++) {
+         tex->src[i-1].src_type = tex->src[i].src_type;
+         nir_instr_move_src(&tex->instr, &tex->src[i-1].src, &tex->src[i].src);
+      }
       tex->num_srcs--;
    }
 

diff --git a/src/glsl/nir/nir_lower_to_source_mods.c b/src/glsl/nir/nir_lower_to_source_mods.c
index 7b4a0f657380c982eed99dfee32f336188e191fd..94c7e36d4d803f22fc84000b67c1f62266ed67f9 100644 (file)
--- a/src/glsl/nir/nir_lower_to_source_mods.c
+++ b/src/glsl/nir/nir_lower_to_source_mods.c
@@ -88,8 +88,8 @@ nir_lower_to_source_mods_block(nir_block *block, void *state)
             alu->src[i].swizzle[j] = parent->src[0].swizzle[alu->src[i].swizzle[j]];
          }
 
-         if (parent->dest.dest.ssa.uses->entries == 0 &&
-             parent->dest.dest.ssa.if_uses->entries == 0)
+         if (list_empty(&parent->dest.dest.ssa.uses) &&
+             list_empty(&parent->dest.dest.ssa.if_uses))
             nir_instr_remove(&parent->instr);
       }
 
@@ -131,13 +131,13 @@ nir_lower_to_source_mods_block(nir_block *block, void *state)
       if (nir_op_infos[alu->op].output_type != nir_type_float)
          continue;
 
-      if (alu->dest.dest.ssa.if_uses->entries != 0)
+      if (!list_empty(&alu->dest.dest.ssa.if_uses))
          continue;
 
       bool all_children_are_sat = true;
-      struct set_entry *entry;
-      set_foreach(alu->dest.dest.ssa.uses, entry) {
-         const nir_instr *child = entry->key;
+      nir_foreach_use(&alu->dest.dest.ssa, child_src) {
+         assert(child_src->is_ssa);
+         nir_instr *child = child_src->parent_instr;
          if (child->type != nir_instr_type_alu) {
             all_children_are_sat = false;
             continue;
@@ -161,8 +161,12 @@ nir_lower_to_source_mods_block(nir_block *block, void *state)
 
       alu->dest.saturate = true;
 
-      set_foreach(alu->dest.dest.ssa.uses, entry) {
-         nir_alu_instr *child_alu = nir_instr_as_alu((nir_instr *)entry->key);
+      nir_foreach_use(&alu->dest.dest.ssa, child_src) {
+         assert(child_src->is_ssa);
+         nir_instr *child = child_src->parent_instr;
+         assert(child->type == nir_instr_type_alu);
+         nir_alu_instr *child_alu = nir_instr_as_alu(child);
+
          child_alu->op = nir_op_fmov;
          child_alu->dest.saturate = false;
          /* We could propagate the dest of our instruction to the

diff --git a/src/glsl/nir/nir_lower_vars_to_ssa.c b/src/glsl/nir/nir_lower_vars_to_ssa.c
index bb60f46012c13b18afacfd4108c516218f111403..ccb8f99dfbaf46df7ceb3322633c9bb595b2d6fb 100644 (file)
--- a/src/glsl/nir/nir_lower_vars_to_ssa.c
+++ b/src/glsl/nir/nir_lower_vars_to_ssa.c
@@ -567,10 +567,11 @@ add_phi_sources(nir_block *block, nir_block *pred,
 
       nir_phi_src *src = ralloc(phi, nir_phi_src);
       src->pred = pred;
+      src->src.parent_instr = &phi->instr;
       src->src.is_ssa = true;
       src->src.ssa = get_ssa_def_for_block(node, pred, state);
 
-      _mesa_set_add(src->src.ssa->uses, instr);
+      list_addtail(&src->src.use_link, &src->src.ssa->uses);
 
       exec_list_push_tail(&phi->srcs, &src->node);
    }

diff --git a/src/glsl/nir/nir_opt_gcm.c b/src/glsl/nir/nir_opt_gcm.c
index b4f5fd3d50aea185b637c4a273a422406f74ed12..44068bf37b8561820d3440667a778f855b4f3496 100644 (file)
--- a/src/glsl/nir/nir_opt_gcm.c
+++ b/src/glsl/nir/nir_opt_gcm.c
@@ -279,9 +279,8 @@ gcm_schedule_late_def(nir_ssa_def *def, void *void_state)
 
    nir_block *lca = NULL;
 
-   struct set_entry *entry;
-   set_foreach(def->uses, entry) {
-      nir_instr *use_instr = (nir_instr *)entry->key;
+   nir_foreach_use(def, use_src) {
+      nir_instr *use_instr = use_src->parent_instr;
 
       gcm_schedule_late_instr(use_instr, state);
 
@@ -304,8 +303,8 @@ gcm_schedule_late_def(nir_ssa_def *def, void *void_state)
       }
    }
 
-   set_foreach(def->if_uses, entry) {
-      nir_if *if_stmt = (nir_if *)entry->key;
+   nir_foreach_if_use(def, use_src) {
+      nir_if *if_stmt = use_src->parent_if;
 
       /* For if statements, we consider the block to be the one immediately
        * preceding the if CF node.
@@ -377,9 +376,8 @@ gcm_place_instr(nir_instr *instr, struct gcm_state *state);
 static bool
 gcm_place_instr_def(nir_ssa_def *def, void *state)
 {
-   struct set_entry *entry;
-   set_foreach(def->uses, entry)
-      gcm_place_instr((nir_instr *)entry->key, state);
+   nir_foreach_use(def, use_src)
+      gcm_place_instr(use_src->parent_instr, state);
 
    return false;
 }

diff --git a/src/glsl/nir/nir_opt_global_to_local.c b/src/glsl/nir/nir_opt_global_to_local.c
index 00db37ba79da6dd05a75777794c85b84e6372366..bccb45b6237f29281db1dc680713ed15f613427e 100644 (file)
--- a/src/glsl/nir/nir_opt_global_to_local.c
+++ b/src/glsl/nir/nir_opt_global_to_local.c
@@ -34,9 +34,8 @@ global_to_local(nir_register *reg)
 
    assert(reg->is_global);
 
-   struct set_entry *entry;
-   set_foreach(reg->defs, entry) {
-      nir_instr *instr = (nir_instr *) entry->key;
+   nir_foreach_def(reg, def_dest) {
+      nir_instr *instr = def_dest->reg.parent_instr;
       nir_function_impl *instr_impl =
          nir_cf_node_get_function(&instr->block->cf_node);
       if (impl != NULL) {
@@ -47,8 +46,8 @@ global_to_local(nir_register *reg)
       }
    }
 
-   set_foreach(reg->uses, entry) {
-      nir_instr *instr = (nir_instr *) entry->key;
+   nir_foreach_use(reg, use_src) {
+      nir_instr *instr = use_src->parent_instr;
       nir_function_impl *instr_impl =
          nir_cf_node_get_function(&instr->block->cf_node);
       if (impl != NULL) {
@@ -59,8 +58,8 @@ global_to_local(nir_register *reg)
       }
    }
 
-   set_foreach(reg->if_uses, entry) {
-      nir_if *if_stmt = (nir_if *) entry->key;
+   nir_foreach_if_use(reg, use_src) {
+      nir_if *if_stmt = use_src->parent_if;
       nir_function_impl *if_impl = nir_cf_node_get_function(&if_stmt->cf_node);
       if (impl != NULL) {
          if (impl != if_impl)

diff --git a/src/glsl/nir/nir_opt_peephole_ffma.c b/src/glsl/nir/nir_opt_peephole_ffma.c
index 9d5646fe65e7a775e7b582d078db3133854d11b3..b430eac8eab2c591bc0418af761744d05a9b0562 100644 (file)
--- a/src/glsl/nir/nir_opt_peephole_ffma.c
+++ b/src/glsl/nir/nir_opt_peephole_ffma.c
@@ -41,12 +41,11 @@ struct peephole_ffma_state {
 static inline bool
 are_all_uses_fadd(nir_ssa_def *def)
 {
-   if (def->if_uses->entries > 0)
+   if (!list_empty(&def->if_uses))
       return false;
 
-   struct set_entry *use_iter;
-   set_foreach(def->uses, use_iter) {
-      nir_instr *use_instr = (nir_instr *)use_iter->key;
+   nir_foreach_use(def, use_src) {
+      nir_instr *use_instr = use_src->parent_instr;
 
       if (use_instr->type != nir_instr_type_alu)
          return false;
@@ -220,7 +219,7 @@ nir_opt_peephole_ffma_block(nir_block *block, void *void_state)
                                state->mem_ctx);
 
       nir_instr_insert_before(&add->instr, &ffma->instr);
-      assert(add->dest.dest.ssa.uses->entries == 0);
+      assert(list_empty(&add->dest.dest.ssa.uses));
       nir_instr_remove(&add->instr);
 
       state->progress = true;

diff --git a/src/glsl/nir/nir_opt_peephole_select.c b/src/glsl/nir/nir_opt_peephole_select.c
index f400cfd66da3ecf4e6656a593f2056922f78b95b..82c65bb442fa1724ff641e5a719ec3f05a45ff30 100644 (file)
--- a/src/glsl/nir/nir_opt_peephole_select.c
+++ b/src/glsl/nir/nir_opt_peephole_select.c
@@ -98,15 +98,13 @@ block_check_for_allowed_instrs(nir_block *block)
             return false;
 
          /* It cannot have any if-uses */
-         if (mov->dest.dest.ssa.if_uses->entries != 0)
+         if (!list_empty(&mov->dest.dest.ssa.if_uses))
             return false;
 
          /* The only uses of this definition must be phi's in the successor */
-         struct set_entry *entry;
-         set_foreach(mov->dest.dest.ssa.uses, entry) {
-            const nir_instr *dest_instr = entry->key;
-            if (dest_instr->type != nir_instr_type_phi ||
-                dest_instr->block != block->successors[0])
+         nir_foreach_use(&mov->dest.dest.ssa, use) {
+            if (use->parent_instr->type != nir_instr_type_phi ||
+                use->parent_instr->block != block->successors[0])
                return false;
          }
          break;

diff --git a/src/glsl/nir/nir_to_ssa.c b/src/glsl/nir/nir_to_ssa.c
index 53ff547669d054e7a5f5fccc1398df284704d2d7..a3c35fa049325ece470cc544d0fb8366480bb9af 100644 (file)
--- a/src/glsl/nir/nir_to_ssa.c
+++ b/src/glsl/nir/nir_to_ssa.c
@@ -89,9 +89,8 @@ insert_phi_nodes(nir_function_impl *impl)
       w_start = w_end = 0;
       iter_count++;
 
-      struct set_entry *entry;
-      set_foreach(reg->defs, entry) {
-         nir_instr *def = (nir_instr *) entry->key;
+      nir_foreach_def(reg, dest) {
+         nir_instr *def = dest->reg.parent_instr;
          if (work[def->block->index] < iter_count)
             W[w_end++] = def->block;
          work[def->block->index] = iter_count;
@@ -99,6 +98,7 @@ insert_phi_nodes(nir_function_impl *impl)
 
       while (w_start != w_end) {
          nir_block *cur = W[w_start++];
+         struct set_entry *entry;
          set_foreach(cur->dom_frontier, entry) {
             nir_block *next = (nir_block *) entry->key;
 
@@ -190,13 +190,12 @@ rewrite_use(nir_src *src, void *_state)
    if (state->states[index].stack == NULL)
       return true;
 
-   src->is_ssa = true;
-   src->ssa = get_ssa_src(src->reg.reg, state);
-
+   nir_ssa_def *def = get_ssa_src(src->reg.reg, state);
    if (state->parent_instr)
-      _mesa_set_add(src->ssa->uses, state->parent_instr);
+      nir_instr_rewrite_src(state->parent_instr, src, nir_src_for_ssa(def));
    else
-      _mesa_set_add(src->ssa->if_uses, state->parent_if);
+      nir_if_rewrite_condition(state->parent_if, nir_src_for_ssa(def));
+
    return true;
 }
 
@@ -219,6 +218,7 @@ rewrite_def_forwards(nir_dest *dest, void *_state)
       name = ralloc_asprintf(state->mem_ctx, "%s_%u", dest->reg.reg->name,
                              state->states[index].num_defs);
 
+   list_del(&dest->reg.def_link);
    nir_ssa_dest_init(state->parent_instr, dest, reg->num_components, name);
 
    /* push our SSA destination on the stack */
@@ -270,6 +270,7 @@ rewrite_alu_instr_forward(nir_alu_instr *instr, rewrite_state *state)
                                 reg->name, state->states[index].num_defs);
 
       instr->dest.write_mask = (1 << num_components) - 1;
+      list_del(&instr->dest.dest.reg.def_link);
       nir_ssa_dest_init(&instr->instr, &instr->dest.dest, num_components, name);
 
       if (nir_op_infos[instr->op].output_size == 0) {
@@ -484,7 +485,7 @@ init_rewrite_state(nir_function_impl *impl, rewrite_state *state)
           * called after phi nodes are inserted so we can count phi node
           * definitions too.
           */
-         unsigned stack_size = reg->defs->entries;
+         unsigned stack_size = list_length(&reg->defs);
 
          state->states[reg->index].stack = ralloc_array(state->states,
                                                         nir_ssa_def *,

diff --git a/src/glsl/nir/nir_validate.c b/src/glsl/nir/nir_validate.c
index 35a853d148419c032612c5f17d8d4e77e4bea7c4..da92ed90472f5048768f1413d55228d75ec43b9d 100644 (file)
--- a/src/glsl/nir/nir_validate.c
+++ b/src/glsl/nir/nir_validate.c
@@ -97,50 +97,47 @@ typedef struct {
 static void validate_src(nir_src *src, validate_state *state);
 
 static void
-validate_reg_src(nir_reg_src *src, validate_state *state)
+validate_reg_src(nir_src *src, validate_state *state)
 {
-   assert(src->reg != NULL);
+   assert(src->reg.reg != NULL);
 
    struct hash_entry *entry;
-   entry = _mesa_hash_table_search(state->regs, src->reg);
+   entry = _mesa_hash_table_search(state->regs, src->reg.reg);
    assert(entry);
 
    reg_validate_state *reg_state = (reg_validate_state *) entry->data;
 
    if (state->instr) {
-      _mesa_set_add(reg_state->uses, state->instr);
-
-      assert(_mesa_set_search(src->reg->uses, state->instr));
+      _mesa_set_add(reg_state->uses, src);
    } else {
       assert(state->if_stmt);
-      _mesa_set_add(reg_state->if_uses, state->if_stmt);
-
-      assert(_mesa_set_search(src->reg->if_uses, state->if_stmt));
+      _mesa_set_add(reg_state->if_uses, src);
    }
 
-   if (!src->reg->is_global) {
+   if (!src->reg.reg->is_global) {
       assert(reg_state->where_defined == state->impl &&
              "using a register declared in a different function");
    }
 
-   assert((src->reg->num_array_elems == 0 ||
-          src->base_offset < src->reg->num_array_elems) &&
+   assert((src->reg.reg->num_array_elems == 0 ||
+          src->reg.base_offset < src->reg.reg->num_array_elems) &&
           "definitely out-of-bounds array access");
 
-   if (src->indirect) {
-      assert(src->reg->num_array_elems != 0);
-      assert((src->indirect->is_ssa || src->indirect->reg.indirect == NULL) &&
+   if (src->reg.indirect) {
+      assert(src->reg.reg->num_array_elems != 0);
+      assert((src->reg.indirect->is_ssa ||
+              src->reg.indirect->reg.indirect == NULL) &&
              "only one level of indirection allowed");
-      validate_src(src->indirect, state);
+      validate_src(src->reg.indirect, state);
    }
 }
 
 static void
-validate_ssa_src(nir_ssa_def *def, validate_state *state)
+validate_ssa_src(nir_src *src, validate_state *state)
 {
-   assert(def != NULL);
+   assert(src->ssa != NULL);
 
-   struct hash_entry *entry = _mesa_hash_table_search(state->ssa_defs, def);
+   struct hash_entry *entry = _mesa_hash_table_search(state->ssa_defs, src->ssa);
 
    assert(entry);
 
@@ -150,14 +147,10 @@ validate_ssa_src(nir_ssa_def *def, validate_state *state)
           "using an SSA value defined in a different function");
 
    if (state->instr) {
-      _mesa_set_add(def_state->uses, state->instr);
-
-      assert(_mesa_set_search(def->uses, state->instr));
+      _mesa_set_add(def_state->uses, src);
    } else {
       assert(state->if_stmt);
-      _mesa_set_add(def_state->if_uses, state->if_stmt);
-
-      assert(_mesa_set_search(def->if_uses, state->if_stmt));
+      _mesa_set_add(def_state->if_uses, src);
    }
 
    /* TODO validate that the use is dominated by the definition */
@@ -166,10 +159,15 @@ validate_ssa_src(nir_ssa_def *def, validate_state *state)
 static void
 validate_src(nir_src *src, validate_state *state)
 {
+   if (state->instr)
+      assert(src->parent_instr == state->instr);
+   else
+      assert(src->parent_if == state->if_stmt);
+
    if (src->is_ssa)
-      validate_ssa_src(src->ssa, state);
+      validate_ssa_src(src, state);
    else
-      validate_reg_src(&src->reg, state);
+      validate_reg_src(src, state);
 }
 
 static void
@@ -201,8 +199,7 @@ validate_reg_dest(nir_reg_dest *dest, validate_state *state)
 {
    assert(dest->reg != NULL);
 
-   struct set_entry *entry = _mesa_set_search(dest->reg->defs, state->instr);
-   assert(entry && "definition not in nir_register.defs");
+   assert(dest->parent_instr == state->instr);
 
    struct hash_entry *entry2;
    entry2 = _mesa_hash_table_search(state->regs, dest->reg);
@@ -210,7 +207,7 @@ validate_reg_dest(nir_reg_dest *dest, validate_state *state)
    assert(entry2);
 
    reg_validate_state *reg_state = (reg_validate_state *) entry2->data;
-   _mesa_set_add(reg_state->defs, state->instr);
+   _mesa_set_add(reg_state->defs, dest);
 
    if (!dest->reg->is_global) {
       assert(reg_state->where_defined == state->impl &&
@@ -240,6 +237,9 @@ validate_ssa_def(nir_ssa_def *def, validate_state *state)
 
    assert(def->num_components <= 4);
 
+   list_validate(&def->uses);
+   list_validate(&def->if_uses);
+
    ssa_def_validate_state *def_state = ralloc(state->ssa_defs,
                                               ssa_def_validate_state);
    def_state->where_defined = state->impl;
@@ -701,6 +701,10 @@ prevalidate_reg_decl(nir_register *reg, bool is_global, validate_state *state)
    assert(!BITSET_TEST(state->regs_found, reg->index));
    BITSET_SET(state->regs_found, reg->index);
 
+   list_validate(&reg->uses);
+   list_validate(&reg->defs);
+   list_validate(&reg->if_uses);
+
    reg_validate_state *reg_state = ralloc(state->regs, reg_validate_state);
    reg_state->uses = _mesa_set_create(reg_state, _mesa_hash_pointer,
                                       _mesa_key_pointer_equal);
@@ -721,47 +725,47 @@ postvalidate_reg_decl(nir_register *reg, validate_state *state)
 
    reg_validate_state *reg_state = (reg_validate_state *) entry->data;
 
-   if (reg_state->uses->entries != reg->uses->entries) {
+   nir_foreach_use(reg, src) {
+      struct set_entry *entry = _mesa_set_search(reg_state->uses, src);
+      assert(entry);
+      _mesa_set_remove(reg_state->uses, entry);
+   }
+
+   if (reg_state->uses->entries != 0) {
       printf("extra entries in register uses:\n");
       struct set_entry *entry;
-      set_foreach(reg->uses, entry) {
-         struct set_entry *entry2 =
-            _mesa_set_search(reg_state->uses, entry->key);
-
-         if (entry2 == NULL) {
-            printf("%p\n", entry->key);
-         }
-      }
+      set_foreach(reg_state->uses, entry)
+         printf("%p\n", entry->key);
 
       abort();
    }
 
-   if (reg_state->if_uses->entries != reg->if_uses->entries) {
+   nir_foreach_if_use(reg, src) {
+      struct set_entry *entry = _mesa_set_search(reg_state->if_uses, src);
+      assert(entry);
+      _mesa_set_remove(reg_state->if_uses, entry);
+   }
+
+   if (reg_state->if_uses->entries != 0) {
       printf("extra entries in register if_uses:\n");
       struct set_entry *entry;
-      set_foreach(reg->if_uses, entry) {
-         struct set_entry *entry2 =
-            _mesa_set_search(reg_state->if_uses, entry->key);
-
-         if (entry2 == NULL) {
-            printf("%p\n", entry->key);
-         }
-      }
+      set_foreach(reg_state->if_uses, entry)
+         printf("%p\n", entry->key);
 
       abort();
    }
 
-   if (reg_state->defs->entries != reg->defs->entries) {
+   nir_foreach_def(reg, src) {
+      struct set_entry *entry = _mesa_set_search(reg_state->defs, src);
+      assert(entry);
+      _mesa_set_remove(reg_state->defs, entry);
+   }
+
+   if (reg_state->defs->entries != 0) {
       printf("extra entries in register defs:\n");
       struct set_entry *entry;
-      set_foreach(reg->defs, entry) {
-         struct set_entry *entry2 =
-            _mesa_set_search(reg_state->defs, entry->key);
-
-         if (entry2 == NULL) {
-            printf("%p\n", entry->key);
-         }
-      }
+      set_foreach(reg_state->defs, entry)
+         printf("%p\n", entry->key);
 
       abort();
    }
@@ -790,32 +794,32 @@ postvalidate_ssa_def(nir_ssa_def *def, void *void_state)
    struct hash_entry *entry = _mesa_hash_table_search(state->ssa_defs, def);
    ssa_def_validate_state *def_state = (ssa_def_validate_state *)entry->data;
 
-   if (def_state->uses->entries != def->uses->entries) {
-      printf("extra entries in SSA def uses:\n");
-      struct set_entry *entry;
-      set_foreach(def->uses, entry) {
-         struct set_entry *entry2 =
-            _mesa_set_search(def_state->uses, entry->key);
+   nir_foreach_use(def, src) {
+      struct set_entry *entry = _mesa_set_search(def_state->uses, src);
+      assert(entry);
+      _mesa_set_remove(def_state->uses, entry);
+   }
 
-         if (entry2 == NULL) {
-            printf("%p\n", entry->key);
-         }
-      }
+   if (def_state->uses->entries != 0) {
+      printf("extra entries in register uses:\n");
+      struct set_entry *entry;
+      set_foreach(def_state->uses, entry)
+         printf("%p\n", entry->key);
 
       abort();
    }
 
-   if (def_state->if_uses->entries != def->if_uses->entries) {
-      printf("extra entries in SSA def uses:\n");
-      struct set_entry *entry;
-      set_foreach(def->if_uses, entry) {
-         struct set_entry *entry2 =
-            _mesa_set_search(def_state->if_uses, entry->key);
+   nir_foreach_if_use(def, src) {
+      struct set_entry *entry = _mesa_set_search(def_state->if_uses, src);
+      assert(entry);
+      _mesa_set_remove(def_state->if_uses, entry);
+   }
 
-         if (entry2 == NULL) {
-            printf("%p\n", entry->key);
-         }
-      }
+   if (def_state->if_uses->entries != 0) {
+      printf("extra entries in register uses:\n");
+      struct set_entry *entry;
+      set_foreach(def_state->if_uses, entry)
+         printf("%p\n", entry->key);
 
       abort();
    }