*/
#include "nir.h"
-#include "nir_control_flow_private.h"
+#include "nir_control_flow.h"
/* Secret Decoder Ring:
* clone_foo():
*/
typedef struct {
+ /* True if we are cloning an entire shader. */
+ bool global_clone;
+
+ /* If true allows the clone operation to fall back to the original pointer
+ * if no clone pointer is found in the remap table. This allows us to
+ * clone a loop body without having to add srcs from outside the loop to
+ * the remap table. This is useful for loop unrolling.
+ */
+ bool allow_remap_fallback;
+
/* maps orig ptr -> cloned ptr: */
- struct hash_table *ptr_table;
+ struct hash_table *remap_table;
/* List of phi sources. */
struct list_head phi_srcs;
} clone_state;
static void
-init_clone_state(clone_state *state)
+init_clone_state(clone_state *state, struct hash_table *remap_table,
+ bool global, bool allow_remap_fallback)
{
- state->ptr_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
- _mesa_key_pointer_equal);
+ state->global_clone = global;
+ state->allow_remap_fallback = allow_remap_fallback;
+
+ if (remap_table) {
+ state->remap_table = remap_table;
+ } else {
+ state->remap_table = _mesa_hash_table_create(NULL, _mesa_hash_pointer,
+ _mesa_key_pointer_equal);
+ }
+
list_inithead(&state->phi_srcs);
}
static void
free_clone_state(clone_state *state)
{
- _mesa_hash_table_destroy(state->ptr_table, NULL);
+ _mesa_hash_table_destroy(state->remap_table, NULL);
}
-static void *
-lookup_ptr(clone_state *state, const void *ptr)
+static inline void *
+_lookup_ptr(clone_state *state, const void *ptr, bool global)
{
struct hash_entry *entry;
if (!ptr)
return NULL;
- entry = _mesa_hash_table_search(state->ptr_table, ptr);
- assert(entry && "Failed to find pointer!");
- if (!entry)
- return NULL;
+ if (!state->global_clone && global)
+ return (void *)ptr;
+
+ entry = _mesa_hash_table_search(state->remap_table, ptr);
+ if (!entry) {
+ assert(state->allow_remap_fallback);
+ return (void *)ptr;
+ }
return entry->data;
}
static void
-store_ptr(clone_state *state, void *nptr, const void *ptr)
+add_remap(clone_state *state, void *nptr, const void *ptr)
+{
+ _mesa_hash_table_insert(state->remap_table, ptr, nptr);
+}
+
+static void *
+remap_local(clone_state *state, const void *ptr)
+{
+ return _lookup_ptr(state, ptr, false);
+}
+
+static void *
+remap_global(clone_state *state, const void *ptr)
+{
+ return _lookup_ptr(state, ptr, true);
+}
+
+static nir_register *
+remap_reg(clone_state *state, const nir_register *reg)
{
- _mesa_hash_table_insert(state->ptr_table, ptr, nptr);
+ return _lookup_ptr(state, reg, reg->is_global);
}
-static nir_constant *
-clone_constant(clone_state *state, const nir_constant *c, nir_variable *nvar)
+static nir_variable *
+remap_var(clone_state *state, const nir_variable *var)
+{
+ return _lookup_ptr(state, var, nir_variable_is_global(var));
+}
+
+nir_constant *
+nir_constant_clone(const nir_constant *c, nir_variable *nvar)
{
nir_constant *nc = ralloc(nvar, nir_constant);
- nc->value = c->value;
+ memcpy(nc->values, c->values, sizeof(nc->values));
nc->num_elements = c->num_elements;
nc->elements = ralloc_array(nvar, nir_constant *, c->num_elements);
for (unsigned i = 0; i < c->num_elements; i++) {
- nc->elements[i] = clone_constant(state, c->elements[i], nvar);
+ nc->elements[i] = nir_constant_clone(c->elements[i], nvar);
}
return nc;
}
-/* NOTE: for cloning nir_variable's, bypass nir_variable_create to avoid
+/* NOTE: for cloning nir_variables, bypass nir_variable_create to avoid
* having to deal with locals and globals separately:
*/
-static nir_variable *
-clone_variable(clone_state *state, const nir_variable *var)
+nir_variable *
+nir_variable_clone(const nir_variable *var, nir_shader *shader)
{
- nir_variable *nvar = rzalloc(state->ns, nir_variable);
- store_ptr(state, nvar, var);
+ nir_variable *nvar = rzalloc(shader, nir_variable);
nvar->type = var->type;
nvar->name = ralloc_strdup(nvar, var->name);
var->num_state_slots * sizeof(nir_state_slot));
if (var->constant_initializer) {
nvar->constant_initializer =
- clone_constant(state, var->constant_initializer, nvar);
+ nir_constant_clone(var->constant_initializer, nvar);
}
nvar->interface_type = var->interface_type;
return nvar;
}
+static nir_variable *
+clone_variable(clone_state *state, const nir_variable *var)
+{
+ nir_variable *nvar = nir_variable_clone(var, state->ns);
+ add_remap(state, nvar, var);
+
+ return nvar;
+}
+
/* clone list of nir_variable: */
static void
clone_var_list(clone_state *state, struct exec_list *dst,
}
}
-/* NOTE: for cloning nir_register's, bypass nir_global/local_reg_create()
+/* NOTE: for cloning nir_registers, bypass nir_global/local_reg_create()
* to avoid having to deal with locals and globals separately:
*/
static nir_register *
clone_register(clone_state *state, const nir_register *reg)
{
nir_register *nreg = rzalloc(state->ns, nir_register);
- store_ptr(state, nreg, reg);
+ add_remap(state, nreg, reg);
nreg->num_components = reg->num_components;
+ nreg->bit_size = reg->bit_size;
nreg->num_array_elems = reg->num_array_elems;
nreg->index = reg->index;
nreg->name = ralloc_strdup(nreg, reg->name);
{
nsrc->is_ssa = src->is_ssa;
if (src->is_ssa) {
- nsrc->ssa = lookup_ptr(state, src->ssa);
+ nsrc->ssa = remap_local(state, src->ssa);
} else {
- nsrc->reg.reg = lookup_ptr(state, src->reg.reg);
+ nsrc->reg.reg = remap_reg(state, src->reg.reg);
if (src->reg.indirect) {
nsrc->reg.indirect = ralloc(ninstr_or_if, nir_src);
__clone_src(state, ninstr_or_if, nsrc->reg.indirect, src->reg.indirect);
{
ndst->is_ssa = dst->is_ssa;
if (dst->is_ssa) {
- nir_ssa_dest_init(ninstr, ndst, dst->ssa.num_components, dst->ssa.name);
- store_ptr(state, &ndst->ssa, &dst->ssa);
+ nir_ssa_dest_init(ninstr, ndst, dst->ssa.num_components,
+ dst->ssa.bit_size, dst->ssa.name);
+ add_remap(state, &ndst->ssa, &dst->ssa);
} else {
- ndst->reg.reg = lookup_ptr(state, dst->reg.reg);
+ ndst->reg.reg = remap_reg(state, dst->reg.reg);
if (dst->reg.indirect) {
ndst->reg.indirect = ralloc(ninstr, nir_src);
__clone_src(state, ninstr, ndst->reg.indirect, dst->reg.indirect);
clone_deref_var(clone_state *state, const nir_deref_var *dvar,
nir_instr *ninstr)
{
- nir_variable *nvar = lookup_ptr(state, dvar->var);
+ nir_variable *nvar = remap_var(state, dvar->var);
nir_deref_var *ndvar = nir_deref_var_create(ninstr, nvar);
if (dvar->deref.child)
clone_alu(clone_state *state, const nir_alu_instr *alu)
{
nir_alu_instr *nalu = nir_alu_instr_create(state->ns, alu->op);
+ nalu->exact = alu->exact;
__clone_dst(state, &nalu->instr, &nalu->dest.dest, &alu->dest.dest);
nalu->dest.saturate = alu->dest.saturate;
return nalu;
}
+static nir_deref_instr *
+clone_deref_instr(clone_state *state, const nir_deref_instr *deref)
+{
+ nir_deref_instr *nderef =
+ nir_deref_instr_create(state->ns, deref->deref_type);
+
+ __clone_dst(state, &nderef->instr, &nderef->dest, &deref->dest);
+
+ nderef->mode = deref->mode;
+ nderef->type = deref->type;
+
+ if (deref->deref_type == nir_deref_type_var) {
+ nderef->var = remap_var(state, deref->var);
+ return nderef;
+ }
+
+ __clone_src(state, &nderef->instr, &nderef->parent, &deref->parent);
+
+ switch (deref->deref_type) {
+ case nir_deref_type_struct:
+ nderef->strct.index = deref->strct.index;
+ break;
+
+ case nir_deref_type_array:
+ __clone_src(state, &nderef->instr,
+ &nderef->arr.index, &deref->arr.index);
+ break;
+
+ case nir_deref_type_array_wildcard:
+ case nir_deref_type_cast:
+ /* Nothing to do */
+ break;
+
+ default:
+ unreachable("Invalid instruction deref type");
+ }
+
+ return nderef;
+}
+
static nir_intrinsic_instr *
clone_intrinsic(clone_state *state, const nir_intrinsic_instr *itr)
{
clone_load_const(clone_state *state, const nir_load_const_instr *lc)
{
nir_load_const_instr *nlc =
- nir_load_const_instr_create(state->ns, lc->def.num_components);
+ nir_load_const_instr_create(state->ns, lc->def.num_components,
+ lc->def.bit_size);
memcpy(&nlc->value, &lc->value, sizeof(nlc->value));
- store_ptr(state, &nlc->def, &lc->def);
+ add_remap(state, &nlc->def, &lc->def);
return nlc;
}
clone_ssa_undef(clone_state *state, const nir_ssa_undef_instr *sa)
{
nir_ssa_undef_instr *nsa =
- nir_ssa_undef_instr_create(state->ns, sa->def.num_components);
+ nir_ssa_undef_instr_create(state->ns, sa->def.num_components,
+ sa->def.bit_size);
- store_ptr(state, &nsa->def, &sa->def);
+ add_remap(state, &nsa->def, &sa->def);
return nsa;
}
static nir_call_instr *
clone_call(clone_state *state, const nir_call_instr *call)
{
- nir_function *ncallee = lookup_ptr(state, call->callee);
+ nir_function *ncallee = remap_global(state, call->callee);
nir_call_instr *ncall = nir_call_instr_create(state->ns, ncallee);
for (unsigned i = 0; i < ncall->num_params; i++)
switch (instr->type) {
case nir_instr_type_alu:
return &clone_alu(state, nir_instr_as_alu(instr))->instr;
+ case nir_instr_type_deref:
+ return &clone_deref_instr(state, nir_instr_as_deref(instr))->instr;
case nir_instr_type_intrinsic:
return &clone_intrinsic(state, nir_instr_as_intrinsic(instr))->instr;
case nir_instr_type_load_const:
assert(exec_list_is_empty(&nblk->instr_list));
/* We need this for phi sources */
- store_ptr(state, nblk, blk);
+ add_remap(state, nblk, blk);
- nir_foreach_instr(blk, instr) {
+ nir_foreach_instr(instr, blk) {
if (instr->type == nir_instr_type_phi) {
/* Phi instructions are a bit of a special case when cloning because
* we don't want inserting the instruction to automatically handle
}
}
+/* After we've cloned almost everything, we have to walk the list of phi
+ * sources and fix them up. Thanks to loops, the block and SSA value for a
+ * phi source may not be defined when we first encounter it. Instead, we
+ * add it to the phi_srcs list and we fix it up here.
+ */
+static void
+fixup_phi_srcs(clone_state *state)
+{
+ list_for_each_entry_safe(nir_phi_src, src, &state->phi_srcs, src.use_link) {
+ src->pred = remap_local(state, src->pred);
+
+ /* Remove from this list */
+ list_del(&src->src.use_link);
+
+ if (src->src.is_ssa) {
+ src->src.ssa = remap_local(state, src->src.ssa);
+ list_addtail(&src->src.use_link, &src->src.ssa->uses);
+ } else {
+ src->src.reg.reg = remap_reg(state, src->src.reg.reg);
+ list_addtail(&src->src.use_link, &src->src.reg.reg->uses);
+ }
+ }
+ assert(list_empty(&state->phi_srcs));
+}
+
+void
+nir_cf_list_clone(nir_cf_list *dst, nir_cf_list *src, nir_cf_node *parent,
+ struct hash_table *remap_table)
+{
+ exec_list_make_empty(&dst->list);
+ dst->impl = src->impl;
+
+ if (exec_list_is_empty(&src->list))
+ return;
+
+ clone_state state;
+ init_clone_state(&state, remap_table, false, true);
+
+ /* We use the same shader */
+ state.ns = src->impl->function->shader;
+
+ /* The control-flow code assumes that the list of cf_nodes always starts
+ * and ends with a block. We start by adding an empty block.
+ */
+ nir_block *nblk = nir_block_create(state.ns);
+ nblk->cf_node.parent = parent;
+ exec_list_push_tail(&dst->list, &nblk->cf_node.node);
+
+ clone_cf_list(&state, &dst->list, &src->list);
+
+ fixup_phi_srcs(&state);
+}
+
static nir_function_impl *
-clone_function_impl(clone_state *state, const nir_function_impl *fi,
- nir_function *nfxn)
+clone_function_impl(clone_state *state, const nir_function_impl *fi)
{
- nir_function_impl *nfi = nir_function_impl_create(nfxn);
+ nir_function_impl *nfi = nir_function_impl_create_bare(state->ns);
clone_var_list(state, &nfi->locals, &fi->locals);
clone_reg_list(state, &nfi->registers, &fi->registers);
nfi->num_params = fi->num_params;
nfi->params = ralloc_array(state->ns, nir_variable *, fi->num_params);
for (unsigned i = 0; i < fi->num_params; i++) {
- nfi->params[i] = lookup_ptr(state, fi->params[i]);
+ nfi->params[i] = clone_variable(state, fi->params[i]);
}
- nfi->return_var = lookup_ptr(state, fi->return_var);
+ if (fi->return_var)
+ nfi->return_var = clone_variable(state, fi->return_var);
assert(list_empty(&state->phi_srcs));
clone_cf_list(state, &nfi->body, &fi->body);
- /* After we've cloned almost everything, we have to walk the list of phi
- * sources and fix them up. Thanks to loops, the block and SSA value for a
- * phi source may not be defined when we first encounter it. Instead, we
- * add it to the phi_srcs list and we fix it up here.
- */
- list_for_each_entry_safe(nir_phi_src, src, &state->phi_srcs, src.use_link) {
- src->pred = lookup_ptr(state, src->pred);
- assert(src->src.is_ssa);
- src->src.ssa = lookup_ptr(state, src->src.ssa);
-
- /* Remove from this list and place in the uses of the SSA def */
- list_del(&src->src.use_link);
- list_addtail(&src->src.use_link, &src->src.ssa->uses);
- }
- assert(list_empty(&state->phi_srcs));
+ fixup_phi_srcs(state);
/* All metadata is invalidated in the cloning process */
nfi->valid_metadata = 0;
return nfi;
}
+nir_function_impl *
+nir_function_impl_clone(const nir_function_impl *fi)
+{
+ clone_state state;
+ init_clone_state(&state, NULL, false, false);
+
+ /* We use the same shader */
+ state.ns = fi->function->shader;
+
+ nir_function_impl *nfi = clone_function_impl(&state, fi);
+
+ free_clone_state(&state);
+
+ return nfi;
+}
+
static nir_function *
clone_function(clone_state *state, const nir_function *fxn, nir_shader *ns)
{
nir_function *nfxn = nir_function_create(ns, fxn->name);
/* Needed for call instructions */
- store_ptr(state, nfxn, fxn);
+ add_remap(state, nfxn, fxn);
nfxn->num_params = fxn->num_params;
nfxn->params = ralloc_array(state->ns, nir_parameter, fxn->num_params);
/* At first glance, it looks like we should clone the function_impl here.
* However, call instructions need to be able to reference at least the
- * function and those will get processed as we clone the function_impl's.
+ * function and those will get processed as we clone the function_impls.
* We stop here and do function_impls as a second pass.
*/
nir_shader_clone(void *mem_ctx, const nir_shader *s)
{
clone_state state;
- init_clone_state(&state);
+ init_clone_state(&state, NULL, true, false);
- nir_shader *ns = nir_shader_create(mem_ctx, s->stage, s->options);
+ nir_shader *ns = nir_shader_create(mem_ctx, s->info.stage, s->options, NULL);
state.ns = ns;
clone_var_list(&state, &ns->uniforms, &s->uniforms);
clone_var_list(&state, &ns->inputs, &s->inputs);
clone_var_list(&state, &ns->outputs, &s->outputs);
+ clone_var_list(&state, &ns->shared, &s->shared);
clone_var_list(&state, &ns->globals, &s->globals);
clone_var_list(&state, &ns->system_values, &s->system_values);
clone_function(&state, fxn, ns);
/* Only after all functions are cloned can we clone the actual function
- * implementations. This is because nir_call_instr's need to reference the
+ * implementations. This is because nir_call_instrs need to reference the
* functions of other functions and we don't know what order the functions
* will have in the list.
*/
- nir_foreach_function(s, fxn) {
- nir_function *nfxn = lookup_ptr(&state, fxn);
- clone_function_impl(&state, fxn->impl, nfxn);
+ nir_foreach_function(fxn, s) {
+ nir_function *nfxn = remap_global(&state, fxn);
+ nfxn->impl = clone_function_impl(&state, fxn->impl);
+ nfxn->impl->function = nfxn;
}
clone_reg_list(&state, &ns->registers, &s->registers);
ns->num_inputs = s->num_inputs;
ns->num_uniforms = s->num_uniforms;
ns->num_outputs = s->num_outputs;
+ ns->num_shared = s->num_shared;
+ ns->lowered_derefs = s->lowered_derefs;
free_clone_state(&state);
projects / mesa.git / blobdiff