#include "nir_builder.h"
#include "util/u_dynarray.h"
-#define HASH(hash, data) _mesa_fnv32_1a_accumulate((hash), (data))
+#define HASH(hash, data) XXH32(&data, sizeof(data), hash)
static uint32_t
hash_src(uint32_t hash, const nir_src *src)
{
assert(src->is_ssa);
+ void *hash_data = nir_src_is_const(*src) ? NULL : src->ssa;
- return HASH(hash, src->ssa);
+ return HASH(hash, hash_data);
}
static uint32_t
static uint32_t
hash_instr(const nir_instr *instr)
{
- uint32_t hash = _mesa_fnv32_1a_offset_bias;
+ uint32_t hash = 0;
switch (instr->type) {
case nir_instr_type_alu:
assert(src1->is_ssa);
assert(src2->is_ssa);
- return src1->ssa == src2->ssa;
+ return src1->ssa == src2->ssa ||
+ nir_src_is_const(*src1) == nir_src_is_const(*src2);
}
static bool
*/
static nir_instr *
-instr_try_combine(nir_instr *instr1, nir_instr *instr2)
+instr_try_combine(struct nir_shader *nir, nir_instr *instr1, nir_instr *instr2,
+ nir_opt_vectorize_cb filter, void *data)
{
assert(instr1->type == nir_instr_type_alu);
assert(instr2->type == nir_instr_type_alu);
if (total_components > 4)
return NULL;
+ if (nir->options->vectorize_vec2_16bit &&
+ (total_components > 2 || alu1->dest.dest.ssa.bit_size != 16))
+ return NULL;
+
+ if (filter && !filter(&alu1->instr, &alu2->instr, data))
+ return NULL;
+
nir_builder b;
nir_builder_init(&b, nir_cf_node_get_function(&instr1->block->cf_node));
b.cursor = nir_after_instr(instr1);
new_alu->dest.write_mask = (1 << total_components) - 1;
for (unsigned i = 0; i < nir_op_infos[alu1->op].num_inputs; i++) {
+ /* handle constant merging case */
+ if (alu1->src[i].src.ssa != alu2->src[i].src.ssa) {
+ nir_const_value *c1 = nir_src_as_const_value(alu1->src[i].src);
+ nir_const_value *c2 = nir_src_as_const_value(alu2->src[i].src);
+ assert(c1 && c2);
+ nir_const_value value[4];
+ unsigned bit_size = alu1->src[i].src.ssa->bit_size;
+
+ for (unsigned j = 0; j < total_components; j++) {
+ value[j].u64 = j < alu1_components ?
+ c1[alu1->src[i].swizzle[j]].u64 :
+ c2[alu2->src[i].swizzle[j - alu1_components]].u64;
+ }
+ nir_ssa_def *def = nir_build_imm(&b, total_components, bit_size, value);
+
+ new_alu->src[i].src = nir_src_for_ssa(def);
+ for (unsigned j = 0; j < total_components; j++)
+ new_alu->src[i].swizzle[j] = j;
+ continue;
+ }
+
new_alu->src[i].src = alu1->src[i].src;
for (unsigned j = 0; j < alu1_components; j++)
nir_if_rewrite_condition(src->parent_if, nir_src_for_ssa(new_alu1));
}
- assert(list_empty(&alu1->dest.dest.ssa.uses));
- assert(list_empty(&alu1->dest.dest.ssa.if_uses));
+ assert(list_is_empty(&alu1->dest.dest.ssa.uses));
+ assert(list_is_empty(&alu1->dest.dest.ssa.if_uses));
nir_foreach_use_safe(src, &alu2->dest.dest.ssa) {
if (src->parent_instr->type == nir_instr_type_alu) {
nir_if_rewrite_condition(src->parent_if, nir_src_for_ssa(new_alu2));
}
- assert(list_empty(&alu2->dest.dest.ssa.uses));
- assert(list_empty(&alu2->dest.dest.ssa.if_uses));
+ assert(list_is_empty(&alu2->dest.dest.ssa.uses));
+ assert(list_is_empty(&alu2->dest.dest.ssa.if_uses));
nir_instr_remove(instr1);
nir_instr_remove(instr2);
/* returns true if we were able to successfully replace the instruction */
static bool
-vec_instr_stack_push(struct util_dynarray *stack, nir_instr *instr)
+vec_instr_stack_push(struct nir_shader *nir, struct util_dynarray *stack,
+ nir_instr *instr,
+ nir_opt_vectorize_cb filter, void *data)
{
/* Walk the stack from child to parent to make live ranges shorter by
* matching the closest thing we can
*/
util_dynarray_foreach_reverse(stack, nir_instr *, stack_instr) {
- nir_instr *new_instr = instr_try_combine(*stack_instr, instr);
+ nir_instr *new_instr = instr_try_combine(nir, *stack_instr, instr,
+ filter, data);
if (new_instr) {
*stack_instr = new_instr;
return true;
static void
vec_instr_stack_pop(struct util_dynarray *stack, nir_instr *instr)
{
- nir_instr *last = util_dynarray_pop(stack, nir_instr *);
+ ASSERTED nir_instr *last = util_dynarray_pop(stack, nir_instr *);
assert(last == instr);
}
}
static bool
-vec_instr_set_add_or_rewrite(struct set *instr_set, nir_instr *instr)
+vec_instr_set_add_or_rewrite(struct nir_shader *nir, struct set *instr_set,
+ nir_instr *instr,
+ nir_opt_vectorize_cb filter, void *data)
{
if (!instr_can_rewrite(instr))
return false;
struct util_dynarray *new_stack = vec_instr_stack_create(instr_set);
- vec_instr_stack_push(new_stack, instr);
+ vec_instr_stack_push(nir, new_stack, instr, filter, data);
struct set_entry *entry = _mesa_set_search(instr_set, new_stack);
if (entry) {
ralloc_free(new_stack);
struct util_dynarray *stack = (struct util_dynarray *) entry->key;
- return vec_instr_stack_push(stack, instr);
+ return vec_instr_stack_push(nir, stack, instr, filter, data);
}
_mesa_set_add(instr_set, new_stack);
}
static void
-vec_instr_set_remove(struct set *instr_set, nir_instr *instr)
+vec_instr_set_remove(struct nir_shader *nir, struct set *instr_set,
+ nir_instr *instr, nir_opt_vectorize_cb filter, void *data)
{
if (!instr_can_rewrite(instr))
return;
* comparison function as well.
*/
struct util_dynarray *temp = vec_instr_stack_create(instr_set);
- vec_instr_stack_push(temp, instr);
+ vec_instr_stack_push(nir, temp, instr, filter, data);
struct set_entry *entry = _mesa_set_search(instr_set, temp);
ralloc_free(temp);
}
static bool
-vectorize_block(nir_block *block, struct set *instr_set)
+vectorize_block(struct nir_shader *nir, nir_block *block,
+ struct set *instr_set,
+ nir_opt_vectorize_cb filter, void *data)
{
bool progress = false;
nir_foreach_instr_safe(instr, block) {
- if (vec_instr_set_add_or_rewrite(instr_set, instr))
+ if (vec_instr_set_add_or_rewrite(nir, instr_set, instr, filter, data))
progress = true;
}
for (unsigned i = 0; i < block->num_dom_children; i++) {
nir_block *child = block->dom_children[i];
- progress |= vectorize_block(child, instr_set);
+ progress |= vectorize_block(nir, child, instr_set, filter, data);
}
nir_foreach_instr_reverse(instr, block)
- vec_instr_set_remove(instr_set, instr);
+ vec_instr_set_remove(nir, instr_set, instr, filter, data);
return progress;
}
static bool
-nir_opt_vectorize_impl(nir_function_impl *impl)
+nir_opt_vectorize_impl(struct nir_shader *nir, nir_function_impl *impl,
+ nir_opt_vectorize_cb filter, void *data)
{
struct set *instr_set = vec_instr_set_create();
nir_metadata_require(impl, nir_metadata_dominance);
- bool progress = vectorize_block(nir_start_block(impl), instr_set);
+ bool progress = vectorize_block(nir, nir_start_block(impl), instr_set,
+ filter, data);
if (progress)
nir_metadata_preserve(impl, nir_metadata_block_index |
}
bool
-nir_opt_vectorize(nir_shader *shader)
+nir_opt_vectorize(nir_shader *shader, nir_opt_vectorize_cb filter,
+ void *data)
{
bool progress = false;
nir_foreach_function(function, shader) {
if (function->impl)
- progress |= nir_opt_vectorize_impl(function->impl);
+ progress |= nir_opt_vectorize_impl(shader, function->impl, filter, data);
}
return progress;
projects / mesa.git / blobdiff