* \file nir_opt_intrinsics.c
*/
-static nir_ssa_def *
-high_subgroup_mask(nir_builder *b,
- nir_ssa_def *count,
- uint64_t base_mask)
-{
- /* group_mask could probably be calculated more efficiently but we want to
- * be sure not to shift by 64 if the subgroup size is 64 because the GLSL
- * shift operator is undefined in that case. In any case if we were worried
- * about efficency this should probably be done further down because the
- * subgroup size is likely to be known at compile time.
- */
- nir_ssa_def *subgroup_size = nir_load_subgroup_size(b);
- nir_ssa_def *all_bits = nir_imm_int64(b, ~0ull);
- nir_ssa_def *shift = nir_isub(b, nir_imm_int(b, 64), subgroup_size);
- nir_ssa_def *group_mask = nir_ushr(b, all_bits, shift);
- nir_ssa_def *higher_bits = nir_ishl(b, nir_imm_int64(b, base_mask), count);
-
- return nir_iand(b, higher_bits, group_mask);
-}
-
static bool
-opt_intrinsics_impl(nir_function_impl *impl)
+opt_intrinsics_impl(nir_function_impl *impl,
+ const struct nir_shader_compiler_options *options)
{
nir_builder b;
nir_builder_init(&b, impl);
switch (intrin->intrinsic) {
case nir_intrinsic_vote_any:
- case nir_intrinsic_vote_all: {
- nir_const_value *val = nir_src_as_const_value(intrin->src[0]);
- if (!val && !b.shader->options->lower_vote_trivial)
- continue;
-
- replacement = nir_ssa_for_src(&b, intrin->src[0], 1);
+ case nir_intrinsic_vote_all:
+ if (nir_src_is_const(intrin->src[0]))
+ replacement = nir_ssa_for_src(&b, intrin->src[0], 1);
break;
- }
- case nir_intrinsic_vote_eq: {
- nir_const_value *val = nir_src_as_const_value(intrin->src[0]);
- if (!val && !b.shader->options->lower_vote_trivial)
- continue;
-
- replacement = nir_imm_int(&b, NIR_TRUE);
+ case nir_intrinsic_vote_feq:
+ case nir_intrinsic_vote_ieq:
+ if (nir_src_is_const(intrin->src[0]))
+ replacement = nir_imm_true(&b);
break;
- }
- case nir_intrinsic_ballot: {
- assert(b.shader->options->max_subgroup_size != 0);
- if (b.shader->options->max_subgroup_size > 32 ||
- intrin->dest.ssa.bit_size <= 32)
+ case nir_intrinsic_load_sample_mask_in:
+ /* Transform:
+ * gl_SampleMaskIn == 0 ---> gl_HelperInvocation
+ * gl_SampleMaskIn != 0 ---> !gl_HelperInvocation
+ */
+ if (!options->optimize_sample_mask_in)
continue;
- nir_intrinsic_instr *ballot =
- nir_intrinsic_instr_create(b.shader, nir_intrinsic_ballot);
- nir_ssa_dest_init(&ballot->instr, &ballot->dest, 1, 32, NULL);
- nir_src_copy(&ballot->src[0], &intrin->src[0], ballot);
-
- nir_builder_instr_insert(&b, &ballot->instr);
-
- replacement = nir_pack_64_2x32_split(&b,
- &ballot->dest.ssa,
- nir_imm_int(&b, 0));
- break;
- }
- case nir_intrinsic_load_subgroup_eq_mask:
- case nir_intrinsic_load_subgroup_ge_mask:
- case nir_intrinsic_load_subgroup_gt_mask:
- case nir_intrinsic_load_subgroup_le_mask:
- case nir_intrinsic_load_subgroup_lt_mask: {
- if (!b.shader->options->lower_subgroup_masks)
- break;
-
- nir_ssa_def *count = nir_load_subgroup_invocation(&b);
-
- switch (intrin->intrinsic) {
- case nir_intrinsic_load_subgroup_eq_mask:
- replacement = nir_ishl(&b, nir_imm_int64(&b, 1ull), count);
- break;
- case nir_intrinsic_load_subgroup_ge_mask:
- replacement = high_subgroup_mask(&b, count, ~0ull);
- break;
- case nir_intrinsic_load_subgroup_gt_mask:
- replacement = high_subgroup_mask(&b, count, ~1ull);
- break;
- case nir_intrinsic_load_subgroup_le_mask:
- replacement = nir_inot(&b, nir_ishl(&b, nir_imm_int64(&b, ~1ull), count));
- break;
- case nir_intrinsic_load_subgroup_lt_mask:
- replacement = nir_inot(&b, nir_ishl(&b, nir_imm_int64(&b, ~0ull), count));
- break;
- default:
- unreachable("you seriously can't tell this is unreachable?");
+ nir_foreach_use_safe(use_src, &intrin->dest.ssa) {
+ if (use_src->parent_instr->type == nir_instr_type_alu) {
+ nir_alu_instr *alu = nir_instr_as_alu(use_src->parent_instr);
+
+ if (alu->op == nir_op_ieq ||
+ alu->op == nir_op_ine) {
+ /* Check for 0 in either operand. */
+ nir_const_value *const_val =
+ nir_src_as_const_value(alu->src[0].src);
+ if (!const_val)
+ const_val = nir_src_as_const_value(alu->src[1].src);
+ if (!const_val || const_val->i32 != 0)
+ continue;
+
+ nir_ssa_def *new_expr = nir_load_helper_invocation(&b, 1);
+
+ if (alu->op == nir_op_ine)
+ new_expr = nir_inot(&b, new_expr);
+
+ nir_ssa_def_rewrite_uses(&alu->dest.dest.ssa,
+ nir_src_for_ssa(new_expr));
+ nir_instr_remove(&alu->instr);
+ continue;
+ }
+ }
}
- break;
- }
+ continue;
default:
break;
}
if (!function->impl)
continue;
- if (opt_intrinsics_impl(function->impl)) {
+ if (opt_intrinsics_impl(function->impl, shader->options)) {
progress = true;
nir_metadata_preserve(function->impl, nir_metadata_block_index |
nir_metadata_dominance);
+ } else {
+ nir_metadata_preserve(function->impl, nir_metadata_all);
}
}
projects / mesa.git / blobdiff