+2020-05-13 Richard Biener <rguenther@suse.de>
+
+ * tree-vectorizer.h (SLP_INSTANCE_GROUP_SIZE): Remove.
+ (_slp_instance::group_size): Likewise.
+ * tree-vect-loop.c (vectorizable_reduction): The group size
+ is the number of lanes in the node.
+ * tree-vect-slp.c (vect_attempt_slp_rearrange_stmts): Likewise.
+ (vect_analyze_slp_instance): Do not set SLP_INSTANCE_GROUP_SIZE,
+ verify it matches the instance trees number of lanes.
+ (vect_slp_analyze_node_operations_1): Use the numer of lanes
+ in the node as group size.
+ (vect_bb_vectorization_profitable_p): Use the instance root
+ number of lanes for the size of life.
+ (vect_schedule_slp_instance): Use the number of lanes as
+ group_size.
+ * tree-vect-stmts.c (vectorizable_load): Remove SLP instance
+ parameter. Use the number of lanes of the load for the group
+ size in the gap adjustment code.
+ (vect_analyze_stmt): Adjust.
+ (vect_transform_stmt): Likewise.
+
2020-05-13 Jakub Jelinek <jakub@redhat.com>
PR debug/95080
which each SLP statement has its own initial value and in which
that value needs to be repeated for every instance of the
statement within the initial vector. */
- unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
+ unsigned int group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
if (!neutral_op
&& !can_duplicate_and_interleave_p (loop_vinfo, group_size,
TREE_TYPE (vectype_out)))
static bool
vect_attempt_slp_rearrange_stmts (slp_instance slp_instn)
{
- unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_instn);
unsigned int i, j;
unsigned int lidx;
slp_tree node, load;
/* Compare all the permutation sequences to the first one. We know
that at least one load is permuted. */
node = SLP_INSTANCE_LOADS (slp_instn)[0];
- if (!node->load_permutation.exists ())
+ if (!SLP_TREE_LOAD_PERMUTATION (node).exists ())
return false;
+ unsigned int group_size = SLP_TREE_LOAD_PERMUTATION (node).length ();
for (i = 1; SLP_INSTANCE_LOADS (slp_instn).iterate (i, &load); ++i)
{
- if (!load->load_permutation.exists ())
+ if (!SLP_TREE_LOAD_PERMUTATION (load).exists ()
+ || SLP_TREE_LOAD_PERMUTATION (load).length () != group_size)
return false;
- FOR_EACH_VEC_ELT (load->load_permutation, j, lidx)
- if (lidx != node->load_permutation[j])
+ FOR_EACH_VEC_ELT (SLP_TREE_LOAD_PERMUTATION (load), j, lidx)
+ if (lidx != SLP_TREE_LOAD_PERMUTATION (node)[j])
return false;
}
/* Create a new SLP instance. */
new_instance = XNEW (class _slp_instance);
SLP_INSTANCE_TREE (new_instance) = node;
- SLP_INSTANCE_GROUP_SIZE (new_instance) = group_size;
SLP_INSTANCE_UNROLLING_FACTOR (new_instance) = unrolling_factor;
SLP_INSTANCE_LOADS (new_instance) = vNULL;
SLP_INSTANCE_ROOT_STMT (new_instance) = constructor ? stmt_info : NULL;
vinfo->slp_instances.safe_push (new_instance);
+ /* ??? We've replaced the old SLP_INSTANCE_GROUP_SIZE with
+ the number of scalar stmts in the root in a few places.
+ Verify that assumption holds. */
+ gcc_assert (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (new_instance))
+ .length () == group_size);
+
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
vf = loop_vinfo->vectorization_factor;
else
vf = 1;
- unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (node_instance);
+ unsigned int group_size = SLP_TREE_SCALAR_STMTS (node).length ();
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
SLP_TREE_NUMBER_OF_VEC_STMTS (node)
= vect_get_num_vectors (vf * group_size, vectype);
FOR_EACH_VEC_ELT (slp_instances, i, instance)
{
auto_vec<bool, 20> life;
- life.safe_grow_cleared (SLP_INSTANCE_GROUP_SIZE (instance));
+ life.safe_grow_cleared
+ (SLP_TREE_SCALAR_STMTS (SLP_INSTANCE_TREE (instance)).length ());
vect_bb_slp_scalar_cost (bb_vinfo,
SLP_INSTANCE_TREE (instance),
&life, &scalar_costs, visited);
/* Generate vector permute statements from a list of loads in DR_CHAIN.
If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
- permute statements for the SLP node NODE of the SLP instance
- SLP_NODE_INSTANCE. */
+ permute statements for the SLP node NODE. */
bool
vect_transform_slp_perm_load (vec_info *vinfo,
/* VECTYPE is the type of the destination. */
vectype = STMT_VINFO_VECTYPE (stmt_info);
poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype);
- group_size = SLP_INSTANCE_GROUP_SIZE (instance);
+ group_size = SLP_TREE_SCALAR_STMTS (node).length ();
gcc_assert (SLP_TREE_NUMBER_OF_VEC_STMTS (node) != 0);
SLP_TREE_VEC_STMTS (node).create (SLP_TREE_NUMBER_OF_VEC_STMTS (node));
vectorizable_load (vec_info *vinfo,
stmt_vec_info stmt_info, gimple_stmt_iterator *gsi,
stmt_vec_info *vec_stmt, slp_tree slp_node,
- slp_instance slp_node_instance,
stmt_vector_for_cost *cost_vec)
{
tree scalar_dest;
unpermuted sequence. In other cases we need to load the
whole group, not only the number of vector stmts the
permutation result fits in. */
+ unsigned scalar_lanes = SLP_TREE_SCALAR_STMTS (slp_node).length ();
if (slp_perm
- && (group_size != SLP_INSTANCE_GROUP_SIZE (slp_node_instance)
+ && (group_size != scalar_lanes
|| !multiple_p (nunits, group_size)))
{
/* We don't yet generate such SLP_TREE_LOAD_PERMUTATIONs for
{
vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node);
group_gap_adj
- = group_size - SLP_INSTANCE_GROUP_SIZE (slp_node_instance);
+ = group_size - scalar_lanes;
}
}
else
NULL, NULL, node, cost_vec)
|| vectorizable_assignment (vinfo, stmt_info,
NULL, NULL, node, cost_vec)
- || vectorizable_load (vinfo, stmt_info,
- NULL, NULL, node, node_instance, cost_vec)
+ || vectorizable_load (vinfo, stmt_info, NULL, NULL, node, cost_vec)
|| vectorizable_store (vinfo, stmt_info, NULL, NULL, node, cost_vec)
|| vectorizable_reduction (as_a <loop_vec_info> (vinfo), stmt_info,
node, node_instance, cost_vec)
|| vectorizable_assignment (vinfo, stmt_info, NULL, NULL, node,
cost_vec)
|| vectorizable_load (vinfo, stmt_info,
- NULL, NULL, node, node_instance, cost_vec)
+ NULL, NULL, node, cost_vec)
|| vectorizable_store (vinfo, stmt_info,
NULL, NULL, node, cost_vec)
|| vectorizable_condition (vinfo, stmt_info,
case load_vec_info_type:
done = vectorizable_load (vinfo, stmt_info, gsi, &vec_stmt, slp_node,
- slp_node_instance, NULL);
+ NULL);
gcc_assert (done);
break;
struct _slp_tree {
/* Nodes that contain def-stmts of this node statements operands. */
vec<slp_tree> children;
+
/* A group of scalar stmts to be vectorized together. */
vec<stmt_vec_info> stmts;
/* A group of scalar operands to be vectorized together. */
vec<tree> ops;
+
/* Load permutation relative to the stores, NULL if there is no
permutation. */
vec<unsigned> load_permutation;
+
/* Vectorized stmt/s. */
vec<stmt_vec_info> vec_stmts;
/* Number of vector stmts that are created to replace the group of scalar
scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
divided by vector size. */
unsigned int vec_stmts_size;
+
/* Reference count in the SLP graph. */
unsigned int refcnt;
/* The maximum number of vector elements for the subtree rooted
from, NULL otherwise. */
stmt_vec_info root_stmt;
- /* Size of groups of scalar stmts that will be replaced by SIMD stmt/s. */
- unsigned int group_size;
-
/* The unrolling factor required to vectorized this SLP instance. */
poly_uint64 unrolling_factor;
/* Access Functions. */
#define SLP_INSTANCE_TREE(S) (S)->root
-#define SLP_INSTANCE_GROUP_SIZE(S) (S)->group_size
#define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
#define SLP_INSTANCE_LOADS(S) (S)->loads
#define SLP_INSTANCE_ROOT_STMT(S) (S)->root_stmt
projects / gcc.git / commitdiff