From: Richard Biener Date: Wed, 16 Sep 2020 09:24:23 +0000 (+0200) Subject: remove STMT_VINFO_NUM_SLP_USES X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=f02b2077c093922faee3b33ad821b717bea80343;p=gcc.git remove STMT_VINFO_NUM_SLP_USES This removes STMT_VINFO_NUM_SLP_USES by pushing the setting of the shared stmt_vec_info vector type to where we actually need it which is alignment analysis and vectorizable_* analysis (where we could eventually elide it for non-load/store operations). In particular "uses" in the cache and in disqualified SLP subgraphs should no longer provide conflicting vector types this way. 2020-09-16 Richard Biener * tree-vectorizer.h (_stmt_vec_info::num_slp_uses): Remove. (STMT_VINFO_NUM_SLP_USES): Likewise. (vect_free_slp_instance): Adjust. (vect_update_shared_vectype): Declare. * tree-vectorizer.c (vec_info::~vec_info): Adjust. * tree-vect-loop.c (vect_analyze_loop_2): Likewise. (vectorizable_live_operation): Use vector type from SLP_TREE_REPRESENTATIVE. (vect_transform_loop): Adjust. * tree-vect-data-refs.c (vect_slp_analyze_node_alignment): Set the shared vector type. * tree-vect-slp.c (vect_free_slp_tree): Remove final_p parameter, remove STMT_VINFO_NUM_SLP_USES updating. (vect_free_slp_instance): Adjust. (vect_create_new_slp_node): Remove STMT_VINFO_NUM_SLP_USES updating. (vect_update_shared_vectype): Always compare with the present vector type, update if NULL. (vect_build_slp_tree_1): Do not update the shared vector type here. (vect_build_slp_tree_2): Adjust. (slp_copy_subtree): Likewise. (vect_attempt_slp_rearrange_stmts): Likewise. (vect_analyze_slp_instance): Likewise. (vect_analyze_slp): Likewise. (vect_slp_analyze_node_operations_1): Update the shared vector type. (vect_slp_analyze_operations): Adjust. (vect_slp_analyze_bb_1): Likewise. --- diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 88c5faed20d..5bf93e2942b 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -2380,6 +2380,11 @@ vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node) if (SLP_TREE_LOAD_PERMUTATION (node).exists ()) first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info); + /* We need to commit to a vector type for the group now. */ + if (is_a (vinfo) + && !vect_update_shared_vectype (first_stmt_info, SLP_TREE_VECTYPE (node))) + return false; + dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info); vect_compute_data_ref_alignment (vinfo, dr_info); /* In several places we need alignment of the first element anyway. */ diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index c95ec5ad267..3af4cf74659 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -2467,7 +2467,7 @@ again: LOOP_VINFO_VECT_FACTOR (loop_vinfo) = saved_vectorization_factor; /* Free the SLP instances. */ FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), j, instance) - vect_free_slp_instance (instance, false); + vect_free_slp_instance (instance); LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release (); /* Reset SLP type to loop_vect on all stmts. */ for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i) @@ -8031,7 +8031,9 @@ vectorizable_live_operation (vec_info *vinfo, loop_vec_info loop_vinfo = dyn_cast (vinfo); imm_use_iterator imm_iter; tree lhs, lhs_type, bitsize, vec_bitsize; - tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree vectype = (slp_node + ? SLP_TREE_VECTYPE (SLP_TREE_REPRESENTATIVE (slp_node)) + : STMT_VINFO_VECTYPE (stmt_info)); poly_uint64 nunits = TYPE_VECTOR_SUBPARTS (vectype); int ncopies; gimple *use_stmt; @@ -9260,7 +9262,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) won't work. */ slp_instance instance; FOR_EACH_VEC_ELT (LOOP_VINFO_SLP_INSTANCES (loop_vinfo), i, instance) - vect_free_slp_instance (instance, true); + vect_free_slp_instance (instance); LOOP_VINFO_SLP_INSTANCES (loop_vinfo).release (); /* Clear-up safelen field since its value is invalid after vectorization since vectorized loop can have loop-carried dependencies. */ diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index f9f4d706f0a..ecce7a982dd 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -84,12 +84,10 @@ _slp_tree::~_slp_tree () SLP_TREE_LANE_PERMUTATION (this).release (); } -/* Recursively free the memory allocated for the SLP tree rooted at NODE. - FINAL_P is true if we have vectorized the instance or if we have - made a final decision not to vectorize the statements in any way. */ +/* Recursively free the memory allocated for the SLP tree rooted at NODE. */ static void -vect_free_slp_tree (slp_tree node, bool final_p) +vect_free_slp_tree (slp_tree node) { int i; slp_tree child; @@ -98,21 +96,7 @@ vect_free_slp_tree (slp_tree node, bool final_p) return; FOR_EACH_VEC_ELT (SLP_TREE_CHILDREN (node), i, child) - vect_free_slp_tree (child, final_p); - - /* Don't update STMT_VINFO_NUM_SLP_USES if it isn't relevant. - Some statements might no longer exist, after having been - removed by vect_transform_stmt. Updating the remaining - statements would be redundant. */ - if (!final_p) - { - stmt_vec_info stmt_info; - FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info) - { - gcc_assert (STMT_VINFO_NUM_SLP_USES (stmt_info) > 0); - STMT_VINFO_NUM_SLP_USES (stmt_info)--; - } - } + vect_free_slp_tree (child); delete node; } @@ -129,14 +113,12 @@ _slp_instance::location () const } -/* Free the memory allocated for the SLP instance. FINAL_P is true if we - have vectorized the instance or if we have made a final decision not - to vectorize the statements in any way. */ +/* Free the memory allocated for the SLP instance. */ void -vect_free_slp_instance (slp_instance instance, bool final_p) +vect_free_slp_instance (slp_instance instance) { - vect_free_slp_tree (SLP_INSTANCE_TREE (instance), final_p); + vect_free_slp_tree (SLP_INSTANCE_TREE (instance)); SLP_INSTANCE_LOADS (instance).release (); instance->subgraph_entries.release (); instance->cost_vec.release (); @@ -155,12 +137,6 @@ vect_create_new_slp_node (vec scalar_stmts, unsigned nops) SLP_TREE_DEF_TYPE (node) = vect_internal_def; SLP_TREE_REPRESENTATIVE (node) = scalar_stmts[0]; SLP_TREE_LANES (node) = scalar_stmts.length (); - - unsigned i; - stmt_vec_info stmt_info; - FOR_EACH_VEC_ELT (scalar_stmts, i, stmt_info) - STMT_VINFO_NUM_SLP_USES (stmt_info)++; - return node; } @@ -614,23 +590,23 @@ again: Return true if we can, meaning that this choice doesn't conflict with existing SLP nodes that use STMT_INFO. */ -static bool +bool vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype) { tree old_vectype = STMT_VINFO_VECTYPE (stmt_info); - if (old_vectype && useless_type_conversion_p (vectype, old_vectype)) - return true; + if (old_vectype) + return useless_type_conversion_p (vectype, old_vectype); - if (STMT_VINFO_GROUPED_ACCESS (stmt_info) - && DR_IS_READ (STMT_VINFO_DATA_REF (stmt_info))) + if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) { /* We maintain the invariant that if any statement in the group is used, all other members of the group have the same vector type. */ stmt_vec_info first_info = DR_GROUP_FIRST_ELEMENT (stmt_info); stmt_vec_info member_info = first_info; for (; member_info; member_info = DR_GROUP_NEXT_ELEMENT (member_info)) - if (STMT_VINFO_NUM_SLP_USES (member_info) > 0 - || is_pattern_stmt_p (member_info)) + if (is_pattern_stmt_p (member_info) + && !useless_type_conversion_p (vectype, + STMT_VINFO_VECTYPE (member_info))) break; if (!member_info) @@ -641,8 +617,7 @@ vect_update_shared_vectype (stmt_vec_info stmt_info, tree vectype) return true; } } - else if (STMT_VINFO_NUM_SLP_USES (stmt_info) == 0 - && !is_pattern_stmt_p (stmt_info)) + else if (!is_pattern_stmt_p (stmt_info)) { STMT_VINFO_VECTYPE (stmt_info) = vectype; return true; @@ -824,10 +799,6 @@ vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap, gcc_assert (vectype); - if (is_a (vinfo) - && !vect_update_shared_vectype (stmt_info, vectype)) - continue; - gcall *call_stmt = dyn_cast (stmt); if (call_stmt) { @@ -1561,7 +1532,7 @@ fail: gcc_assert (child == NULL); FOR_EACH_VEC_ELT (children, j, child) - vect_free_slp_tree (child, false); + vect_free_slp_tree (child); vect_free_oprnd_info (oprnds_info); return NULL; } @@ -1589,7 +1560,7 @@ fail: /* Roll back. */ matches[0] = false; FOR_EACH_VEC_ELT (children, j, child) - vect_free_slp_tree (child, false); + vect_free_slp_tree (child); if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -1833,12 +1804,7 @@ slp_copy_subtree (slp_tree node, hash_map &map) copy->max_nunits = node->max_nunits; copy->refcnt = 0; if (SLP_TREE_SCALAR_STMTS (node).exists ()) - { - SLP_TREE_SCALAR_STMTS (copy) = SLP_TREE_SCALAR_STMTS (node).copy (); - stmt_vec_info stmt_info; - FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt_info) - STMT_VINFO_NUM_SLP_USES (stmt_info)++; - } + SLP_TREE_SCALAR_STMTS (copy) = SLP_TREE_SCALAR_STMTS (node).copy (); if (SLP_TREE_SCALAR_OPS (node).exists ()) SLP_TREE_SCALAR_OPS (copy) = SLP_TREE_SCALAR_OPS (node).copy (); if (SLP_TREE_LOAD_PERMUTATION (node).exists ()) @@ -1968,7 +1934,7 @@ vect_attempt_slp_rearrange_stmts (slp_instance slp_instn) /* We have to unshare the SLP tree we modify. */ hash_map map; slp_tree unshared = slp_copy_subtree (SLP_INSTANCE_TREE (slp_instn), map); - vect_free_slp_tree (SLP_INSTANCE_TREE (slp_instn), false); + vect_free_slp_tree (SLP_INSTANCE_TREE (slp_instn)); unshared->refcnt++; SLP_INSTANCE_TREE (slp_instn) = unshared; FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node) @@ -2280,7 +2246,7 @@ vect_analyze_slp_instance (vec_info *vinfo, "Build SLP failed: store group " "size not a multiple of the vector size " "in basic block SLP\n"); - vect_free_slp_tree (node, false); + vect_free_slp_tree (node); return false; } /* Fatal mismatch. */ @@ -2290,7 +2256,7 @@ vect_analyze_slp_instance (vec_info *vinfo, "splitting\n"); matches[0] = true; matches[group_size / const_max_nunits * const_max_nunits] = false; - vect_free_slp_tree (node, false); + vect_free_slp_tree (node); } else { @@ -2351,7 +2317,7 @@ vect_analyze_slp_instance (vec_info *vinfo, dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, "Built SLP cancelled: can use " "load/store-lanes\n"); - vect_free_slp_instance (new_instance, false); + vect_free_slp_instance (new_instance); return false; } } @@ -2517,7 +2483,7 @@ vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size) for (scalar_stmts_to_slp_tree_map_t::iterator it = bst_map->begin (); it != bst_map->end (); ++it) if ((*it).second) - vect_free_slp_tree ((*it).second, false); + vect_free_slp_tree ((*it).second); delete bst_map; /* Optimize permutations in SLP reductions. */ @@ -2839,6 +2805,10 @@ vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node, if (SLP_TREE_CODE (node) == VEC_PERM_EXPR) return vectorizable_slp_permutation (vinfo, NULL, node, cost_vec); + if (is_a (vinfo) + && !vect_update_shared_vectype (stmt_info, SLP_TREE_VECTYPE (node))) + return false; + bool dummy; return vect_analyze_stmt (vinfo, stmt_info, &dummy, node, node_instance, cost_vec); @@ -3164,7 +3134,7 @@ vect_slp_analyze_operations (vec_info *vinfo) dump_printf_loc (MSG_NOTE, vect_location, "removing SLP instance operations starting from: %G", stmt_info->stmt); - vect_free_slp_instance (instance, false); + vect_free_slp_instance (instance); vinfo->slp_instances.ordered_remove (i); cost_vec.release (); } @@ -3590,7 +3560,7 @@ vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal, dump_printf_loc (MSG_NOTE, vect_location, "removing SLP instance operations starting from: %G", stmt_info->stmt); - vect_free_slp_instance (instance, false); + vect_free_slp_instance (instance); BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i); continue; } diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index bbe2de56365..37e36f556e5 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -473,7 +473,7 @@ vec_info::~vec_info () unsigned int i; FOR_EACH_VEC_ELT (slp_instances, i, instance) - vect_free_slp_instance (instance, true); + vect_free_slp_instance (instance); destroy_cost_data (target_cost_data); free_stmt_vec_infos (); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 6c29ee6cfed..9dffc5570e5 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1223,9 +1223,6 @@ public: /* Whether on this stmt reduction meta is recorded. */ bool is_reduc_info; - /* The number of scalar stmt references from active SLP instances. */ - unsigned int num_slp_uses; - /* If nonzero, the lhs of the statement could be truncated to this many bits without affecting any users of the result. */ unsigned int min_output_precision; @@ -1340,7 +1337,6 @@ struct gather_scatter_info { #define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part #define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist -#define STMT_VINFO_NUM_SLP_USES(S) (S)->num_slp_uses #define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type #define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code #define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn @@ -2024,7 +2020,7 @@ extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, extern tree cse_and_gimplify_to_preheader (loop_vec_info, tree); /* In tree-vect-slp.c. */ -extern void vect_free_slp_instance (slp_instance, bool); +extern void vect_free_slp_instance (slp_instance); extern bool vect_transform_slp_perm_load (vec_info *, slp_tree, vec, gimple_stmt_iterator *, poly_uint64, bool, unsigned *); @@ -2047,6 +2043,7 @@ extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree, extern void duplicate_and_interleave (vec_info *, gimple_seq *, tree, vec, unsigned int, vec &); extern int vect_get_place_in_interleaving_chain (stmt_vec_info, stmt_vec_info); +extern bool vect_update_shared_vectype (stmt_vec_info, tree); /* In tree-vect-patterns.c. */ /* Pattern recognition functions.