From a5b50aa1fb086c90dd0119f8cb9e0a88ea9cce58 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Wed, 11 Nov 2015 14:40:36 +0000 Subject: [PATCH] tree-vectorizer.h (vect_slp_analyze_and_verify_instance_alignment): Declare. 2015-11-11 Richard Biener * tree-vectorizer.h (vect_slp_analyze_and_verify_instance_alignment): Declare. (vect_analyze_data_refs_alignment): Make loop vect specific. (vect_verify_datarefs_alignment): Likewise. * tree-vect-data-refs.c (vect_slp_analyze_data_ref_dependences): Add missing continue. (vect_compute_data_ref_alignment): Export. (vect_compute_data_refs_alignment): Merge into... (vect_analyze_data_refs_alignment): ... this. (verify_data_ref_alignment): Split out from ... (vect_verify_datarefs_alignment): ... here. (vect_slp_analyze_and_verify_node_alignment): New function. (vect_slp_analyze_and_verify_instance_alignment): Likewise. * tree-vect-slp.c (vect_supported_load_permutation_p): Remove misplaced checks on alignment. (vect_slp_analyze_bb_1): Add fatal output parameter. Do alignment analysis after SLP discovery and do it per instance. (vect_slp_bb): When vect_slp_analyze_bb_1 fatally failed do not bother to re-try using different vector sizes. From-SVN: r230173 --- gcc/ChangeLog | 22 ++++ gcc/tree-vect-data-refs.c | 241 +++++++++++++++++++++++--------------- gcc/tree-vect-slp.c | 110 ++++++++--------- gcc/tree-vectorizer.h | 5 +- 4 files changed, 225 insertions(+), 153 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index b3bbc21c87a..ba87dcb4bd7 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2015-11-11 Richard Biener + + * tree-vectorizer.h (vect_slp_analyze_and_verify_instance_alignment): + Declare. + (vect_analyze_data_refs_alignment): Make loop vect specific. + (vect_verify_datarefs_alignment): Likewise. + * tree-vect-data-refs.c (vect_slp_analyze_data_ref_dependences): + Add missing continue. + (vect_compute_data_ref_alignment): Export. + (vect_compute_data_refs_alignment): Merge into... + (vect_analyze_data_refs_alignment): ... this. + (verify_data_ref_alignment): Split out from ... + (vect_verify_datarefs_alignment): ... here. + (vect_slp_analyze_and_verify_node_alignment): New function. + (vect_slp_analyze_and_verify_instance_alignment): Likewise. + * tree-vect-slp.c (vect_supported_load_permutation_p): Remove + misplaced checks on alignment. + (vect_slp_analyze_bb_1): Add fatal output parameter. Do + alignment analysis after SLP discovery and do it per instance. + (vect_slp_bb): When vect_slp_analyze_bb_1 fatally failed do not + bother to re-try using different vector sizes. + 2015-11-11 Nathan Sidwell Cesar Philippidis diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 87936ddee1d..f7471b8a75e 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -645,6 +645,7 @@ vect_slp_analyze_data_ref_dependences (bb_vec_info bb_vinfo) (SLP_INSTANCE_TREE (instance))[0], 0); vect_free_slp_instance (instance); BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i); + continue; } i++; } @@ -668,7 +669,7 @@ vect_slp_analyze_data_ref_dependences (bb_vec_info bb_vinfo) FOR NOW: No analysis is actually performed. Misalignment is calculated only for trivial cases. TODO. */ -static bool +bool vect_compute_data_ref_alignment (struct data_reference *dr) { gimple *stmt = DR_STMT (dr); @@ -838,45 +839,6 @@ vect_compute_data_ref_alignment (struct data_reference *dr) } -/* Function vect_compute_data_refs_alignment - - Compute the misalignment of data references in the loop. - Return FALSE if a data reference is found that cannot be vectorized. */ - -static bool -vect_compute_data_refs_alignment (vec_info *vinfo) -{ - vec datarefs = vinfo->datarefs; - struct data_reference *dr; - unsigned int i; - - FOR_EACH_VEC_ELT (datarefs, i, dr) - { - stmt_vec_info stmt_info = vinfo_for_stmt (DR_STMT (dr)); - if (STMT_VINFO_VECTORIZABLE (stmt_info) - && !vect_compute_data_ref_alignment (dr)) - { - /* Strided accesses perform only component accesses, misalignment - information is irrelevant for them. */ - if (STMT_VINFO_STRIDED_P (stmt_info) - && !STMT_VINFO_GROUPED_ACCESS (stmt_info)) - continue; - - if (is_a (vinfo)) - { - /* Mark unsupported statement as unvectorizable. */ - STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (DR_STMT (dr))) = false; - continue; - } - else - return false; - } - } - - return true; -} - - /* Function vect_update_misalignment_for_peel DR - the data reference whose misalignment is to be adjusted. @@ -936,63 +898,76 @@ vect_update_misalignment_for_peel (struct data_reference *dr, } +/* Function verify_data_ref_alignment + + Return TRUE if DR can be handled with respect to alignment. */ + +static bool +verify_data_ref_alignment (data_reference_p dr) +{ + enum dr_alignment_support supportable_dr_alignment; + gimple *stmt = DR_STMT (dr); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + + if (!STMT_VINFO_RELEVANT_P (stmt_info)) + return true; + + /* For interleaving, only the alignment of the first access matters. + Skip statements marked as not vectorizable. */ + if ((STMT_VINFO_GROUPED_ACCESS (stmt_info) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt) + || !STMT_VINFO_VECTORIZABLE (stmt_info)) + return true; + + /* Strided accesses perform only component accesses, alignment is + irrelevant for them. */ + if (STMT_VINFO_STRIDED_P (stmt_info) + && !STMT_VINFO_GROUPED_ACCESS (stmt_info)) + return true; + + supportable_dr_alignment = vect_supportable_dr_alignment (dr, false); + if (!supportable_dr_alignment) + { + if (dump_enabled_p ()) + { + if (DR_IS_READ (dr)) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: unsupported unaligned load."); + else + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: unsupported unaligned " + "store."); + + dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, + DR_REF (dr)); + dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); + } + return false; + } + + if (supportable_dr_alignment != dr_aligned && dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "Vectorizing an unaligned access.\n"); + + return true; +} + /* Function vect_verify_datarefs_alignment Return TRUE if all data references in the loop can be handled with respect to alignment. */ bool -vect_verify_datarefs_alignment (vec_info *vinfo) +vect_verify_datarefs_alignment (loop_vec_info vinfo) { vec datarefs = vinfo->datarefs; struct data_reference *dr; - enum dr_alignment_support supportable_dr_alignment; unsigned int i; FOR_EACH_VEC_ELT (datarefs, i, dr) - { - gimple *stmt = DR_STMT (dr); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - - if (!STMT_VINFO_RELEVANT_P (stmt_info)) - continue; - - /* For interleaving, only the alignment of the first access matters. - Skip statements marked as not vectorizable. */ - if ((STMT_VINFO_GROUPED_ACCESS (stmt_info) - && GROUP_FIRST_ELEMENT (stmt_info) != stmt) - || !STMT_VINFO_VECTORIZABLE (stmt_info)) - continue; - - /* Strided accesses perform only component accesses, alignment is - irrelevant for them. */ - if (STMT_VINFO_STRIDED_P (stmt_info) - && !STMT_VINFO_GROUPED_ACCESS (stmt_info)) - continue; - - supportable_dr_alignment = vect_supportable_dr_alignment (dr, false); - if (!supportable_dr_alignment) - { - if (dump_enabled_p ()) - { - if (DR_IS_READ (dr)) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: unsupported unaligned load."); - else - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: unsupported unaligned " - "store."); + if (! verify_data_ref_alignment (dr)) + return false; - dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_SLIM, - DR_REF (dr)); - dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); - } - return false; - } - if (supportable_dr_alignment != dr_aligned && dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "Vectorizing an unaligned access.\n"); - } return true; } @@ -2064,7 +2039,7 @@ vect_find_same_alignment_drs (struct data_dependence_relation *ddr, Return FALSE if a data reference is found that cannot be vectorized. */ bool -vect_analyze_data_refs_alignment (vec_info *vinfo) +vect_analyze_data_refs_alignment (loop_vec_info vinfo) { if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -2072,28 +2047,100 @@ vect_analyze_data_refs_alignment (vec_info *vinfo) /* Mark groups of data references with same alignment using data dependence information. */ - if (is_a (vinfo)) + vec ddrs = vinfo->ddrs; + struct data_dependence_relation *ddr; + unsigned int i; + + FOR_EACH_VEC_ELT (ddrs, i, ddr) + vect_find_same_alignment_drs (ddr, vinfo); + + vec datarefs = vinfo->datarefs; + struct data_reference *dr; + + FOR_EACH_VEC_ELT (datarefs, i, dr) { - vec ddrs = vinfo->ddrs; - struct data_dependence_relation *ddr; - unsigned int i; + stmt_vec_info stmt_info = vinfo_for_stmt (DR_STMT (dr)); + if (STMT_VINFO_VECTORIZABLE (stmt_info) + && !vect_compute_data_ref_alignment (dr)) + { + /* Strided accesses perform only component accesses, misalignment + information is irrelevant for them. */ + if (STMT_VINFO_STRIDED_P (stmt_info) + && !STMT_VINFO_GROUPED_ACCESS (stmt_info)) + continue; + + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: can't calculate alignment " + "for data ref.\n"); - FOR_EACH_VEC_ELT (ddrs, i, ddr) - vect_find_same_alignment_drs (ddr, as_a (vinfo)); + return false; + } } - if (!vect_compute_data_refs_alignment (vinfo)) + return true; +} + + +/* Analyze alignment of DRs of stmts in NODE. */ + +static bool +vect_slp_analyze_and_verify_node_alignment (slp_tree node) +{ + unsigned i; + gimple *stmt; + FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (node), i, stmt) { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: can't calculate alignment " - "for data ref.\n"); - return false; + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + + /* Strided accesses perform only component accesses, misalignment + information is irrelevant for them. */ + if (STMT_VINFO_STRIDED_P (stmt_info) + && !STMT_VINFO_GROUPED_ACCESS (stmt_info)) + continue; + + data_reference_p dr = STMT_VINFO_DATA_REF (stmt_info); + if (! vect_compute_data_ref_alignment (dr) + || ! verify_data_ref_alignment (dr)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: bad data alignment in basic " + "block.\n"); + return false; + } } return true; } +/* Function vect_slp_analyze_instance_alignment + + Analyze the alignment of the data-references in the SLP instance. + Return FALSE if a data reference is found that cannot be vectorized. */ + +bool +vect_slp_analyze_and_verify_instance_alignment (slp_instance instance) +{ + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "=== vect_slp_analyze_and_verify_instance_alignment ===\n"); + + slp_tree node; + unsigned i; + FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node) + if (! vect_slp_analyze_and_verify_node_alignment (node)) + return false; + + node = SLP_INSTANCE_TREE (instance); + if (STMT_VINFO_DATA_REF (vinfo_for_stmt (SLP_TREE_SCALAR_STMTS (node)[0])) + && ! vect_slp_analyze_and_verify_node_alignment + (SLP_INSTANCE_TREE (instance))) + return false; + + return true; +} + /* Analyze groups of accesses: check that DR belongs to a group of accesses of legal size, step, etc. Detect gaps, single element diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 9d971405e07..75875f333ad 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -1282,8 +1282,7 @@ vect_supported_load_permutation_p (slp_instance slp_instn) unsigned int group_size = SLP_INSTANCE_GROUP_SIZE (slp_instn); unsigned int i, j, k, next; slp_tree node; - gimple *stmt, *load, *next_load, *first_load; - struct data_reference *dr; + gimple *stmt, *load, *next_load; if (dump_enabled_p ()) { @@ -1365,33 +1364,6 @@ vect_supported_load_permutation_p (slp_instance slp_instn) } } } - - /* Check that the alignment of the first load in every subchain, i.e., - the first statement in every load node, is supported. - ??? This belongs in alignment checking. */ - FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (slp_instn), i, node) - { - first_load = SLP_TREE_SCALAR_STMTS (node)[0]; - if (first_load != GROUP_FIRST_ELEMENT (vinfo_for_stmt (first_load))) - { - dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_load)); - if (vect_supportable_dr_alignment (dr, false) - == dr_unaligned_unsupported) - { - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_MISSED_OPTIMIZATION, - vect_location, - "unsupported unaligned load "); - dump_gimple_stmt (MSG_MISSED_OPTIMIZATION, TDF_SLIM, - first_load, 0); - dump_printf (MSG_MISSED_OPTIMIZATION, "\n"); - } - return false; - } - } - } - return true; } @@ -2311,12 +2283,15 @@ vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo) return true; } -/* Check if the basic block can be vectorized. */ +/* Check if the basic block can be vectorized. Returns a bb_vec_info + if so and sets fatal to true if failure is independent of + current_vector_size. */ static bb_vec_info vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, gimple_stmt_iterator region_end, - vec datarefs, int n_stmts) + vec datarefs, int n_stmts, + bool &fatal) { bb_vec_info bb_vinfo; vec slp_instances; @@ -2324,6 +2299,9 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, int i; int min_vf = 2; + /* The first group of checks is independent of the vector size. */ + fatal = true; + if (n_stmts > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB)) { if (dump_enabled_p ()) @@ -2375,19 +2353,25 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, return NULL; } - vect_pattern_recog (bb_vinfo); - - if (!vect_analyze_data_refs_alignment (bb_vinfo)) + /* If there are no grouped stores in the region there is no need + to continue with pattern recog as vect_analyze_slp will fail + anyway. */ + if (bb_vinfo->grouped_stores.is_empty ()) { if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: bad data alignment in basic " - "block.\n"); + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: no grouped stores in " + "basic block.\n"); destroy_bb_vec_info (bb_vinfo); return NULL; } + /* While the rest of the analysis below depends on it in some way. */ + fatal = false; + + vect_pattern_recog (bb_vinfo); + /* Check the SLP opportunities in the basic block, analyze and build SLP trees. */ if (!vect_analyze_slp (bb_vinfo, n_stmts)) @@ -2405,6 +2389,30 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, return NULL; } + /* Analyze and verify the alignment of data references in the SLP + instances. */ + for (i = 0; BB_VINFO_SLP_INSTANCES (bb_vinfo).iterate (i, &instance); ) + { + if (! vect_slp_analyze_and_verify_instance_alignment (instance)) + { + dump_printf_loc (MSG_NOTE, vect_location, + "removing SLP instance operations starting from: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, + SLP_TREE_SCALAR_STMTS + (SLP_INSTANCE_TREE (instance))[0], 0); + vect_free_slp_instance (instance); + BB_VINFO_SLP_INSTANCES (bb_vinfo).ordered_remove (i); + continue; + } + i++; + } + + if (! BB_VINFO_SLP_INSTANCES (bb_vinfo).length ()) + { + destroy_bb_vec_info (bb_vinfo); + return NULL; + } + slp_instances = BB_VINFO_SLP_INSTANCES (bb_vinfo); /* Mark all the statements that we want to vectorize as pure SLP and @@ -2427,23 +2435,13 @@ vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin, /* Analyze dependences. At this point all stmts not participating in vectorization have to be marked. Dependence analysis assumes that we either vectorize all SLP instances or none at all. */ - if (!vect_slp_analyze_data_ref_dependences (bb_vinfo)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: unhandled data dependence " - "in basic block.\n"); - - destroy_bb_vec_info (bb_vinfo); - return NULL; - } - - if (!vect_verify_datarefs_alignment (bb_vinfo)) + if (! vect_slp_analyze_data_ref_dependences (bb_vinfo)) { if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "not vectorized: unsupported alignment in basic " - "block.\n"); + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "not vectorized: unhandled data dependence " + "in basic block.\n"); + destroy_bb_vec_info (bb_vinfo); return NULL; } @@ -2533,8 +2531,9 @@ vect_slp_bb (basic_block bb) gimple_stmt_iterator region_end = gsi; bool vectorized = false; + bool fatal = false; bb_vinfo = vect_slp_analyze_bb_1 (region_begin, region_end, - datarefs, insns); + datarefs, insns, fatal); if (bb_vinfo && dbg_cnt (vect_slp)) { @@ -2559,7 +2558,10 @@ vect_slp_bb (basic_block bb) vector_sizes &= ~current_vector_size; if (vectorized || vector_sizes == 0 - || current_vector_size == 0) + || current_vector_size == 0 + /* If vect_slp_analyze_bb_1 signaled that analysis for all + vector sizes will fail do not bother iterating. */ + || fatal) { if (gsi_end_p (region_end)) break; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 6ad0cc452f1..eaeeb05f2a2 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1011,8 +1011,9 @@ extern tree vect_get_smallest_scalar_type (gimple *, HOST_WIDE_INT *, extern bool vect_analyze_data_ref_dependences (loop_vec_info, int *); extern bool vect_slp_analyze_data_ref_dependences (bb_vec_info); extern bool vect_enhance_data_refs_alignment (loop_vec_info); -extern bool vect_analyze_data_refs_alignment (vec_info *); -extern bool vect_verify_datarefs_alignment (vec_info *); +extern bool vect_analyze_data_refs_alignment (loop_vec_info); +extern bool vect_verify_datarefs_alignment (loop_vec_info); +extern bool vect_slp_analyze_and_verify_instance_alignment (slp_instance); extern bool vect_analyze_data_ref_accesses (vec_info *); extern bool vect_prune_runtime_alias_test_list (loop_vec_info); extern tree vect_check_gather_scatter (gimple *, loop_vec_info, tree *, tree *, -- 2.30.2