From 1a58f770e487e94227fc75a42eee385d86b7bee9 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Tue, 30 May 2017 13:21:44 +0000 Subject: [PATCH] tree-vectorizer.h (struct _stmt_vec_info): Add reduc_type and reduc_def fields. 2017-05-30 Richard Biener * tree-vectorizer.h (struct _stmt_vec_info): Add reduc_type and reduc_def fields. (STMT_VINFO_REDUC_TYPE): New define. (STMT_VINFO_REDUC_DEF): Likewise. (vect_force_simple_reduction): Adjust prototype. * tree-vect-loop.c (vect_analyze_scalar_cycles_1): Adjust. (vect_is_simple_reduction): Remove check_reduction argument. (vect_force_simple_reduction): Adjust and set STMT_VINFO_REDUC_TYPE and STMT_VINFO_REDUC_DEF. (vectorizable_reduction): Do not re-do reduction analysis but use STMT_VINFO_REDUC_TYPE and STMT_VINFO_REDUC_DEF. * tree-parloops.c (gather_scalar_reductions): Adjust. From-SVN: r248685 --- gcc/ChangeLog | 15 ++++++++++++++ gcc/tree-parloops.c | 4 ++-- gcc/tree-vect-loop.c | 46 +++++++++++++++++++++++-------------------- gcc/tree-vectorizer.h | 11 ++++++++++- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 932ab6930a3..8598574542c 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,18 @@ +2017-05-30 Richard Biener + + * tree-vectorizer.h (struct _stmt_vec_info): Add reduc_type + and reduc_def fields. + (STMT_VINFO_REDUC_TYPE): New define. + (STMT_VINFO_REDUC_DEF): Likewise. + (vect_force_simple_reduction): Adjust prototype. + * tree-vect-loop.c (vect_analyze_scalar_cycles_1): Adjust. + (vect_is_simple_reduction): Remove check_reduction argument. + (vect_force_simple_reduction): Adjust and set + STMT_VINFO_REDUC_TYPE and STMT_VINFO_REDUC_DEF. + (vectorizable_reduction): Do not re-do reduction analysis + but use STMT_VINFO_REDUC_TYPE and STMT_VINFO_REDUC_DEF. + * tree-parloops.c (gather_scalar_reductions): Adjust. + 2017-05-30 Richard Biener PR middle-end/80901 diff --git a/gcc/tree-parloops.c b/gcc/tree-parloops.c index 83d3c803080..641f675edf7 100644 --- a/gcc/tree-parloops.c +++ b/gcc/tree-parloops.c @@ -2543,7 +2543,7 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list continue; gimple *reduc_stmt - = vect_force_simple_reduction (simple_loop_info, phi, true, + = vect_force_simple_reduction (simple_loop_info, phi, &double_reduc, true); if (!reduc_stmt) continue; @@ -2589,7 +2589,7 @@ gather_scalar_reductions (loop_p loop, reduction_info_table_type *reduction_list gimple *inner_reduc_stmt = vect_force_simple_reduction (simple_loop_info, inner_phi, - true, &double_reduc, true); + &double_reduc, true); gcc_assert (!double_reduc); if (inner_reduc_stmt == NULL) continue; diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index b41abf3335c..3133a968d69 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -844,7 +844,6 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop) tree def = PHI_RESULT (phi); stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi); gimple *reduc_stmt; - bool nested_cycle; if (dump_enabled_p ()) { @@ -855,8 +854,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop) gcc_assert (!virtual_operand_p (def) && STMT_VINFO_DEF_TYPE (stmt_vinfo) == vect_unknown_def_type); - nested_cycle = (loop != LOOP_VINFO_LOOP (loop_vinfo)); - reduc_stmt = vect_force_simple_reduction (loop_vinfo, phi, !nested_cycle, + reduc_stmt = vect_force_simple_reduction (loop_vinfo, phi, &double_reduc, false); if (reduc_stmt) { @@ -872,7 +870,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop) } else { - if (nested_cycle) + if (loop != LOOP_VINFO_LOOP (loop_vinfo)) { if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -2633,7 +2631,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi, } -/* Function vect_is_simple_reduction_1 +/* Function vect_is_simple_reduction (1) Detect a cross-iteration def-use cycle that represents a simple reduction computation. We look for the following pattern: @@ -2652,7 +2650,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi, such that: 1. operation is commutative and associative and it is safe to - change the order of the computation (if CHECK_REDUCTION is true) + change the order of the computation 2. no uses for a2 in the loop (a2 is used out of the loop) 3. no uses of a1 in the loop besides the reduction operation 4. no uses of a1 outside the loop. @@ -2661,7 +2659,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi, Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized. (2) Detect a cross-iteration def-use cycle in nested loops, i.e., - nested cycles, if CHECK_REDUCTION is false. + nested cycles. (3) Detect cycles of phi nodes in outer-loop vectorization, i.e., double reductions: @@ -2679,7 +2677,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi, static gimple * vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi, - bool check_reduction, bool *double_reduc, + bool *double_reduc, bool need_wrapping_integral_overflow, enum vect_reduction_type *v_reduc_type) { @@ -2700,8 +2698,8 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi, *double_reduc = false; *v_reduc_type = TREE_CODE_REDUCTION; - /* If CHECK_REDUCTION is true, we assume inner-most loop vectorization, - otherwise, we assume outer loop vectorization. */ + /* Check validity of the reduction only for the innermost loop. */ + bool check_reduction = ! flow_loop_nested_p (vect_loop, loop); gcc_assert ((check_reduction && loop == vect_loop) || (!check_reduction && flow_loop_nested_p (vect_loop, loop))); @@ -3120,20 +3118,26 @@ vect_is_simple_reduction (loop_vec_info loop_info, gimple *phi, return NULL; } -/* Wrapper around vect_is_simple_reduction_1, which will modify code +/* Wrapper around vect_is_simple_reduction, which will modify code in-place if it enables detection of more reductions. Arguments as there. */ gimple * vect_force_simple_reduction (loop_vec_info loop_info, gimple *phi, - bool check_reduction, bool *double_reduc, + bool *double_reduc, bool need_wrapping_integral_overflow) { enum vect_reduction_type v_reduc_type; - return vect_is_simple_reduction (loop_info, phi, check_reduction, - double_reduc, - need_wrapping_integral_overflow, - &v_reduc_type); + gimple *def = vect_is_simple_reduction (loop_info, phi, double_reduc, + need_wrapping_integral_overflow, + &v_reduc_type); + if (def) + { + stmt_vec_info reduc_def_info = vinfo_for_stmt (phi); + STMT_VINFO_REDUC_TYPE (reduc_def_info) = v_reduc_type; + STMT_VINFO_REDUC_DEF (reduc_def_info) = def; + } + return def; } /* Calculate cost of peeling the loop PEEL_ITERS_PROLOGUE times. */ @@ -5501,7 +5505,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, tree ops[3]; bool nested_cycle = false, found_nested_cycle_def = false; gimple *reduc_def_stmt = NULL; - bool double_reduc = false, dummy; + bool double_reduc = false; basic_block def_bb; struct loop * def_stmt_loop, *outer_loop = NULL; tree def_arg; @@ -5702,10 +5706,10 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, return false; } - enum vect_reduction_type v_reduc_type; - gimple *tmp = vect_is_simple_reduction (loop_vinfo, reduc_def_stmt, - !nested_cycle, &dummy, false, - &v_reduc_type); + stmt_vec_info reduc_def_info = vinfo_for_stmt (reduc_def_stmt); + enum vect_reduction_type v_reduc_type + = STMT_VINFO_REDUC_TYPE (reduc_def_info); + gimple *tmp = STMT_VINFO_REDUC_DEF (reduc_def_info); STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) = v_reduc_type; /* If we have a condition reduction, see if we can simplify it further. */ diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index dfbe59eb10d..a7f5c6e6f13 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -643,6 +643,13 @@ typedef struct _stmt_vec_info { /* For CONST_COND_REDUCTION, record the reduc code. */ enum tree_code const_cond_reduc_code; + /* On a reduction PHI the reduction type as detected by + vect_force_simple_reduction. */ + enum vect_reduction_type reduc_type; + + /* On a reduction PHI the def returned by vect_force_simple_reduction. */ + gimple *reduc_def; + /* The number of scalar stmt references from active SLP instances. */ unsigned int num_slp_uses; } *stmt_vec_info; @@ -722,6 +729,8 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo) #define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part #define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist #define STMT_VINFO_NUM_SLP_USES(S) (S)->num_slp_uses +#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type +#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def #define GROUP_FIRST_ELEMENT(S) (S)->first_element #define GROUP_NEXT_ELEMENT(S) (S)->next_element @@ -1137,7 +1146,7 @@ extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, /* In tree-vect-loop.c. */ /* FORNOW: Used in tree-parloops.c. */ extern void destroy_loop_vec_info (loop_vec_info, bool); -extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *, bool, +extern gimple *vect_force_simple_reduction (loop_vec_info, gimple *, bool *, bool); /* Drive for loop analysis stage. */ extern loop_vec_info vect_analyze_loop (struct loop *, loop_vec_info); -- 2.30.2