From f78347996e02a8a767a525bfb764e769afe29d67 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Wed, 9 Oct 2019 07:02:22 +0000 Subject: [PATCH] tree-vectorizer.h (_stmt_vec_info::reduc_vectype_in): New. 2019-10-08 Richard Biener * tree-vectorizer.h (_stmt_vec_info::reduc_vectype_in): New. (_stmt_vec_info::force_single_cycle): Likewise. (STMT_VINFO_FORCE_SINGLE_CYCLE): New. (STMT_VINFO_REDUC_VECTYPE_IN): Likewise. * tree-vect-loop.c (vectorizable_reduction): Set STMT_VINFO_REDUC_VECTYPE_IN and STMT_VINFO_FORCE_SINGLE_CYCLE. (vect_transform_reduction): Use them to remove redundant code. (vect_transform_cycle_phi): Likewise. From-SVN: r276752 --- gcc/ChangeLog | 11 +++ gcc/tree-vect-loop.c | 188 +++++++----------------------------------- gcc/tree-vectorizer.h | 8 ++ 3 files changed, 49 insertions(+), 158 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 1045a47b6b0..02143107211 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,14 @@ +2019-10-08 Richard Biener + + * tree-vectorizer.h (_stmt_vec_info::reduc_vectype_in): New. + (_stmt_vec_info::force_single_cycle): Likewise. + (STMT_VINFO_FORCE_SINGLE_CYCLE): New. + (STMT_VINFO_REDUC_VECTYPE_IN): Likewise. + * tree-vect-loop.c (vectorizable_reduction): Set + STMT_VINFO_REDUC_VECTYPE_IN and STMT_VINFO_FORCE_SINGLE_CYCLE. + (vect_transform_reduction): Use them to remove redundant code. + (vect_transform_cycle_phi): Likewise. + 2019-10-08 Dmitrij Pochepko PR tree-optimization/90836 diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index f63bb855618..a9ea0caf218 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -5983,9 +5983,13 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, } } } - if (!vectype_in) vectype_in = vectype_out; + STMT_VINFO_REDUC_VECTYPE_IN (reduc_info) = vectype_in; + /* For the SSA cycle we store on each participating stmt the operand index + where the cycle continues. Store the one relevant for the actual + operation in the reduction meta. */ + STMT_VINFO_REDUC_IDX (reduc_info) = reduc_index; /* When vectorizing a reduction chain w/o SLP the reduction PHI is not directy used in stmt. */ @@ -6457,7 +6461,7 @@ vectorizable_reduction (stmt_vec_info stmt_info, slp_tree slp_node, && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info) || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info)) && vect_stmt_to_vectorize (use_stmt_info) == stmt_info) - single_defuse_cycle = true; + STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info) = single_defuse_cycle = true; if (single_defuse_cycle || code == DOT_PROD_EXPR @@ -6584,17 +6588,11 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, stmt_vec_info *vec_stmt, slp_tree slp_node) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); - tree vectype_in = NULL_TREE; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - enum tree_code code; - int op_type; - bool is_simple_use; int i; int ncopies; - bool single_defuse_cycle = false; int j; - tree ops[3]; int vec_num; stmt_vec_info reduc_info = info_for_reduction (stmt_info); @@ -6607,30 +6605,20 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, } gassign *stmt = as_a (stmt_info->stmt); + enum tree_code code = gimple_assign_rhs_code (stmt); + int op_type = TREE_CODE_LENGTH (code); /* Flatten RHS. */ - switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt))) + tree ops[3]; + switch (get_gimple_rhs_class (code)) { - case GIMPLE_BINARY_RHS: - code = gimple_assign_rhs_code (stmt); - op_type = TREE_CODE_LENGTH (code); - gcc_assert (op_type == binary_op); - ops[0] = gimple_assign_rhs1 (stmt); - ops[1] = gimple_assign_rhs2 (stmt); - break; - case GIMPLE_TERNARY_RHS: - code = gimple_assign_rhs_code (stmt); - op_type = TREE_CODE_LENGTH (code); - gcc_assert (op_type == ternary_op); + ops[2] = gimple_assign_rhs3 (stmt); + /* Fall thru. */ + case GIMPLE_BINARY_RHS: ops[0] = gimple_assign_rhs1 (stmt); ops[1] = gimple_assign_rhs2 (stmt); - ops[2] = gimple_assign_rhs3 (stmt); break; - - case GIMPLE_UNARY_RHS: - return false; - default: gcc_unreachable (); } @@ -6641,110 +6629,19 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, reduction variable. */ stmt_vec_info phi_info = STMT_VINFO_REDUC_DEF (vect_orig_stmt (stmt_info)); gphi *reduc_def_phi = as_a (phi_info->stmt); - tree reduc_def = PHI_RESULT (reduc_def_phi); - int reduc_index = -1; - for (i = 0; i < op_type; i++) - { - /* The condition of COND_EXPR is checked in vectorizable_condition(). */ - if (i == 0 && code == COND_EXPR) - continue; - - stmt_vec_info def_stmt_info; - enum vect_def_type dt; - tree tem; - is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, &dt, &tem, - &def_stmt_info); - gcc_assert (is_simple_use); - if (dt == vect_reduction_def - && ops[i] == reduc_def) - { - reduc_index = i; - continue; - } - else if (tem) - { - /* To properly compute ncopies we are interested in the widest - input type in case we're looking at a widening accumulation. */ - if (!vectype_in - || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in))) - < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (tem))))) - vectype_in = tem; - } - - if (dt == vect_nested_cycle - && ops[i] == reduc_def) - { - reduc_index = i; - } - } - - if (!vectype_in) - vectype_in = vectype_out; + int reduc_index = STMT_VINFO_REDUC_IDX (reduc_info); + tree vectype_in = STMT_VINFO_REDUC_VECTYPE_IN (reduc_info); if (slp_node) - ncopies = 1; - else - ncopies = vect_get_num_copies (loop_vinfo, vectype_in); - - vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info); - - /* In case of widenning multiplication by a constant, we update the type - of the constant to be the type of the other operand. We check that the - constant fits the type in the pattern recognition pass. */ - if (code == DOT_PROD_EXPR - && !types_compatible_p (TREE_TYPE (ops[0]), TREE_TYPE (ops[1]))) { - gcc_unreachable (); - /* No testcase for this. PR49478. */ - if (TREE_CODE (ops[0]) == INTEGER_CST) - ops[0] = fold_convert (TREE_TYPE (ops[1]), ops[0]); - else if (TREE_CODE (ops[1]) == INTEGER_CST) - ops[1] = fold_convert (TREE_TYPE (ops[0]), ops[1]); + ncopies = 1; + vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); } - - /* In case the vectorization factor (VF) is bigger than the number - of elements that we can fit in a vectype (nunits), we have to generate - more than one vector stmt - i.e - we need to "unroll" the - vector stmt by a factor VF/nunits. For more details see documentation - in vectorizable_operation. */ - - /* If the reduction is used in an outer loop we need to generate - VF intermediate results, like so (e.g. for ncopies=2): - r0 = phi (init, r0) - r1 = phi (init, r1) - r0 = x0 + r0; - r1 = x1 + r1; - (i.e. we generate VF results in 2 registers). - In this case we have a separate def-use cycle for each copy, and therefore - for each copy we get the vector def for the reduction variable from the - respective phi node created for this copy. - - Otherwise (the reduction is unused in the loop nest), we can combine - together intermediate results, like so (e.g. for ncopies=2): - r = phi (init, r) - r = x0 + r; - r = x1 + r; - (i.e. we generate VF/2 results in a single register). - In this case for each copy we get the vector def for the reduction variable - from the vectorized reduction operation generated in the previous iteration. - - This only works when we see both the reduction PHI and its only consumer - in vectorizable_reduction and there are no intermediate stmts - participating. */ - stmt_vec_info use_stmt_info; - tree reduc_phi_result = gimple_phi_result (reduc_def_phi); - if (ncopies > 1 - && (STMT_VINFO_RELEVANT (stmt_info) <= vect_used_only_live) - && (use_stmt_info = loop_vinfo->lookup_single_use (reduc_phi_result)) - && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info) - || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info)) - && vect_stmt_to_vectorize (use_stmt_info) == stmt_info) - single_defuse_cycle = true; - - if (slp_node) - vec_num = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); else - vec_num = 1; + { + ncopies = vect_get_num_copies (loop_vinfo, vectype_in); + vec_num = 1; + } internal_fn cond_fn = get_conditional_internal_fn (code); vec_loop_masks *masks = &LOOP_VINFO_MASKS (loop_vinfo); @@ -6768,6 +6665,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, bool masked_loop_p = LOOP_VINFO_FULLY_MASKED_P (loop_vinfo); + vect_reduction_type reduction_type = STMT_VINFO_REDUC_TYPE (reduc_info); if (reduction_type == FOLD_LEFT_REDUCTION) { internal_fn reduc_fn = STMT_VINFO_REDUC_FN (reduc_info); @@ -6776,6 +6674,7 @@ vect_transform_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, reduc_fn, ops, vectype_in, reduc_index, masks); } + bool single_defuse_cycle = STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info); gcc_assert (single_defuse_cycle || code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR @@ -6935,10 +6834,8 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, slp_tree slp_node, slp_instance slp_node_instance) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); - tree vectype_in = NULL_TREE; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); - enum vect_def_type dt; int i; int ncopies; stmt_vec_info prev_phi_info; @@ -6952,9 +6849,6 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, nested_cycle = true; } - gphi *phi = as_a (stmt_info->stmt); - tree phi_result = gimple_phi_result (phi); - stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info); reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info); stmt_vec_info reduc_info = info_for_reduction (stmt_info); @@ -6965,26 +6859,8 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, /* Leave the scalar phi in place. */ return true; - if (gassign *reduc_stmt = dyn_cast (reduc_stmt_info->stmt)) - for (unsigned k = 1; k < gimple_num_ops (reduc_stmt); ++k) - { - tree op = gimple_op (reduc_stmt, k); - if (op == phi_result) - continue; - if (k == 1 && gimple_assign_rhs_code (reduc_stmt) == COND_EXPR) - continue; - bool is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt); - gcc_assert (is_simple_use); - if (dt == vect_constant_def || dt == vect_external_def) - continue; - if (!vectype_in - || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in))) - < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (op))))) - vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op)); - break; - } - /* For a nested cycle we might end up with an operation like - phi_result * phi_result. */ + tree vectype_in = STMT_VINFO_REDUC_VECTYPE_IN (reduc_info); + /* For a nested cycle we do not fill the above. */ if (!vectype_in) vectype_in = STMT_VINFO_VECTYPE (stmt_info); gcc_assert (vectype_in); @@ -7003,19 +6879,15 @@ vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, ncopies = vect_get_num_copies (loop_vinfo, vectype_in); } - /* Check whether we can use a single PHI node and accumulate + /* Check whether we should use a single PHI node and accumulate vectors to one before the backedge. */ - stmt_vec_info use_stmt_info; - if (ncopies > 1 - && STMT_VINFO_RELEVANT (reduc_stmt_info) <= vect_used_only_live - && (use_stmt_info = loop_vinfo->lookup_single_use (phi_result)) - && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info) - || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info)) - && vect_stmt_to_vectorize (use_stmt_info) == reduc_stmt_info) + if (STMT_VINFO_FORCE_SINGLE_CYCLE (reduc_info)) ncopies = 1; /* Create the destination vector */ - tree vec_dest = vect_create_destination_var (phi_result, vectype_out); + gphi *phi = as_a (stmt_info->stmt); + tree vec_dest = vect_create_destination_var (gimple_phi_result (phi), + vectype_out); /* Get the loop-entry arguments. */ tree vec_initial_def; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 71ea6adde04..484de4f6e11 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -961,6 +961,12 @@ public: corresponding PHI. */ stmt_vec_info reduc_def; + /* The vector input type relevant for reduction vectorization. */ + tree reduc_vectype_in; + + /* Whether we force a single cycle PHI during reduction vectorization. */ + bool force_single_cycle; + /* Whether on this stmt reduction meta is recorded. */ bool is_reduc_info; @@ -1050,6 +1056,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo) #define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val #define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment #define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx +#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle #define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop #define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address @@ -1083,6 +1090,7 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo) #define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code #define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn #define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def +#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in #define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p #define DR_GROUP_FIRST_ELEMENT(S) \ -- 2.30.2