From 291fa23ac04e317877c1e102937532f080180bb2 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Wed, 2 Oct 2019 09:21:57 +0000 Subject: [PATCH] tree-vectorizer.h (stmt_vec_info_type::cycle_phi_info_type): New. 2019-10-02 Richard Biener * tree-vectorizer.h (stmt_vec_info_type::cycle_phi_info_type): New. (vect_transform_cycle_phi): Declare. * tree-vect-stmts.c (vect_transform_stmt): Call vect_transform_cycle_phi. * tree-vect-loop.c (vectorizable_reduction): Split out PHI transformation stage to ... (vect_transform_cycle_phi): ... here. From-SVN: r276441 --- gcc/ChangeLog | 11 ++ gcc/tree-vect-loop.c | 418 ++++++++++++++++++++++-------------------- gcc/tree-vect-stmts.c | 6 + gcc/tree-vectorizer.h | 3 + 4 files changed, 239 insertions(+), 199 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 69f5d7153c9..b7830a44794 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,14 @@ +2019-10-02 Richard Biener + + * tree-vectorizer.h (stmt_vec_info_type::cycle_phi_info_type): + New. + (vect_transform_cycle_phi): Declare. + * tree-vect-stmts.c (vect_transform_stmt): Call + vect_transform_cycle_phi. + * tree-vect-loop.c (vectorizable_reduction): Split out + PHI transformation stage to ... + (vect_transform_cycle_phi): ... here. + 2019-10-02 Richard Sandiford PR middle-end/91957 diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 350cee58246..a3fd011e6c4 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -5783,7 +5783,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, bool is_simple_use; int i; int ncopies; - stmt_vec_info prev_phi_info; bool single_defuse_cycle = false; int j; tree ops[3]; @@ -5811,207 +5810,15 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, gcc_assert (slp_node && REDUC_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info); - if (gphi *phi = dyn_cast (stmt_info->stmt)) + if (is_a (stmt_info->stmt)) { - tree phi_result = gimple_phi_result (phi); /* Analysis is fully done on the reduction stmt invocation. */ - if (! vec_stmt) - { - if (slp_node) - slp_node_instance->reduc_phis = slp_node; - - STMT_VINFO_TYPE (stmt_info) = reduc_vec_info_type; - return true; - } - - if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION) - /* Leave the scalar phi in place. Note that checking - STMT_VINFO_VEC_REDUCTION_TYPE (as below) only works - for reductions involving a single statement. */ - return true; - - stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info); - reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info); - - if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) - == EXTRACT_LAST_REDUCTION) - /* Leave the scalar phi in place. */ - return true; - - if (gassign *reduc_stmt = dyn_cast (reduc_stmt_info->stmt)) - for (unsigned k = 1; k < gimple_num_ops (reduc_stmt); ++k) - { - tree op = gimple_op (reduc_stmt, k); - if (op == phi_result) - continue; - if (k == 1 && gimple_assign_rhs_code (reduc_stmt) == COND_EXPR) - continue; - bool is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt); - gcc_assert (is_simple_use); - if (dt == vect_constant_def || dt == vect_external_def) - continue; - if (!vectype_in - || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in))) - < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (op))))) - vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op)); - break; - } - /* For a nested cycle we might end up with an operation like - phi_result * phi_result. */ - if (!vectype_in) - vectype_in = STMT_VINFO_VECTYPE (stmt_info); - gcc_assert (vectype_in); + gcc_assert (! vec_stmt); if (slp_node) - { - /* The size vect_schedule_slp_instance computes is off for us. */ - vec_num = vect_get_num_vectors - (LOOP_VINFO_VECT_FACTOR (loop_vinfo) - * SLP_TREE_SCALAR_STMTS (slp_node).length (), vectype_in); - ncopies = 1; - } - else - { - vec_num = 1; - ncopies = vect_get_num_copies (loop_vinfo, vectype_in); - } - - /* Check whether we can use a single PHI node and accumulate - vectors to one before the backedge. */ - stmt_vec_info use_stmt_info; - if (ncopies > 1 - && STMT_VINFO_RELEVANT (reduc_stmt_info) <= vect_used_only_live - && (use_stmt_info = loop_vinfo->lookup_single_use (phi_result)) - && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info) - || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info)) - && vect_stmt_to_vectorize (use_stmt_info) == reduc_stmt_info) - { - single_defuse_cycle = true; - ncopies = 1; - } - - /* Create the destination vector */ - tree vec_dest = vect_create_destination_var (phi_result, vectype_out); - - /* Get the loop-entry arguments. */ - tree vec_initial_def; - auto_vec vec_initial_defs; - if (slp_node) - { - vec_initial_defs.reserve (vec_num); - gcc_assert (slp_node == slp_node_instance->reduc_phis); - stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info); - tree neutral_op - = neutral_op_for_slp_reduction (slp_node, - STMT_VINFO_REDUC_CODE - (first ? first : reduc_stmt_info), - first != NULL); - get_initial_defs_for_reduction (slp_node_instance->reduc_phis, - &vec_initial_defs, vec_num, - first != NULL, neutral_op); - } - else - { - /* Get at the scalar def before the loop, that defines the initial - value of the reduction variable. */ - tree initial_def = PHI_ARG_DEF_FROM_EDGE (phi, - loop_preheader_edge (loop)); - /* Optimize: if initial_def is for REDUC_MAX smaller than the base - and we can't use zero for induc_val, use initial_def. Similarly - for REDUC_MIN and initial_def larger than the base. */ - if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) - == INTEGER_INDUC_COND_REDUCTION) - { - tree induc_val - = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info); - if (TREE_CODE (initial_def) == INTEGER_CST - && (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) - == INTEGER_INDUC_COND_REDUCTION) - && !integer_zerop (induc_val) - && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info) - == MAX_EXPR) - && tree_int_cst_lt (initial_def, induc_val)) - || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info) - == MIN_EXPR) - && tree_int_cst_lt (induc_val, initial_def)))) - { - induc_val = initial_def; - /* Communicate we used the initial_def to epilouge - generation. */ - STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info) - = NULL_TREE; - } - vec_initial_def = build_vector_from_val (vectype_out, induc_val); - } - else if (nested_cycle) - { - /* Do not use an adjustment def as that case is not supported - correctly if ncopies is not one. */ - vec_initial_def = vect_get_vec_def_for_operand (initial_def, - reduc_stmt_info); - } - else - { - tree adjustment_def = NULL_TREE; - tree *adjustment_defp = &adjustment_def; - enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_stmt_info); - /* ??? For the outer loop PHI we have to do a bit of searching - to find the stmt with the code. reduc_stmt_info here is the - loop-closed PHI of the inner reduction which means we can look - at its single-arg def. */ - if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def) - { - tree def = gimple_phi_arg_def - (as_a (reduc_stmt_info->stmt), 0); - code = STMT_VINFO_REDUC_CODE - (vect_stmt_to_vectorize (loop_vinfo->lookup_def (def))); - adjustment_defp = NULL; - } - vec_initial_def - = get_initial_def_for_reduction (reduc_stmt_info, code, - initial_def, adjustment_defp); - STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_stmt_info) - = adjustment_def; - } - vec_initial_defs.create (1); - vec_initial_defs.quick_push (vec_initial_def); - } - - /* Generate the reduction PHIs upfront. */ - prev_phi_info = NULL; - for (i = 0; i < vec_num; i++) - { - tree vec_init_def = vec_initial_defs[i]; - for (j = 0; j < ncopies; j++) - { - /* Create the reduction-phi that defines the reduction - operand. */ - gphi *new_phi = create_phi_node (vec_dest, loop->header); - stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); - - /* Set the loop-entry arg of the reduction-phi. */ - if (j != 0 && nested_cycle) - vec_init_def = vect_get_vec_def_for_stmt_copy (loop_vinfo, - vec_init_def); - add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop), - UNKNOWN_LOCATION); - - /* The loop-latch arg is set in epilogue processing. */ - - if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); - else - { - if (j == 0) - STMT_VINFO_VEC_STMT (stmt_info) - = *vec_stmt = new_phi_info; - else - STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info; - prev_phi_info = new_phi_info; - } - } - } + slp_node_instance->reduc_phis = slp_node; + STMT_VINFO_TYPE (stmt_info) = cycle_phi_info_type; return true; } @@ -6841,7 +6648,6 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, tree vec_dest = vect_create_destination_var (scalar_dest, vectype_out); prev_stmt_info = NULL; - prev_phi_info = NULL; if (!slp_node) { vec_oprnds0.create (1); @@ -6998,7 +6804,221 @@ vectorizable_reduction (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, return true; } -/* Vectorizes LC PHIs of nested cycles (sofar). */ +/* Transform phase of a cycle PHI. */ + +bool +vect_transform_cycle_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, + slp_tree slp_node, slp_instance slp_node_instance) +{ + tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); + tree vectype_in = NULL_TREE; + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + enum vect_def_type dt; + int i; + int ncopies; + stmt_vec_info prev_phi_info; + int j; + bool nested_cycle = false; + int vec_num; + + if (nested_in_vect_loop_p (loop, stmt_info)) + { + loop = loop->inner; + nested_cycle = true; + } + + gphi *phi = as_a (stmt_info->stmt); + tree phi_result = gimple_phi_result (phi); + + if (STMT_VINFO_REDUC_TYPE (stmt_info) == FOLD_LEFT_REDUCTION) + /* Leave the scalar phi in place. Note that checking + STMT_VINFO_VEC_REDUCTION_TYPE (as below) only works + for reductions involving a single statement. */ + return true; + + stmt_vec_info reduc_stmt_info = STMT_VINFO_REDUC_DEF (stmt_info); + reduc_stmt_info = vect_stmt_to_vectorize (reduc_stmt_info); + + if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) + == EXTRACT_LAST_REDUCTION) + /* Leave the scalar phi in place. */ + return true; + + if (gassign *reduc_stmt = dyn_cast (reduc_stmt_info->stmt)) + for (unsigned k = 1; k < gimple_num_ops (reduc_stmt); ++k) + { + tree op = gimple_op (reduc_stmt, k); + if (op == phi_result) + continue; + if (k == 1 && gimple_assign_rhs_code (reduc_stmt) == COND_EXPR) + continue; + bool is_simple_use = vect_is_simple_use (op, loop_vinfo, &dt); + gcc_assert (is_simple_use); + if (dt == vect_constant_def || dt == vect_external_def) + continue; + if (!vectype_in + || (GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (vectype_in))) + < GET_MODE_SIZE (SCALAR_TYPE_MODE (TREE_TYPE (op))))) + vectype_in = get_vectype_for_scalar_type (TREE_TYPE (op)); + break; + } + /* For a nested cycle we might end up with an operation like + phi_result * phi_result. */ + if (!vectype_in) + vectype_in = STMT_VINFO_VECTYPE (stmt_info); + gcc_assert (vectype_in); + + if (slp_node) + { + /* The size vect_schedule_slp_instance computes is off for us. */ + vec_num = vect_get_num_vectors + (LOOP_VINFO_VECT_FACTOR (loop_vinfo) + * SLP_TREE_SCALAR_STMTS (slp_node).length (), vectype_in); + ncopies = 1; + } + else + { + vec_num = 1; + ncopies = vect_get_num_copies (loop_vinfo, vectype_in); + } + + /* Check whether we can use a single PHI node and accumulate + vectors to one before the backedge. */ + stmt_vec_info use_stmt_info; + if (ncopies > 1 + && STMT_VINFO_RELEVANT (reduc_stmt_info) <= vect_used_only_live + && (use_stmt_info = loop_vinfo->lookup_single_use (phi_result)) + && (!STMT_VINFO_IN_PATTERN_P (use_stmt_info) + || !STMT_VINFO_PATTERN_DEF_SEQ (use_stmt_info)) + && vect_stmt_to_vectorize (use_stmt_info) == reduc_stmt_info) + ncopies = 1; + + /* Create the destination vector */ + tree vec_dest = vect_create_destination_var (phi_result, vectype_out); + + /* Get the loop-entry arguments. */ + tree vec_initial_def; + auto_vec vec_initial_defs; + if (slp_node) + { + vec_initial_defs.reserve (vec_num); + gcc_assert (slp_node == slp_node_instance->reduc_phis); + stmt_vec_info first = REDUC_GROUP_FIRST_ELEMENT (reduc_stmt_info); + tree neutral_op + = neutral_op_for_slp_reduction (slp_node, + STMT_VINFO_REDUC_CODE + (first ? first : reduc_stmt_info), + first != NULL); + get_initial_defs_for_reduction (slp_node_instance->reduc_phis, + &vec_initial_defs, vec_num, + first != NULL, neutral_op); + } + else + { + /* Get at the scalar def before the loop, that defines the initial + value of the reduction variable. */ + tree initial_def = PHI_ARG_DEF_FROM_EDGE (phi, + loop_preheader_edge (loop)); + /* Optimize: if initial_def is for REDUC_MAX smaller than the base + and we can't use zero for induc_val, use initial_def. Similarly + for REDUC_MIN and initial_def larger than the base. */ + if (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + { + tree induc_val + = STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info); + if (TREE_CODE (initial_def) == INTEGER_CST + && (STMT_VINFO_VEC_REDUCTION_TYPE (reduc_stmt_info) + == INTEGER_INDUC_COND_REDUCTION) + && !integer_zerop (induc_val) + && (((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info) + == MAX_EXPR) + && tree_int_cst_lt (initial_def, induc_val)) + || ((STMT_VINFO_VEC_COND_REDUC_CODE (reduc_stmt_info) + == MIN_EXPR) + && tree_int_cst_lt (induc_val, initial_def)))) + { + induc_val = initial_def; + /* Communicate we used the initial_def to epilouge + generation. */ + STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL (reduc_stmt_info) + = NULL_TREE; + } + vec_initial_def = build_vector_from_val (vectype_out, induc_val); + } + else if (nested_cycle) + { + /* Do not use an adjustment def as that case is not supported + correctly if ncopies is not one. */ + vec_initial_def = vect_get_vec_def_for_operand (initial_def, + reduc_stmt_info); + } + else + { + tree adjustment_def = NULL_TREE; + tree *adjustment_defp = &adjustment_def; + enum tree_code code = STMT_VINFO_REDUC_CODE (reduc_stmt_info); + /* ??? For the outer loop PHI we have to do a bit of searching + to find the stmt with the code. reduc_stmt_info here is the + loop-closed PHI of the inner reduction which means we can look + at its single-arg def. */ + if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def) + { + tree def = gimple_phi_arg_def + (as_a (reduc_stmt_info->stmt), 0); + code = STMT_VINFO_REDUC_CODE + (vect_stmt_to_vectorize (loop_vinfo->lookup_def (def))); + adjustment_defp = NULL; + } + vec_initial_def + = get_initial_def_for_reduction (reduc_stmt_info, code, + initial_def, adjustment_defp); + STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT (reduc_stmt_info) + = adjustment_def; + } + vec_initial_defs.create (1); + vec_initial_defs.quick_push (vec_initial_def); + } + + /* Generate the reduction PHIs upfront. */ + prev_phi_info = NULL; + for (i = 0; i < vec_num; i++) + { + tree vec_init_def = vec_initial_defs[i]; + for (j = 0; j < ncopies; j++) + { + /* Create the reduction-phi that defines the reduction + operand. */ + gphi *new_phi = create_phi_node (vec_dest, loop->header); + stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); + + /* Set the loop-entry arg of the reduction-phi. */ + if (j != 0 && nested_cycle) + vec_init_def = vect_get_vec_def_for_stmt_copy (loop_vinfo, + vec_init_def); + add_phi_arg (new_phi, vec_init_def, loop_preheader_edge (loop), + UNKNOWN_LOCATION); + + /* The loop-latch arg is set in epilogue processing. */ + + if (slp_node) + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + else + { + if (j == 0) + STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi_info; + else + STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info; + prev_phi_info = new_phi_info; + } + } + } + + return true; +} + +/* Vectorizes LC PHIs. */ bool vectorizable_lc_phi (stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 7fcd2fa19a8..ddc2f7408d8 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -10824,6 +10824,12 @@ vect_transform_stmt (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, gcc_assert (done); break; + case cycle_phi_info_type: + done = vect_transform_cycle_phi (stmt_info, &vec_stmt, slp_node, + slp_node_instance); + gcc_assert (done); + break; + case lc_phi_info_type: done = vectorizable_lc_phi (stmt_info, &vec_stmt, slp_node); gcc_assert (done); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 462a968f2ba..eae9d4c9a18 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -694,6 +694,7 @@ enum stmt_vec_info_type { type_promotion_vec_info_type, type_demotion_vec_info_type, type_conversion_vec_info_type, + cycle_phi_info_type, lc_phi_info_type, loop_exit_ctrl_vec_info_type }; @@ -1658,6 +1659,8 @@ extern bool vectorizable_reduction (stmt_vec_info, gimple_stmt_iterator *, extern bool vectorizable_induction (stmt_vec_info, gimple_stmt_iterator *, stmt_vec_info *, slp_tree, stmt_vector_for_cost *); +extern bool vect_transform_cycle_phi (stmt_vec_info, stmt_vec_info *, + slp_tree, slp_instance); extern bool vectorizable_lc_phi (stmt_vec_info, stmt_vec_info *, slp_tree); extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, -- 2.30.2