From 62cf73356918189f84ad9fcc83c1ffe8f86d7595 Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Wed, 28 Jun 2017 13:02:01 +0000 Subject: [PATCH] tree-vect-loop.c (vectorizable_reduction): Move special cond reduction IV var creation ... 2017-06-28 Richard Biener * tree-vect-loop.c (vectorizable_reduction): Move special cond reduction IV var creation ... (vect_create_epilog_for_reduction): ... here. Remove induction_index parameter. Use STMT_VINFO_VECTYPE. * tree-vect-slp.c (vect_get_constant_vectors): Properly reset constant_p. From-SVN: r249736 --- gcc/ChangeLog | 9 ++ gcc/tree-vect-loop.c | 191 ++++++++++++++++++++++--------------------- gcc/tree-vect-slp.c | 10 +-- 3 files changed, 108 insertions(+), 102 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 5fc16ca8afd..dc94164a21c 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,12 @@ +2017-06-28 Richard Biener + + * tree-vect-loop.c (vectorizable_reduction): Move special + cond reduction IV var creation ... + (vect_create_epilog_for_reduction): ... here. Remove induction_index + parameter. Use STMT_VINFO_VECTYPE. + * tree-vect-slp.c (vect_get_constant_vectors): Properly reset + constant_p. + 2017-06-28 Martin Liska PR ipa/81128 diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 0e725aedd8f..a0558360a96 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4263,8 +4263,6 @@ get_initial_defs_for_reduction (slp_tree slp_node, DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled. SLP_NODE is an SLP node containing a group of reduction statements. The first one in this group is STMT. - INDUCTION_INDEX is the index of the loop for condition reductions. - Otherwise it is undefined. This function: 1. Creates the reduction def-use cycles: sets the arguments for @@ -4310,7 +4308,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, int ncopies, enum tree_code reduc_code, vec reduction_phis, int reduc_index, bool double_reduc, - slp_tree slp_node, tree induction_index) + slp_tree slp_node) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); stmt_vec_info prev_phi_info; @@ -4331,7 +4329,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree bitsize; tree adjustment_def = NULL; tree vec_initial_def = NULL; - tree reduction_op, expr, def, initial_def = NULL; + tree expr, def, initial_def = NULL; tree orig_name, scalar_result; imm_use_iterator imm_iter, phi_imm_iter; use_operand_p use_p, phi_use_p; @@ -4348,6 +4346,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, bool slp_reduc = false; tree new_phi_result; gimple *inner_phi = NULL; + tree induction_index = NULL_TREE; if (slp_node) group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); @@ -4360,9 +4359,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, gcc_assert (!slp_node); } - reduction_op = get_reduction_op (stmt, reduc_index); - - vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op)); + vectype = STMT_VINFO_VECTYPE (stmt_info); gcc_assert (vectype); mode = TYPE_MODE (vectype); @@ -4396,6 +4393,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, { /* Get at the scalar def before the loop, that defines the initial value of the reduction variable. */ + tree reduction_op = get_reduction_op (stmt, reduc_index); gimple *def_stmt = SSA_NAME_DEF_STMT (reduction_op); initial_def = PHI_ARG_DEF_FROM_EDGE (def_stmt, loop_preheader_edge (loop)); @@ -4465,6 +4463,95 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, } } + /* For cond reductions we want to create a new vector (INDEX_COND_EXPR) + which is updated with the current index of the loop for every match of + the original loop's cond_expr (VEC_STMT). This results in a vector + containing the last time the condition passed for that vector lane. + The first match will be a 1 to allow 0 to be used for non-matching + indexes. If there are no matches at all then the vector will be all + zeroes. */ + if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION) + { + tree indx_before_incr, indx_after_incr; + int nunits_out = TYPE_VECTOR_SUBPARTS (vectype); + int k; + + gimple *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info); + gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR); + + int scalar_precision + = GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (vectype))); + tree cr_index_scalar_type = make_unsigned_type (scalar_precision); + tree cr_index_vector_type = build_vector_type + (cr_index_scalar_type, TYPE_VECTOR_SUBPARTS (vectype)); + + /* First we create a simple vector induction variable which starts + with the values {1,2,3,...} (SERIES_VECT) and increments by the + vector size (STEP). */ + + /* Create a {1,2,3,...} vector. */ + tree *vtemp = XALLOCAVEC (tree, nunits_out); + for (k = 0; k < nunits_out; ++k) + vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1); + tree series_vect = build_vector (cr_index_vector_type, vtemp); + + /* Create a vector of the step value. */ + tree step = build_int_cst (cr_index_scalar_type, nunits_out); + tree vec_step = build_vector_from_val (cr_index_vector_type, step); + + /* Create an induction variable. */ + gimple_stmt_iterator incr_gsi; + bool insert_after; + standard_iv_increment_position (loop, &incr_gsi, &insert_after); + create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi, + insert_after, &indx_before_incr, &indx_after_incr); + + /* Next create a new phi node vector (NEW_PHI_TREE) which starts + filled with zeros (VEC_ZERO). */ + + /* Create a vector of 0s. */ + tree zero = build_zero_cst (cr_index_scalar_type); + tree vec_zero = build_vector_from_val (cr_index_vector_type, zero); + + /* Create a vector phi node. */ + tree new_phi_tree = make_ssa_name (cr_index_vector_type); + new_phi = create_phi_node (new_phi_tree, loop->header); + set_vinfo_for_stmt (new_phi, + new_stmt_vec_info (new_phi, loop_vinfo)); + add_phi_arg (as_a (new_phi), vec_zero, + loop_preheader_edge (loop), UNKNOWN_LOCATION); + + /* Now take the condition from the loops original cond_expr + (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for + every match uses values from the induction variable + (INDEX_BEFORE_INCR) otherwise uses values from the phi node + (NEW_PHI_TREE). + Finally, we update the phi (NEW_PHI_TREE) to take the value of + the new cond_expr (INDEX_COND_EXPR). */ + + /* Duplicate the condition from vec_stmt. */ + tree ccompare = unshare_expr (gimple_assign_rhs1 (vec_stmt)); + + /* Create a conditional, where the condition is taken from vec_stmt + (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and + else is the phi (NEW_PHI_TREE). */ + tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type, + ccompare, indx_before_incr, + new_phi_tree); + induction_index = make_ssa_name (cr_index_vector_type); + gimple *index_condition = gimple_build_assign (induction_index, + index_cond_expr); + gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT); + stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition, + loop_vinfo); + STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type; + set_vinfo_for_stmt (index_condition, index_vec_info); + + /* Update the phi with the vec cond. */ + add_phi_arg (as_a (new_phi), induction_index, + loop_latch_edge (loop), UNKNOWN_LOCATION); + } + /* 2. Create epilog code. The reduction epilog code operates across the elements of the vector of partial results computed by the vectorized loop. @@ -6248,100 +6335,14 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, prev_phi_info = vinfo_for_stmt (new_phi); } - tree indx_before_incr, indx_after_incr, cond_name = NULL; - /* Finalize the reduction-phi (set its arguments) and create the epilog reduction code. */ if ((!single_defuse_cycle || code == COND_EXPR) && !slp_node) - { - new_temp = gimple_assign_lhs (*vec_stmt); - vect_defs[0] = new_temp; - - /* For cond reductions we want to create a new vector (INDEX_COND_EXPR) - which is updated with the current index of the loop for every match of - the original loop's cond_expr (VEC_STMT). This results in a vector - containing the last time the condition passed for that vector lane. - The first match will be a 1 to allow 0 to be used for non-matching - indexes. If there are no matches at all then the vector will be all - zeroes. */ - if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION) - { - int nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out); - int k; - - gcc_assert (gimple_assign_rhs_code (*vec_stmt) == VEC_COND_EXPR); - - /* First we create a simple vector induction variable which starts - with the values {1,2,3,...} (SERIES_VECT) and increments by the - vector size (STEP). */ - - /* Create a {1,2,3,...} vector. */ - tree *vtemp = XALLOCAVEC (tree, nunits_out); - for (k = 0; k < nunits_out; ++k) - vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1); - tree series_vect = build_vector (cr_index_vector_type, vtemp); - - /* Create a vector of the step value. */ - tree step = build_int_cst (cr_index_scalar_type, nunits_out); - tree vec_step = build_vector_from_val (cr_index_vector_type, step); - - /* Create an induction variable. */ - gimple_stmt_iterator incr_gsi; - bool insert_after; - standard_iv_increment_position (loop, &incr_gsi, &insert_after); - create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi, - insert_after, &indx_before_incr, &indx_after_incr); - - /* Next create a new phi node vector (NEW_PHI_TREE) which starts - filled with zeros (VEC_ZERO). */ - - /* Create a vector of 0s. */ - tree zero = build_zero_cst (cr_index_scalar_type); - tree vec_zero = build_vector_from_val (cr_index_vector_type, zero); - - /* Create a vector phi node. */ - tree new_phi_tree = make_ssa_name (cr_index_vector_type); - new_phi = create_phi_node (new_phi_tree, loop->header); - set_vinfo_for_stmt (new_phi, - new_stmt_vec_info (new_phi, loop_vinfo)); - add_phi_arg (new_phi, vec_zero, loop_preheader_edge (loop), - UNKNOWN_LOCATION); - - /* Now take the condition from the loops original cond_expr - (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for - every match uses values from the induction variable - (INDEX_BEFORE_INCR) otherwise uses values from the phi node - (NEW_PHI_TREE). - Finally, we update the phi (NEW_PHI_TREE) to take the value of - the new cond_expr (INDEX_COND_EXPR). */ - - /* Duplicate the condition from vec_stmt. */ - tree ccompare = unshare_expr (gimple_assign_rhs1 (*vec_stmt)); - - /* Create a conditional, where the condition is taken from vec_stmt - (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and - else is the phi (NEW_PHI_TREE). */ - tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type, - ccompare, indx_before_incr, - new_phi_tree); - cond_name = make_ssa_name (cr_index_vector_type); - gimple *index_condition = gimple_build_assign (cond_name, - index_cond_expr); - gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT); - stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition, - loop_vinfo); - STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type; - set_vinfo_for_stmt (index_condition, index_vec_info); - - /* Update the phi with the vec cond. */ - add_phi_arg (new_phi, cond_name, loop_latch_edge (loop), - UNKNOWN_LOCATION); - } - } + vect_defs[0] = gimple_assign_lhs (*vec_stmt); vect_create_epilog_for_reduction (vect_defs, stmt, epilog_copies, epilog_reduc_code, phis, reduc_index, - double_reduc, slp_node, cond_name); + double_reduc, slp_node); return true; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index b755562ed76..eb7bc55c43f 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -3016,11 +3016,6 @@ vect_get_constant_vectors (tree op, slp_tree slp_node, gcc_assert (op); - if (CONSTANT_CLASS_P (op)) - constant_p = true; - else - constant_p = false; - /* NUMBER_OF_COPIES is the number of times we need to use the same values in created vectors. It is greater than 1 if unrolling is performed. @@ -3040,6 +3035,7 @@ vect_get_constant_vectors (tree op, slp_tree slp_node, number_of_copies = nunits * number_of_vectors / group_size; number_of_places_left_in_vector = nunits; + constant_p = true; elts = XALLOCAVEC (tree, nunits); bool place_after_defs = false; for (j = 0; j < number_of_copies; j++) @@ -3156,8 +3152,6 @@ vect_get_constant_vectors (tree op, slp_tree slp_node, if (number_of_places_left_in_vector == 0) { - number_of_places_left_in_vector = nunits; - if (constant_p) vec_cst = build_vector (vector_type, elts); else @@ -3188,6 +3182,8 @@ vect_get_constant_vectors (tree op, slp_tree slp_node, } voprnds.quick_push (init); place_after_defs = false; + number_of_places_left_in_vector = nunits; + constant_p = true; } } } -- 2.30.2