From e7baeb396d313ef141f972cdedc19b12ef1b9cfe Mon Sep 17 00:00:00 2001 From: Richard Biener Date: Tue, 6 Jun 2017 07:37:14 +0000 Subject: [PATCH] re PR tree-optimization/80928 (SLP vectorization does not handle induction in outer loop vectorization) 2017-06-06 Richard Biener PR tree-optimization/80928 * tree-vect-loop.c (vect_update_vf_for_slp): Amend dumps. (vect_analyze_loop_operations): Properly guard analysis for pure SLP case. (vect_transform_loop): Likewise. (vect_analyze_loop_2): Also reset SLP type on PHIs. (vect_model_induction_cost): Do not cost for pure SLP. (vectorizable_induction): Pass in SLP node, implement SLP vectorization of induction in inner loop vectorization. * tree-vect-slp.c (vect_create_new_slp_node): Handle PHIs. (vect_get_and_check_slp_defs): Handle vect_induction_def. (vect_build_slp_tree): Likewise. Handle PHIs as terminating the recursion. (vect_analyze_slp_cost_1): Cost induction. (vect_detect_hybrid_slp_stmts): Handle PHIs. (vect_get_slp_vect_defs): Likewise. * tree-vect-stmts.c (vect_analyze_stmt): Handle induction. (vect_transform_stmt): Handle SLP reductions. * tree-vectorizer.h (vectorizable_induction): Adjust. * gcc.dg/vect/pr80928.c: New testcase. * gcc.dg/vect/slp-13-big-array.c: Remove XFAILs. * gcc.dg/vect/slp-13.c: Likewise. * gcc.dg/vect/slp-perm-9.c: Prevent vectorization of check loop. From-SVN: r248909 --- gcc/ChangeLog | 22 +++ gcc/testsuite/ChangeLog | 8 + gcc/testsuite/gcc.dg/vect/pr80928.c | 37 ++++ gcc/testsuite/gcc.dg/vect/slp-13-big-array.c | 11 +- gcc/testsuite/gcc.dg/vect/slp-13.c | 11 +- gcc/testsuite/gcc.dg/vect/slp-perm-9.c | 1 + gcc/tree-vect-loop.c | 185 +++++++++++++++++-- gcc/tree-vect-slp.c | 69 ++++++- gcc/tree-vect-stmts.c | 7 +- gcc/tree-vectorizer.h | 3 +- 10 files changed, 316 insertions(+), 38 deletions(-) create mode 100644 gcc/testsuite/gcc.dg/vect/pr80928.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 1b105b2a969..4a729fe4c75 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2017-06-06 Richard Biener + + PR tree-optimization/80928 + * tree-vect-loop.c (vect_update_vf_for_slp): Amend dumps. + (vect_analyze_loop_operations): Properly guard analysis for + pure SLP case. + (vect_transform_loop): Likewise. + (vect_analyze_loop_2): Also reset SLP type on PHIs. + (vect_model_induction_cost): Do not cost for pure SLP. + (vectorizable_induction): Pass in SLP node, implement SLP vectorization + of induction in inner loop vectorization. + * tree-vect-slp.c (vect_create_new_slp_node): Handle PHIs. + (vect_get_and_check_slp_defs): Handle vect_induction_def. + (vect_build_slp_tree): Likewise. Handle PHIs as terminating the + recursion. + (vect_analyze_slp_cost_1): Cost induction. + (vect_detect_hybrid_slp_stmts): Handle PHIs. + (vect_get_slp_vect_defs): Likewise. + * tree-vect-stmts.c (vect_analyze_stmt): Handle induction. + (vect_transform_stmt): Handle SLP reductions. + * tree-vectorizer.h (vectorizable_induction): Adjust. + 2017-06-05 Michael Meissner * config/rs6000/rs6000.c (make_resolver_func): Update diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index f1bb404a956..35d36c49798 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,11 @@ +2017-06-06 Richard Biener + + PR tree-optimization/80928 + * gcc.dg/vect/pr80928.c: New testcase. + * gcc.dg/vect/slp-13-big-array.c: Remove XFAILs. + * gcc.dg/vect/slp-13.c: Likewise. + * gcc.dg/vect/slp-perm-9.c: Prevent vectorization of check loop. + 2017-06-05 David Malcolm * g++.dg/plugin/comment_plugin.c: New test plugin. diff --git a/gcc/testsuite/gcc.dg/vect/pr80928.c b/gcc/testsuite/gcc.dg/vect/pr80928.c new file mode 100644 index 00000000000..e6c1f1ab5a7 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/pr80928.c @@ -0,0 +1,37 @@ +/* { dg-do compile } */ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" + +int a[1020]; + +void __attribute__((noinline)) +foo () +{ + for (int i = 0; i < 1020; i += 5) + { + a[i] = i; + a[i+1] = i; + a[i+2] = i; + a[i+3] = i; + a[i+4] = i; + } +} + +int main () +{ + check_vect (); + + foo (); + + /* check results */ + for (int i = 0; i < 1020; ++i) + if (a[i] != ((i + 4) / 5) * 5) + abort (); + + return 0; +} + +/* Make sure we are not triggering hybrid SLP due to the IV update. */ +/* { dg-final { scan-tree-dump "Loop contains only SLP stmts" "vect" } } */ +/* { dg-final { scan-tree-dump "vectorized 1 loops" "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/slp-13-big-array.c b/gcc/testsuite/gcc.dg/vect/slp-13-big-array.c index 4520fb15f20..635ca47cd2e 100644 --- a/gcc/testsuite/gcc.dg/vect/slp-13-big-array.c +++ b/gcc/testsuite/gcc.dg/vect/slp-13-big-array.c @@ -22,7 +22,7 @@ main1 () abort (); } - /* Induction is not SLPable yet. */ + /* Induction is SLPable. */ for (i = 0; i < N; i++) { out[i*8] = in[i*8] + i; @@ -49,8 +49,7 @@ main1 () abort (); } - /* Induction is not SLPable yet and strided group size must be a power of 2 - to get vectorized. */ + /* Induction is SLPable. */ for (i = 0; i < N/2; i++) { out2[i*12] = in2[i*12] + i; @@ -134,7 +133,5 @@ int main (void) return 0; } -/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */ -/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" { xfail *-*-* } } } */ -/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" } } */ - +/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */ +/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/slp-13.c b/gcc/testsuite/gcc.dg/vect/slp-13.c index 2394874d36c..084858db8cb 100644 --- a/gcc/testsuite/gcc.dg/vect/slp-13.c +++ b/gcc/testsuite/gcc.dg/vect/slp-13.c @@ -14,7 +14,7 @@ main1 () unsigned int in2[N*8] = {0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63}; unsigned int out2[N*8]; - /* Induction is not SLPable yet. */ + /* Induction is SLPable. */ for (i = 0; i < N; i++) { out[i*8] = in[i*8] + i; @@ -41,8 +41,7 @@ main1 () abort (); } - /* Induction is not SLPable yet and strided group size must be a power of 2 - to get vectorized. */ + /* Induction is SLPable. */ for (i = 0; i < N/2; i++) { out2[i*12] = in2[i*12] + i; @@ -126,7 +125,5 @@ int main (void) return 0; } -/* { dg-final { scan-tree-dump-times "vectorized 2 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */ -/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" { xfail *-*-* } } } */ -/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 1 "vect" } } */ - +/* { dg-final { scan-tree-dump-times "vectorized 3 loops" 1 "vect" { target { vect_interleave && vect_extract_even_odd } } } } */ +/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 3 "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/slp-perm-9.c b/gcc/testsuite/gcc.dg/vect/slp-perm-9.c index 781096e9f66..4d9c11dcc47 100644 --- a/gcc/testsuite/gcc.dg/vect/slp-perm-9.c +++ b/gcc/testsuite/gcc.dg/vect/slp-perm-9.c @@ -42,6 +42,7 @@ int main (int argc, const char* argv[]) check_results[3*i] = 9 * i + 6; check_results[3*i+1] = 9 * i + 15; check_results[3*i+2] = 9 * i + 4; + __asm__ volatile ("" : : : "memory"); } foo (input, output); diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 28f4349a3c8..f81eb6f3f20 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -1649,11 +1649,19 @@ vect_update_vf_for_slp (loop_vec_info loop_vinfo) } if (only_slp_in_loop) - vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo); + { + dump_printf_loc (MSG_NOTE, vect_location, + "Loop contains only SLP stmts\n"); + vectorization_factor = LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo); + } else - vectorization_factor - = least_common_multiple (vectorization_factor, - LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo)); + { + dump_printf_loc (MSG_NOTE, vect_location, + "Loop contains SLP and non-SLP stmts\n"); + vectorization_factor + = least_common_multiple (vectorization_factor, + LOOP_VINFO_SLP_UNROLLING_FACTOR (loop_vinfo)); + } LOOP_VINFO_VECT_FACTOR (loop_vinfo) = vectorization_factor; if (dump_enabled_p ()) @@ -1765,8 +1773,9 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo) if (STMT_VINFO_RELEVANT_P (stmt_info)) { need_to_vectorize = true; - if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) - ok = vectorizable_induction (phi, NULL, NULL); + if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def + && ! PURE_SLP_STMT (stmt_info)) + ok = vectorizable_induction (phi, NULL, NULL, NULL); } if (ok && STMT_VINFO_LIVE_P (stmt_info)) @@ -2275,6 +2284,12 @@ again: for (i = 0; i < LOOP_VINFO_LOOP (loop_vinfo)->num_nodes; ++i) { basic_block bb = LOOP_VINFO_BBS (loop_vinfo)[i]; + for (gimple_stmt_iterator si = gsi_start_phis (bb); + !gsi_end_p (si); gsi_next (&si)) + { + stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si)); + STMT_SLP_TYPE (stmt_info) = loop_vect; + } for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { @@ -3797,6 +3812,9 @@ vect_model_induction_cost (stmt_vec_info stmt_info, int ncopies) void *target_cost_data = LOOP_VINFO_TARGET_COST_DATA (loop_vinfo); unsigned inside_cost, prologue_cost; + if (PURE_SLP_STMT (stmt_info)) + return; + /* loop cost for vec_loop. */ inside_cost = add_stmt_cost (target_cost_data, ncopies, vector_stmt, stmt_info, 0, vect_body); @@ -6086,7 +6104,7 @@ vect_min_worthwhile_factor (enum tree_code code) bool vectorizable_induction (gimple *phi, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED, - gimple **vec_stmt) + gimple **vec_stmt, slp_tree slp_node) { stmt_vec_info stmt_info = vinfo_for_stmt (phi); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); @@ -6125,14 +6143,13 @@ vectorizable_induction (gimple *phi, if (STMT_VINFO_DEF_TYPE (stmt_info) != vect_induction_def) return false; - /* FORNOW: SLP not supported. */ - if (STMT_SLP_TYPE (stmt_info)) - return false; - tree vectype = STMT_VINFO_VECTYPE (stmt_info); unsigned nunits = TYPE_VECTOR_SUBPARTS (vectype); - ncopies = vf / nunits; + if (slp_node) + ncopies = 1; + else + ncopies = vf / nunits; gcc_assert (ncopies >= 1); /* FORNOW. These restrictions should be relaxed. */ @@ -6234,6 +6251,147 @@ vectorizable_induction (gimple *phi, /* Find the first insertion point in the BB. */ si = gsi_after_labels (bb); + /* For SLP induction we have to generate several IVs as for example + with group size 3 we need [i, i, i, i + S] [i + S, i + S, i + 2*S, i + 2*S] + [i + 2*S, i + 3*S, i + 3*S, i + 3*S]. The step is the same uniform + [VF*S, VF*S, VF*S, VF*S] for all. */ + if (slp_node) + { + /* Convert the init to the desired type. */ + stmts = NULL; + init_expr = gimple_convert (&stmts, TREE_TYPE (vectype), init_expr); + if (stmts) + { + new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); + gcc_assert (!new_bb); + } + + /* Generate [VF*S, VF*S, ... ]. */ + if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))) + { + expr = build_int_cst (integer_type_node, vf); + expr = fold_convert (TREE_TYPE (step_expr), expr); + } + else + expr = build_int_cst (TREE_TYPE (step_expr), vf); + new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr), + expr, step_expr); + if (! CONSTANT_CLASS_P (new_name)) + new_name = vect_init_vector (phi, new_name, + TREE_TYPE (step_expr), NULL); + new_vec = build_vector_from_val (vectype, new_name); + vec_step = vect_init_vector (phi, new_vec, vectype, NULL); + + /* Now generate the IVs. */ + unsigned group_size = SLP_TREE_SCALAR_STMTS (slp_node).length (); + unsigned nvects = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); + unsigned elts = nunits * nvects; + unsigned nivs = least_common_multiple (group_size, nunits) / nunits; + gcc_assert (elts % group_size == 0); + tree elt = init_expr; + unsigned ivn; + for (ivn = 0; ivn < nivs; ++ivn) + { + tree *elts = XALLOCAVEC (tree, nunits); + bool constant_p = true; + for (unsigned eltn = 0; eltn < nunits; ++eltn) + { + if (ivn*nunits + eltn >= group_size + && (ivn*nunits + eltn) % group_size == 0) + { + stmts = NULL; + elt = gimple_build (&stmts, PLUS_EXPR, TREE_TYPE (elt), + elt, step_expr); + if (stmts) + { + new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts); + gcc_assert (!new_bb); + } + } + if (! CONSTANT_CLASS_P (elt)) + constant_p = false; + elts[eltn] = elt; + } + if (constant_p) + new_vec = build_vector (vectype, elts); + else + { + vec *v; + vec_alloc (v, nunits); + for (i = 0; i < nunits; ++i) + CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, elts[i]); + new_vec = build_constructor (vectype, v); + } + vec_init = vect_init_vector (phi, new_vec, vectype, NULL); + + /* Create the induction-phi that defines the induction-operand. */ + vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); + induction_phi = create_phi_node (vec_dest, iv_loop->header); + set_vinfo_for_stmt (induction_phi, + new_stmt_vec_info (induction_phi, loop_vinfo)); + induc_def = PHI_RESULT (induction_phi); + + /* Create the iv update inside the loop */ + vec_def = make_ssa_name (vec_dest); + new_stmt = gimple_build_assign (vec_def, PLUS_EXPR, induc_def, vec_step); + gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); + set_vinfo_for_stmt (new_stmt, new_stmt_vec_info (new_stmt, loop_vinfo)); + + /* Set the arguments of the phi node: */ + add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION); + add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), + UNKNOWN_LOCATION); + + SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi); + } + + /* Re-use IVs when we can. */ + if (ivn < nvects) + { + unsigned vfp + = least_common_multiple (group_size, nunits) / group_size; + /* Generate [VF'*S, VF'*S, ... ]. */ + if (SCALAR_FLOAT_TYPE_P (TREE_TYPE (step_expr))) + { + expr = build_int_cst (integer_type_node, vfp); + expr = fold_convert (TREE_TYPE (step_expr), expr); + } + else + expr = build_int_cst (TREE_TYPE (step_expr), vfp); + new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr), + expr, step_expr); + if (! CONSTANT_CLASS_P (new_name)) + new_name = vect_init_vector (phi, new_name, + TREE_TYPE (step_expr), NULL); + new_vec = build_vector_from_val (vectype, new_name); + vec_step = vect_init_vector (phi, new_vec, vectype, NULL); + for (; ivn < nvects; ++ivn) + { + gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; + tree def; + if (gimple_code (iv) == GIMPLE_PHI) + def = gimple_phi_result (iv); + else + def = gimple_assign_lhs (iv); + new_stmt = gimple_build_assign (make_ssa_name (vectype), + PLUS_EXPR, + def, vec_step); + if (gimple_code (iv) == GIMPLE_PHI) + gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); + else + { + gimple_stmt_iterator tgsi = gsi_for_stmt (iv); + gsi_insert_after (&tgsi, new_stmt, GSI_CONTINUE_LINKING); + } + set_vinfo_for_stmt (new_stmt, + new_stmt_vec_info (new_stmt, loop_vinfo)); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + } + } + + return true; + } + /* Create the vector that holds the initial_value of the induction. */ if (nested_in_vect_loop) { @@ -6841,7 +6999,8 @@ vect_transform_loop (loop_vec_info loop_vinfo) && dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n"); - if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def) + if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_induction_def + && ! PURE_SLP_STMT (stmt_info)) { if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "transform phi.\n"); diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 31194b823d0..0800a3f6b23 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -100,6 +100,8 @@ vect_create_new_slp_node (vec scalar_stmts) if (gimple_assign_rhs_code (stmt) == COND_EXPR) nops++; } + else if (gimple_code (stmt) == GIMPLE_PHI) + nops = 0; else return NULL; @@ -401,9 +403,10 @@ again: { case vect_constant_def: case vect_external_def: - case vect_reduction_def: + case vect_reduction_def: break; + case vect_induction_def: case vect_internal_def: oprnd_info->def_stmts.quick_push (def_stmt); break; @@ -935,9 +938,23 @@ vect_build_slp_tree (vec_info *vinfo, if (gimple_assign_rhs_code (stmt) == COND_EXPR) nops++; } + else if (gimple_code (stmt) == GIMPLE_PHI) + nops = 0; else return NULL; + /* If the SLP node is a PHI (induction), terminate the recursion. */ + if (gimple_code (stmt) == GIMPLE_PHI) + { + FOR_EACH_VEC_ELT (stmts, i, stmt) + if (stmt != stmts[0]) + /* Induction from different IVs is not supported. */ + return NULL; + node = vect_create_new_slp_node (stmts); + return node; + } + + bool two_operators = false; unsigned char *swap = XALLOCAVEC (unsigned char, group_size); if (!vect_build_slp_tree_1 (vinfo, swap, @@ -987,7 +1004,8 @@ vect_build_slp_tree (vec_info *vinfo, unsigned old_tree_size = this_tree_size; unsigned int j; - if (oprnd_info->first_dt != vect_internal_def) + if (oprnd_info->first_dt != vect_internal_def + && oprnd_info->first_dt != vect_induction_def) continue; if (++this_tree_size > max_tree_size) @@ -1611,6 +1629,28 @@ vect_analyze_slp_cost_1 (slp_instance instance, slp_tree node, return; } } + else if (STMT_VINFO_TYPE (stmt_info) == induc_vec_info_type) + { + /* ncopies_for_cost is the number of IVs we generate. */ + record_stmt_cost (body_cost_vec, ncopies_for_cost, vector_stmt, + stmt_info, 0, vect_body); + + /* Prologue cost for the initial values and step vector. */ + record_stmt_cost (prologue_cost_vec, ncopies_for_cost, + CONSTANT_CLASS_P + (STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED + (stmt_info)) + ? vector_load : vec_construct, + stmt_info, 0, vect_prologue); + record_stmt_cost (prologue_cost_vec, 1, + CONSTANT_CLASS_P + (STMT_VINFO_LOOP_PHI_EVOLUTION_PART (stmt_info)) + ? vector_load : vec_construct, + stmt_info, 0, vect_prologue); + + /* ??? No easy way to get at the actual number of vector stmts + to be geneated and thus the derived IVs. */ + } else { record_stmt_cost (body_cost_vec, ncopies_for_cost, vector_stmt, @@ -2169,8 +2209,13 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype) if (! STMT_VINFO_IN_PATTERN_P (stmt_vinfo) && STMT_VINFO_RELATED_STMT (stmt_vinfo)) stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo); - if (TREE_CODE (gimple_op (stmt, 0)) == SSA_NAME) - FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, gimple_op (stmt, 0)) + tree def; + if (gimple_code (stmt) == GIMPLE_PHI) + def = gimple_phi_result (stmt); + else + def = SINGLE_SSA_TREE_OPERAND (stmt, SSA_OP_DEF); + if (def) + FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, def) { if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))) continue; @@ -3277,7 +3322,10 @@ vect_get_slp_vect_defs (slp_tree slp_node, vec *vec_oprnds) FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt) { gcc_assert (vec_def_stmt); - vec_oprnd = gimple_get_lhs (vec_def_stmt); + if (gimple_code (vec_def_stmt) == GIMPLE_PHI) + vec_oprnd = gimple_phi_result (vec_def_stmt); + else + vec_oprnd = gimple_get_lhs (vec_def_stmt); vec_oprnds->quick_push (vec_oprnd); } } @@ -3331,8 +3379,13 @@ vect_get_slp_defs (vec ops, slp_tree slp_node, gimple *first_def = SLP_TREE_SCALAR_STMTS (child)[0]; gimple *related = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (first_def)); + tree first_def_op; - if (operand_equal_p (oprnd, gimple_get_lhs (first_def), 0) + if (gimple_code (first_def) == GIMPLE_PHI) + first_def_op = gimple_phi_result (first_def); + else + first_def_op = gimple_get_lhs (first_def); + if (operand_equal_p (oprnd, first_def_op, 0) || (related && operand_equal_p (oprnd, gimple_get_lhs (related), 0))) { @@ -3372,9 +3425,9 @@ vect_get_slp_defs (vec ops, slp_tree slp_node, /* The defs are already vectorized. */ vect_get_slp_vect_defs (child, &vec_defs); else - /* Build vectors from scalar defs. */ + /* Build vectors from scalar defs. */ vect_get_constant_vectors (oprnd, slp_node, &vec_defs, i, - number_of_vects, reduc_index); + number_of_vects, reduc_index); vec_oprnds->quick_push (vec_defs); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 1e9d42cd53f..d1d97cb662e 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -8520,6 +8520,9 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) break; case vect_induction_def: + gcc_assert (!bb_vinfo); + break; + case vect_constant_def: case vect_external_def: case vect_unknown_def_type: @@ -8598,6 +8601,7 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) || vectorizable_call (stmt, NULL, NULL, node) || vectorizable_store (stmt, NULL, NULL, node) || vectorizable_reduction (stmt, NULL, NULL, node) + || vectorizable_induction (stmt, NULL, NULL, node) || vectorizable_condition (stmt, NULL, NULL, NULL, 0, node) || vectorizable_comparison (stmt, NULL, NULL, NULL, node)); else @@ -8681,8 +8685,7 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, break; case induc_vec_info_type: - gcc_assert (!slp_node); - done = vectorizable_induction (stmt, gsi, &vec_stmt); + done = vectorizable_induction (stmt, gsi, &vec_stmt, slp_node); gcc_assert (done); break; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index df8da9eb1fd..3dacb1aee47 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -1159,7 +1159,8 @@ extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *, slp_tree, int, gimple **); extern bool vectorizable_reduction (gimple *, gimple_stmt_iterator *, gimple **, slp_tree); -extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, gimple **); +extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, + gimple **, slp_tree); extern tree get_initial_def_for_reduction (gimple *, tree, tree *); extern int vect_min_worthwhile_factor (enum tree_code); extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, -- 2.30.2