From b28ead45fe630b9e018a81cb46b7e1f971e4dc17 Mon Sep 17 00:00:00 2001 From: Alan Hayward Date: Fri, 3 Jun 2016 13:00:06 +0000 Subject: [PATCH] [2/3] Vectorize inductions that are live after the loop 2016-06-03 Alan Hayward [2/3] Vectorize inductions that are live after the loop gcc/ * tree-vect-loop.c (vect_analyze_loop_operations): Allow live stmts. (vectorizable_reduction): Check for new relevant state. (vectorizable_live_operation): vectorize live stmts using BIT_FIELD_REF. Remove special case for gimple assigns stmts. * tree-vect-stmts.c (is_simple_and_all_uses_invariant): New function. (vect_stmt_relevant_p): Check for stmts which are only used live. (process_use): Use of a stmt does not inherit it's live value. (vect_mark_stmts_to_be_vectorized): Simplify relevance inheritance. (vect_analyze_stmt): Check for new relevant state. * tree-vectorizer.h (vect_relevant): New entry for a stmt which is used outside the loop, but not inside it. testsuite/ * gcc.dg/tree-ssa/pr64183.c: Ensure test does not vectorize. * testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c: Remove xfail. * gcc.dg/vect/vect-live-1.c: New test. * gcc.dg/vect/vect-live-2.c: New test. * gcc.dg/vect/vect-live-3.c: New test. * gcc.dg/vect/vect-live-4.c: New test. * gcc.dg/vect/vect-live-5.c: New test. * gcc.dg/vect/vect-live-slp-1.c: New test. * gcc.dg/vect/vect-live-slp-2.c: New test. * gcc.dg/vect/vect-live-slp-3.c: New test. From-SVN: r237064 --- gcc/ChangeLog | 14 ++ gcc/testsuite/ChangeLog | 13 ++ gcc/testsuite/gcc.dg/tree-ssa/pr64183.c | 2 +- .../gcc.dg/vect/no-scevccp-vect-iv-2.c | 6 +- gcc/testsuite/gcc.dg/vect/vect-live-1.c | 46 +++++ gcc/testsuite/gcc.dg/vect/vect-live-2.c | 55 ++++++ gcc/testsuite/gcc.dg/vect/vect-live-3.c | 53 ++++++ gcc/testsuite/gcc.dg/vect/vect-live-4.c | 50 ++++++ gcc/testsuite/gcc.dg/vect/vect-live-5.c | 50 ++++++ gcc/testsuite/gcc.dg/vect/vect-live-slp-1.c | 70 ++++++++ gcc/testsuite/gcc.dg/vect/vect-live-slp-2.c | 64 +++++++ gcc/testsuite/gcc.dg/vect/vect-live-slp-3.c | 71 ++++++++ gcc/tree-vect-loop.c | 157 ++++++++++-------- gcc/tree-vect-stmts.c | 148 +++++++++++------ gcc/tree-vectorizer.h | 6 +- 15 files changed, 678 insertions(+), 127 deletions(-) create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-1.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-2.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-3.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-4.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-5.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-slp-1.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-slp-2.c create mode 100644 gcc/testsuite/gcc.dg/vect/vect-live-slp-3.c diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9d75bca4c1c..a7476c64d5a 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,17 @@ +2016-06-03 Alan Hayward + + * tree-vect-loop.c (vect_analyze_loop_operations): Allow live stmts. + (vectorizable_reduction): Check for new relevant state. + (vectorizable_live_operation): vectorize live stmts using + BIT_FIELD_REF. Remove special case for gimple assigns stmts. + * tree-vect-stmts.c (is_simple_and_all_uses_invariant): New function. + (vect_stmt_relevant_p): Check for stmts which are only used live. + (process_use): Use of a stmt does not inherit it's live value. + (vect_mark_stmts_to_be_vectorized): Simplify relevance inheritance. + (vect_analyze_stmt): Check for new relevant state. + * tree-vectorizer.h (vect_relevant): New entry for a stmt which is used + outside the loop, but not inside it. + 2016-06-03 Alan Hayward * tree-vectorizer.h (vect_get_vec_def_for_operand_1): New diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index c6d6392b144..0c976b67901 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,16 @@ +2016-06-03 Alan Hayward + + * gcc.dg/tree-ssa/pr64183.c: Ensure test does not vectorize. + * testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c: Remove xfail. + * gcc.dg/vect/vect-live-1.c: New test. + * gcc.dg/vect/vect-live-2.c: New test. + * gcc.dg/vect/vect-live-3.c: New test. + * gcc.dg/vect/vect-live-4.c: New test. + * gcc.dg/vect/vect-live-5.c: New test. + * gcc.dg/vect/vect-live-slp-1.c: New test. + * gcc.dg/vect/vect-live-slp-2.c: New test. + * gcc.dg/vect/vect-live-slp-3.c: New test. + 2016-06-03 Jakub Jelinek PR middle-end/71387 diff --git a/gcc/testsuite/gcc.dg/tree-ssa/pr64183.c b/gcc/testsuite/gcc.dg/tree-ssa/pr64183.c index 1c87d6ac586..8a3fadc7602 100644 --- a/gcc/testsuite/gcc.dg/tree-ssa/pr64183.c +++ b/gcc/testsuite/gcc.dg/tree-ssa/pr64183.c @@ -1,5 +1,5 @@ /* { dg-do compile } */ -/* { dg-options "-O3 -fdump-tree-cunroll-details" } */ +/* { dg-options "-O3 -fno-tree-vectorize -fdump-tree-cunroll-details" } */ int bits; unsigned int size; diff --git a/gcc/testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c b/gcc/testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c index e541bb65276..131d2d9e03f 100644 --- a/gcc/testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c +++ b/gcc/testsuite/gcc.dg/vect/no-scevccp-vect-iv-2.c @@ -12,9 +12,7 @@ int main1 () int k = 0; int m = 3, i = 0; - /* Vectorization of induction that is used after the loop. - Currently vectorizable because scev_ccp disconnects the - use-after-the-loop from the iv def inside the loop. */ + /* Vectorization of induction that is used after the loop. */ do { k = k + 2; @@ -46,4 +44,4 @@ int main (void) return 0; } -/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { xfail *-*-* } } } */ +/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" } } */ diff --git a/gcc/testsuite/gcc.dg/vect/vect-live-1.c b/gcc/testsuite/gcc.dg/vect/vect-live-1.c new file mode 100644 index 00000000000..3f3a44f8bfb --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-live-1.c @@ -0,0 +1,46 @@ +/* { dg-require-effective-target vect_int } */ +/* { dg-options "-O2 -ftree-vectorize -fno-tree-scev-cprop -fdump-tree-vect-details" } */ + +#include "tree-vect.h" + +/* Statement used outside the loop. + NOTE: SCEV disabled to ensure the live operation is not removed before + vectorization. */ +__attribute__ ((noinline)) int +liveloop (int start, int n, int *x) +{ + int i = start; + int j; + + for (j = 0; j < n; ++j) + { + i += 1; + x[j] = i; + } + return i; +} + +#define MAX 62 +#define START 27 + +int +main (void) +{ + int a[MAX]; + int i; + + int ret = liveloop (START, MAX, a); + + if (ret != MAX + START) + abort (); + + for (i=0; i vec_oprnds; gcc_assert (STMT_VINFO_LIVE_P (stmt_info)); if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def) return false; - if (!is_gimple_assign (stmt)) + /* FORNOW. CHECKME. */ + if (nested_in_vect_loop_p (loop, stmt)) + return false; + + /* If STMT is a simple assignment and its inputs are invariant, then it can + remain in place, unvectorized. The original last scalar value that it + computes will be used. */ + if (is_simple_and_all_uses_invariant (stmt, loop_vinfo)) { - if (gimple_call_internal_p (stmt) - && gimple_call_internal_fn (stmt) == IFN_GOMP_SIMD_LANE - && gimple_call_lhs (stmt) - && loop->simduid - && TREE_CODE (gimple_call_arg (stmt, 0)) == SSA_NAME - && loop->simduid - == SSA_NAME_VAR (gimple_call_arg (stmt, 0))) - { - edge e = single_exit (loop); - basic_block merge_bb = e->dest; - imm_use_iterator imm_iter; - use_operand_p use_p; - tree lhs = gimple_call_lhs (stmt); + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "statement is simple and uses invariant. Leaving in " + "place.\n"); + return true; + } - FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs) - { - gimple *use_stmt = USE_STMT (use_p); - if (gimple_code (use_stmt) == GIMPLE_PHI - && gimple_bb (use_stmt) == merge_bb) - { - if (vec_stmt) - { - tree vfm1 - = build_int_cst (unsigned_type_node, - loop_vinfo->vectorization_factor - 1); - SET_PHI_ARG_DEF (use_stmt, e->dest_idx, vfm1); - } - return true; - } - } - } + if (!vec_stmt) + /* No transformation required. */ + return true; - return false; - } + /* If stmt has a related stmt, then use that for getting the lhs. */ + if (is_pattern_stmt_p (stmt_info)) + stmt = STMT_VINFO_RELATED_STMT (stmt_info); - if (TREE_CODE (gimple_assign_lhs (stmt)) != SSA_NAME) - return false; + lhs = (is_a (stmt)) ? gimple_phi_result (stmt) + : gimple_get_lhs (stmt); + lhs_type = TREE_TYPE (lhs); - /* FORNOW. CHECKME. */ - if (nested_in_vect_loop_p (loop, stmt)) - return false; + /* Find all uses of STMT outside the loop - there should be exactly one. */ + auto_vec worklist; + FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, lhs) + if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))) + worklist.safe_push (use_stmt); + gcc_assert (worklist.length () == 1); + + bitsize = TYPE_SIZE (lhs_type); + vec_bitsize = TYPE_SIZE (vectype); - /* FORNOW: support only if all uses are invariant. This means - that the scalar operations can remain in place, unvectorized. - The original last scalar value that they compute will be used. */ - FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE) + /* Get the vectorized lhs of STMT and the lane to use (counted in bits). */ + tree vec_lhs, bitstart; + if (slp_node) { - enum vect_def_type dt = vect_uninitialized_def; + gcc_assert (slp_index >= 0); - if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "use not simple.\n"); - return false; - } + int num_scalar = SLP_TREE_SCALAR_STMTS (slp_node).length (); + int num_vec = SLP_TREE_NUMBER_OF_VEC_STMTS (slp_node); + int scalar_per_vec = num_scalar / num_vec; - if (dt != vect_external_def && dt != vect_constant_def) - return false; + /* There are three possibilites here: + 1: All scalar stmts fit in a single vector. + 2: All scalar stmts fit multiple times into a single vector. + We must choose the last occurence of stmt in the vector. + 3: Scalar stmts are split across multiple vectors. + We must choose the correct vector and mod the lane accordingly. */ + + /* Get the correct slp vectorized stmt. */ + int vec_entry = slp_index / scalar_per_vec; + vec_lhs = gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[vec_entry]); + + /* Get entry to use. */ + bitstart = build_int_cst (unsigned_type_node, + scalar_per_vec - (slp_index % scalar_per_vec)); + bitstart = int_const_binop (MULT_EXPR, bitsize, bitstart); + bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitstart); + } + else + { + enum vect_def_type dt = STMT_VINFO_DEF_TYPE (stmt_info); + vec_lhs = vect_get_vec_def_for_operand_1 (stmt, dt); + + /* For multiple copies, get the last copy. */ + for (int i = 1; i < ncopies; ++i) + vec_lhs = vect_get_vec_def_for_stmt_copy (vect_unknown_def_type, + vec_lhs); + + /* Get the last lane in the vector. */ + bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitsize); } - /* No transformation is required for the cases we currently support. */ + /* Create a new vectorized stmt for the uses of STMT and insert outside the + loop. */ + tree new_name = make_ssa_name (lhs_type); + tree new_tree = build3 (BIT_FIELD_REF, lhs_type, vec_lhs, bitsize, bitstart); + gimple *new_stmt = gimple_build_assign (new_name, new_tree); + gsi_insert_on_edge_immediate (single_exit (loop), new_stmt); + + /* Replace all uses of the USE_STMT in the worklist with the newly inserted + statement. */ + use_stmt = worklist.pop (); + replace_uses_by (gimple_phi_result (use_stmt), new_name); + update_stmt (use_stmt); + return true; } diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 231bf4e0e1c..5554fe8a46d 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -236,6 +236,38 @@ vect_mark_relevant (vec *worklist, gimple *stmt, } +/* Function is_simple_and_all_uses_invariant + + Return true if STMT is simple and all uses of it are invariant. */ + +bool +is_simple_and_all_uses_invariant (gimple *stmt, loop_vec_info loop_vinfo) +{ + tree op; + gimple *def_stmt; + ssa_op_iter iter; + + if (!is_gimple_assign (stmt)) + return false; + + FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE) + { + enum vect_def_type dt = vect_uninitialized_def; + + if (!vect_is_simple_use (op, loop_vinfo, &def_stmt, &dt)) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "use not simple.\n"); + return false; + } + + if (dt != vect_external_def && dt != vect_constant_def) + return false; + } + return true; +} + /* Function vect_stmt_relevant_p. Return true if STMT in loop that is represented by LOOP_VINFO is @@ -303,6 +335,14 @@ vect_stmt_relevant_p (gimple *stmt, loop_vec_info loop_vinfo, } } + if (*live_p && *relevant == vect_unused_in_scope) + { + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, + "vec_stmt_relevant_p: stmt live but not relevant.\n"); + *relevant = vect_used_only_live; + } + return (*live_p || *relevant); } @@ -377,7 +417,7 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) Inputs: - a USE in STMT in a loop represented by LOOP_VINFO - - LIVE_P, RELEVANT - enum values to be set in the STMT_VINFO of the stmt + - RELEVANT - enum value to be set in the STMT_VINFO of the stmt that defined USE. This is done by calling mark_relevant and passing it the WORKLIST (to add DEF_STMT to the WORKLIST in case it is relevant). - FORCE is true if exist_non_indexing_operands_for_use_p check shouldn't @@ -400,7 +440,7 @@ exist_non_indexing_operands_for_use_p (tree use, gimple *stmt) Return true if everything is as expected. Return false otherwise. */ static bool -process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, bool live_p, +process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, enum vect_relevant relevant, vec *worklist, bool force) { @@ -519,6 +559,7 @@ process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, bool live_p, break; case vect_used_by_reduction: + case vect_used_only_live: relevant = vect_used_in_outer_by_reduction; break; @@ -531,7 +572,7 @@ process_use (gimple *stmt, tree use, loop_vec_info loop_vinfo, bool live_p, } } - vect_mark_relevant (worklist, def_stmt, relevant, live_p); + vect_mark_relevant (worklist, def_stmt, relevant, false); return true; } @@ -565,8 +606,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) basic_block bb; gimple *phi; bool live_p; - enum vect_relevant relevant, tmp_relevant; - enum vect_def_type def_type; + enum vect_relevant relevant; if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -618,57 +658,42 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) } /* Examine the USEs of STMT. For each USE, mark the stmt that defines it - (DEF_STMT) as relevant/irrelevant and live/dead according to the - liveness and relevance properties of STMT. */ + (DEF_STMT) as relevant/irrelevant according to the relevance property + of STMT. */ stmt_vinfo = vinfo_for_stmt (stmt); relevant = STMT_VINFO_RELEVANT (stmt_vinfo); - live_p = STMT_VINFO_LIVE_P (stmt_vinfo); - /* Generally, the liveness and relevance properties of STMT are - propagated as is to the DEF_STMTs of its USEs: - live_p <-- STMT_VINFO_LIVE_P (STMT_VINFO) - relevant <-- STMT_VINFO_RELEVANT (STMT_VINFO) + /* Generally, the relevance property of STMT (in STMT_VINFO_RELEVANT) is + propagated as is to the DEF_STMTs of its USEs. One exception is when STMT has been identified as defining a reduction - variable; in this case we set the liveness/relevance as follows: - live_p = false - relevant = vect_used_by_reduction + variable; in this case we set the relevance to vect_used_by_reduction. This is because we distinguish between two kinds of relevant stmts - those that are used by a reduction computation, and those that are (also) used by a regular computation. This allows us later on to identify stmts that are used solely by a reduction, and therefore the order of the results that they produce does not have to be kept. */ - def_type = STMT_VINFO_DEF_TYPE (stmt_vinfo); - tmp_relevant = relevant; - switch (def_type) + switch (STMT_VINFO_DEF_TYPE (stmt_vinfo)) { case vect_reduction_def: - switch (tmp_relevant) + gcc_assert (relevant != vect_unused_in_scope); + if (relevant != vect_unused_in_scope + && relevant != vect_used_in_scope + && relevant != vect_used_by_reduction + && relevant != vect_used_only_live) { - case vect_unused_in_scope: - relevant = vect_used_by_reduction; - break; - - case vect_used_by_reduction: - if (gimple_code (stmt) == GIMPLE_PHI) - break; - /* fall through */ - - default: - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "unsupported use of reduction.\n"); - return false; + if (dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "unsupported use of reduction.\n"); + return false; } - - live_p = false; break; case vect_nested_cycle: - if (tmp_relevant != vect_unused_in_scope - && tmp_relevant != vect_used_in_outer_by_reduction - && tmp_relevant != vect_used_in_outer) + if (relevant != vect_unused_in_scope + && relevant != vect_used_in_outer_by_reduction + && relevant != vect_used_in_outer) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -676,13 +701,12 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) return false; } - - live_p = false; break; case vect_double_reduction_def: - if (tmp_relevant != vect_unused_in_scope - && tmp_relevant != vect_used_by_reduction) + if (relevant != vect_unused_in_scope + && relevant != vect_used_by_reduction + && relevant != vect_used_only_live) { if (dump_enabled_p ()) dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, @@ -690,8 +714,6 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) return false; } - - live_p = false; break; default: @@ -712,9 +734,9 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) if (rhs_code == COND_EXPR && COMPARISON_CLASS_P (op)) { if (!process_use (stmt, TREE_OPERAND (op, 0), loop_vinfo, - live_p, relevant, &worklist, false) + relevant, &worklist, false) || !process_use (stmt, TREE_OPERAND (op, 1), loop_vinfo, - live_p, relevant, &worklist, false)) + relevant, &worklist, false)) return false; i = 2; } @@ -722,7 +744,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) { op = gimple_op (stmt, i); if (TREE_CODE (op) == SSA_NAME - && !process_use (stmt, op, loop_vinfo, live_p, relevant, + && !process_use (stmt, op, loop_vinfo, relevant, &worklist, false)) return false; } @@ -732,7 +754,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) for (i = 0; i < gimple_call_num_args (stmt); i++) { tree arg = gimple_call_arg (stmt, i); - if (!process_use (stmt, arg, loop_vinfo, live_p, relevant, + if (!process_use (stmt, arg, loop_vinfo, relevant, &worklist, false)) return false; } @@ -742,7 +764,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) FOR_EACH_PHI_OR_STMT_USE (use_p, stmt, iter, SSA_OP_USE) { tree op = USE_FROM_PTR (use_p); - if (!process_use (stmt, op, loop_vinfo, live_p, relevant, + if (!process_use (stmt, op, loop_vinfo, relevant, &worklist, false)) return false; } @@ -752,8 +774,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) tree off; tree decl = vect_check_gather_scatter (stmt, loop_vinfo, NULL, &off, NULL); gcc_assert (decl); - if (!process_use (stmt, off, loop_vinfo, live_p, relevant, - &worklist, true)) + if (!process_use (stmt, off, loop_vinfo, relevant, &worklist, true)) return false; } } /* while worklist */ @@ -8001,7 +8022,8 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) && (relevance == vect_used_in_outer || relevance == vect_used_in_outer_by_reduction || relevance == vect_used_by_reduction - || relevance == vect_unused_in_scope)); + || relevance == vect_unused_in_scope + || relevance == vect_used_only_live)); break; case vect_induction_def: @@ -8115,7 +8137,7 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node) need extra handling, except for vectorizable reductions. */ if (STMT_VINFO_LIVE_P (stmt_info) && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type) - ok = vectorizable_live_operation (stmt, NULL, NULL); + ok = vectorizable_live_operation (stmt, NULL, NULL, -1, NULL); if (!ok) { @@ -8291,10 +8313,26 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, /* Handle stmts whose DEF is used outside the loop-nest that is being vectorized. */ - if (STMT_VINFO_LIVE_P (stmt_info) + if (slp_node) + { + gimple *slp_stmt; + int i; + FOR_EACH_VEC_ELT (SLP_TREE_SCALAR_STMTS (slp_node), i, slp_stmt) + { + stmt_vec_info slp_stmt_info = vinfo_for_stmt (slp_stmt); + if (STMT_VINFO_LIVE_P (slp_stmt_info) + && STMT_VINFO_TYPE (slp_stmt_info) != reduc_vec_info_type) + { + done = vectorizable_live_operation (slp_stmt, gsi, slp_node, i, + &vec_stmt); + gcc_assert (done); + } + } + } + else if (STMT_VINFO_LIVE_P (stmt_info) && STMT_VINFO_TYPE (stmt_info) != reduc_vec_info_type) { - done = vectorizable_live_operation (stmt, gsi, &vec_stmt); + done = vectorizable_live_operation (stmt, gsi, slp_node, -1, &vec_stmt); gcc_assert (done); } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 18f9f7a9ef9..6d52b5531a0 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -442,6 +442,9 @@ enum stmt_vec_info_type { block. */ enum vect_relevant { vect_unused_in_scope = 0, + + /* The def is only used outside the loop. */ + vect_used_only_live, /* The def is in the inner loop, and the use is in the outer loop, and the use is a reduction stmt. */ vect_used_in_outer_by_reduction, @@ -1072,7 +1075,7 @@ extern loop_vec_info vect_analyze_loop (struct loop *); extern void vect_transform_loop (loop_vec_info); extern loop_vec_info vect_analyze_loop_form (struct loop *); extern bool vectorizable_live_operation (gimple *, gimple_stmt_iterator *, - gimple **); + slp_tree, int, gimple **); extern bool vectorizable_reduction (gimple *, gimple_stmt_iterator *, gimple **, slp_tree); extern bool vectorizable_induction (gimple *, gimple_stmt_iterator *, gimple **); @@ -1098,6 +1101,7 @@ extern void vect_get_slp_defs (vec , slp_tree, vec > *, int); extern bool vect_slp_bb (basic_block); extern gimple *vect_find_last_scalar_stmt_in_slp (slp_tree); +extern bool is_simple_and_all_uses_invariant (gimple *, loop_vec_info); /* In tree-vect-patterns.c. */ /* Pattern recognition functions. -- 2.30.2