From: Ira Rosen Date: Thu, 30 Jun 2011 06:37:41 +0000 (+0000) Subject: tree-vect-loop.c (vect_determine_vectorization_factor): Handle both pattern and origi... X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=83197f37339cecf1c89724341a85aae539751f84;p=gcc.git tree-vect-loop.c (vect_determine_vectorization_factor): Handle both pattern and original statements if necessary. * tree-vect-loop.c (vect_determine_vectorization_factor): Handle both pattern and original statements if necessary. (vect_transform_loop): Likewise. * tree-vect-patterns.c (vect_pattern_recog): Update documentation. * tree-vect-stmts.c (vect_mark_relevant): Add new argument. Mark the pattern statement only if the original statement doesn't have its own uses. (process_use): Call vect_mark_relevant with additional parameter. (vect_mark_stmts_to_be_vectorized): Likewise. (vect_get_vec_def_for_operand): Use vectorized pattern statement. (vect_analyze_stmt): Handle both pattern and original statements if necessary. (vect_transform_stmt): Don't store vectorized pattern statement in the original statement. (vect_is_simple_use_1): Use related pattern statement only if the original statement is irrelevant. * tree-vect-slp.c (vect_get_and_check_slp_defs): Likewise. From-SVN: r175681 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 97e17fabff6..9db422d5772 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,23 @@ +2011-06-30 Ira Rosen + + * tree-vect-loop.c (vect_determine_vectorization_factor): Handle + both pattern and original statements if necessary. + (vect_transform_loop): Likewise. + * tree-vect-patterns.c (vect_pattern_recog): Update documentation. + * tree-vect-stmts.c (vect_mark_relevant): Add new argument. + Mark the pattern statement only if the original statement doesn't + have its own uses. + (process_use): Call vect_mark_relevant with additional parameter. + (vect_mark_stmts_to_be_vectorized): Likewise. + (vect_get_vec_def_for_operand): Use vectorized pattern statement. + (vect_analyze_stmt): Handle both pattern and original statements + if necessary. + (vect_transform_stmt): Don't store vectorized pattern statement + in the original statement. + (vect_is_simple_use_1): Use related pattern statement only if the + original statement is irrelevant. + * tree-vect-slp.c (vect_get_and_check_slp_defs): Likewise. + 2011-06-29 Changpeng Fang * config/i386/i386.opt (mprefer-avx128): Redefine the flag as a Mask option. diff --git a/gcc/testsuite/ChangeLog b/gcc/testsuite/ChangeLog index 7afaafb1f31..ac678156cc3 100644 --- a/gcc/testsuite/ChangeLog +++ b/gcc/testsuite/ChangeLog @@ -1,3 +1,8 @@ +2011-06-30 Ira Rosen + + * gcc.dg/vect/slp-widen-mult-half.c: New test. + * gcc.dg/vect/vect-widen-mult-half.c: New test. + 2011-06-29 Jason Merrill PR c++/49216 diff --git a/gcc/testsuite/gcc.dg/vect/slp-widen-mult-half.c b/gcc/testsuite/gcc.dg/vect/slp-widen-mult-half.c new file mode 100644 index 00000000000..e7a0f2a75d4 --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/slp-widen-mult-half.c @@ -0,0 +1,52 @@ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" +#include + +#define N 32 +#define COEF 32470 +#define COEF2 324700 + +unsigned char in[N]; +int out[N]; +int out2[N]; + +__attribute__ ((noinline)) void +foo () +{ + int i; + + for (i = 0; i < N/2; i++) + { + out[2*i] = in[2*i] * COEF; + out2[2*i] = in[2*i] + COEF2; + out[2*i+1] = in[2*i+1] * COEF; + out2[2*i+1] = in[2*i+1] + COEF2; + } +} + +int main (void) +{ + int i; + + for (i = 0; i < N; i++) + { + in[i] = i; + __asm__ volatile (""); + } + + foo (); + + for (i = 0; i < N; i++) + if (out[i] != in[i] * COEF || out2[i] != in[i] + COEF2) + abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target vect_widen_mult_hi_to_si } } } */ +/* { dg-final { scan-tree-dump-times "vectorizing stmts using SLP" 2 "vect" { target vect_widen_mult_hi_to_si } } } */ +/* { dg-final { scan-tree-dump-times "vect_recog_widen_mult_pattern: detected" 2 "vect" { target vect_widen_mult_hi_to_si_pattern } } } */ +/* { dg-final { scan-tree-dump-times "pattern recognized" 2 "vect" { target vect_widen_mult_hi_to_si_pattern } } } */ +/* { dg-final { cleanup-tree-dump "vect" } } */ + diff --git a/gcc/testsuite/gcc.dg/vect/vect-widen-mult-half.c b/gcc/testsuite/gcc.dg/vect/vect-widen-mult-half.c new file mode 100644 index 00000000000..3fb162de79a --- /dev/null +++ b/gcc/testsuite/gcc.dg/vect/vect-widen-mult-half.c @@ -0,0 +1,49 @@ +/* { dg-require-effective-target vect_int } */ + +#include "tree-vect.h" +#include + +#define N 32 +#define COEF 32470 +#define COEF2 324700 + +unsigned char in[N]; +int out[N]; +int out2[N]; + +__attribute__ ((noinline)) void +foo (int a) +{ + int i; + + for (i = 0; i < N; i++) + { + out[i] = in[i] * COEF; + out2[i] = in[i] + a; + } +} + +int main (void) +{ + int i; + + for (i = 0; i < N; i++) + { + in[i] = i; + __asm__ volatile (""); + } + + foo (COEF2); + + for (i = 0; i < N; i++) + if (out[i] != in[i] * COEF || out2[i] != in[i] + COEF2) + abort (); + + return 0; +} + +/* { dg-final { scan-tree-dump-times "vectorized 1 loops" 1 "vect" { target vect_widen_mult_hi_to_si } } } */ +/* { dg-final { scan-tree-dump-times "vect_recog_widen_mult_pattern: detected" 1 "vect" { target vect_widen_mult_hi_to_si_pattern } } } */ +/* { dg-final { scan-tree-dump-times "pattern recognized" 1 "vect" { target vect_widen_mult_hi_to_si_pattern } } } */ +/* { dg-final { cleanup-tree-dump "vect" } } */ + diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 76912673882..0352607732b 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -181,6 +181,8 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) stmt_vec_info stmt_info; int i; HOST_WIDE_INT dummy; + gimple stmt, pattern_stmt = NULL; + bool analyze_pattern_stmt = false; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "=== vect_determine_vectorization_factor ==="); @@ -241,11 +243,19 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) } } - for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) + for (si = gsi_start_bb (bb); !gsi_end_p (si) || analyze_pattern_stmt;) { - tree vf_vectype; - gimple stmt = gsi_stmt (si), pattern_stmt; - stmt_info = vinfo_for_stmt (stmt); + tree vf_vectype; + + if (analyze_pattern_stmt) + { + stmt = pattern_stmt; + analyze_pattern_stmt = false; + } + else + stmt = gsi_stmt (si); + + stmt_info = vinfo_for_stmt (stmt); if (vect_print_dump_info (REPORT_DETAILS)) { @@ -276,9 +286,15 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "skip."); + gsi_next (&si); continue; } } + else if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) + && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) + analyze_pattern_stmt = true; if (gimple_get_lhs (stmt) == NULL_TREE) { @@ -383,6 +399,9 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) if (!vectorization_factor || (nunits > vectorization_factor)) vectorization_factor = nunits; + + if (!analyze_pattern_stmt) + gsi_next (&si); } } @@ -5057,6 +5076,8 @@ vect_transform_loop (loop_vec_info loop_vinfo) tree cond_expr = NULL_TREE; gimple_seq cond_expr_stmt_list = NULL; bool do_peeling_for_loop_bound; + gimple stmt, pattern_stmt; + bool transform_pattern_stmt = false; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "=== vec_transform_loop ==="); @@ -5144,11 +5165,19 @@ vect_transform_loop (loop_vec_info loop_vinfo) } } - for (si = gsi_start_bb (bb); !gsi_end_p (si);) + pattern_stmt = NULL; + for (si = gsi_start_bb (bb); !gsi_end_p (si) || transform_pattern_stmt;) { - gimple stmt = gsi_stmt (si), pattern_stmt; bool is_store; + if (transform_pattern_stmt) + { + stmt = pattern_stmt; + transform_pattern_stmt = false; + } + else + stmt = gsi_stmt (si); + if (vect_print_dump_info (REPORT_DETAILS)) { fprintf (vect_dump, "------>vectorizing statement: "); @@ -5186,6 +5215,11 @@ vect_transform_loop (loop_vec_info loop_vinfo) continue; } } + else if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) + && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) + transform_pattern_stmt = true; gcc_assert (STMT_VINFO_VECTYPE (stmt_info)); nunits = (unsigned int) TYPE_VECTOR_SUBPARTS ( @@ -5214,8 +5248,9 @@ vect_transform_loop (loop_vec_info loop_vinfo) /* Hybrid SLP stmts must be vectorized in addition to SLP. */ if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info)) { - gsi_next (&si); - continue; + if (!transform_pattern_stmt) + gsi_next (&si); + continue; } } @@ -5234,7 +5269,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) the chain. */ vect_remove_stores (GROUP_FIRST_ELEMENT (stmt_info)); gsi_remove (&si, true); - continue; + continue; } else { @@ -5244,7 +5279,9 @@ vect_transform_loop (loop_vec_info loop_vinfo) continue; } } - gsi_next (&si); + + if (!transform_pattern_stmt) + gsi_next (&si); } /* stmts in BB */ } /* BBs in loop */ diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 429dd87cfff..758d4bcbaed 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -1016,10 +1016,8 @@ vect_pattern_recog_1 ( If vectorization succeeds, vect_transform_stmt will skip over {S1,S2,S3} (because they are marked as irrelevant). It will vectorize S6, and record - a pointer to the new vector stmt VS6 both from S6 (as usual), and also - from S4. We do that so that when we get to vectorizing stmts that use the - def of S4 (like S5 that uses a_0), we'll know where to take the relevant - vector-def from. S4 will be skipped, and S5 will be vectorized as usual: + a pointer to the new vector stmt VS6 from S6 (as usual). + S4 will be skipped, and S5 will be vectorized as usual: in_pattern_p related_stmt vec_stmt S1: a_i = .... - - - @@ -1035,7 +1033,21 @@ vect_pattern_recog_1 ( elsewhere), and we'll end up with: VS6: va_new = .... - VS5: ... = ..vuse(va_new).. */ + VS5: ... = ..vuse(va_new).. + + In case of more than one pattern statements, e.g., widen-mult with + intermediate type: + + S1 a_t = ; + S2 a_T = (TYPE) a_t; + '--> S3: a_it = (interm_type) a_t; + S4 prod_T = a_T * CONST; + '--> S5: prod_T' = a_it w* CONST; + + there may be other users of a_T outside the pattern. In that case S2 will + be marked as relevant (as well as S3), and both S2 and S3 will be analyzed + and vectorized. The vector stmt VS2 will be recorded in S2, and VS3 will + be recorded in S3. */ void vect_pattern_recog (loop_vec_info loop_vinfo) diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index efedf66e2ec..60bc475c75a 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -152,7 +152,9 @@ vect_get_and_check_slp_defs (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, if (loop && def_stmt && gimple_bb (def_stmt) && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)) && vinfo_for_stmt (def_stmt) - && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt))) + && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt)) + && !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt)) + && !STMT_VINFO_LIVE_P (vinfo_for_stmt (def_stmt))) { if (!*first_stmt_dt0) *pattern0 = true; diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 8ac21255662..1d334b01b90 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -126,33 +126,72 @@ create_array_ref (tree type, tree ptr, struct data_reference *first_dr) static void vect_mark_relevant (VEC(gimple,heap) **worklist, gimple stmt, - enum vect_relevant relevant, bool live_p) + enum vect_relevant relevant, bool live_p, + bool used_in_pattern) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info); bool save_live_p = STMT_VINFO_LIVE_P (stmt_info); + gimple pattern_stmt; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "mark relevant %d, live %d.", relevant, live_p); + /* If this stmt is an original stmt in a pattern, we might need to mark its + related pattern stmt instead of the original stmt. However, such stmts + may have their own uses that are not in any pattern, in such cases the + stmt itself should be marked. */ if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { - gimple pattern_stmt; + bool found = false; + if (!used_in_pattern) + { + imm_use_iterator imm_iter; + use_operand_p use_p; + gimple use_stmt; + tree lhs; - /* This is the last stmt in a sequence that was detected as a - pattern that can potentially be vectorized. Don't mark the stmt - as relevant/live because it's not going to be vectorized. - Instead mark the pattern-stmt that replaces it. */ + if (is_gimple_assign (stmt)) + lhs = gimple_assign_lhs (stmt); + else + lhs = gimple_call_lhs (stmt); - pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); + /* This use is out of pattern use, if LHS has other uses that are + pattern uses, we should mark the stmt itself, and not the pattern + stmt. */ + FOR_EACH_IMM_USE_FAST (use_p, imm_iter, lhs) + { + if (is_gimple_debug (USE_STMT (use_p))) + continue; + use_stmt = USE_STMT (use_p); - if (vect_print_dump_info (REPORT_DETAILS)) - fprintf (vect_dump, "last stmt in pattern. don't mark relevant/live."); - stmt_info = vinfo_for_stmt (pattern_stmt); - gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt); - save_relevant = STMT_VINFO_RELEVANT (stmt_info); - save_live_p = STMT_VINFO_LIVE_P (stmt_info); - stmt = pattern_stmt; + if (vinfo_for_stmt (use_stmt) + && STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (use_stmt))) + { + found = true; + break; + } + } + } + + if (!found) + { + /* This is the last stmt in a sequence that was detected as a + pattern that can potentially be vectorized. Don't mark the stmt + as relevant/live because it's not going to be vectorized. + Instead mark the pattern-stmt that replaces it. */ + + pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); + + if (vect_print_dump_info (REPORT_DETAILS)) + fprintf (vect_dump, "last stmt in pattern. don't mark" + " relevant/live."); + stmt_info = vinfo_for_stmt (pattern_stmt); + gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt); + save_relevant = STMT_VINFO_RELEVANT (stmt_info); + save_live_p = STMT_VINFO_LIVE_P (stmt_info); + stmt = pattern_stmt; + } } STMT_VINFO_LIVE_P (stmt_info) |= live_p; @@ -437,7 +476,8 @@ process_use (gimple stmt, tree use, loop_vec_info loop_vinfo, bool live_p, } } - vect_mark_relevant (worklist, def_stmt, relevant, live_p); + vect_mark_relevant (worklist, def_stmt, relevant, live_p, + is_pattern_stmt_p (stmt_vinfo)); return true; } @@ -494,7 +534,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) } if (vect_stmt_relevant_p (phi, loop_vinfo, &relevant, &live_p)) - vect_mark_relevant (&worklist, phi, relevant, live_p); + vect_mark_relevant (&worklist, phi, relevant, live_p, false); } for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { @@ -506,7 +546,7 @@ vect_mark_stmts_to_be_vectorized (loop_vec_info loop_vinfo) } if (vect_stmt_relevant_p (stmt, loop_vinfo, &relevant, &live_p)) - vect_mark_relevant (&worklist, stmt, relevant, live_p); + vect_mark_relevant (&worklist, stmt, relevant, live_p, false); } } @@ -1184,7 +1224,14 @@ vect_get_vec_def_for_operand (tree op, gimple stmt, tree *scalar_def) /* Get the def from the vectorized stmt. */ def_stmt_info = vinfo_for_stmt (def_stmt); + vec_stmt = STMT_VINFO_VEC_STMT (def_stmt_info); + /* Get vectorized pattern statement. */ + if (!vec_stmt + && STMT_VINFO_IN_PATTERN_P (def_stmt_info) + && !STMT_VINFO_RELEVANT (def_stmt_info)) + vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt ( + STMT_VINFO_RELATED_STMT (def_stmt_info))); gcc_assert (vec_stmt); if (gimple_code (vec_stmt) == GIMPLE_PHI) vec_oprnd = PHI_RESULT (vec_stmt); @@ -4863,6 +4910,7 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node) enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; tree scalar_type, vectype; + gimple pattern_stmt; if (vect_print_dump_info (REPORT_DETAILS)) { @@ -4884,16 +4932,22 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node) - any LABEL_EXPRs in the loop - computations that are used only for array indexing or loop control. In basic blocks we only analyze statements that are a part of some SLP - instance, therefore, all the statements are relevant. */ + instance, therefore, all the statements are relevant. + Pattern statement need to be analyzed instead of the original statement + if the original statement is not relevant. Otherwise, we analyze both + statements. */ + + pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); if (!STMT_VINFO_RELEVANT_P (stmt_info) && !STMT_VINFO_LIVE_P (stmt_info)) { - gimple pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && pattern_stmt && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) { + /* Analyze PATTERN_STMT instead of the original stmt. */ stmt = pattern_stmt; stmt_info = vinfo_for_stmt (pattern_stmt); if (vect_print_dump_info (REPORT_DETAILS)) @@ -4910,6 +4964,21 @@ vect_analyze_stmt (gimple stmt, bool *need_to_vectorize, slp_tree node) return true; } } + else if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && pattern_stmt + && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) + { + /* Analyze PATTERN_STMT too. */ + if (vect_print_dump_info (REPORT_DETAILS)) + { + fprintf (vect_dump, "==> examining pattern statement: "); + print_gimple_stmt (vect_dump, stmt, 0, TDF_SLIM); + } + + if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node)) + return false; + } switch (STMT_VINFO_DEF_TYPE (stmt_info)) { @@ -5043,7 +5112,6 @@ vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi, bool is_store = false; gimple vec_stmt = NULL; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - gimple orig_stmt_in_pattern, orig_scalar_stmt = stmt; bool done; switch (STMT_VINFO_TYPE (stmt_info)) @@ -5182,25 +5250,7 @@ vect_transform_stmt (gimple stmt, gimple_stmt_iterator *gsi, } if (vec_stmt) - { - STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt; - orig_stmt_in_pattern = STMT_VINFO_RELATED_STMT (stmt_info); - if (orig_stmt_in_pattern) - { - stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt_in_pattern); - /* STMT was inserted by the vectorizer to replace a computation idiom. - ORIG_STMT_IN_PATTERN is a stmt in the original sequence that - computed this idiom. We need to record a pointer to VEC_STMT in - the stmt_info of ORIG_STMT_IN_PATTERN. See more details in the - documentation of vect_pattern_recog. */ - if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo)) - { - gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) - == orig_scalar_stmt); - STMT_VINFO_VEC_STMT (stmt_vinfo) = vec_stmt; - } - } - } + STMT_VINFO_VEC_STMT (stmt_info) = vec_stmt; return is_store; } @@ -5587,8 +5637,12 @@ vect_is_simple_use_1 (tree operand, loop_vec_info loop_vinfo, || *dt == vect_nested_cycle) { stmt_vec_info stmt_info = vinfo_for_stmt (*def_stmt); - if (STMT_VINFO_IN_PATTERN_P (stmt_info)) + + if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && !STMT_VINFO_RELEVANT (stmt_info) + && !STMT_VINFO_LIVE_P (stmt_info)) stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + *vectype = STMT_VINFO_VECTYPE (stmt_info); gcc_assert (*vectype != NULL_TREE); }