From d54a098e48987e7368ff190b703efd72aba9e6d9 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Wed, 20 Jun 2018 08:05:41 +0000 Subject: [PATCH] [1/n] PR85694: Allow pattern definition statements to be reused This patch is the first part of a series to fix to PR85694. Later patches can make the pattern for a statement S2 reuse the results of a PATTERN_DEF_SEQ statement attached to an earlier statement S1. Although vect_mark_stmts_to_be_vectorized handled this fine, vect_analyze_stmt and vect_transform_loop both skipped the PATTERN_DEF_SEQ for S1 if S1's main pattern wasn't live or relevant. I couldn't wrap my head around the flow in vect_transform_loop, so ended up moving the per-statement handling into a subroutine. That makes the patch look bigger than it actually is. 2018-06-20 Richard Sandiford gcc/ * tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern definition statements before the early exit for statements that aren't live or relevant. * tree-vect-loop.c (vect_transform_loop_stmt): New function, split out from... (vect_transform_loop): ...here. Process pattern definition statements without first checking whether the main pattern statement is live or relevant. From-SVN: r261784 --- gcc/ChangeLog | 11 ++ gcc/tree-vect-loop.c | 289 +++++++++++++++++------------------------- gcc/tree-vect-stmts.c | 56 ++++---- 3 files changed, 152 insertions(+), 204 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 8698635eaa4..983143f226b 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,14 @@ +2018-06-20 Richard Sandiford + + * tree-vect-stmts.c (vect_analyze_stmt): Move the handling of pattern + definition statements before the early exit for statements that aren't + live or relevant. + * tree-vect-loop.c (vect_transform_loop_stmt): New function, + split out from... + (vect_transform_loop): ...here. Process pattern definition + statements without first checking whether the main pattern + statement is live or relevant. + 2018-06-19 Eric Botcazou * tree-cfgcleanup.c (tree_forwarder_block_p): Do not return false at diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 8e45aecfc76..b07fcdecb72 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -8290,6 +8290,74 @@ scale_profile_for_vect_loop (struct loop *loop, unsigned vf) scale_bbs_frequencies (&loop->latch, 1, exit_l->probability / prob); } +/* Vectorize STMT if relevant, inserting any new instructions before GSI. + When vectorizing STMT as a store, set *SEEN_STORE to its stmt_vec_info. + *SLP_SCHEDULE is a running record of whether we have called + vect_schedule_slp. */ + +static void +vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt, + gimple_stmt_iterator *gsi, + stmt_vec_info *seen_store, bool *slp_scheduled) +{ + struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); + poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + if (!stmt_info) + return; + + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "------>vectorizing statement: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); + } + + if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info)) + vect_loop_kill_debug_uses (loop, stmt); + + if (!STMT_VINFO_RELEVANT_P (stmt_info) + && !STMT_VINFO_LIVE_P (stmt_info)) + return; + + if (STMT_VINFO_VECTYPE (stmt_info)) + { + poly_uint64 nunits + = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); + if (!STMT_SLP_TYPE (stmt_info) + && maybe_ne (nunits, vf) + && dump_enabled_p ()) + /* For SLP VF is set according to unrolling factor, and not + to vector size, hence for SLP this print is not valid. */ + dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n"); + } + + /* SLP. Schedule all the SLP instances when the first SLP stmt is + reached. */ + if (STMT_SLP_TYPE (stmt_info)) + { + if (!*slp_scheduled) + { + *slp_scheduled = true; + + DUMP_VECT_SCOPE ("scheduling SLP instances"); + + vect_schedule_slp (loop_vinfo); + } + + /* Hybrid SLP stmts must be vectorized in addition to SLP. */ + if (PURE_SLP_STMT (stmt_info)) + return; + } + + if (dump_enabled_p ()) + dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n"); + + bool grouped_store = false; + if (vect_transform_stmt (stmt, gsi, &grouped_store, NULL, NULL)) + *seen_store = stmt_info; +} + /* Function vect_transform_loop. The analysis phase has determined that the loop is vectorizable. @@ -8310,12 +8378,8 @@ vect_transform_loop (loop_vec_info loop_vinfo) tree niters_vector_mult_vf = NULL_TREE; poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); unsigned int lowest_vf = constant_lower_bound (vf); - bool grouped_store; bool slp_scheduled = false; - gimple *stmt, *pattern_stmt; - gimple_seq pattern_def_seq = NULL; - gimple_stmt_iterator pattern_def_si = gsi_none (); - bool transform_pattern_stmt = false; + gimple *stmt; bool check_profitability = false; unsigned int th; @@ -8466,194 +8530,67 @@ vect_transform_loop (loop_vec_info loop_vinfo) } } - pattern_stmt = NULL; for (gimple_stmt_iterator si = gsi_start_bb (bb); - !gsi_end_p (si) || transform_pattern_stmt;) + !gsi_end_p (si);) { - bool is_store; - - if (transform_pattern_stmt) - stmt = pattern_stmt; - else + stmt = gsi_stmt (si); + /* During vectorization remove existing clobber stmts. */ + if (gimple_clobber_p (stmt)) { - stmt = gsi_stmt (si); - /* During vectorization remove existing clobber stmts. */ - if (gimple_clobber_p (stmt)) - { - unlink_stmt_vdef (stmt); - gsi_remove (&si, true); - release_defs (stmt); - continue; - } + unlink_stmt_vdef (stmt); + gsi_remove (&si, true); + release_defs (stmt); } - - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, - "------>vectorizing statement: "); - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); - } - - stmt_info = vinfo_for_stmt (stmt); - - /* vector stmts created in the outer-loop during vectorization of - stmts in an inner-loop may not have a stmt_info, and do not - need to be vectorized. */ - if (!stmt_info) + else { - gsi_next (&si); - continue; - } - - if (MAY_HAVE_DEBUG_BIND_STMTS && !STMT_VINFO_LIVE_P (stmt_info)) - vect_loop_kill_debug_uses (loop, stmt); + stmt_info = vinfo_for_stmt (stmt); - if (!STMT_VINFO_RELEVANT_P (stmt_info) - && !STMT_VINFO_LIVE_P (stmt_info)) - { - if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) - && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) - { - stmt = pattern_stmt; - stmt_info = vinfo_for_stmt (stmt); - } - else - { - gsi_next (&si); - continue; - } - } - else if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && (pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info)) - && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) - transform_pattern_stmt = true; - - /* If pattern statement has def stmts, vectorize them too. */ - if (is_pattern_stmt_p (stmt_info)) - { - if (pattern_def_seq == NULL) + /* vector stmts created in the outer-loop during vectorization of + stmts in an inner-loop may not have a stmt_info, and do not + need to be vectorized. */ + stmt_vec_info seen_store = NULL; + if (stmt_info) { - pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); - pattern_def_si = gsi_start (pattern_def_seq); - } - else if (!gsi_end_p (pattern_def_si)) - gsi_next (&pattern_def_si); - if (pattern_def_seq != NULL) - { - gimple *pattern_def_stmt = NULL; - stmt_vec_info pattern_def_stmt_info = NULL; - - while (!gsi_end_p (pattern_def_si)) + if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { - pattern_def_stmt = gsi_stmt (pattern_def_si); - pattern_def_stmt_info - = vinfo_for_stmt (pattern_def_stmt); - if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info) - || STMT_VINFO_LIVE_P (pattern_def_stmt_info)) - break; - gsi_next (&pattern_def_si); - } - - if (!gsi_end_p (pattern_def_si)) - { - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, - "==> vectorizing pattern def " - "stmt: "); - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, - pattern_def_stmt, 0); - } - - stmt = pattern_def_stmt; - stmt_info = pattern_def_stmt_info; - } - else - { - pattern_def_si = gsi_none (); - transform_pattern_stmt = false; + gimple *def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); + for (gimple_stmt_iterator subsi = gsi_start (def_seq); + !gsi_end_p (subsi); gsi_next (&subsi)) + vect_transform_loop_stmt (loop_vinfo, + gsi_stmt (subsi), &si, + &seen_store, + &slp_scheduled); + gimple *pat_stmt = STMT_VINFO_RELATED_STMT (stmt_info); + vect_transform_loop_stmt (loop_vinfo, pat_stmt, &si, + &seen_store, &slp_scheduled); } + vect_transform_loop_stmt (loop_vinfo, stmt, &si, + &seen_store, &slp_scheduled); } - else - transform_pattern_stmt = false; - } - - if (STMT_VINFO_VECTYPE (stmt_info)) - { - poly_uint64 nunits - = TYPE_VECTOR_SUBPARTS (STMT_VINFO_VECTYPE (stmt_info)); - if (!STMT_SLP_TYPE (stmt_info) - && maybe_ne (nunits, vf) - && dump_enabled_p ()) - /* For SLP VF is set according to unrolling factor, and not - to vector size, hence for SLP this print is not valid. */ - dump_printf_loc (MSG_NOTE, vect_location, "multiple-types.\n"); - } - - /* SLP. Schedule all the SLP instances when the first SLP stmt is - reached. */ - if (STMT_SLP_TYPE (stmt_info)) - { - if (!slp_scheduled) + if (seen_store) { - slp_scheduled = true; - - DUMP_VECT_SCOPE ("scheduling SLP instances"); - - vect_schedule_slp (loop_vinfo); - } - - /* Hybrid SLP stmts must be vectorized in addition to SLP. */ - if (!vinfo_for_stmt (stmt) || PURE_SLP_STMT (stmt_info)) - { - if (!transform_pattern_stmt && gsi_end_p (pattern_def_si)) + if (STMT_VINFO_GROUPED_ACCESS (seen_store)) { - pattern_def_seq = NULL; + /* Interleaving. If IS_STORE is TRUE, the + vectorization of the interleaving chain was + completed - free all the stores in the chain. */ gsi_next (&si); + vect_remove_stores (DR_GROUP_FIRST_ELEMENT (seen_store)); + } + else + { + /* Free the attached stmt_vec_info and remove the + stmt. */ + free_stmt_vec_info (stmt); + unlink_stmt_vdef (stmt); + gsi_remove (&si, true); + release_defs (stmt); } - continue; - } - } - - /* -------- vectorize statement ------------ */ - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, "transform statement.\n"); - - grouped_store = false; - is_store = vect_transform_stmt (stmt, &si, &grouped_store, NULL, NULL); - if (is_store) - { - if (STMT_VINFO_GROUPED_ACCESS (stmt_info)) - { - /* Interleaving. If IS_STORE is TRUE, the vectorization of the - interleaving chain was completed - free all the stores in - the chain. */ - gsi_next (&si); - vect_remove_stores (DR_GROUP_FIRST_ELEMENT (stmt_info)); } else - { - /* Free the attached stmt_vec_info and remove the stmt. */ - gimple *store = gsi_stmt (si); - free_stmt_vec_info (store); - unlink_stmt_vdef (store); - gsi_remove (&si, true); - release_defs (store); - } - - /* Stores can only appear at the end of pattern statements. */ - gcc_assert (!transform_pattern_stmt); - pattern_def_seq = NULL; + gsi_next (&si); } - else if (!transform_pattern_stmt && gsi_end_p (pattern_def_si)) - { - pattern_def_seq = NULL; - gsi_next (&si); - } - } /* stmts in BB */ + } /* Stub out scalar statements that must not survive vectorization. Doing this here helps with grouped statements, or statements that diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index f2f91df935c..047edcdae5f 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -9393,6 +9393,34 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, return false; } + if (STMT_VINFO_IN_PATTERN_P (stmt_info) + && node == NULL + && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))) + { + gimple_stmt_iterator si; + + for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) + { + gimple *pattern_def_stmt = gsi_stmt (si); + if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt)) + || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))) + { + /* Analyze def stmt of STMT if it's a pattern stmt. */ + if (dump_enabled_p ()) + { + dump_printf_loc (MSG_NOTE, vect_location, + "==> examining pattern def statement: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0); + } + + if (!vect_analyze_stmt (pattern_def_stmt, + need_to_vectorize, node, node_instance, + cost_vec)) + return false; + } + } + } + /* Skip stmts that do not need to be vectorized. In loops this is expected to include: - the COND_EXPR which is the loop exit condition @@ -9453,34 +9481,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, return false; } - if (is_pattern_stmt_p (stmt_info) - && node == NULL - && (pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info))) - { - gimple_stmt_iterator si; - - for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) - { - gimple *pattern_def_stmt = gsi_stmt (si); - if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))) - { - /* Analyze def stmt of STMT if it's a pattern stmt. */ - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, - "==> examining pattern def statement: "); - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, pattern_def_stmt, 0); - } - - if (!vect_analyze_stmt (pattern_def_stmt, - need_to_vectorize, node, node_instance, - cost_vec)) - return false; - } - } - } - switch (STMT_VINFO_DEF_TYPE (stmt_info)) { case vect_internal_def: -- 2.30.2