From 10681ce8cb6227ae5c11cc74ddf48f2fc5e6f87e Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Tue, 31 Jul 2018 14:22:09 +0000 Subject: [PATCH] [13/46] Make STMT_VINFO_RELATED_STMT a stmt_vec_info This patch changes STMT_VINFO_RELATED_STMT from a gimple stmt to a stmt_vec_info. 2018-07-31 Richard Sandiford gcc/ * tree-vectorizer.h (_stmt_vec_info::related_stmt): Change from a gimple stmt to a stmt_vec_info. (is_pattern_stmt_p): Update accordingly. * tree-vect-data-refs.c (vect_preserves_scalar_order_p): Likewise. (vect_record_grouped_load_vectors): Likewise. * tree-vect-loop.c (vect_determine_vf_for_stmt): Likewise. (vect_fixup_reduc_chain, vect_update_vf_for_slp): Likewise. (vect_model_reduction_cost): Likewise. (vect_create_epilog_for_reduction): Likewise. (vectorizable_reduction, vectorizable_induction): Likewise. * tree-vect-patterns.c (vect_init_pattern_stmt): Likewise. Return the stmt_vec_info for the pattern statement. (vect_set_pattern_stmt): Update use of STMT_VINFO_RELATED_STMT. (vect_split_statement, vect_mark_pattern_stmts): Likewise. * tree-vect-slp.c (vect_detect_hybrid_slp_stmts): Likewise. (vect_detect_hybrid_slp, vect_get_slp_defs): Likewise. * tree-vect-stmts.c (vect_mark_relevant): Likewise. (vect_get_vec_def_for_operand_1, vectorizable_call): Likewise. (vectorizable_simd_clone_call, vect_analyze_stmt, new_stmt_vec_info) (free_stmt_vec_info, vect_is_simple_use): Likewise. From-SVN: r263128 --- gcc/ChangeLog | 23 ++++++++++ gcc/tree-vect-data-refs.c | 31 +++++++------ gcc/tree-vect-loop.c | 96 ++++++++++++++++++--------------------- gcc/tree-vect-patterns.c | 22 ++++----- gcc/tree-vect-slp.c | 11 +++-- gcc/tree-vect-stmts.c | 56 ++++++++++------------- gcc/tree-vectorizer.h | 14 ++---- 7 files changed, 128 insertions(+), 125 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 16a9bb360f5..cd6f8212a66 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,26 @@ +2018-07-31 Richard Sandiford + + * tree-vectorizer.h (_stmt_vec_info::related_stmt): Change from + a gimple stmt to a stmt_vec_info. + (is_pattern_stmt_p): Update accordingly. + * tree-vect-data-refs.c (vect_preserves_scalar_order_p): Likewise. + (vect_record_grouped_load_vectors): Likewise. + * tree-vect-loop.c (vect_determine_vf_for_stmt): Likewise. + (vect_fixup_reduc_chain, vect_update_vf_for_slp): Likewise. + (vect_model_reduction_cost): Likewise. + (vect_create_epilog_for_reduction): Likewise. + (vectorizable_reduction, vectorizable_induction): Likewise. + * tree-vect-patterns.c (vect_init_pattern_stmt): Likewise. + Return the stmt_vec_info for the pattern statement. + (vect_set_pattern_stmt): Update use of STMT_VINFO_RELATED_STMT. + (vect_split_statement, vect_mark_pattern_stmts): Likewise. + * tree-vect-slp.c (vect_detect_hybrid_slp_stmts): Likewise. + (vect_detect_hybrid_slp, vect_get_slp_defs): Likewise. + * tree-vect-stmts.c (vect_mark_relevant): Likewise. + (vect_get_vec_def_for_operand_1, vectorizable_call): Likewise. + (vectorizable_simd_clone_call, vect_analyze_stmt, new_stmt_vec_info) + (free_stmt_vec_info, vect_is_simple_use): Likewise. + 2018-07-31 Richard Sandiford * tree-vectorizer.h (vect_finish_replace_stmt): Return a stmt_vec_info diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 65b49bfc2a6..70dd4667b1d 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -213,10 +213,10 @@ vect_preserves_scalar_order_p (gimple *stmt_a, gimple *stmt_b) current position (but could happen earlier). Reordering is therefore only possible if the first access is a write. */ if (is_pattern_stmt_p (stmtinfo_a)) - stmt_a = STMT_VINFO_RELATED_STMT (stmtinfo_a); + stmtinfo_a = STMT_VINFO_RELATED_STMT (stmtinfo_a); if (is_pattern_stmt_p (stmtinfo_b)) - stmt_b = STMT_VINFO_RELATED_STMT (stmtinfo_b); - gimple *earlier_stmt = get_earlier_stmt (stmt_a, stmt_b); + stmtinfo_b = STMT_VINFO_RELATED_STMT (stmtinfo_b); + gimple *earlier_stmt = get_earlier_stmt (stmtinfo_a, stmtinfo_b); return !DR_IS_WRITE (STMT_VINFO_DATA_REF (vinfo_for_stmt (earlier_stmt))); } @@ -6359,8 +6359,10 @@ vect_transform_grouped_load (gimple *stmt, vec dr_chain, int size, void vect_record_grouped_load_vectors (gimple *stmt, vec result_chain) { - gimple *first_stmt = DR_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)); - gimple *next_stmt, *new_stmt; + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + vec_info *vinfo = stmt_info->vinfo; + gimple *first_stmt = DR_GROUP_FIRST_ELEMENT (stmt_info); + gimple *next_stmt; unsigned int i, gap_count; tree tmp_data_ref; @@ -6389,29 +6391,28 @@ vect_record_grouped_load_vectors (gimple *stmt, vec result_chain) while (next_stmt) { - new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref); + stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref); /* We assume that if VEC_STMT is not NULL, this is a case of multiple copies, and we put the new vector statement in the first available RELATED_STMT. */ if (!STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt))) - STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)) = new_stmt; + STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)) = new_stmt_info; else { if (!DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) { gimple *prev_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)); - gimple *rel_stmt = - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (prev_stmt)); - while (rel_stmt) + stmt_vec_info rel_stmt_info + = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (prev_stmt)); + while (rel_stmt_info) { - prev_stmt = rel_stmt; - rel_stmt = - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (rel_stmt)); + prev_stmt = rel_stmt_info; + rel_stmt_info = STMT_VINFO_RELATED_STMT (rel_stmt_info); } - STMT_VINFO_RELATED_STMT (vinfo_for_stmt (prev_stmt)) = - new_stmt; + STMT_VINFO_RELATED_STMT (vinfo_for_stmt (prev_stmt)) + = new_stmt_info; } } diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index ff4a8cfa58a..bf9e205b2dc 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -226,7 +226,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf, && STMT_VINFO_RELATED_STMT (stmt_info)) { gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); - stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); /* If a pattern statement has def stmts, analyze them too. */ for (gimple_stmt_iterator si = gsi_start (pattern_def_seq); @@ -654,23 +654,23 @@ vect_analyze_scalar_cycles (loop_vec_info loop_vinfo) static void vect_fixup_reduc_chain (gimple *stmt) { - gimple *firstp = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); - gimple *stmtp; - gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (firstp)) - && REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt))); - REDUC_GROUP_SIZE (vinfo_for_stmt (firstp)) - = REDUC_GROUP_SIZE (vinfo_for_stmt (stmt)); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info firstp = STMT_VINFO_RELATED_STMT (stmt_info); + stmt_vec_info stmtp; + gcc_assert (!REDUC_GROUP_FIRST_ELEMENT (firstp) + && REDUC_GROUP_FIRST_ELEMENT (stmt_info)); + REDUC_GROUP_SIZE (firstp) = REDUC_GROUP_SIZE (stmt_info); do { stmtp = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); - REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmtp)) = firstp; + REDUC_GROUP_FIRST_ELEMENT (stmtp) = firstp; stmt = REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)); if (stmt) - REDUC_GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmtp)) + REDUC_GROUP_NEXT_ELEMENT (stmtp) = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); } while (stmt); - STMT_VINFO_DEF_TYPE (vinfo_for_stmt (stmtp)) = vect_reduction_def; + STMT_VINFO_DEF_TYPE (stmtp) = vect_reduction_def; } /* Fixup scalar cycles that now have their stmts detected as patterns. */ @@ -1436,14 +1436,10 @@ vect_update_vf_for_slp (loop_vec_info loop_vinfo) for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { - gimple *stmt = gsi_stmt (si); stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); if (STMT_VINFO_IN_PATTERN_P (stmt_info) && STMT_VINFO_RELATED_STMT (stmt_info)) - { - stmt = STMT_VINFO_RELATED_STMT (stmt_info); - stmt_info = vinfo_for_stmt (stmt); - } + stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); if ((STMT_VINFO_RELEVANT_P (stmt_info) || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (stmt_info))) && !PURE_SLP_STMT (stmt_info)) @@ -2247,7 +2243,7 @@ again: if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { gimple *pattern_def_seq = STMT_VINFO_PATTERN_DEF_SEQ (stmt_info); - stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); STMT_SLP_TYPE (stmt_info) = loop_vect; for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq); !gsi_end_p (pi); gsi_next (&pi)) @@ -3836,7 +3832,6 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn, enum tree_code code; optab optab; tree vectype; - gimple *orig_stmt; machine_mode mode; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); struct loop *loop = NULL; @@ -3852,12 +3847,12 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn, vectype = STMT_VINFO_VECTYPE (stmt_info); mode = TYPE_MODE (vectype); - orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info); + stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); - if (!orig_stmt) - orig_stmt = STMT_VINFO_STMT (stmt_info); + if (!orig_stmt_info) + orig_stmt_info = stmt_info; - code = gimple_assign_rhs_code (orig_stmt); + code = gimple_assign_rhs_code (orig_stmt_info->stmt); if (reduction_type == EXTRACT_LAST_REDUCTION || reduction_type == FOLD_LEFT_REDUCTION) @@ -3902,7 +3897,7 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn, We have a reduction operator that will reduce the vector in one statement. Also requires scalar extract. */ - if (!loop || !nested_in_vect_loop_p (loop, orig_stmt)) + if (!loop || !nested_in_vect_loop_p (loop, orig_stmt_info)) { if (reduc_fn != IFN_LAST) { @@ -3953,7 +3948,7 @@ vect_model_reduction_cost (stmt_vec_info stmt_info, internal_fn reduc_fn, { int vec_size_in_bits = tree_to_uhwi (TYPE_SIZE (vectype)); tree bitsize = - TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt))); + TYPE_SIZE (TREE_TYPE (gimple_assign_lhs (orig_stmt_info->stmt))); int element_bitsize = tree_to_uhwi (bitsize); int nelements = vec_size_in_bits / element_bitsize; @@ -4447,7 +4442,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree orig_name, scalar_result; imm_use_iterator imm_iter, phi_imm_iter; use_operand_p use_p, phi_use_p; - gimple *use_stmt, *orig_stmt, *reduction_phi = NULL; + gimple *use_stmt, *reduction_phi = NULL; bool nested_in_vect_loop = false; auto_vec new_phis; auto_vec inner_phis; @@ -4726,7 +4721,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, else { def = vect_get_vec_def_for_stmt_copy (dt, def); - STMT_VINFO_RELATED_STMT (prev_phi_info) = phi; + STMT_VINFO_RELATED_STMT (prev_phi_info) = phi_info; } SET_PHI_ARG_DEF (phi, single_exit (loop)->dest_idx, def); @@ -4758,7 +4753,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, SET_PHI_ARG_DEF (outer_phi, single_exit (loop)->dest_idx, PHI_RESULT (phi)); stmt_vec_info outer_phi_info = loop_vinfo->add_stmt (outer_phi); - STMT_VINFO_RELATED_STMT (prev_phi_info) = outer_phi; + STMT_VINFO_RELATED_STMT (prev_phi_info) = outer_phi_info; prev_phi_info = outer_phi_info; } } @@ -4775,27 +4770,26 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, Otherwise (it is a regular reduction) - the tree-code and scalar-def are taken from STMT. */ - orig_stmt = STMT_VINFO_RELATED_STMT (stmt_info); - if (!orig_stmt) + stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + if (!orig_stmt_info) { /* Regular reduction */ - orig_stmt = stmt; + orig_stmt_info = stmt_info; } else { /* Reduction pattern */ - stmt_vec_info stmt_vinfo = vinfo_for_stmt (orig_stmt); - gcc_assert (STMT_VINFO_IN_PATTERN_P (stmt_vinfo)); - gcc_assert (STMT_VINFO_RELATED_STMT (stmt_vinfo) == stmt); + gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)); + gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) == stmt_info); } - code = gimple_assign_rhs_code (orig_stmt); + code = gimple_assign_rhs_code (orig_stmt_info->stmt); /* For MINUS_EXPR the initial vector is [init_val,0,...,0], therefore, partial results are added and not subtracted. */ if (code == MINUS_EXPR) code = PLUS_EXPR; - scalar_dest = gimple_assign_lhs (orig_stmt); + scalar_dest = gimple_assign_lhs (orig_stmt_info->stmt); scalar_type = TREE_TYPE (scalar_dest); scalar_results.create (group_size); new_scalar_dest = vect_create_destination_var (scalar_dest, NULL); @@ -5613,10 +5607,11 @@ vect_finalize_reduction: { gimple *current_stmt = SLP_TREE_SCALAR_STMTS (slp_node)[k]; - orig_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (current_stmt)); - /* SLP statements can't participate in patterns. */ - gcc_assert (!orig_stmt); - scalar_dest = gimple_assign_lhs (current_stmt); + orig_stmt_info + = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (current_stmt)); + /* SLP statements can't participate in patterns. */ + gcc_assert (!orig_stmt_info); + scalar_dest = gimple_assign_lhs (current_stmt); } phis.create (3); @@ -6097,8 +6092,6 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, enum tree_code cond_reduc_op_code = ERROR_MARK; tree scalar_type; bool is_simple_use; - gimple *orig_stmt; - stmt_vec_info orig_stmt_info = NULL; int i; int ncopies; int epilog_copies; @@ -6229,7 +6222,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, if (j == 0) STMT_VINFO_VEC_STMT (stmt_info) = *vec_stmt = new_phi; else - STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi; + STMT_VINFO_RELATED_STMT (prev_phi_info) = new_phi_info; prev_phi_info = new_phi_info; } } @@ -6259,10 +6252,9 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, the STMT_VINFO_RELATED_STMT field records the last stmt in the original sequence that constitutes the pattern. */ - orig_stmt = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (stmt)); - if (orig_stmt) + stmt_vec_info orig_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + if (orig_stmt_info) { - orig_stmt_info = vinfo_for_stmt (orig_stmt); gcc_assert (STMT_VINFO_IN_PATTERN_P (orig_stmt_info)); gcc_assert (!STMT_VINFO_IN_PATTERN_P (stmt_info)); } @@ -6393,7 +6385,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, return false; } - if (orig_stmt) + if (orig_stmt_info) reduc_def_stmt = STMT_VINFO_REDUC_DEF (orig_stmt_info); else reduc_def_stmt = STMT_VINFO_REDUC_DEF (stmt_info); @@ -6414,7 +6406,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, /* For pattern recognized stmts, orig_stmt might be a reduction, but some helper statements for the pattern might not, or might be COND_EXPRs with reduction uses in the condition. */ - gcc_assert (orig_stmt); + gcc_assert (orig_stmt_info); return false; } @@ -6548,10 +6540,10 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, } } - if (orig_stmt) - gcc_assert (tmp == orig_stmt + if (orig_stmt_info) + gcc_assert (tmp == orig_stmt_info || (REDUC_GROUP_FIRST_ELEMENT (vinfo_for_stmt (tmp)) - == orig_stmt)); + == orig_stmt_info)); else /* We changed STMT to be the first stmt in reduction chain, hence we check that in this case the first element in the chain is STMT. */ @@ -6673,13 +6665,13 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, vect_reduction_type reduction_type = STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info); - if (orig_stmt + if (orig_stmt_info && (reduction_type == TREE_CODE_REDUCTION || reduction_type == FOLD_LEFT_REDUCTION)) { /* This is a reduction pattern: get the vectype from the type of the reduction variable, and get the tree-code from orig_stmt. */ - orig_code = gimple_assign_rhs_code (orig_stmt); + orig_code = gimple_assign_rhs_code (orig_stmt_info->stmt); gcc_assert (vectype_out); vec_mode = TYPE_MODE (vectype_out); } @@ -7757,7 +7749,7 @@ vectorizable_induction (gimple *phi, gsi_insert_before (&si, new_stmt, GSI_SAME_STMT); new_stmt_info = loop_vinfo->add_stmt (new_stmt); - STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt; + STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt_info; prev_stmt_vinfo = new_stmt_info; } } diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 4ac8f9b4b39..6fce980e601 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -94,10 +94,11 @@ vect_pattern_detected (const char *name, gimple *stmt) } } -/* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO. - Set its vector type to VECTYPE if it doesn't have one already. */ +/* Associate pattern statement PATTERN_STMT with ORIG_STMT_INFO and + return the pattern statement's stmt_vec_info. Set its vector type to + VECTYPE if it doesn't have one already. */ -static void +static stmt_vec_info vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info, tree vectype) { @@ -107,11 +108,12 @@ vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info, pattern_stmt_info = orig_stmt_info->vinfo->add_stmt (pattern_stmt); gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt)); - STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info->stmt; + STMT_VINFO_RELATED_STMT (pattern_stmt_info) = orig_stmt_info; STMT_VINFO_DEF_TYPE (pattern_stmt_info) = STMT_VINFO_DEF_TYPE (orig_stmt_info); if (!STMT_VINFO_VECTYPE (pattern_stmt_info)) STMT_VINFO_VECTYPE (pattern_stmt_info) = vectype; + return pattern_stmt_info; } /* Set the pattern statement of ORIG_STMT_INFO to PATTERN_STMT. @@ -123,8 +125,8 @@ vect_set_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info, tree vectype) { STMT_VINFO_IN_PATTERN_P (orig_stmt_info) = true; - STMT_VINFO_RELATED_STMT (orig_stmt_info) = pattern_stmt; - vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype); + STMT_VINFO_RELATED_STMT (orig_stmt_info) + = vect_init_pattern_stmt (pattern_stmt, orig_stmt_info, vectype); } /* Add NEW_STMT to STMT_INFO's pattern definition statements. If VECTYPE @@ -634,8 +636,7 @@ vect_split_statement (stmt_vec_info stmt2_info, tree new_rhs, { /* STMT2_INFO is part of a pattern. Get the statement to which the pattern is attached. */ - stmt_vec_info orig_stmt2_info - = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt2_info)); + stmt_vec_info orig_stmt2_info = STMT_VINFO_RELATED_STMT (stmt2_info); vect_init_pattern_stmt (stmt1, orig_stmt2_info, vectype); if (dump_enabled_p ()) @@ -659,7 +660,7 @@ vect_split_statement (stmt_vec_info stmt2_info, tree new_rhs, } gimple_seq *def_seq = &STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt2_info); - if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info->stmt) + if (STMT_VINFO_RELATED_STMT (orig_stmt2_info) == stmt2_info) /* STMT2_INFO is the actual pattern statement. Add STMT1 to the end of the definition sequence. */ gimple_seq_add_stmt_without_update (def_seq, stmt1); @@ -4754,8 +4755,7 @@ vect_mark_pattern_stmts (gimple *orig_stmt, gimple *pattern_stmt, } /* Switch to the statement that ORIG replaces. */ - orig_stmt_info - = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (orig_stmt_info)); + orig_stmt_info = STMT_VINFO_RELATED_STMT (orig_stmt_info); /* We shouldn't be replacing the main pattern statement. */ gcc_assert (STMT_VINFO_RELATED_STMT (orig_stmt_info) != orig_stmt); diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index fbb419f3ee8..c7af18b9b9a 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -2327,7 +2327,7 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype) original stmt for immediate uses. */ if (! STMT_VINFO_IN_PATTERN_P (stmt_vinfo) && STMT_VINFO_RELATED_STMT (stmt_vinfo)) - stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo); + stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo)->stmt; tree def; if (gimple_code (stmt) == GIMPLE_PHI) def = gimple_phi_result (stmt); @@ -2341,7 +2341,7 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype) continue; if (STMT_VINFO_IN_PATTERN_P (use_vinfo) && STMT_VINFO_RELATED_STMT (use_vinfo)) - use_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (use_vinfo)); + use_vinfo = STMT_VINFO_RELATED_STMT (use_vinfo); if (!STMT_SLP_TYPE (use_vinfo) && (STMT_VINFO_RELEVANT (use_vinfo) || VECTORIZABLE_CYCLE_DEF (STMT_VINFO_DEF_TYPE (use_vinfo))) @@ -2446,7 +2446,7 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo) memset (&wi, 0, sizeof (wi)); wi.info = loop_vinfo; gimple_stmt_iterator gsi2 - = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)->stmt); walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2, vect_detect_hybrid_slp_1, &wi); walk_gimple_seq (STMT_VINFO_PATTERN_DEF_SEQ (stmt_info), @@ -3612,7 +3612,7 @@ vect_get_slp_defs (vec ops, slp_tree slp_node, if (SLP_TREE_DEF_TYPE (child) == vect_internal_def) { gimple *first_def = SLP_TREE_SCALAR_STMTS (child)[0]; - gimple *related + stmt_vec_info related = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (first_def)); tree first_def_op; @@ -3622,7 +3622,8 @@ vect_get_slp_defs (vec ops, slp_tree slp_node, first_def_op = gimple_get_lhs (first_def); if (operand_equal_p (oprnd, first_def_op, 0) || (related - && operand_equal_p (oprnd, gimple_get_lhs (related), 0))) + && operand_equal_p (oprnd, + gimple_get_lhs (related->stmt), 0))) { /* The number of vector defs is determined by the number of vector statements in the node from which we get those diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index f228301f4ff..8bddeca0315 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -202,7 +202,6 @@ vect_mark_relevant (vec *worklist, gimple *stmt, stmt_vec_info stmt_info = vinfo_for_stmt (stmt); enum vect_relevant save_relevant = STMT_VINFO_RELEVANT (stmt_info); bool save_live_p = STMT_VINFO_LIVE_P (stmt_info); - gimple *pattern_stmt; if (dump_enabled_p ()) { @@ -222,17 +221,16 @@ vect_mark_relevant (vec *worklist, gimple *stmt, as relevant/live because it's not going to be vectorized. Instead mark the pattern-stmt that replaces it. */ - pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); - if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "last stmt in pattern. don't mark" " relevant/live.\n"); - stmt_info = vinfo_for_stmt (pattern_stmt); - gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == stmt); + stmt_vec_info old_stmt_info = stmt_info; + stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + gcc_assert (STMT_VINFO_RELATED_STMT (stmt_info) == old_stmt_info); save_relevant = STMT_VINFO_RELEVANT (stmt_info); save_live_p = STMT_VINFO_LIVE_P (stmt_info); - stmt = pattern_stmt; + stmt = stmt_info->stmt; } STMT_VINFO_LIVE_P (stmt_info) |= live_p; @@ -1489,8 +1487,8 @@ vect_get_vec_def_for_operand_1 (gimple *def_stmt, enum vect_def_type dt) if (!vec_stmt && STMT_VINFO_IN_PATTERN_P (def_stmt_info) && !STMT_VINFO_RELEVANT (def_stmt_info)) - vec_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt ( - STMT_VINFO_RELATED_STMT (def_stmt_info))); + vec_stmt = (STMT_VINFO_VEC_STMT + (STMT_VINFO_RELATED_STMT (def_stmt_info))); gcc_assert (vec_stmt); if (gimple_code (vec_stmt) == GIMPLE_PHI) vec_oprnd = PHI_RESULT (vec_stmt); @@ -3635,7 +3633,7 @@ vectorizable_call (gimple *gs, gimple_stmt_iterator *gsi, gimple **vec_stmt, return true; if (is_pattern_stmt_p (stmt_info)) - stmt_info = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); + stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); lhs = gimple_get_lhs (stmt_info->stmt); gassign *new_stmt @@ -4370,7 +4368,7 @@ vectorizable_simd_clone_call (gimple *stmt, gimple_stmt_iterator *gsi, { type = TREE_TYPE (scalar_dest); if (is_pattern_stmt_p (stmt_info)) - lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info)); + lhs = gimple_call_lhs (STMT_VINFO_RELATED_STMT (stmt_info)->stmt); else lhs = gimple_call_lhs (stmt); new_stmt = gimple_build_assign (lhs, build_zero_cst (type)); @@ -9420,7 +9418,6 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; - gimple *pattern_stmt; gimple_seq pattern_def_seq; if (dump_enabled_p ()) @@ -9482,18 +9479,18 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, traversal, don't analyze pattern stmts instead, the pattern stmts already will be part of SLP instance. */ - pattern_stmt = STMT_VINFO_RELATED_STMT (stmt_info); + stmt_vec_info pattern_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); if (!STMT_VINFO_RELEVANT_P (stmt_info) && !STMT_VINFO_LIVE_P (stmt_info)) { if (STMT_VINFO_IN_PATTERN_P (stmt_info) - && pattern_stmt - && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) + && pattern_stmt_info + && (STMT_VINFO_RELEVANT_P (pattern_stmt_info) + || STMT_VINFO_LIVE_P (pattern_stmt_info))) { /* Analyze PATTERN_STMT instead of the original stmt. */ - stmt = pattern_stmt; - stmt_info = vinfo_for_stmt (pattern_stmt); + stmt = pattern_stmt_info->stmt; + stmt_info = pattern_stmt_info; if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, @@ -9511,9 +9508,9 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, } else if (STMT_VINFO_IN_PATTERN_P (stmt_info) && node == NULL - && pattern_stmt - && (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_stmt)))) + && pattern_stmt_info + && (STMT_VINFO_RELEVANT_P (pattern_stmt_info) + || STMT_VINFO_LIVE_P (pattern_stmt_info))) { /* Analyze PATTERN_STMT too. */ if (dump_enabled_p ()) @@ -9523,7 +9520,7 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, dump_gimple_stmt (MSG_NOTE, TDF_SLIM, stmt, 0); } - if (!vect_analyze_stmt (pattern_stmt, need_to_vectorize, node, + if (!vect_analyze_stmt (pattern_stmt_info, need_to_vectorize, node, node_instance, cost_vec)) return false; } @@ -9855,7 +9852,6 @@ new_stmt_vec_info (gimple *stmt, vec_info *vinfo) STMT_VINFO_VEC_STMT (res) = NULL; STMT_VINFO_VECTORIZABLE (res) = true; STMT_VINFO_IN_PATTERN_P (res) = false; - STMT_VINFO_RELATED_STMT (res) = NULL; STMT_VINFO_PATTERN_DEF_SEQ (res) = NULL; STMT_VINFO_DATA_REF (res) = NULL; STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION; @@ -9936,16 +9932,14 @@ free_stmt_vec_info (gimple *stmt) release_ssa_name (lhs); free_stmt_vec_info (seq_stmt); } - stmt_vec_info patt_info - = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); - if (patt_info) + stmt_vec_info patt_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + if (patt_stmt_info) { - gimple *patt_stmt = STMT_VINFO_STMT (patt_info); - gimple_set_bb (patt_stmt, NULL); - tree lhs = gimple_get_lhs (patt_stmt); + gimple_set_bb (patt_stmt_info->stmt, NULL); + tree lhs = gimple_get_lhs (patt_stmt_info->stmt); if (lhs && TREE_CODE (lhs) == SSA_NAME) release_ssa_name (lhs); - free_stmt_vec_info (patt_stmt); + free_stmt_vec_info (patt_stmt_info); } } @@ -10143,8 +10137,8 @@ vect_is_simple_use (tree operand, vec_info *vinfo, enum vect_def_type *dt, { if (STMT_VINFO_IN_PATTERN_P (stmt_vinfo)) { - def_stmt = STMT_VINFO_RELATED_STMT (stmt_vinfo); - stmt_vinfo = vinfo_for_stmt (def_stmt); + stmt_vinfo = STMT_VINFO_RELATED_STMT (stmt_vinfo); + def_stmt = stmt_vinfo->stmt; } switch (gimple_code (def_stmt)) { diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 4109cd9d73a..fb7b592f07e 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -847,7 +847,7 @@ struct _stmt_vec_info { related_stmt of the "pattern stmt" points back to this stmt (which is the last stmt in the original sequence of stmts that constitutes the pattern). */ - gimple *related_stmt; + stmt_vec_info related_stmt; /* Used to keep a sequence of def stmts of a pattern stmt if such exists. The sequence is attached to the original statement rather than the @@ -1189,16 +1189,8 @@ get_later_stmt (gimple *stmt1, gimple *stmt2) static inline bool is_pattern_stmt_p (stmt_vec_info stmt_info) { - gimple *related_stmt; - stmt_vec_info related_stmt_info; - - related_stmt = STMT_VINFO_RELATED_STMT (stmt_info); - if (related_stmt - && (related_stmt_info = vinfo_for_stmt (related_stmt)) - && STMT_VINFO_IN_PATTERN_P (related_stmt_info)) - return true; - - return false; + stmt_vec_info related_stmt_info = STMT_VINFO_RELATED_STMT (stmt_info); + return related_stmt_info && STMT_VINFO_IN_PATTERN_P (related_stmt_info); } /* Return true if BB is a loop header. */ -- 2.30.2