From: Richard Sandiford Date: Tue, 31 Jul 2018 14:21:41 +0000 (+0000) Subject: [07/46] Add vec_info::lookup_stmt X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=6585ff8f3a55bbfed6a4f2c2addac7a27ed087d3;p=gcc.git [07/46] Add vec_info::lookup_stmt This patch adds a vec_info replacement for vinfo_for_stmt. The main difference is that the new routine can cope with arbitrary statements, so there's no need to call vect_stmt_in_region_p first. The patch only converts calls that are still needed at the end of the series. Later patches get rid of most other calls to vinfo_for_stmt. 2018-07-31 Richard Sandiford gcc/ * tree-vectorizer.h (vec_info::lookup_stmt): Declare. * tree-vectorizer.c (vec_info::lookup_stmt): New function. * tree-vect-loop.c (vect_determine_vf_for_stmt): Use it instead of vinfo_for_stmt. (vect_determine_vectorization_factor, vect_analyze_scalar_cycles_1) (vect_compute_single_scalar_iteration_cost, vect_analyze_loop_form) (vect_update_vf_for_slp, vect_analyze_loop_operations) (vect_is_slp_reduction, vectorizable_induction) (vect_transform_loop_stmt, vect_transform_loop): Likewise. * tree-vect-patterns.c (vect_init_pattern_stmt): (vect_determine_min_output_precision_1, vect_determine_precisions) (vect_pattern_recog): Likewise. * tree-vect-stmts.c (vect_analyze_stmt, vect_transform_stmt): Likewise. * config/powerpcspe/powerpcspe.c (rs6000_density_test): Likewise. * config/rs6000/rs6000.c (rs6000_density_test): Likewise. * tree-vect-slp.c (vect_detect_hybrid_slp_stmts): Likewise. (vect_detect_hybrid_slp_1, vect_detect_hybrid_slp_2) (vect_detect_hybrid_slp): Likewise. Change the walk_stmt_info info field from a loop to a loop_vec_info. From-SVN: r263122 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 95cddbb6b4a..2f244c42d88 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,25 @@ +2018-07-31 Richard Sandiford + + * tree-vectorizer.h (vec_info::lookup_stmt): Declare. + * tree-vectorizer.c (vec_info::lookup_stmt): New function. + * tree-vect-loop.c (vect_determine_vf_for_stmt): Use it instead + of vinfo_for_stmt. + (vect_determine_vectorization_factor, vect_analyze_scalar_cycles_1) + (vect_compute_single_scalar_iteration_cost, vect_analyze_loop_form) + (vect_update_vf_for_slp, vect_analyze_loop_operations) + (vect_is_slp_reduction, vectorizable_induction) + (vect_transform_loop_stmt, vect_transform_loop): Likewise. + * tree-vect-patterns.c (vect_init_pattern_stmt): + (vect_determine_min_output_precision_1, vect_determine_precisions) + (vect_pattern_recog): Likewise. + * tree-vect-stmts.c (vect_analyze_stmt, vect_transform_stmt): Likewise. + * config/powerpcspe/powerpcspe.c (rs6000_density_test): Likewise. + * config/rs6000/rs6000.c (rs6000_density_test): Likewise. + * tree-vect-slp.c (vect_detect_hybrid_slp_stmts): Likewise. + (vect_detect_hybrid_slp_1, vect_detect_hybrid_slp_2) + (vect_detect_hybrid_slp): Likewise. Change the walk_stmt_info + info field from a loop to a loop_vec_info. + 2018-07-31 Richard Sandiford * tree-vectorizer.h (stmt_vec_info): Move typedef earlier in file. diff --git a/gcc/config/powerpcspe/powerpcspe.c b/gcc/config/powerpcspe/powerpcspe.c index 7dd24e47641..dea1eab1188 100644 --- a/gcc/config/powerpcspe/powerpcspe.c +++ b/gcc/config/powerpcspe/powerpcspe.c @@ -6030,6 +6030,7 @@ rs6000_density_test (rs6000_cost_data *data) struct loop *loop = data->loop_info; basic_block *bbs = get_loop_body (loop); int nbbs = loop->num_nodes; + loop_vec_info loop_vinfo = loop_vec_info_for_loop (data->loop_info); int vec_cost = data->cost[vect_body], not_vec_cost = 0; int i, density_pct; @@ -6041,7 +6042,7 @@ rs6000_density_test (rs6000_cost_data *data) for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) { gimple *stmt = gsi_stmt (gsi); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (!STMT_VINFO_RELEVANT_P (stmt_info) && !STMT_VINFO_IN_PATTERN_P (stmt_info)) diff --git a/gcc/config/rs6000/rs6000.c b/gcc/config/rs6000/rs6000.c index cbe18680115..30403828c46 100644 --- a/gcc/config/rs6000/rs6000.c +++ b/gcc/config/rs6000/rs6000.c @@ -5325,6 +5325,7 @@ rs6000_density_test (rs6000_cost_data *data) struct loop *loop = data->loop_info; basic_block *bbs = get_loop_body (loop); int nbbs = loop->num_nodes; + loop_vec_info loop_vinfo = loop_vec_info_for_loop (data->loop_info); int vec_cost = data->cost[vect_body], not_vec_cost = 0; int i, density_pct; @@ -5336,7 +5337,7 @@ rs6000_density_test (rs6000_cost_data *data) for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) { gimple *stmt = gsi_stmt (gsi); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (!STMT_VINFO_RELEVANT_P (stmt_info) && !STMT_VINFO_IN_PATTERN_P (stmt_info)) diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index c011011dcde..50103fc1dc5 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -213,6 +213,7 @@ static bool vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf, vec *mask_producers) { + vec_info *vinfo = stmt_info->vinfo; if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: "); @@ -231,7 +232,7 @@ vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf, for (gimple_stmt_iterator si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) { - stmt_vec_info def_stmt_info = vinfo_for_stmt (gsi_stmt (si)); + stmt_vec_info def_stmt_info = vinfo->lookup_stmt (gsi_stmt (si)); if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, @@ -306,7 +307,7 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) gsi_next (&si)) { phi = si.phi (); - stmt_info = vinfo_for_stmt (phi); + stmt_info = loop_vinfo->lookup_stmt (phi); if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, "==> examining phi: "); @@ -366,7 +367,7 @@ vect_determine_vectorization_factor (loop_vec_info loop_vinfo) for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { - stmt_info = vinfo_for_stmt (gsi_stmt (si)); + stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); if (!vect_determine_vf_for_stmt (stmt_info, &vectorization_factor, &mask_producers)) return false; @@ -487,7 +488,7 @@ vect_analyze_scalar_cycles_1 (loop_vec_info loop_vinfo, struct loop *loop) gphi *phi = gsi.phi (); tree access_fn = NULL; tree def = PHI_RESULT (phi); - stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi); + stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi); if (dump_enabled_p ()) { @@ -1101,7 +1102,7 @@ vect_compute_single_scalar_iteration_cost (loop_vec_info loop_vinfo) for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { gimple *stmt = gsi_stmt (si); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (!is_gimple_assign (stmt) && !is_gimple_call (stmt)) continue; @@ -1390,10 +1391,14 @@ vect_analyze_loop_form (struct loop *loop, vec_info_shared *shared) } } - STMT_VINFO_TYPE (vinfo_for_stmt (loop_cond)) = loop_exit_ctrl_vec_info_type; + stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (loop_cond); + STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type; if (inner_loop_cond) - STMT_VINFO_TYPE (vinfo_for_stmt (inner_loop_cond)) - = loop_exit_ctrl_vec_info_type; + { + stmt_vec_info inner_loop_cond_info + = loop_vinfo->lookup_stmt (inner_loop_cond); + STMT_VINFO_TYPE (inner_loop_cond_info) = loop_exit_ctrl_vec_info_type; + } gcc_assert (!loop->aux); loop->aux = loop_vinfo; @@ -1432,7 +1437,7 @@ vect_update_vf_for_slp (loop_vec_info loop_vinfo) gsi_next (&si)) { gimple *stmt = gsi_stmt (si); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); if (STMT_VINFO_IN_PATTERN_P (stmt_info) && STMT_VINFO_RELATED_STMT (stmt_info)) { @@ -1532,7 +1537,7 @@ vect_analyze_loop_operations (loop_vec_info loop_vinfo) gphi *phi = si.phi (); ok = true; - stmt_info = vinfo_for_stmt (phi); + stmt_info = loop_vinfo->lookup_stmt (phi); if (dump_enabled_p ()) { dump_printf_loc (MSG_NOTE, vect_location, "examining phi: "); @@ -2238,13 +2243,13 @@ again: for (gimple_stmt_iterator si = gsi_start_phis (bb); !gsi_end_p (si); gsi_next (&si)) { - stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si)); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); STMT_SLP_TYPE (stmt_info) = loop_vect; } for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si)) { - stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si)); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si)); STMT_SLP_TYPE (stmt_info) = loop_vect; if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { @@ -2253,10 +2258,8 @@ again: STMT_SLP_TYPE (stmt_info) = loop_vect; for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq); !gsi_end_p (pi); gsi_next (&pi)) - { - gimple *pstmt = gsi_stmt (pi); - STMT_SLP_TYPE (vinfo_for_stmt (pstmt)) = loop_vect; - } + STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi))) + = loop_vect; } } } @@ -2602,7 +2605,7 @@ vect_is_slp_reduction (loop_vec_info loop_info, gimple *phi, return false; /* Insert USE_STMT into reduction chain. */ - use_stmt_info = vinfo_for_stmt (loop_use_stmt); + use_stmt_info = loop_info->lookup_stmt (loop_use_stmt); if (current_stmt) { current_stmt_info = vinfo_for_stmt (current_stmt); @@ -5549,7 +5552,7 @@ vect_finalize_reduction: { stmt_vec_info epilog_stmt_info = loop_vinfo->add_stmt (epilog_stmt); STMT_VINFO_RELATED_STMT (epilog_stmt_info) - = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_phi)); + = STMT_VINFO_RELATED_STMT (loop_vinfo->lookup_stmt (new_phi)); if (!double_reduc) scalar_results.quick_push (new_temp); @@ -5653,7 +5656,8 @@ vect_finalize_reduction: { if (outer_loop) { - stmt_vec_info exit_phi_vinfo = vinfo_for_stmt (exit_phi); + stmt_vec_info exit_phi_vinfo + = loop_vinfo->lookup_stmt (exit_phi); gphi *vect_phi; /* FORNOW. Currently not supporting the case that an inner-loop @@ -5700,7 +5704,7 @@ vect_finalize_reduction: || gimple_phi_num_args (use_stmt) != 2 || bb->loop_father != outer_loop) continue; - use_stmt_vinfo = vinfo_for_stmt (use_stmt); + use_stmt_vinfo = loop_vinfo->lookup_stmt (use_stmt); if (!use_stmt_vinfo || STMT_VINFO_DEF_TYPE (use_stmt_vinfo) != vect_double_reduction_def) @@ -7377,7 +7381,7 @@ vectorizable_induction (gimple *phi, } if (exit_phi) { - stmt_vec_info exit_phi_vinfo = vinfo_for_stmt (exit_phi); + stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi); if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo) && !STMT_VINFO_LIVE_P (exit_phi_vinfo))) { @@ -7801,7 +7805,7 @@ vectorizable_induction (gimple *phi, } if (exit_phi) { - stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi); + stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (exit_phi); /* FORNOW. Currently not supporting the case that an inner-loop induction is not used in the outer-loop (i.e. only outside the outer-loop). */ gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo) @@ -8260,7 +8264,7 @@ vect_transform_loop_stmt (loop_vec_info loop_vinfo, gimple *stmt, { struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (!stmt_info) return; @@ -8463,7 +8467,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) "------>vectorizing phi: "); dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0); } - stmt_info = vinfo_for_stmt (phi); + stmt_info = loop_vinfo->lookup_stmt (phi); if (!stmt_info) continue; @@ -8504,7 +8508,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) } else { - stmt_info = vinfo_for_stmt (stmt); + stmt_info = loop_vinfo->lookup_stmt (stmt); /* vector stmts created in the outer-loop during vectorization of stmts in an inner-loop may not have a stmt_info, and do not diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 1a431e7f15e..de0454b31ae 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -101,7 +101,8 @@ static void vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info, tree vectype) { - stmt_vec_info pattern_stmt_info = vinfo_for_stmt (pattern_stmt); + vec_info *vinfo = orig_stmt_info->vinfo; + stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt); if (pattern_stmt_info == NULL) pattern_stmt_info = orig_stmt_info->vinfo->add_stmt (pattern_stmt); gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt)); @@ -4401,6 +4402,7 @@ static bool vect_determine_min_output_precision_1 (stmt_vec_info stmt_info, tree lhs) { /* Take the maximum precision required by users of the result. */ + vec_info *vinfo = stmt_info->vinfo; unsigned int precision = 0; imm_use_iterator iter; use_operand_p use; @@ -4409,10 +4411,8 @@ vect_determine_min_output_precision_1 (stmt_vec_info stmt_info, tree lhs) gimple *use_stmt = USE_STMT (use); if (is_gimple_debug (use_stmt)) continue; - if (!vect_stmt_in_region_p (stmt_info->vinfo, use_stmt)) - return false; - stmt_vec_info use_stmt_info = vinfo_for_stmt (use_stmt); - if (!use_stmt_info->min_input_precision) + stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt); + if (!use_stmt_info || !use_stmt_info->min_input_precision) return false; precision = MAX (precision, use_stmt_info->min_input_precision); } @@ -4657,7 +4657,8 @@ vect_determine_precisions (vec_info *vinfo) basic_block bb = bbs[nbbs - i - 1]; for (gimple_stmt_iterator si = gsi_last_bb (bb); !gsi_end_p (si); gsi_prev (&si)) - vect_determine_stmt_precisions (vinfo_for_stmt (gsi_stmt (si))); + vect_determine_stmt_precisions + (vinfo->lookup_stmt (gsi_stmt (si))); } } else @@ -4672,7 +4673,7 @@ vect_determine_precisions (vec_info *vinfo) else gsi_prev (&si); stmt = gsi_stmt (si); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt); if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info)) vect_determine_stmt_precisions (stmt_info); } @@ -4971,7 +4972,7 @@ vect_pattern_recog (vec_info *vinfo) gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si)) { gimple *stmt = gsi_stmt (si); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = bb_vinfo->lookup_stmt (stmt); if (stmt_info && !STMT_VINFO_VECTORIZABLE (stmt_info)) continue; diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index fbce211d117..631a076219c 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -2315,7 +2315,6 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype) stmt_vec_info use_vinfo, stmt_vinfo = vinfo_for_stmt (stmt); slp_tree child; loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo); - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); int j; /* Propagate hybrid down the SLP tree. */ @@ -2340,9 +2339,9 @@ vect_detect_hybrid_slp_stmts (slp_tree node, unsigned i, slp_vect_type stype) if (def) FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, def) { - if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))) + use_vinfo = loop_vinfo->lookup_stmt (use_stmt); + if (!use_vinfo) continue; - use_vinfo = vinfo_for_stmt (use_stmt); if (STMT_VINFO_IN_PATTERN_P (use_vinfo) && STMT_VINFO_RELATED_STMT (use_vinfo)) use_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (use_vinfo)); @@ -2385,25 +2384,23 @@ static tree vect_detect_hybrid_slp_1 (tree *tp, int *, void *data) { walk_stmt_info *wi = (walk_stmt_info *)data; - struct loop *loopp = (struct loop *)wi->info; + loop_vec_info loop_vinfo = (loop_vec_info) wi->info; if (wi->is_lhs) return NULL_TREE; + stmt_vec_info def_stmt_info; if (TREE_CODE (*tp) == SSA_NAME - && !SSA_NAME_IS_DEFAULT_DEF (*tp)) + && !SSA_NAME_IS_DEFAULT_DEF (*tp) + && (def_stmt_info = loop_vinfo->lookup_stmt (SSA_NAME_DEF_STMT (*tp))) + && PURE_SLP_STMT (def_stmt_info)) { - gimple *def_stmt = SSA_NAME_DEF_STMT (*tp); - if (flow_bb_inside_loop_p (loopp, gimple_bb (def_stmt)) - && PURE_SLP_STMT (vinfo_for_stmt (def_stmt))) + if (dump_enabled_p ()) { - if (dump_enabled_p ()) - { - dump_printf_loc (MSG_NOTE, vect_location, "marking hybrid: "); - dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0); - } - STMT_SLP_TYPE (vinfo_for_stmt (def_stmt)) = hybrid; + dump_printf_loc (MSG_NOTE, vect_location, "marking hybrid: "); + dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt_info->stmt, 0); } + STMT_SLP_TYPE (def_stmt_info) = hybrid; } return NULL_TREE; @@ -2411,9 +2408,10 @@ vect_detect_hybrid_slp_1 (tree *tp, int *, void *data) static tree vect_detect_hybrid_slp_2 (gimple_stmt_iterator *gsi, bool *handled, - walk_stmt_info *) + walk_stmt_info *wi) { - stmt_vec_info use_vinfo = vinfo_for_stmt (gsi_stmt (*gsi)); + loop_vec_info loop_vinfo = (loop_vec_info) wi->info; + stmt_vec_info use_vinfo = loop_vinfo->lookup_stmt (gsi_stmt (*gsi)); /* If the stmt is in a SLP instance then this isn't a reason to mark use definitions in other SLP instances as hybrid. */ if (! STMT_SLP_TYPE (use_vinfo) @@ -2447,12 +2445,12 @@ vect_detect_hybrid_slp (loop_vec_info loop_vinfo) gsi_next (&gsi)) { gimple *stmt = gsi_stmt (gsi); - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt); if (STMT_VINFO_IN_PATTERN_P (stmt_info)) { walk_stmt_info wi; memset (&wi, 0, sizeof (wi)); - wi.info = LOOP_VINFO_LOOP (loop_vinfo); + wi.info = loop_vinfo; gimple_stmt_iterator gsi2 = gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info)); walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2, diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 21de238dc59..2aca038d38c 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -9377,6 +9377,7 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, slp_instance node_instance, stmt_vector_for_cost *cost_vec) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + vec_info *vinfo = stmt_info->vinfo; bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info); enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info); bool ok; @@ -9407,8 +9408,10 @@ vect_analyze_stmt (gimple *stmt, bool *need_to_vectorize, slp_tree node, for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si)) { gimple *pattern_def_stmt = gsi_stmt (si); - if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt)) - || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt))) + stmt_vec_info pattern_def_stmt_info + = vinfo->lookup_stmt (gsi_stmt (si)); + if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info) + || STMT_VINFO_LIVE_P (pattern_def_stmt_info)) { /* Analyze def stmt of STMT if it's a pattern stmt. */ if (dump_enabled_p ()) @@ -9605,9 +9608,10 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, bool *grouped_store, slp_tree slp_node, slp_instance slp_node_instance) { + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + vec_info *vinfo = stmt_info->vinfo; bool is_store = false; gimple *vec_stmt = NULL; - stmt_vec_info stmt_info = vinfo_for_stmt (stmt); bool done; gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info)); @@ -9728,7 +9732,6 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, imm_use_iterator imm_iter; use_operand_p use_p; tree scalar_dest; - gimple *exit_phi; if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, @@ -9743,13 +9746,12 @@ vect_transform_stmt (gimple *stmt, gimple_stmt_iterator *gsi, scalar_dest = gimple_assign_lhs (stmt); FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest) - { - if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p)))) - { - exit_phi = USE_STMT (use_p); - STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt; - } - } + if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p)))) + { + stmt_vec_info exit_phi_info + = vinfo->lookup_stmt (USE_STMT (use_p)); + STMT_VINFO_VEC_STMT (exit_phi_info) = vec_stmt; + } } /* Handle stmts whose DEF is used outside the loop-nest that is diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index 01bb372df76..9273f4d067d 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -518,6 +518,23 @@ vec_info::add_stmt (gimple *stmt) return res; } +/* If STMT has an associated stmt_vec_info, return that vec_info, otherwise + return null. It is safe to call this function on any statement, even if + it might not be part of the vectorizable region. */ + +stmt_vec_info +vec_info::lookup_stmt (gimple *stmt) +{ + unsigned int uid = gimple_uid (stmt); + if (uid > 0 && uid - 1 < stmt_vec_infos.length ()) + { + stmt_vec_info res = stmt_vec_infos[uid - 1]; + if (res && res->stmt == stmt) + return res; + } + return NULL; +} + /* A helper function to free scev and LOOP niter information, as well as clear loop constraint LOOP_C_FINITE. */ diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 67494179e2b..3b55a65f97f 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -218,6 +218,7 @@ struct vec_info { ~vec_info (); stmt_vec_info add_stmt (gimple *); + stmt_vec_info lookup_stmt (gimple *); /* The type of vectorization. */ vec_kind kind;