+2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
+
+ * tree-vectorizer.h (vec_info::lookup_stmt): Declare.
+ * tree-vectorizer.c (vec_info::lookup_stmt): New function.
+ * tree-vect-loop.c (vect_determine_vf_for_stmt): Use it instead
+ of vinfo_for_stmt.
+ (vect_determine_vectorization_factor, vect_analyze_scalar_cycles_1)
+ (vect_compute_single_scalar_iteration_cost, vect_analyze_loop_form)
+ (vect_update_vf_for_slp, vect_analyze_loop_operations)
+ (vect_is_slp_reduction, vectorizable_induction)
+ (vect_transform_loop_stmt, vect_transform_loop): Likewise.
+ * tree-vect-patterns.c (vect_init_pattern_stmt):
+ (vect_determine_min_output_precision_1, vect_determine_precisions)
+ (vect_pattern_recog): Likewise.
+ * tree-vect-stmts.c (vect_analyze_stmt, vect_transform_stmt): Likewise.
+ * config/powerpcspe/powerpcspe.c (rs6000_density_test): Likewise.
+ * config/rs6000/rs6000.c (rs6000_density_test): Likewise.
+ * tree-vect-slp.c (vect_detect_hybrid_slp_stmts): Likewise.
+ (vect_detect_hybrid_slp_1, vect_detect_hybrid_slp_2)
+ (vect_detect_hybrid_slp): Likewise. Change the walk_stmt_info
+ info field from a loop to a loop_vec_info.
+
2018-07-31 Richard Sandiford <richard.sandiford@arm.com>
* tree-vectorizer.h (stmt_vec_info): Move typedef earlier in file.
struct loop *loop = data->loop_info;
basic_block *bbs = get_loop_body (loop);
int nbbs = loop->num_nodes;
+ loop_vec_info loop_vinfo = loop_vec_info_for_loop (data->loop_info);
int vec_cost = data->cost[vect_body], not_vec_cost = 0;
int i, density_pct;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_IN_PATTERN_P (stmt_info))
struct loop *loop = data->loop_info;
basic_block *bbs = get_loop_body (loop);
int nbbs = loop->num_nodes;
+ loop_vec_info loop_vinfo = loop_vec_info_for_loop (data->loop_info);
int vec_cost = data->cost[vect_body], not_vec_cost = 0;
int i, density_pct;
for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
if (!STMT_VINFO_RELEVANT_P (stmt_info)
&& !STMT_VINFO_IN_PATTERN_P (stmt_info))
vect_determine_vf_for_stmt (stmt_vec_info stmt_info, poly_uint64 *vf,
vec<stmt_vec_info > *mask_producers)
{
+ vec_info *vinfo = stmt_info->vinfo;
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "==> examining statement: ");
for (gimple_stmt_iterator si = gsi_start (pattern_def_seq);
!gsi_end_p (si); gsi_next (&si))
{
- stmt_vec_info def_stmt_info = vinfo_for_stmt (gsi_stmt (si));
+ stmt_vec_info def_stmt_info = vinfo->lookup_stmt (gsi_stmt (si));
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
gsi_next (&si))
{
phi = si.phi ();
- stmt_info = vinfo_for_stmt (phi);
+ stmt_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "==> examining phi: ");
for (gimple_stmt_iterator si = gsi_start_bb (bb); !gsi_end_p (si);
gsi_next (&si))
{
- stmt_info = vinfo_for_stmt (gsi_stmt (si));
+ stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
if (!vect_determine_vf_for_stmt (stmt_info, &vectorization_factor,
&mask_producers))
return false;
gphi *phi = gsi.phi ();
tree access_fn = NULL;
tree def = PHI_RESULT (phi);
- stmt_vec_info stmt_vinfo = vinfo_for_stmt (phi);
+ stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
{
for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
if (!is_gimple_assign (stmt) && !is_gimple_call (stmt))
continue;
}
}
- STMT_VINFO_TYPE (vinfo_for_stmt (loop_cond)) = loop_exit_ctrl_vec_info_type;
+ stmt_vec_info loop_cond_info = loop_vinfo->lookup_stmt (loop_cond);
+ STMT_VINFO_TYPE (loop_cond_info) = loop_exit_ctrl_vec_info_type;
if (inner_loop_cond)
- STMT_VINFO_TYPE (vinfo_for_stmt (inner_loop_cond))
- = loop_exit_ctrl_vec_info_type;
+ {
+ stmt_vec_info inner_loop_cond_info
+ = loop_vinfo->lookup_stmt (inner_loop_cond);
+ STMT_VINFO_TYPE (inner_loop_cond_info) = loop_exit_ctrl_vec_info_type;
+ }
gcc_assert (!loop->aux);
loop->aux = loop_vinfo;
gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
if (STMT_VINFO_IN_PATTERN_P (stmt_info)
&& STMT_VINFO_RELATED_STMT (stmt_info))
{
gphi *phi = si.phi ();
ok = true;
- stmt_info = vinfo_for_stmt (phi);
+ stmt_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location, "examining phi: ");
for (gimple_stmt_iterator si = gsi_start_phis (bb);
!gsi_end_p (si); gsi_next (&si))
{
- stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si));
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
STMT_SLP_TYPE (stmt_info) = loop_vect;
}
for (gimple_stmt_iterator si = gsi_start_bb (bb);
!gsi_end_p (si); gsi_next (&si))
{
- stmt_vec_info stmt_info = vinfo_for_stmt (gsi_stmt (si));
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (gsi_stmt (si));
STMT_SLP_TYPE (stmt_info) = loop_vect;
if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
STMT_SLP_TYPE (stmt_info) = loop_vect;
for (gimple_stmt_iterator pi = gsi_start (pattern_def_seq);
!gsi_end_p (pi); gsi_next (&pi))
- {
- gimple *pstmt = gsi_stmt (pi);
- STMT_SLP_TYPE (vinfo_for_stmt (pstmt)) = loop_vect;
- }
+ STMT_SLP_TYPE (loop_vinfo->lookup_stmt (gsi_stmt (pi)))
+ = loop_vect;
}
}
}
return false;
/* Insert USE_STMT into reduction chain. */
- use_stmt_info = vinfo_for_stmt (loop_use_stmt);
+ use_stmt_info = loop_info->lookup_stmt (loop_use_stmt);
if (current_stmt)
{
current_stmt_info = vinfo_for_stmt (current_stmt);
{
stmt_vec_info epilog_stmt_info = loop_vinfo->add_stmt (epilog_stmt);
STMT_VINFO_RELATED_STMT (epilog_stmt_info)
- = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_phi));
+ = STMT_VINFO_RELATED_STMT (loop_vinfo->lookup_stmt (new_phi));
if (!double_reduc)
scalar_results.quick_push (new_temp);
{
if (outer_loop)
{
- stmt_vec_info exit_phi_vinfo = vinfo_for_stmt (exit_phi);
+ stmt_vec_info exit_phi_vinfo
+ = loop_vinfo->lookup_stmt (exit_phi);
gphi *vect_phi;
/* FORNOW. Currently not supporting the case that an inner-loop
|| gimple_phi_num_args (use_stmt) != 2
|| bb->loop_father != outer_loop)
continue;
- use_stmt_vinfo = vinfo_for_stmt (use_stmt);
+ use_stmt_vinfo = loop_vinfo->lookup_stmt (use_stmt);
if (!use_stmt_vinfo
|| STMT_VINFO_DEF_TYPE (use_stmt_vinfo)
!= vect_double_reduction_def)
}
if (exit_phi)
{
- stmt_vec_info exit_phi_vinfo = vinfo_for_stmt (exit_phi);
+ stmt_vec_info exit_phi_vinfo = loop_vinfo->lookup_stmt (exit_phi);
if (!(STMT_VINFO_RELEVANT_P (exit_phi_vinfo)
&& !STMT_VINFO_LIVE_P (exit_phi_vinfo)))
{
}
if (exit_phi)
{
- stmt_vec_info stmt_vinfo = vinfo_for_stmt (exit_phi);
+ stmt_vec_info stmt_vinfo = loop_vinfo->lookup_stmt (exit_phi);
/* FORNOW. Currently not supporting the case that an inner-loop induction
is not used in the outer-loop (i.e. only outside the outer-loop). */
gcc_assert (STMT_VINFO_RELEVANT_P (stmt_vinfo)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
if (!stmt_info)
return;
"------>vectorizing phi: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
}
- stmt_info = vinfo_for_stmt (phi);
+ stmt_info = loop_vinfo->lookup_stmt (phi);
if (!stmt_info)
continue;
}
else
{
- stmt_info = vinfo_for_stmt (stmt);
+ stmt_info = loop_vinfo->lookup_stmt (stmt);
/* vector stmts created in the outer-loop during vectorization of
stmts in an inner-loop may not have a stmt_info, and do not
vect_init_pattern_stmt (gimple *pattern_stmt, stmt_vec_info orig_stmt_info,
tree vectype)
{
- stmt_vec_info pattern_stmt_info = vinfo_for_stmt (pattern_stmt);
+ vec_info *vinfo = orig_stmt_info->vinfo;
+ stmt_vec_info pattern_stmt_info = vinfo->lookup_stmt (pattern_stmt);
if (pattern_stmt_info == NULL)
pattern_stmt_info = orig_stmt_info->vinfo->add_stmt (pattern_stmt);
gimple_set_bb (pattern_stmt, gimple_bb (orig_stmt_info->stmt));
vect_determine_min_output_precision_1 (stmt_vec_info stmt_info, tree lhs)
{
/* Take the maximum precision required by users of the result. */
+ vec_info *vinfo = stmt_info->vinfo;
unsigned int precision = 0;
imm_use_iterator iter;
use_operand_p use;
gimple *use_stmt = USE_STMT (use);
if (is_gimple_debug (use_stmt))
continue;
- if (!vect_stmt_in_region_p (stmt_info->vinfo, use_stmt))
- return false;
- stmt_vec_info use_stmt_info = vinfo_for_stmt (use_stmt);
- if (!use_stmt_info->min_input_precision)
+ stmt_vec_info use_stmt_info = vinfo->lookup_stmt (use_stmt);
+ if (!use_stmt_info || !use_stmt_info->min_input_precision)
return false;
precision = MAX (precision, use_stmt_info->min_input_precision);
}
basic_block bb = bbs[nbbs - i - 1];
for (gimple_stmt_iterator si = gsi_last_bb (bb);
!gsi_end_p (si); gsi_prev (&si))
- vect_determine_stmt_precisions (vinfo_for_stmt (gsi_stmt (si)));
+ vect_determine_stmt_precisions
+ (vinfo->lookup_stmt (gsi_stmt (si)));
}
}
else
else
gsi_prev (&si);
stmt = gsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
if (stmt_info && STMT_VINFO_VECTORIZABLE (stmt_info))
vect_determine_stmt_precisions (stmt_info);
}
gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = bb_vinfo->lookup_stmt (stmt);
if (stmt_info && !STMT_VINFO_VECTORIZABLE (stmt_info))
continue;
stmt_vec_info use_vinfo, stmt_vinfo = vinfo_for_stmt (stmt);
slp_tree child;
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
int j;
/* Propagate hybrid down the SLP tree. */
if (def)
FOR_EACH_IMM_USE_STMT (use_stmt, imm_iter, def)
{
- if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ use_vinfo = loop_vinfo->lookup_stmt (use_stmt);
+ if (!use_vinfo)
continue;
- use_vinfo = vinfo_for_stmt (use_stmt);
if (STMT_VINFO_IN_PATTERN_P (use_vinfo)
&& STMT_VINFO_RELATED_STMT (use_vinfo))
use_vinfo = vinfo_for_stmt (STMT_VINFO_RELATED_STMT (use_vinfo));
vect_detect_hybrid_slp_1 (tree *tp, int *, void *data)
{
walk_stmt_info *wi = (walk_stmt_info *)data;
- struct loop *loopp = (struct loop *)wi->info;
+ loop_vec_info loop_vinfo = (loop_vec_info) wi->info;
if (wi->is_lhs)
return NULL_TREE;
+ stmt_vec_info def_stmt_info;
if (TREE_CODE (*tp) == SSA_NAME
- && !SSA_NAME_IS_DEFAULT_DEF (*tp))
+ && !SSA_NAME_IS_DEFAULT_DEF (*tp)
+ && (def_stmt_info = loop_vinfo->lookup_stmt (SSA_NAME_DEF_STMT (*tp)))
+ && PURE_SLP_STMT (def_stmt_info))
{
- gimple *def_stmt = SSA_NAME_DEF_STMT (*tp);
- if (flow_bb_inside_loop_p (loopp, gimple_bb (def_stmt))
- && PURE_SLP_STMT (vinfo_for_stmt (def_stmt)))
+ if (dump_enabled_p ())
{
- if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "marking hybrid: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt, 0);
- }
- STMT_SLP_TYPE (vinfo_for_stmt (def_stmt)) = hybrid;
+ dump_printf_loc (MSG_NOTE, vect_location, "marking hybrid: ");
+ dump_gimple_stmt (MSG_NOTE, TDF_SLIM, def_stmt_info->stmt, 0);
}
+ STMT_SLP_TYPE (def_stmt_info) = hybrid;
}
return NULL_TREE;
static tree
vect_detect_hybrid_slp_2 (gimple_stmt_iterator *gsi, bool *handled,
- walk_stmt_info *)
+ walk_stmt_info *wi)
{
- stmt_vec_info use_vinfo = vinfo_for_stmt (gsi_stmt (*gsi));
+ loop_vec_info loop_vinfo = (loop_vec_info) wi->info;
+ stmt_vec_info use_vinfo = loop_vinfo->lookup_stmt (gsi_stmt (*gsi));
/* If the stmt is in a SLP instance then this isn't a reason
to mark use definitions in other SLP instances as hybrid. */
if (! STMT_SLP_TYPE (use_vinfo)
gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ stmt_vec_info stmt_info = loop_vinfo->lookup_stmt (stmt);
if (STMT_VINFO_IN_PATTERN_P (stmt_info))
{
walk_stmt_info wi;
memset (&wi, 0, sizeof (wi));
- wi.info = LOOP_VINFO_LOOP (loop_vinfo);
+ wi.info = loop_vinfo;
gimple_stmt_iterator gsi2
= gsi_for_stmt (STMT_VINFO_RELATED_STMT (stmt_info));
walk_gimple_stmt (&gsi2, vect_detect_hybrid_slp_2,
slp_instance node_instance, stmt_vector_for_cost *cost_vec)
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ vec_info *vinfo = stmt_info->vinfo;
bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_info);
enum vect_relevant relevance = STMT_VINFO_RELEVANT (stmt_info);
bool ok;
for (si = gsi_start (pattern_def_seq); !gsi_end_p (si); gsi_next (&si))
{
gimple *pattern_def_stmt = gsi_stmt (si);
- if (STMT_VINFO_RELEVANT_P (vinfo_for_stmt (pattern_def_stmt))
- || STMT_VINFO_LIVE_P (vinfo_for_stmt (pattern_def_stmt)))
+ stmt_vec_info pattern_def_stmt_info
+ = vinfo->lookup_stmt (gsi_stmt (si));
+ if (STMT_VINFO_RELEVANT_P (pattern_def_stmt_info)
+ || STMT_VINFO_LIVE_P (pattern_def_stmt_info))
{
/* Analyze def stmt of STMT if it's a pattern stmt. */
if (dump_enabled_p ())
bool *grouped_store, slp_tree slp_node,
slp_instance slp_node_instance)
{
+ stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
+ vec_info *vinfo = stmt_info->vinfo;
bool is_store = false;
gimple *vec_stmt = NULL;
- stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
bool done;
gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info));
imm_use_iterator imm_iter;
use_operand_p use_p;
tree scalar_dest;
- gimple *exit_phi;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
scalar_dest = gimple_assign_lhs (stmt);
FOR_EACH_IMM_USE_FAST (use_p, imm_iter, scalar_dest)
- {
- if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
- {
- exit_phi = USE_STMT (use_p);
- STMT_VINFO_VEC_STMT (vinfo_for_stmt (exit_phi)) = vec_stmt;
- }
- }
+ if (!flow_bb_inside_loop_p (innerloop, gimple_bb (USE_STMT (use_p))))
+ {
+ stmt_vec_info exit_phi_info
+ = vinfo->lookup_stmt (USE_STMT (use_p));
+ STMT_VINFO_VEC_STMT (exit_phi_info) = vec_stmt;
+ }
}
/* Handle stmts whose DEF is used outside the loop-nest that is
return res;
}
+/* If STMT has an associated stmt_vec_info, return that vec_info, otherwise
+ return null. It is safe to call this function on any statement, even if
+ it might not be part of the vectorizable region. */
+
+stmt_vec_info
+vec_info::lookup_stmt (gimple *stmt)
+{
+ unsigned int uid = gimple_uid (stmt);
+ if (uid > 0 && uid - 1 < stmt_vec_infos.length ())
+ {
+ stmt_vec_info res = stmt_vec_infos[uid - 1];
+ if (res && res->stmt == stmt)
+ return res;
+ }
+ return NULL;
+}
+
/* A helper function to free scev and LOOP niter information, as well as
clear loop constraint LOOP_C_FINITE. */
~vec_info ();
stmt_vec_info add_stmt (gimple *);
+ stmt_vec_info lookup_stmt (gimple *);
/* The type of vectorization. */
vec_kind kind;
projects / gcc.git / commitdiff