vect_same_loop_or_bb_p (gimple *stmt1, gimple *stmt2)
{
stmt_vec_info stmt_vinfo = vinfo_for_stmt (stmt1);
- loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
- bb_vec_info bb_vinfo = STMT_VINFO_BB_VINFO (stmt_vinfo);
-
- if (!gimple_bb (stmt2))
- return false;
-
- if (loop_vinfo)
- {
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- if (!flow_bb_inside_loop_p (loop, gimple_bb (stmt2)))
- return false;
- }
- else
- {
- if (gimple_bb (stmt2) != BB_VINFO_BB (bb_vinfo)
- || gimple_code (stmt2) == GIMPLE_PHI)
- return false;
- }
-
- gcc_assert (vinfo_for_stmt (stmt2));
- return true;
+ return vect_stmt_in_region_p (stmt_vinfo->vinfo, stmt2);
}
/* If the LHS of DEF_STMT has a single use, and that statement is
loop = LOOP_VINFO_LOOP (loop_vinfo);
bbs = LOOP_VINFO_BBS (loop_vinfo);
nbbs = loop->num_nodes;
+
+ /* Scan through the loop stmts, applying the pattern recognition
+ functions starting at each stmt visited: */
+ for (i = 0; i < nbbs; i++)
+ {
+ basic_block bb = bbs[i];
+ for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+ {
+ /* Scan over all generic vect_recog_xxx_pattern functions. */
+ for (j = 0; j < NUM_PATTERNS; j++)
+ {
+ vect_recog_func = vect_vect_recog_func_ptrs[j];
+ vect_pattern_recog_1 (vect_recog_func, si,
+ &stmts_to_replace);
+ }
+ }
+ }
}
else
{
- bbs = &as_a <bb_vec_info> (vinfo)->bb;
- nbbs = 1;
- }
-
- /* Scan through the loop stmts, applying the pattern recognition
- functions starting at each stmt visited: */
- for (i = 0; i < nbbs; i++)
- {
- basic_block bb = bbs[i];
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
- {
- if (is_a <bb_vec_info> (vinfo)
- && (stmt = gsi_stmt (si))
+ bb_vec_info bb_vinfo = as_a <bb_vec_info> (vinfo);
+ for (si = bb_vinfo->region_begin;
+ gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
+ {
+ if ((stmt = gsi_stmt (si))
&& vinfo_for_stmt (stmt)
&& !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)))
- continue;
+ continue;
- /* Scan over all generic vect_recog_xxx_pattern functions. */
- for (j = 0; j < NUM_PATTERNS; j++)
- {
+ /* Scan over all generic vect_recog_xxx_pattern functions. */
+ for (j = 0; j < NUM_PATTERNS; j++)
+ {
vect_recog_func = vect_vect_recog_func_ptrs[j];
vect_pattern_recog_1 (vect_recog_func, si,
&stmts_to_replace);
- }
- }
+ }
+ }
}
}
unsigned int i, number_of_oprnds;
gimple *def_stmt;
enum vect_def_type dt = vect_uninitialized_def;
- struct loop *loop = NULL;
bool pattern = false;
slp_oprnd_info oprnd_info;
int first_op_idx = 1;
bool first = stmt_num == 0;
bool second = stmt_num == 1;
- if (is_a <loop_vec_info> (vinfo))
- loop = LOOP_VINFO_LOOP (as_a <loop_vec_info> (vinfo));
-
if (is_gimple_call (stmt))
{
number_of_oprnds = gimple_call_num_args (stmt);
from the pattern. Check that all the stmts of the node are in the
pattern. */
if (def_stmt && gimple_bb (def_stmt)
- && ((is_a <loop_vec_info> (vinfo)
- && flow_bb_inside_loop_p (loop, gimple_bb (def_stmt)))
- || (is_a <bb_vec_info> (vinfo)
- && gimple_bb (def_stmt) == as_a <bb_vec_info> (vinfo)->bb
- && gimple_code (def_stmt) != GIMPLE_PHI))
+ && vect_stmt_in_region_p (vinfo, def_stmt)
&& vinfo_for_stmt (def_stmt)
&& STMT_VINFO_IN_PATTERN_P (vinfo_for_stmt (def_stmt))
&& !STMT_VINFO_RELEVANT (vinfo_for_stmt (def_stmt))
stmt_vec_info structs for all the stmts in it. */
static bb_vec_info
-new_bb_vec_info (basic_block bb)
+new_bb_vec_info (gimple_stmt_iterator region_begin,
+ gimple_stmt_iterator region_end)
{
+ basic_block bb = gsi_bb (region_begin);
bb_vec_info res = NULL;
gimple_stmt_iterator gsi;
res = (bb_vec_info) xcalloc (1, sizeof (struct _bb_vec_info));
res->kind = vec_info::bb;
BB_VINFO_BB (res) = bb;
+ res->region_begin = region_begin;
+ res->region_end = region_end;
- for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
+ for (gsi = region_begin; gsi_stmt (gsi) != gsi_stmt (region_end);
+ gsi_next (&gsi))
{
gimple *stmt = gsi_stmt (gsi);
gimple_set_uid (stmt, 0);
bb = BB_VINFO_BB (bb_vinfo);
- for (si = gsi_start_bb (bb); !gsi_end_p (si); gsi_next (&si))
+ for (si = bb_vinfo->region_begin;
+ gsi_stmt (si) != gsi_stmt (bb_vinfo->region_end); gsi_next (&si))
{
gimple *stmt = gsi_stmt (si);
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
if (stmt_info)
/* Free stmt_vec_info. */
free_stmt_vec_info (stmt);
+
+ /* Reset region marker. */
+ gimple_set_uid (stmt, -1);
}
vect_destroy_datarefs (bb_vinfo);
gimple *use_stmt;
FOR_EACH_IMM_USE_STMT (use_stmt, use_iter, DEF_FROM_PTR (def_p))
if (!is_gimple_debug (use_stmt)
- && (gimple_code (use_stmt) == GIMPLE_PHI
- || gimple_bb (use_stmt) != bb
+ && (! vect_stmt_in_region_p (vinfo_for_stmt (stmt)->vinfo,
+ use_stmt)
|| !STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (use_stmt))))
{
(*life)[i] = true;
/* Check if the basic block can be vectorized. */
static bb_vec_info
-vect_slp_analyze_bb_1 (basic_block bb)
+vect_slp_analyze_bb_1 (gimple_stmt_iterator region_begin,
+ gimple_stmt_iterator region_end,
+ vec<data_reference_p> datarefs, int n_stmts)
{
bb_vec_info bb_vinfo;
vec<slp_instance> slp_instances;
slp_instance instance;
int i;
int min_vf = 2;
- unsigned n_stmts = 0;
- bb_vinfo = new_bb_vec_info (bb);
+ if (n_stmts > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "not vectorized: too many instructions in "
+ "basic block.\n");
+ free_data_refs (datarefs);
+ return NULL;
+ }
+
+ bb_vinfo = new_bb_vec_info (region_begin, region_end);
if (!bb_vinfo)
return NULL;
- /* Gather all data references in the basic-block. */
-
- for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
- !gsi_end_p (gsi); gsi_next (&gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- if (is_gimple_debug (stmt))
- continue;
- ++n_stmts;
- if (!find_data_references_in_stmt (NULL, stmt,
- &BB_VINFO_DATAREFS (bb_vinfo)))
- {
- /* Mark the rest of the basic-block as unvectorizable. */
- for (; !gsi_end_p (gsi); gsi_next (&gsi))
- {
- stmt = gsi_stmt (gsi);
- STMT_VINFO_VECTORIZABLE (vinfo_for_stmt (stmt)) = false;
- }
- break;
- }
- }
+ BB_VINFO_DATAREFS (bb_vinfo) = datarefs;
/* Analyze the data references. */
}
/* Mark all the statements that we do not want to vectorize. */
- for (gimple_stmt_iterator gsi = gsi_start_bb (BB_VINFO_BB (bb_vinfo));
- !gsi_end_p (gsi); gsi_next (&gsi))
+ for (gimple_stmt_iterator gsi = bb_vinfo->region_begin;
+ gsi_stmt (gsi) != gsi_stmt (bb_vinfo->region_end); gsi_next (&gsi))
{
stmt_vec_info vinfo = vinfo_for_stmt (gsi_stmt (gsi));
if (STMT_SLP_TYPE (vinfo) != pure_slp)
vect_slp_bb (basic_block bb)
{
bb_vec_info bb_vinfo;
- int insns = 0;
gimple_stmt_iterator gsi;
unsigned int vector_sizes;
+ bool any_vectorized = false;
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location, "===vect_slp_analyze_bb===\n");
- for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- gimple *stmt = gsi_stmt (gsi);
- if (!is_gimple_debug (stmt)
- && !gimple_nop_p (stmt)
- && gimple_code (stmt) != GIMPLE_LABEL)
- insns++;
- if (gimple_location (stmt) != UNKNOWN_LOCATION)
- vect_location = gimple_location (stmt);
- }
-
- if (insns > PARAM_VALUE (PARAM_SLP_MAX_INSNS_IN_BB))
- {
- if (dump_enabled_p ())
- dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
- "not vectorized: too many instructions in "
- "basic block.\n");
-
- return false;
- }
-
/* Autodetect first vector size we try. */
current_vector_size = 0;
vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
+ gsi = gsi_start_bb (bb);
+
while (1)
{
- bb_vinfo = vect_slp_analyze_bb_1 (bb);
- if (bb_vinfo)
+ if (gsi_end_p (gsi))
+ break;
+
+ gimple_stmt_iterator region_begin = gsi;
+ vec<data_reference_p> datarefs = vNULL;
+ int insns = 0;
+
+ for (; !gsi_end_p (gsi); gsi_next (&gsi))
{
- if (!dbg_cnt (vect_slp))
- {
- destroy_bb_vec_info (bb_vinfo);
- return false;
- }
+ gimple *stmt = gsi_stmt (gsi);
+ if (is_gimple_debug (stmt))
+ continue;
+ insns++;
+
+ if (gimple_location (stmt) != UNKNOWN_LOCATION)
+ vect_location = gimple_location (stmt);
+
+ if (!find_data_references_in_stmt (NULL, stmt, &datarefs))
+ break;
+ }
+
+ /* Skip leading unhandled stmts. */
+ if (gsi_stmt (region_begin) == gsi_stmt (gsi))
+ {
+ gsi_next (&gsi);
+ continue;
+ }
+ gimple_stmt_iterator region_end = gsi;
+
+ bool vectorized = false;
+ bb_vinfo = vect_slp_analyze_bb_1 (region_begin, region_end,
+ datarefs, insns);
+ if (bb_vinfo
+ && dbg_cnt (vect_slp))
+ {
if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB\n");
+ dump_printf_loc (MSG_NOTE, vect_location, "SLPing BB part\n");
vect_schedule_slp (bb_vinfo);
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
- "BASIC BLOCK VECTORIZED\n");
+ "basic block part vectorized\n");
destroy_bb_vec_info (bb_vinfo);
- return true;
+ vectorized = true;
}
+ else
+ destroy_bb_vec_info (bb_vinfo);
- destroy_bb_vec_info (bb_vinfo);
+ any_vectorized |= vectorized;
vector_sizes &= ~current_vector_size;
- if (vector_sizes == 0
- || current_vector_size == 0)
- return false;
+ if (vectorized
+ || vector_sizes == 0
+ || current_vector_size == 0)
+ {
+ if (gsi_end_p (region_end))
+ break;
- /* Try the next biggest vector size. */
- current_vector_size = 1 << floor_log2 (vector_sizes);
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "***** Re-trying analysis with "
- "vector size %d\n", current_vector_size);
+ /* Skip the unhandled stmt. */
+ gsi_next (&gsi);
+
+ /* And reset vector sizes. */
+ current_vector_size = 0;
+ vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
+ }
+ else
+ {
+ /* Try the next biggest vector size. */
+ current_vector_size = 1 << floor_log2 (vector_sizes);
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "***** Re-trying analysis with "
+ "vector size %d\n", current_vector_size);
+
+ /* Start over. */
+ gsi = region_begin;
+ }
}
+
+ return any_vectorized;
}
projects / gcc.git / commitdiff