#include "optabs.h"
#include "params.h"
#include "diagnostic-core.h"
-#include "toplev.h"
#include "tree-chrec.h"
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
/* Analyze the evolution function. */
access_fn = analyze_scalar_evolution (loop, def);
+ if (access_fn)
+ STRIP_NOPS (access_fn);
if (access_fn && vect_print_dump_info (REPORT_DETAILS))
{
fprintf (vect_dump, "Access function of PHI: ");
LOOP_VINFO_VECTORIZABLE_P (res) = 0;
LOOP_PEELING_FOR_ALIGNMENT (res) = 0;
LOOP_VINFO_VECT_FACTOR (res) = 0;
+ LOOP_VINFO_LOOP_NEST (res) = VEC_alloc (loop_p, heap, 3);
LOOP_VINFO_DATAREFS (res) = VEC_alloc (data_reference_p, heap, 10);
LOOP_VINFO_DDRS (res) = VEC_alloc (ddr_p, heap, 10 * 10);
LOOP_VINFO_UNALIGNED_DR (res) = NULL;
free (LOOP_VINFO_BBS (loop_vinfo));
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
+ VEC_free (loop_p, heap, LOOP_VINFO_LOOP_NEST (loop_vinfo));
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
+ VEC_free (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
free (loop_vinfo);
loop->aux = NULL;
free (LOOP_VINFO_BBS (loop_vinfo));
free_data_refs (LOOP_VINFO_DATAREFS (loop_vinfo));
free_dependence_relations (LOOP_VINFO_DDRS (loop_vinfo));
+ VEC_free (loop_p, heap, LOOP_VINFO_LOOP_NEST (loop_vinfo));
VEC_free (gimple, heap, LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo));
VEC_free (ddr_p, heap, LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo));
slp_instances = LOOP_VINFO_SLP_INSTANCES (loop_vinfo);
}
-/* Function vect_analyze_loop.
+/* Function vect_analyze_loop_2.
Apply a set of analyses on LOOP, and create a loop_vec_info struct
for it. The different analyses will record information in the
loop_vec_info struct. */
-loop_vec_info
-vect_analyze_loop (struct loop *loop)
+static bool
+vect_analyze_loop_2 (loop_vec_info loop_vinfo)
{
bool ok, dummy;
- loop_vec_info loop_vinfo;
int max_vf = MAX_VECTORIZATION_FACTOR;
int min_vf = 2;
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "===== analyze_loop_nest =====");
-
- if (loop_outer (loop)
- && loop_vec_info_for_loop (loop_outer (loop))
- && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "outer-loop already vectorized.");
- return NULL;
- }
-
- /* Check the CFG characteristics of the loop (nesting, entry/exit, etc. */
-
- loop_vinfo = vect_analyze_loop_form (loop);
- if (!loop_vinfo)
- {
- if (vect_print_dump_info (REPORT_DETAILS))
- fprintf (vect_dump, "bad loop form.");
- return NULL;
- }
-
/* Find all data references in the loop (which correspond to vdefs/vuses)
and analyze their evolution in the loop. Also adjust the minimal
vectorization factor according to the loads and stores.
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data references.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Classify all cross-iteration scalar data-flow cycles.
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "unexpected pattern.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Analyze data dependences between the data-refs in the loop
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data dependence.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
ok = vect_determine_vectorization_factor (loop_vinfo);
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "can't determine vectorization factor.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
if (max_vf < LOOP_VINFO_VECT_FACTOR (loop_vinfo))
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data dependence.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Analyze the alignment of the data-refs in the loop.
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data alignment.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Analyze the access patterns of the data-refs in the loop (consecutive,
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data access.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Prune the list of ddrs to be tested at run-time by versioning for alias.
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "too long list of versioning for alias "
"run-time tests.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* This pass will decide on using loop versioning and/or loop peeling in
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad data alignment.");
- destroy_loop_vec_info (loop_vinfo, true);
- return NULL;
+ return false;
}
/* Check the SLP opportunities in the loop, analyze and build SLP trees. */
{
if (vect_print_dump_info (REPORT_DETAILS))
fprintf (vect_dump, "bad operation or unsupported loop bound.");
- destroy_loop_vec_info (loop_vinfo, true);
+ return false;
+ }
+
+ return true;
+}
+
+/* Function vect_analyze_loop.
+
+ Apply a set of analyses on LOOP, and create a loop_vec_info struct
+ for it. The different analyses will record information in the
+ loop_vec_info struct. */
+loop_vec_info
+vect_analyze_loop (struct loop *loop)
+{
+ loop_vec_info loop_vinfo;
+ unsigned int vector_sizes;
+
+ /* Autodetect first vector size we try. */
+ current_vector_size = 0;
+ vector_sizes = targetm.vectorize.autovectorize_vector_sizes ();
+
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "===== analyze_loop_nest =====");
+
+ if (loop_outer (loop)
+ && loop_vec_info_for_loop (loop_outer (loop))
+ && LOOP_VINFO_VECTORIZABLE_P (loop_vec_info_for_loop (loop_outer (loop))))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "outer-loop already vectorized.");
return NULL;
}
- LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
+ while (1)
+ {
+ /* Check the CFG characteristics of the loop (nesting, entry/exit). */
+ loop_vinfo = vect_analyze_loop_form (loop);
+ if (!loop_vinfo)
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "bad loop form.");
+ return NULL;
+ }
- return loop_vinfo;
+ if (vect_analyze_loop_2 (loop_vinfo))
+ {
+ LOOP_VINFO_VECTORIZABLE_P (loop_vinfo) = 1;
+
+ return loop_vinfo;
+ }
+
+ destroy_loop_vec_info (loop_vinfo, true);
+
+ vector_sizes &= ~current_vector_size;
+ if (vector_sizes == 0
+ || current_vector_size == 0)
+ return NULL;
+
+ /* Try the next biggest vector size. */
+ current_vector_size = 1 << floor_log2 (vector_sizes);
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "***** Re-trying analysis with "
+ "vector size %d\n", current_vector_size);
+ }
}
1. operation is commutative and associative and it is safe to
change the order of the computation (if CHECK_REDUCTION is true)
2. no uses for a2 in the loop (a2 is used out of the loop)
- 3. no uses of a1 in the loop besides the reduction operation.
+ 3. no uses of a1 in the loop besides the reduction operation
+ 4. no uses of a1 outside the loop.
- Condition 1 is tested here.
+ Conditions 1,4 are tested here.
Conditions 2,3 are tested in vect_mark_stmts_to_be_vectorized.
(2) Detect a cross-iteration def-use cycle in nested loops, i.e.,
gimple use_stmt = USE_STMT (use_p);
if (is_gimple_debug (use_stmt))
continue;
- if (flow_bb_inside_loop_p (loop, gimple_bb (use_stmt))
- && vinfo_for_stmt (use_stmt)
+
+ if (!flow_bb_inside_loop_p (loop, gimple_bb (use_stmt)))
+ {
+ if (vect_print_dump_info (REPORT_DETAILS))
+ fprintf (vect_dump, "intermediate value used outside loop.");
+
+ return NULL;
+ }
+
+ if (vinfo_for_stmt (use_stmt)
&& !is_pattern_stmt_p (vinfo_for_stmt (use_stmt)))
nloop_uses++;
if (nloop_uses > 1)
simply rewriting this into "res += -x[i]". Avoid changing
gimple instruction for the first simple tests and only do this
if we're allowed to change code at all. */
- if (code == MINUS_EXPR && modify)
+ if (code == MINUS_EXPR
+ && modify
+ && (op1 = gimple_assign_rhs1 (def_stmt))
+ && TREE_CODE (op1) == SSA_NAME
+ && SSA_NAME_DEF_STMT (op1) == phi)
code = PLUS_EXPR;
if (check_reduction
&& (code == COND_EXPR
|| (def1 && flow_bb_inside_loop_p (loop, gimple_bb (def1))
&& (is_gimple_assign (def1)
+ || is_gimple_call (def1)
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def1))
== vect_induction_def
|| (gimple_code (def1) == GIMPLE_PHI
&& (code == COND_EXPR
|| (def2 && flow_bb_inside_loop_p (loop, gimple_bb (def2))
&& (is_gimple_assign (def2)
+ || is_gimple_call (def2)
|| STMT_VINFO_DEF_TYPE (vinfo_for_stmt (def2))
== vect_induction_def
|| (gimple_code (def2) == GIMPLE_PHI
stmt_vec_info stmt_vinfo = vinfo_for_stmt (iv_phi);
loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_vinfo);
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- tree scalar_type = TREE_TYPE (gimple_phi_result (iv_phi));
+ tree scalar_type;
tree vectype;
int nunits;
edge pe = loop_preheader_edge (loop);
gimple_stmt_iterator si;
basic_block bb = gimple_bb (iv_phi);
tree stepvectype;
-
- vectype = get_vectype_for_scalar_type (scalar_type);
- gcc_assert (vectype);
- nunits = TYPE_VECTOR_SUBPARTS (vectype);
- ncopies = vf / nunits;
-
- gcc_assert (phi_info);
- gcc_assert (ncopies >= 1);
-
- /* Find the first insertion point in the BB. */
- si = gsi_after_labels (bb);
-
- if (INTEGRAL_TYPE_P (scalar_type))
- step_expr = build_int_cst (scalar_type, 0);
- else if (POINTER_TYPE_P (scalar_type))
- step_expr = size_zero_node;
- else
- step_expr = build_real (scalar_type, dconst0);
+ tree resvectype;
/* Is phi in an inner-loop, while vectorizing an enclosing outer-loop? */
if (nested_in_vect_loop_p (loop, iv_phi))
access_fn = analyze_scalar_evolution (iv_loop, PHI_RESULT (iv_phi));
gcc_assert (access_fn);
+ STRIP_NOPS (access_fn);
ok = vect_is_simple_iv_evolution (iv_loop->num, access_fn,
&init_expr, &step_expr);
gcc_assert (ok);
pe = loop_preheader_edge (iv_loop);
+ scalar_type = TREE_TYPE (init_expr);
+ vectype = get_vectype_for_scalar_type (scalar_type);
+ resvectype = get_vectype_for_scalar_type (TREE_TYPE (PHI_RESULT (iv_phi)));
+ gcc_assert (vectype);
+ nunits = TYPE_VECTOR_SUBPARTS (vectype);
+ ncopies = vf / nunits;
+
+ gcc_assert (phi_info);
+ gcc_assert (ncopies >= 1);
+
+ /* Find the first insertion point in the BB. */
+ si = gsi_after_labels (bb);
+
/* Create the vector that holds the initial_value of the induction. */
if (nested_in_vect_loop)
{
}
t = NULL_TREE;
- t = tree_cons (NULL_TREE, init_expr, t);
+ t = tree_cons (NULL_TREE, new_name, t);
for (i = 1; i < nunits; i++)
{
/* Create: new_name_i = new_name + step_expr */
expr, step_expr);
}
- t = NULL_TREE;
- for (i = 0; i < nunits; i++)
- t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ t = unshare_expr (new_name);
gcc_assert (CONSTANT_CLASS_P (new_name));
stepvectype = get_vectype_for_scalar_type (TREE_TYPE (new_name));
gcc_assert (stepvectype);
- vec = build_vector (stepvectype, t);
+ vec = build_vector_from_val (stepvectype, t);
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
expr = build_int_cst (TREE_TYPE (step_expr), nunits);
new_name = fold_build2 (MULT_EXPR, TREE_TYPE (step_expr),
expr, step_expr);
- t = NULL_TREE;
- for (i = 0; i < nunits; i++)
- t = tree_cons (NULL_TREE, unshare_expr (new_name), t);
+ t = unshare_expr (new_name);
gcc_assert (CONSTANT_CLASS_P (new_name));
- vec = build_vector (stepvectype, t);
+ vec = build_vector_from_val (stepvectype, t);
vec_step = vect_init_vector (iv_phi, vec, stepvectype, NULL);
vec_def = induc_def;
gimple_assign_set_lhs (new_stmt, vec_def);
gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
+ if (!useless_type_conversion_p (resvectype, vectype))
+ {
+ new_stmt = gimple_build_assign_with_ops
+ (VIEW_CONVERT_EXPR,
+ vect_get_new_vect_var (resvectype, vect_simple_var,
+ "vec_iv_"),
+ build1 (VIEW_CONVERT_EXPR, resvectype,
+ gimple_assign_lhs (new_stmt)), NULL_TREE);
+ gimple_assign_set_lhs (new_stmt,
+ make_ssa_name
+ (gimple_assign_lhs (new_stmt), new_stmt));
+ gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
+ }
set_vinfo_for_stmt (new_stmt,
new_stmt_vec_info (new_stmt, loop_vinfo, NULL));
STMT_VINFO_RELATED_STMT (prev_stmt_vinfo) = new_stmt;
}
STMT_VINFO_VEC_STMT (phi_info) = induction_phi;
+ if (!useless_type_conversion_p (resvectype, vectype))
+ {
+ new_stmt = gimple_build_assign_with_ops
+ (VIEW_CONVERT_EXPR,
+ vect_get_new_vect_var (resvectype, vect_simple_var, "vec_iv_"),
+ build1 (VIEW_CONVERT_EXPR, resvectype, induc_def), NULL_TREE);
+ induc_def = make_ssa_name (gimple_assign_lhs (new_stmt), new_stmt);
+ gimple_assign_set_lhs (new_stmt, induc_def);
+ si = gsi_start_bb (bb);
+ gsi_insert_before (&si, new_stmt, GSI_SAME_STMT);
+ set_vinfo_for_stmt (new_stmt,
+ new_stmt_vec_info (new_stmt, loop_vinfo, NULL));
+ STMT_VINFO_RELATED_STMT (vinfo_for_stmt (new_stmt))
+ = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (induction_phi));
+ }
+
return induc_def;
}
break;
}
- for (i = nunits - 1; i >= 0; --i)
- t = tree_cons (NULL_TREE, init_value, t);
-
- if (TREE_CONSTANT (init_val))
- init_def = build_vector (vectype, t);
- else
- init_def = build_constructor_from_list (vectype, t);
-
+ init_def = build_vector_from_val (vectype, init_value);
break;
default:
/* Get the loop-entry arguments. */
if (slp_node)
- vect_get_slp_defs (slp_node, &vec_initial_defs, NULL, reduc_index);
+ vect_get_slp_defs (reduction_op, NULL_TREE, slp_node, &vec_initial_defs,
+ NULL, reduc_index);
else
{
vec_initial_defs = VEC_alloc (tree, heap, 1);
VEC (tree, heap) *vec_oprnds0 = NULL, *vec_oprnds1 = NULL, *vect_defs = NULL;
VEC (gimple, heap) *phis = NULL;
int vec_num;
- tree def0, def1;
+ tree def0, def1, tem;
if (nested_in_vect_loop_p (loop, stmt))
{
gcc_assert (is_gimple_assign (stmt));
- /* Flatten RHS */
+ /* Flatten RHS. */
switch (get_gimple_rhs_class (gimple_assign_rhs_code (stmt)))
{
case GIMPLE_SINGLE_RHS:
reduction variable. */
for (i = 0; i < op_type-1; i++)
{
- tree tem;
-
/* The condition of COND_EXPR is checked in vectorizable_condition(). */
if (i == 0 && code == COND_EXPR)
continue;
}
}
- is_simple_use = vect_is_simple_use (ops[i], loop_vinfo, NULL, &def_stmt,
- &def, &dt);
+ is_simple_use = vect_is_simple_use_1 (ops[i], loop_vinfo, NULL, &def_stmt,
+ &def, &dt, &tem);
+ if (!vectype_in)
+ vectype_in = tem;
gcc_assert (is_simple_use);
gcc_assert (dt == vect_reduction_def
|| dt == vect_nested_cycle
/* Handle uses. */
if (j == 0)
{
+ tree op0, op1 = NULL_TREE;
+
+ op0 = ops[!reduc_index];
+ if (op_type == ternary_op)
+ {
+ if (reduc_index == 0)
+ op1 = ops[2];
+ else
+ op1 = ops[1];
+ }
+
if (slp_node)
- vect_get_slp_defs (slp_node, &vec_oprnds0, &vec_oprnds1, -1);
+ vect_get_slp_defs (op0, op1, slp_node, &vec_oprnds0, &vec_oprnds1,
+ -1);
else
{
loop_vec_def0 = vect_get_vec_def_for_operand (ops[!reduc_index],
VEC_quick_push (tree, vec_oprnds0, loop_vec_def0);
if (op_type == ternary_op)
{
- if (reduc_index == 0)
- loop_vec_def1 = vect_get_vec_def_for_operand (ops[2], stmt,
- NULL);
- else
- loop_vec_def1 = vect_get_vec_def_for_operand (ops[1], stmt,
- NULL);
-
+ loop_vec_def1 = vect_get_vec_def_for_operand (op1, stmt,
+ NULL);
VEC_quick_push (tree, vec_oprnds1, loop_vec_def1);
}
}
projects / gcc.git / blobdiff