{
tree src = src_rgm->masks[i / 2];
tree dest = dest_rgm->masks[i];
- tree_code code = (i & 1 ? VEC_UNPACK_HI_EXPR
+ tree_code code = ((i & 1) == (BYTES_BIG_ENDIAN ? 0 : 1)
+ ? VEC_UNPACK_HI_EXPR
: VEC_UNPACK_LO_EXPR);
gassign *stmt;
if (dest_masktype == unpack_masktype)
loop_cond_gsi);
/* Remove old loop exit test. */
- gsi_remove (&loop_cond_gsi, true);
- free_stmt_vec_info (orig_cond);
+ stmt_vec_info orig_cond_info;
+ if (loop_vinfo
+ && (orig_cond_info = loop_vinfo->lookup_stmt (orig_cond)))
+ loop_vinfo->remove_stmt (orig_cond_info);
+ else
+ gsi_remove (&loop_cond_gsi, true);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "New loop exit condition: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, cond_stmt, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "New loop exit condition: %G",
+ cond_stmt);
}
/* Helper routine of slpeel_tree_duplicate_loop_to_edge_cfg.
}
if (TREE_CODE (from_arg) != SSA_NAME)
gcc_assert (operand_equal_p (from_arg, to_arg, 0));
- else
+ else if (TREE_CODE (to_arg) == SSA_NAME
+ && from_arg != to_arg)
{
if (get_current_def (to_arg) == NULL_TREE)
- set_current_def (to_arg, get_current_def (from_arg));
+ {
+ gcc_assert (types_compatible_p (TREE_TYPE (to_arg),
+ TREE_TYPE (get_current_def
+ (from_arg))));
+ set_current_def (to_arg, get_current_def (from_arg));
+ }
}
gsi_next (&gsi_from);
gsi_next (&gsi_to);
location is calculated.
Return the loop location if succeed and NULL if not. */
-source_location
+dump_user_location_t
find_loop_location (struct loop *loop)
{
gimple *stmt = NULL;
gimple_stmt_iterator si;
if (!loop)
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
stmt = get_loop_exit_condition (loop);
if (stmt
&& LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
- return gimple_location (stmt);
+ return stmt;
/* If we got here the loop is probably not "well formed",
try to estimate the loop location */
if (!loop->header)
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
bb = loop->header;
{
stmt = gsi_stmt (si);
if (LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
- return gimple_location (stmt);
+ return stmt;
}
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
}
-/* Return true if PHI defines an IV of the loop to be vectorized. */
+/* Return true if the phi described by STMT_INFO defines an IV of the
+ loop to be vectorized. */
static bool
-iv_phi_p (gphi *phi)
+iv_phi_p (stmt_vec_info stmt_info)
{
+ gphi *phi = as_a <gphi *> (stmt_info->stmt);
if (virtual_operand_p (PHI_RESULT (phi)))
return false;
- stmt_vec_info stmt_info = vinfo_for_stmt (phi);
- gcc_assert (stmt_info != NULL);
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
|| STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
return false;
tree evolution_part;
gphi *phi = gsi.phi ();
+ stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
+ phi_info->stmt);
/* Skip virtual phi's. The data dependences that are associated with
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere.
Skip reduction phis. */
- if (!iv_phi_p (phi))
+ if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
/* Analyze the evolution function. */
- evolution_part
- = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (vinfo_for_stmt (phi));
+ evolution_part = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
if (evolution_part == NULL_TREE)
{
if (dump_enabled_p ())
gphi *phi = gsi.phi ();
gphi *phi1 = gsi1.phi ();
+ stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "vect_update_ivs_after_vectorizer: phi: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "vect_update_ivs_after_vectorizer: phi: %G", phi);
/* Skip reduction and virtual phis. */
- if (!iv_phi_p (phi))
+ if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
}
type = TREE_TYPE (gimple_phi_result (phi));
- step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (vinfo_for_stmt (phi));
+ step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
step_expr = unshare_expr (step_expr);
/* FORNOW: We do not support IVs whose evolution function is a polynomial
static tree
get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
{
- struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
- gimple *dr_stmt = DR_STMT (dr);
- stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
- gcc_assert (target_align != 0);
+ poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr_info);
+ unsigned HOST_WIDE_INT target_align_c;
+ tree target_align_minus_1;
- bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
+ bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+ size_zero_node) < 0;
tree offset = (negative
? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1)
: size_zero_node);
- tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt, seq,
+ tree start_addr = vect_create_addr_base_for_vector_ref (stmt_info, seq,
offset);
tree type = unsigned_type_for (TREE_TYPE (start_addr));
- tree target_align_minus_1 = build_int_cst (type, target_align - 1);
+ if (target_align.is_constant (&target_align_c))
+ target_align_minus_1 = build_int_cst (type, target_align_c - 1);
+ else
+ {
+ tree vla = build_int_cst (type, target_align);
+ tree vla_align = fold_build2 (BIT_AND_EXPR, type, vla,
+ fold_build2 (MINUS_EXPR, type,
+ build_int_cst (type, 0), vla));
+ target_align_minus_1 = fold_build2 (MINUS_EXPR, type, vla_align,
+ build_int_cst (type, 1));
+ }
+
HOST_WIDE_INT elem_size
= int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
tree elem_size_log = build_int_cst (type, exact_log2 (elem_size));
vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo,
basic_block bb, int *bound)
{
- struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
tree var;
tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
gimple_seq stmts = NULL, new_stmts = NULL;
tree iters, iters_name;
- gimple *dr_stmt = DR_STMT (dr);
- stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
+ stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
+ poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr_info);
if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
{
tree type = TREE_TYPE (misalign_in_elems);
HOST_WIDE_INT elem_size
= int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
- HOST_WIDE_INT align_in_elems = target_align / elem_size;
- tree align_in_elems_minus_1 = build_int_cst (type, align_in_elems - 1);
+ /* We only do prolog peeling if the target alignment is known at compile
+ time. */
+ poly_uint64 align_in_elems =
+ exact_div (target_align, elem_size);
+ tree align_in_elems_minus_1 =
+ build_int_cst (type, align_in_elems - 1);
tree align_in_elems_tree = build_int_cst (type, align_in_elems);
/* Create: (niters_type) ((align_in_elems - misalign_in_elems)
& (align_in_elems - 1)). */
- bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
+ bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+ size_zero_node) < 0;
if (negative)
iters = fold_build2 (MINUS_EXPR, type, misalign_in_elems,
align_in_elems_tree);
misalign_in_elems);
iters = fold_build2 (BIT_AND_EXPR, type, iters, align_in_elems_minus_1);
iters = fold_convert (niters_type, iters);
- *bound = align_in_elems - 1;
+ unsigned HOST_WIDE_INT align_in_elems_c;
+ if (align_in_elems.is_constant (&align_in_elems_c))
+ *bound = align_in_elems_c - 1;
+ else
+ *bound = -1;
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "niters for prolog loop: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, iters);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "niters for prolog loop: %T\n", iters);
var = create_tmp_var (niters_type, "prolog_loop_niters");
iters_name = force_gimple_operand (iters, &new_stmts, false, var);
vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
struct data_reference *dr;
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "=== vect_update_inits_of_dr ===\n");
+ DUMP_VECT_SCOPE ("vect_update_inits_of_dr");
/* Adjust niters to sizetype and insert stmts on loop preheader edge. */
if (!types_compatible_p (sizetype, TREE_TYPE (niters)))
}
FOR_EACH_VEC_ELT (datarefs, i, dr)
- vect_update_init_of_dr (dr, niters, code);
+ {
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt))
+ vect_update_init_of_dr (dr, niters, code);
+ }
}
/* For the information recorded in LOOP_VINFO prepare the loop for peeling
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "misalignment for fully-masked loop: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, misalign_in_elems);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "misalignment for fully-masked loop: %T\n",
+ misalign_in_elems);
LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo) = misalign_in_elems;
tree arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, first_latch_e);
/* Generate lcssa PHI node for the first loop. */
gphi *vect_phi = (loop == first) ? orig_phi : update_phi;
- if (create_lcssa_for_iv_phis || !iv_phi_p (vect_phi))
+ stmt_vec_info vect_phi_info = loop_vinfo->lookup_stmt (vect_phi);
+ if (create_lcssa_for_iv_phis || !iv_phi_p (vect_phi_info))
{
tree new_res = copy_ssa_name (PHI_RESULT (orig_phi));
gphi *lcssa_phi = create_phi_node (new_res, between_bb);
struct loop *update_loop,
edge guard_edge, edge merge_edge)
{
- source_location merge_loc, guard_loc;
+ location_t merge_loc, guard_loc;
edge orig_e = loop_preheader_edge (skip_loop);
edge update_e = loop_preheader_edge (update_loop);
gphi_iterator gsi_orig, gsi_update;
profile_probability prob_prolog, prob_vector, prob_epilog;
int estimated_vf;
int prolog_peeling = 0;
+ /* We currently do not support prolog peeling if the target alignment is not
+ known at compile time. 'vect_gen_prolog_loop_niters' depends on the
+ target alignment being constant. */
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ if (dr_info && !DR_TARGET_ALIGNMENT (dr_info).is_constant ())
+ return NULL;
+
if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
}
}
- source_location loop_loc = find_loop_location (loop);
+ dump_user_location_t loop_loc = find_loop_location (loop);
struct loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
if (prolog_peeling)
{
/* Function vect_create_cond_for_niters_checks.
Create a conditional expression that represents the run-time checks for
- loop's niter. The loop is guaranteed to to terminate if the run-time
+ loop's niter. The loop is guaranteed to terminate if the run-time
checks hold.
Input:
tree *cond_expr,
gimple_seq *cond_expr_stmt_list)
{
- vec<gimple *> may_misalign_stmts
+ vec<stmt_vec_info> may_misalign_stmts
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
- gimple *ref_stmt;
+ stmt_vec_info stmt_info;
int mask = LOOP_VINFO_PTR_MASK (loop_vinfo);
tree mask_cst;
unsigned int i;
/* Create expression (mask & (dr_1 || ... || dr_n)) where dr_i is the address
of the first vector of the i'th data reference. */
- FOR_EACH_VEC_ELT (may_misalign_stmts, i, ref_stmt)
+ FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
{
gimple_seq new_stmt_list = NULL;
tree addr_base;
tree addr_tmp_name;
tree new_or_tmp_name;
gimple *addr_stmt, *or_stmt;
- stmt_vec_info stmt_vinfo = vinfo_for_stmt (ref_stmt);
- tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
bool negative = tree_int_cst_compare
- (DR_STEP (STMT_VINFO_DATA_REF (stmt_vinfo)), size_zero_node) < 0;
+ (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)), size_zero_node) < 0;
tree offset = negative
? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node;
/* create: addr_tmp = (int)(address_of_first_vector) */
addr_base =
- vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list,
+ vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
offset);
if (new_stmt_list != NULL)
gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list);
while we need to move it above LOOP's preheader. */
e = loop_preheader_edge (loop);
scalar_e = loop_preheader_edge (scalar_loop);
- gcc_assert (empty_block_p (e->src)
- && single_pred_p (e->src));
+ /* The vector loop preheader might not be empty, since new
+ invariants could have been created while analyzing the loop. */
+ gcc_assert (single_pred_p (e->src));
gcc_assert (empty_block_p (scalar_e->src)
&& single_pred_p (scalar_e->src));
gcc_assert (single_pred_p (condition_bb));
loop_constraint_set (loop, LOOP_C_INFINITE);
}
- if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
+ if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
&& dump_enabled_p ())
{
if (version_alias)
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
+ vect_location,
"loop versioned for vectorization because of "
"possible aliasing\n");
if (version_align)
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
+ vect_location,
"loop versioned for vectorization to enhance "
"alignment\n");
projects / gcc.git / blobdiff