/* Vectorizer Specific Loop Manipulations
- Copyright (C) 2003-2018 Free Software Foundation, Inc.
+ Copyright (C) 2003-2019 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
ssa_op_iter iter;
edge e;
edge_iterator ei;
- struct loop *loop = bb->loop_father;
- struct loop *outer_loop = NULL;
+ class loop *loop = bb->loop_father;
+ class loop *outer_loop = NULL;
if (rename_from_outer_loop)
{
value that it should have on subsequent iterations. */
static void
-vect_set_loop_mask (struct loop *loop, tree mask, tree init_mask,
+vect_set_loop_mask (class loop *loop, tree mask, tree init_mask,
tree next_mask)
{
gphi *phi = create_phi_node (mask, loop->header);
/* Add SEQ to the end of LOOP's preheader block. */
static void
-add_preheader_seq (struct loop *loop, gimple_seq seq)
+add_preheader_seq (class loop *loop, gimple_seq seq)
{
if (seq)
{
/* Add SEQ to the beginning of LOOP's header block. */
static void
-add_header_seq (struct loop *loop, gimple_seq seq)
+add_header_seq (class loop *loop, gimple_seq seq)
{
if (seq)
{
{
tree src = src_rgm->masks[i / 2];
tree dest = dest_rgm->masks[i];
- tree_code code = (i & 1 ? VEC_UNPACK_HI_EXPR
+ tree_code code = ((i & 1) == (BYTES_BIG_ENDIAN ? 0 : 1)
+ ? VEC_UNPACK_HI_EXPR
: VEC_UNPACK_LO_EXPR);
gassign *stmt;
if (dest_masktype == unpack_masktype)
Use LOOP_COND_GSI to insert code before the exit gcond.
RGM belongs to loop LOOP. The loop originally iterated NITERS
- times and has been vectorized according to LOOP_VINFO. Each iteration
- of the vectorized loop handles VF iterations of the scalar loop.
+ times and has been vectorized according to LOOP_VINFO.
+
+ If NITERS_SKIP is nonnull, the first iteration of the vectorized loop
+ starts with NITERS_SKIP dummy iterations of the scalar loop before
+ the real work starts. The mask elements for these dummy iterations
+ must be 0, to ensure that the extra iterations do not have an effect.
It is known that:
might overflow before hitting a value above:
- NITERS * RGM->max_nscalars_per_iter
+ (NITERS + NITERS_SKIP) * RGM->max_nscalars_per_iter
This means that we cannot guarantee that such an induction variable
would ever hit a value that produces a set of all-false masks for RGM. */
static tree
-vect_set_loop_masks_directly (struct loop *loop, loop_vec_info loop_vinfo,
+vect_set_loop_masks_directly (class loop *loop, loop_vec_info loop_vinfo,
gimple_seq *preheader_seq,
gimple_stmt_iterator loop_cond_gsi,
- rgroup_masks *rgm, tree vf,
- tree niters, bool might_wrap_p)
+ rgroup_masks *rgm, tree niters, tree niters_skip,
+ bool might_wrap_p)
{
tree compare_type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo);
+ tree iv_type = LOOP_VINFO_MASK_IV_TYPE (loop_vinfo);
tree mask_type = rgm->mask_type;
unsigned int nscalars_per_iter = rgm->max_nscalars_per_iter;
poly_uint64 nscalars_per_mask = TYPE_VECTOR_SUBPARTS (mask_type);
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
/* Calculate the maximum number of scalar values that the rgroup
- handles in total and the number that it handles for each iteration
- of the vector loop. */
+ handles in total, the number that it handles for each iteration
+ of the vector loop, and the number that it should skip during the
+ first iteration of the vector loop. */
tree nscalars_total = niters;
- tree nscalars_step = vf;
+ tree nscalars_step = build_int_cst (iv_type, vf);
+ tree nscalars_skip = niters_skip;
if (nscalars_per_iter != 1)
{
/* We checked before choosing to use a fully-masked loop that these
multiplications don't overflow. */
- tree factor = build_int_cst (compare_type, nscalars_per_iter);
+ tree compare_factor = build_int_cst (compare_type, nscalars_per_iter);
+ tree iv_factor = build_int_cst (iv_type, nscalars_per_iter);
nscalars_total = gimple_build (preheader_seq, MULT_EXPR, compare_type,
- nscalars_total, factor);
- nscalars_step = gimple_build (preheader_seq, MULT_EXPR, compare_type,
- nscalars_step, factor);
+ nscalars_total, compare_factor);
+ nscalars_step = gimple_build (preheader_seq, MULT_EXPR, iv_type,
+ nscalars_step, iv_factor);
+ if (nscalars_skip)
+ nscalars_skip = gimple_build (preheader_seq, MULT_EXPR, compare_type,
+ nscalars_skip, compare_factor);
}
/* Create an induction variable that counts the number of scalars
tree index_before_incr, index_after_incr;
gimple_stmt_iterator incr_gsi;
bool insert_after;
- tree zero_index = build_int_cst (compare_type, 0);
standard_iv_increment_position (loop, &incr_gsi, &insert_after);
- create_iv (zero_index, nscalars_step, NULL_TREE, loop, &incr_gsi,
- insert_after, &index_before_incr, &index_after_incr);
+ create_iv (build_int_cst (iv_type, 0), nscalars_step, NULL_TREE, loop,
+ &incr_gsi, insert_after, &index_before_incr, &index_after_incr);
- tree test_index, test_limit;
+ tree zero_index = build_int_cst (compare_type, 0);
+ tree test_index, test_limit, first_limit;
gimple_stmt_iterator *test_gsi;
if (might_wrap_p)
{
/* In principle the loop should stop iterating once the incremented
- IV reaches a value greater than or equal to NSCALAR_TOTAL.
- However, there's no guarantee that the IV hits a value above
- this value before wrapping around. We therefore adjust the
- limit down by one IV step:
+ IV reaches a value greater than or equal to:
- NSCALARS_TOTAL -[infinite-prec] NSCALARS_STEP
+ NSCALARS_TOTAL +[infinite-prec] NSCALARS_SKIP
+
+ However, there's no guarantee that this addition doesn't overflow
+ the comparison type, or that the IV hits a value above it before
+ wrapping around. We therefore adjust the limit down by one
+ IV step:
+
+ (NSCALARS_TOTAL +[infinite-prec] NSCALARS_SKIP)
+ -[infinite-prec] NSCALARS_STEP
and compare the IV against this limit _before_ incrementing it.
Since the comparison type is unsigned, we actually want the
subtraction to saturate at zero:
- NSCALARS_TOTAL -[sat] NSCALARS_STEP. */
+ (NSCALARS_TOTAL +[infinite-prec] NSCALARS_SKIP)
+ -[sat] NSCALARS_STEP
+
+ And since NSCALARS_SKIP < NSCALARS_STEP, we can reassociate this as:
+
+ NSCALARS_TOTAL -[sat] (NSCALARS_STEP - NSCALARS_SKIP)
+
+ where the rightmost subtraction can be done directly in
+ COMPARE_TYPE. */
test_index = index_before_incr;
+ tree adjust = gimple_convert (preheader_seq, compare_type,
+ nscalars_step);
+ if (nscalars_skip)
+ adjust = gimple_build (preheader_seq, MINUS_EXPR, compare_type,
+ adjust, nscalars_skip);
test_limit = gimple_build (preheader_seq, MAX_EXPR, compare_type,
- nscalars_total, nscalars_step);
+ nscalars_total, adjust);
test_limit = gimple_build (preheader_seq, MINUS_EXPR, compare_type,
- test_limit, nscalars_step);
+ test_limit, adjust);
test_gsi = &incr_gsi;
+
+ /* Get a safe limit for the first iteration. */
+ if (nscalars_skip)
+ {
+ /* The first vector iteration can handle at most NSCALARS_STEP
+ scalars. NSCALARS_STEP <= CONST_LIMIT, and adding
+ NSCALARS_SKIP to that cannot overflow. */
+ tree const_limit = build_int_cst (compare_type,
+ LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ * nscalars_per_iter);
+ first_limit = gimple_build (preheader_seq, MIN_EXPR, compare_type,
+ nscalars_total, const_limit);
+ first_limit = gimple_build (preheader_seq, PLUS_EXPR, compare_type,
+ first_limit, nscalars_skip);
+ }
+ else
+ /* For the first iteration it doesn't matter whether the IV hits
+ a value above NSCALARS_TOTAL. That only matters for the latch
+ condition. */
+ first_limit = nscalars_total;
}
else
{
the bound before wrapping. */
test_index = index_after_incr;
test_limit = nscalars_total;
+ if (nscalars_skip)
+ test_limit = gimple_build (preheader_seq, PLUS_EXPR, compare_type,
+ test_limit, nscalars_skip);
test_gsi = &loop_cond_gsi;
+
+ first_limit = test_limit;
}
+ /* Convert the IV value to the comparison type (either a no-op or
+ a demotion). */
+ gimple_seq test_seq = NULL;
+ test_index = gimple_convert (&test_seq, compare_type, test_index);
+ gsi_insert_seq_before (test_gsi, test_seq, GSI_SAME_STMT);
+
/* Provide a definition of each mask in the group. */
tree next_mask = NULL_TREE;
tree mask;
to have a full mask. */
poly_uint64 const_limit;
bool first_iteration_full
- = (poly_int_tree_p (nscalars_total, &const_limit)
+ = (poly_int_tree_p (first_limit, &const_limit)
&& known_ge (const_limit, (i + 1) * nscalars_per_mask));
/* Rather than have a new IV that starts at BIAS and goes up to
bias_tree);
}
- /* Create the initial mask. */
+ /* Create the initial mask. First include all scalars that
+ are within the loop limit. */
tree init_mask = NULL_TREE;
if (!first_iteration_full)
{
tree start, end;
- if (nscalars_total == test_limit)
+ if (first_limit == test_limit)
{
/* Use a natural test between zero (the initial IV value)
and the loop limit. The "else" block would be valid too,
}
else
{
+ /* FIRST_LIMIT is the maximum number of scalars handled by the
+ first iteration of the vector loop. Test the portion
+ associated with this mask. */
start = bias_tree;
- end = nscalars_total;
+ end = first_limit;
}
init_mask = make_temp_ssa_name (mask_type, NULL, "max_mask");
gimple_seq_add_stmt (preheader_seq, tmp_stmt);
}
+ /* Now AND out the bits that are within the number of skipped
+ scalars. */
+ poly_uint64 const_skip;
+ if (nscalars_skip
+ && !(poly_int_tree_p (nscalars_skip, &const_skip)
+ && known_le (const_skip, bias)))
+ {
+ tree unskipped_mask = vect_gen_while_not (preheader_seq, mask_type,
+ bias_tree, nscalars_skip);
+ if (init_mask)
+ init_mask = gimple_build (preheader_seq, BIT_AND_EXPR, mask_type,
+ init_mask, unskipped_mask);
+ else
+ init_mask = unskipped_mask;
+ }
+
if (!init_mask)
/* First iteration is full. */
init_mask = build_minus_one_cst (mask_type);
/* Make LOOP iterate NITERS times using masking and WHILE_ULT calls.
LOOP_VINFO describes the vectorization of LOOP. NITERS is the
- number of iterations of the original scalar loop. NITERS_MAYBE_ZERO
- and FINAL_IV are as for vect_set_loop_condition.
+ number of iterations of the original scalar loop that should be
+ handled by the vector loop. NITERS_MAYBE_ZERO and FINAL_IV are
+ as for vect_set_loop_condition.
Insert the branch-back condition before LOOP_COND_GSI and return the
final gcond. */
static gcond *
-vect_set_loop_condition_masked (struct loop *loop, loop_vec_info loop_vinfo,
+vect_set_loop_condition_masked (class loop *loop, loop_vec_info loop_vinfo,
tree niters, tree final_iv,
bool niters_maybe_zero,
gimple_stmt_iterator loop_cond_gsi)
tree compare_type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo);
unsigned int compare_precision = TYPE_PRECISION (compare_type);
- unsigned HOST_WIDE_INT max_vf = vect_max_vf (loop_vinfo);
tree orig_niters = niters;
/* Type of the initial value of NITERS. */
tree ni_actual_type = TREE_TYPE (niters);
unsigned int ni_actual_precision = TYPE_PRECISION (ni_actual_type);
+ tree niters_skip = LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo);
/* Convert NITERS to the same size as the compare. */
if (compare_precision > ni_actual_precision
else
niters = gimple_convert (&preheader_seq, compare_type, niters);
- /* Now calculate the value that the induction variable must be able
- to hit in order to ensure that we end the loop with an all-false mask.
- This involves adding the maximum number of inactive trailing scalar
- iterations. */
- widest_int iv_limit;
- bool known_max_iters = max_loop_iterations (loop, &iv_limit);
- if (known_max_iters)
- {
- /* IV_LIMIT is the maximum number of latch iterations, which is also
- the maximum in-range IV value. Round this value down to the previous
- vector alignment boundary and then add an extra full iteration. */
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- iv_limit = (iv_limit & -(int) known_alignment (vf)) + max_vf;
- }
-
- /* Get the vectorization factor in tree form. */
- tree vf = build_int_cst (compare_type,
- LOOP_VINFO_VECT_FACTOR (loop_vinfo));
+ widest_int iv_limit = vect_iv_limit_for_full_masking (loop_vinfo);
/* Iterate over all the rgroups and fill in their masks. We could use
the first mask from any rgroup for the loop condition; here we
/* See whether zero-based IV would ever generate all-false masks
before wrapping around. */
bool might_wrap_p
- = (!known_max_iters
+ = (iv_limit == -1
|| (wi::min_precision (iv_limit * rgm->max_nscalars_per_iter,
UNSIGNED)
> compare_precision));
/* Set up all masks for this group. */
test_mask = vect_set_loop_masks_directly (loop, loop_vinfo,
&preheader_seq,
- loop_cond_gsi, rgm, vf,
- niters, might_wrap_p);
+ loop_cond_gsi, rgm,
+ niters, niters_skip,
+ might_wrap_p);
}
/* Emit all accumulated statements. */
are no loop masks. */
static gcond *
-vect_set_loop_condition_unmasked (struct loop *loop, tree niters,
+vect_set_loop_condition_unmasked (class loop *loop, tree niters,
tree step, tree final_iv,
bool niters_maybe_zero,
gimple_stmt_iterator loop_cond_gsi)
Assumption: the exit-condition of LOOP is the last stmt in the loop. */
void
-vect_set_loop_condition (struct loop *loop, loop_vec_info loop_vinfo,
+vect_set_loop_condition (class loop *loop, loop_vec_info loop_vinfo,
tree niters, tree step, tree final_iv,
bool niters_maybe_zero)
{
loop_cond_gsi);
/* Remove old loop exit test. */
- gsi_remove (&loop_cond_gsi, true);
- free_stmt_vec_info (orig_cond);
+ stmt_vec_info orig_cond_info;
+ if (loop_vinfo
+ && (orig_cond_info = loop_vinfo->lookup_stmt (orig_cond)))
+ loop_vinfo->remove_stmt (orig_cond_info);
+ else
+ gsi_remove (&loop_cond_gsi, true);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "New loop exit condition: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, cond_stmt, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "New loop exit condition: %G",
+ cond_stmt);
}
/* Helper routine of slpeel_tree_duplicate_loop_to_edge_cfg.
}
if (TREE_CODE (from_arg) != SSA_NAME)
gcc_assert (operand_equal_p (from_arg, to_arg, 0));
- else
+ else if (TREE_CODE (to_arg) == SSA_NAME
+ && from_arg != to_arg)
{
if (get_current_def (to_arg) == NULL_TREE)
- set_current_def (to_arg, get_current_def (from_arg));
+ {
+ gcc_assert (types_compatible_p (TREE_TYPE (to_arg),
+ TREE_TYPE (get_current_def
+ (from_arg))));
+ set_current_def (to_arg, get_current_def (from_arg));
+ }
}
gsi_next (&gsi_from);
gsi_next (&gsi_to);
basic blocks from SCALAR_LOOP instead of LOOP, but to either the
entry or exit of LOOP. */
-struct loop *
-slpeel_tree_duplicate_loop_to_edge_cfg (struct loop *loop,
- struct loop *scalar_loop, edge e)
+class loop *
+slpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop,
+ class loop *scalar_loop, edge e)
{
- struct loop *new_loop;
+ class loop *new_loop;
basic_block *new_bbs, *bbs, *pbbs;
bool at_exit;
bool was_imm_dom;
*/
bool
-slpeel_can_duplicate_loop_p (const struct loop *loop, const_edge e)
+slpeel_can_duplicate_loop_p (const class loop *loop, const_edge e)
{
edge exit_e = single_exit (loop);
edge entry_e = loop_preheader_edge (loop);
uses should be renamed. */
static void
-create_lcssa_for_virtual_phi (struct loop *loop)
+create_lcssa_for_virtual_phi (class loop *loop)
{
gphi_iterator gsi;
edge exit_e = single_exit (loop);
location is calculated.
Return the loop location if succeed and NULL if not. */
-source_location
-find_loop_location (struct loop *loop)
+dump_user_location_t
+find_loop_location (class loop *loop)
{
gimple *stmt = NULL;
basic_block bb;
gimple_stmt_iterator si;
if (!loop)
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
stmt = get_loop_exit_condition (loop);
if (stmt
&& LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
- return gimple_location (stmt);
+ return stmt;
/* If we got here the loop is probably not "well formed",
try to estimate the loop location */
if (!loop->header)
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
bb = loop->header;
{
stmt = gsi_stmt (si);
if (LOCATION_LOCUS (gimple_location (stmt)) > BUILTINS_LOCATION)
- return gimple_location (stmt);
+ return stmt;
}
- return UNKNOWN_LOCATION;
+ return dump_user_location_t ();
}
-/* Return true if PHI defines an IV of the loop to be vectorized. */
+/* Return true if the phi described by STMT_INFO defines an IV of the
+ loop to be vectorized. */
static bool
-iv_phi_p (gphi *phi)
+iv_phi_p (stmt_vec_info stmt_info)
{
+ gphi *phi = as_a <gphi *> (stmt_info->stmt);
if (virtual_operand_p (PHI_RESULT (phi)))
return false;
- stmt_vec_info stmt_info = vinfo_for_stmt (phi);
- gcc_assert (stmt_info != NULL);
if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
|| STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
return false;
bool
vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block bb = loop->header;
gphi_iterator gsi;
tree evolution_part;
gphi *phi = gsi.phi ();
+ stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: %G",
+ phi_info->stmt);
/* Skip virtual phi's. The data dependences that are associated with
virtual defs/uses (i.e., memory accesses) are analyzed elsewhere.
Skip reduction phis. */
- if (!iv_phi_p (phi))
+ if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
/* Analyze the evolution function. */
- evolution_part
- = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (vinfo_for_stmt (phi));
+ evolution_part = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
if (evolution_part == NULL_TREE)
{
if (dump_enabled_p ())
tree niters, edge update_e)
{
gphi_iterator gsi, gsi1;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block update_bb = update_e->dest;
basic_block exit_bb = single_exit (loop)->dest;
gphi *phi = gsi.phi ();
gphi *phi1 = gsi1.phi ();
+ stmt_vec_info phi_info = loop_vinfo->lookup_stmt (phi);
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "vect_update_ivs_after_vectorizer: phi: ");
- dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "vect_update_ivs_after_vectorizer: phi: %G", phi);
/* Skip reduction and virtual phis. */
- if (!iv_phi_p (phi))
+ if (!iv_phi_p (phi_info))
{
if (dump_enabled_p ())
dump_printf_loc (MSG_NOTE, vect_location,
}
type = TREE_TYPE (gimple_phi_result (phi));
- step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (vinfo_for_stmt (phi));
+ step_expr = STMT_VINFO_LOOP_PHI_EVOLUTION_PART (phi_info);
step_expr = unshare_expr (step_expr);
/* FORNOW: We do not support IVs whose evolution function is a polynomial
}
}
+/* Return a gimple value containing the misalignment (measured in vector
+ elements) for the loop described by LOOP_VINFO, i.e. how many elements
+ it is away from a perfectly aligned address. Add any new statements
+ to SEQ. */
+
+static tree
+get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
+{
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ stmt_vec_info stmt_info = dr_info->stmt;
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
+
+ poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr_info);
+ unsigned HOST_WIDE_INT target_align_c;
+ tree target_align_minus_1;
+
+ bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+ size_zero_node) < 0;
+ tree offset = (negative
+ ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1)
+ : size_zero_node);
+ tree start_addr = vect_create_addr_base_for_vector_ref (stmt_info, seq,
+ offset);
+ tree type = unsigned_type_for (TREE_TYPE (start_addr));
+ if (target_align.is_constant (&target_align_c))
+ target_align_minus_1 = build_int_cst (type, target_align_c - 1);
+ else
+ {
+ tree vla = build_int_cst (type, target_align);
+ tree vla_align = fold_build2 (BIT_AND_EXPR, type, vla,
+ fold_build2 (MINUS_EXPR, type,
+ build_int_cst (type, 0), vla));
+ target_align_minus_1 = fold_build2 (MINUS_EXPR, type, vla_align,
+ build_int_cst (type, 1));
+ }
+
+ HOST_WIDE_INT elem_size
+ = int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
+ tree elem_size_log = build_int_cst (type, exact_log2 (elem_size));
+
+ /* Create: misalign_in_bytes = addr & (target_align - 1). */
+ tree int_start_addr = fold_convert (type, start_addr);
+ tree misalign_in_bytes = fold_build2 (BIT_AND_EXPR, type, int_start_addr,
+ target_align_minus_1);
+
+ /* Create: misalign_in_elems = misalign_in_bytes / element_size. */
+ tree misalign_in_elems = fold_build2 (RSHIFT_EXPR, type, misalign_in_bytes,
+ elem_size_log);
+
+ return misalign_in_elems;
+}
+
/* Function vect_gen_prolog_loop_niters
Generate the number of iterations which should be peeled as prolog for the
If the misalignment of DR is known at compile time:
addr_mis = int mis = DR_MISALIGNMENT (dr);
Else, compute address misalignment in bytes:
- addr_mis = addr & (vectype_align - 1)
+ addr_mis = addr & (target_align - 1)
prolog_niters = ((VF - addr_mis/elem_size)&(VF-1))/step
vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo,
basic_block bb, int *bound)
{
- struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
tree var;
tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo));
gimple_seq stmts = NULL, new_stmts = NULL;
tree iters, iters_name;
- gimple *dr_stmt = DR_STMT (dr);
- stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
+ stmt_vec_info stmt_info = dr_info->stmt;
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
- unsigned int target_align = DR_TARGET_ALIGNMENT (dr);
+ poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr_info);
if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
{
}
else
{
- bool negative = tree_int_cst_compare (DR_STEP (dr), size_zero_node) < 0;
- tree offset = negative
- ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node;
- tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt,
- &stmts, offset);
- tree type = unsigned_type_for (TREE_TYPE (start_addr));
- tree target_align_minus_1 = build_int_cst (type, target_align - 1);
+ tree misalign_in_elems = get_misalign_in_elems (&stmts, loop_vinfo);
+ tree type = TREE_TYPE (misalign_in_elems);
HOST_WIDE_INT elem_size
= int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
- tree elem_size_log = build_int_cst (type, exact_log2 (elem_size));
- HOST_WIDE_INT align_in_elems = target_align / elem_size;
- tree align_in_elems_minus_1 = build_int_cst (type, align_in_elems - 1);
+ /* We only do prolog peeling if the target alignment is known at compile
+ time. */
+ poly_uint64 align_in_elems =
+ exact_div (target_align, elem_size);
+ tree align_in_elems_minus_1 =
+ build_int_cst (type, align_in_elems - 1);
tree align_in_elems_tree = build_int_cst (type, align_in_elems);
- tree misalign_in_bytes;
- tree misalign_in_elems;
-
- /* Create: misalign_in_bytes = addr & (target_align - 1). */
- misalign_in_bytes
- = fold_build2 (BIT_AND_EXPR, type, fold_convert (type, start_addr),
- target_align_minus_1);
-
- /* Create: misalign_in_elems = misalign_in_bytes / element_size. */
- misalign_in_elems
- = fold_build2 (RSHIFT_EXPR, type, misalign_in_bytes, elem_size_log);
/* Create: (niters_type) ((align_in_elems - misalign_in_elems)
& (align_in_elems - 1)). */
+ bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
+ size_zero_node) < 0;
if (negative)
iters = fold_build2 (MINUS_EXPR, type, misalign_in_elems,
align_in_elems_tree);
misalign_in_elems);
iters = fold_build2 (BIT_AND_EXPR, type, iters, align_in_elems_minus_1);
iters = fold_convert (niters_type, iters);
- *bound = align_in_elems - 1;
+ unsigned HOST_WIDE_INT align_in_elems_c;
+ if (align_in_elems.is_constant (&align_in_elems_c))
+ *bound = align_in_elems_c - 1;
+ else
+ *bound = -1;
}
if (dump_enabled_p ())
- {
- dump_printf_loc (MSG_NOTE, vect_location,
- "niters for prolog loop: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, iters);
- dump_printf (MSG_NOTE, "\n");
- }
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "niters for prolog loop: %T\n", iters);
var = create_tmp_var (niters_type, "prolog_loop_niters");
iters_name = force_gimple_operand (iters, &new_stmts, false, var);
/* Function vect_update_init_of_dr
- NITERS iterations were peeled from LOOP. DR represents a data reference
- in LOOP. This function updates the information recorded in DR to
- account for the fact that the first NITERS iterations had already been
- executed. Specifically, it updates the OFFSET field of DR. */
+ If CODE is PLUS, the vector loop starts NITERS iterations after the
+ scalar one, otherwise CODE is MINUS and the vector loop starts NITERS
+ iterations before the scalar one (using masking to skip inactive
+ elements). This function updates the information recorded in DR to
+ account for the difference. Specifically, it updates the OFFSET
+ field of DR. */
static void
-vect_update_init_of_dr (struct data_reference *dr, tree niters)
+vect_update_init_of_dr (struct data_reference *dr, tree niters, tree_code code)
{
tree offset = DR_OFFSET (dr);
niters = fold_build2 (MULT_EXPR, sizetype,
fold_convert (sizetype, niters),
fold_convert (sizetype, DR_STEP (dr)));
- offset = fold_build2 (PLUS_EXPR, sizetype,
+ offset = fold_build2 (code, sizetype,
fold_convert (sizetype, offset), niters);
DR_OFFSET (dr) = offset;
}
/* Function vect_update_inits_of_drs
- NITERS iterations were peeled from the loop represented by LOOP_VINFO.
- This function updates the information recorded for the data references in
- the loop to account for the fact that the first NITERS iterations had
- already been executed. Specifically, it updates the initial_condition of
- the access_function of all the data_references in the loop. */
+ Apply vect_update_inits_of_dr to all accesses in LOOP_VINFO.
+ CODE and NITERS are as for vect_update_inits_of_dr. */
static void
-vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters)
+vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters,
+ tree_code code)
{
unsigned int i;
vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
struct data_reference *dr;
- if (dump_enabled_p ())
- dump_printf_loc (MSG_NOTE, vect_location,
- "=== vect_update_inits_of_dr ===\n");
+ DUMP_VECT_SCOPE ("vect_update_inits_of_dr");
/* Adjust niters to sizetype and insert stmts on loop preheader edge. */
if (!types_compatible_p (sizetype, TREE_TYPE (niters)))
}
FOR_EACH_VEC_ELT (datarefs, i, dr)
- vect_update_init_of_dr (dr, niters);
+ {
+ dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
+ if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt))
+ vect_update_init_of_dr (dr, niters, code);
+ }
}
+/* For the information recorded in LOOP_VINFO prepare the loop for peeling
+ by masking. This involves calculating the number of iterations to
+ be peeled and then aligning all memory references appropriately. */
+
+void
+vect_prepare_for_masked_peels (loop_vec_info loop_vinfo)
+{
+ tree misalign_in_elems;
+ tree type = LOOP_VINFO_MASK_COMPARE_TYPE (loop_vinfo);
+
+ gcc_assert (vect_use_loop_mask_for_alignment_p (loop_vinfo));
+
+ /* From the information recorded in LOOP_VINFO get the number of iterations
+ that need to be skipped via masking. */
+ if (LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) > 0)
+ {
+ poly_int64 misalign = (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
+ - LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo));
+ misalign_in_elems = build_int_cst (type, misalign);
+ }
+ else
+ {
+ gimple_seq seq1 = NULL, seq2 = NULL;
+ misalign_in_elems = get_misalign_in_elems (&seq1, loop_vinfo);
+ misalign_in_elems = fold_convert (type, misalign_in_elems);
+ misalign_in_elems = force_gimple_operand (misalign_in_elems,
+ &seq2, true, NULL_TREE);
+ gimple_seq_add_seq (&seq1, seq2);
+ if (seq1)
+ {
+ edge pe = loop_preheader_edge (LOOP_VINFO_LOOP (loop_vinfo));
+ basic_block new_bb = gsi_insert_seq_on_edge_immediate (pe, seq1);
+ gcc_assert (!new_bb);
+ }
+ }
+
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "misalignment for fully-masked loop: %T\n",
+ misalign_in_elems);
+
+ LOOP_VINFO_MASK_SKIP_NITERS (loop_vinfo) = misalign_in_elems;
+
+ vect_update_inits_of_drs (loop_vinfo, misalign_in_elems, MINUS_EXPR);
+}
/* This function builds ni_name = number of iterations. Statements
are emitted on the loop preheader edge. If NEW_VAR_P is not NULL, set
/* Calculate the number of iterations above which vectorized loop will be
preferred than scalar loop. NITERS_PROLOG is the number of iterations
of prolog loop. If it's integer const, the integer number is also passed
- in INT_NITERS_PROLOG. BOUND_PROLOG is the upper bound (included) of
- number of iterations of prolog loop. VFM1 is vector factor minus one.
- If CHECK_PROFITABILITY is true, TH is the threshold below which scalar
- (rather than vectorized) loop will be executed. This function stores
- upper bound (included) of the result in BOUND_SCALAR. */
+ in INT_NITERS_PROLOG. BOUND_PROLOG is the upper bound (inclusive) of the
+ number of iterations of the prolog loop. BOUND_EPILOG is the corresponding
+ value for the epilog loop. If CHECK_PROFITABILITY is true, TH is the
+ threshold below which the scalar (rather than vectorized) loop will be
+ executed. This function stores the upper bound (inclusive) of the result
+ in BOUND_SCALAR. */
static tree
vect_gen_scalar_loop_niters (tree niters_prolog, int int_niters_prolog,
- int bound_prolog, poly_int64 vfm1, int th,
+ int bound_prolog, poly_int64 bound_epilog, int th,
poly_uint64 *bound_scalar,
bool check_profitability)
{
tree type = TREE_TYPE (niters_prolog);
tree niters = fold_build2 (PLUS_EXPR, type, niters_prolog,
- build_int_cst (type, vfm1));
+ build_int_cst (type, bound_epilog));
- *bound_scalar = vfm1 + bound_prolog;
+ *bound_scalar = bound_prolog + bound_epilog;
if (check_profitability)
{
/* TH indicates the minimum niters of vectorized loop, while we
/* Peeling for constant times. */
if (int_niters_prolog >= 0)
{
- *bound_scalar = upper_bound (int_niters_prolog + vfm1, th);
+ *bound_scalar = upper_bound (int_niters_prolog + bound_epilog, th);
return build_int_cst (type, *bound_scalar);
}
- /* Peeling for unknown times. Note BOUND_PROLOG is the upper
- bound (inlcuded) of niters of prolog loop. */
- if (known_ge (th, vfm1 + bound_prolog))
+ /* Peeling an unknown number of times. Note that both BOUND_PROLOG
+ and BOUND_EPILOG are inclusive upper bounds. */
+ if (known_ge (th, bound_prolog + bound_epilog))
{
*bound_scalar = th;
return build_int_cst (type, th);
}
/* Need to do runtime comparison. */
- else if (maybe_gt (th, vfm1))
+ else if (maybe_gt (th, bound_epilog))
{
*bound_scalar = upper_bound (*bound_scalar, th);
return fold_build2 (MAX_EXPR, type,
{
/* We should be using a step_vector of VF if VF is variable. */
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo).to_constant ();
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
tree type = TREE_TYPE (niters_vector);
tree log_vf = build_int_cst (type, exact_log2 (vf));
basic_block exit_bb = single_exit (loop)->dest;
static void
slpeel_update_phi_nodes_for_loops (loop_vec_info loop_vinfo,
- struct loop *first, struct loop *second,
+ class loop *first, class loop *second,
bool create_lcssa_for_iv_phis)
{
gphi_iterator gsi_update, gsi_orig;
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
edge first_latch_e = EDGE_SUCC (first->latch, 0);
edge second_preheader_e = loop_preheader_edge (second);
tree arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, first_latch_e);
/* Generate lcssa PHI node for the first loop. */
gphi *vect_phi = (loop == first) ? orig_phi : update_phi;
- if (create_lcssa_for_iv_phis || !iv_phi_p (vect_phi))
+ stmt_vec_info vect_phi_info = loop_vinfo->lookup_stmt (vect_phi);
+ if (create_lcssa_for_iv_phis || !iv_phi_p (vect_phi_info))
{
tree new_res = copy_ssa_name (PHI_RESULT (orig_phi));
gphi *lcssa_phi = create_phi_node (new_res, between_bb);
in the update_loop's PHI node with the result of new PHI result. */
static void
-slpeel_update_phi_nodes_for_guard1 (struct loop *skip_loop,
- struct loop *update_loop,
+slpeel_update_phi_nodes_for_guard1 (class loop *skip_loop,
+ class loop *update_loop,
edge guard_edge, edge merge_edge)
{
- source_location merge_loc, guard_loc;
+ location_t merge_loc, guard_loc;
edge orig_e = loop_preheader_edge (skip_loop);
edge update_e = loop_preheader_edge (update_loop);
gphi_iterator gsi_orig, gsi_update;
NULL. */
static tree
-find_guard_arg (struct loop *loop, struct loop *epilog ATTRIBUTE_UNUSED,
+find_guard_arg (class loop *loop, class loop *epilog ATTRIBUTE_UNUSED,
gphi *lcssa_phi)
{
gphi_iterator gsi;
in exit_bb will also be updated. */
static void
-slpeel_update_phi_nodes_for_guard2 (struct loop *loop, struct loop *epilog,
+slpeel_update_phi_nodes_for_guard2 (class loop *loop, class loop *epilog,
edge guard_edge, edge merge_edge)
{
gphi_iterator gsi;
the arg of its loop closed ssa PHI needs to be updated. */
static void
-slpeel_update_phi_nodes_for_lcssa (struct loop *epilog)
+slpeel_update_phi_nodes_for_lcssa (class loop *epilog)
{
gphi_iterator gsi;
basic_block exit_bb = single_exit (epilog)->dest;
versioning conditions if loop versioning is needed. */
-struct loop *
+class loop *
vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1,
tree *niters_vector, tree *step_vector,
tree *niters_vector_mult_vf_var, int th,
tree type = TREE_TYPE (niters), guard_cond;
basic_block guard_bb, guard_to;
profile_probability prob_prolog, prob_vector, prob_epilog;
- int bound_prolog = 0;
- poly_uint64 bound_scalar = 0;
int estimated_vf;
- int prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
- bool epilog_peeling = (LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo)
- || LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo));
+ int prolog_peeling = 0;
+ /* We currently do not support prolog peeling if the target alignment is not
+ known at compile time. 'vect_gen_prolog_loop_niters' depends on the
+ target alignment being constant. */
+ dr_vec_info *dr_info = LOOP_VINFO_UNALIGNED_DR (loop_vinfo);
+ if (dr_info && !DR_TARGET_ALIGNMENT (dr_info).is_constant ())
+ return NULL;
+
+ if (!vect_use_loop_mask_for_alignment_p (loop_vinfo))
+ prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
+
+ poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
+ poly_uint64 bound_epilog = 0;
+ if (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
+ && LOOP_VINFO_PEELING_FOR_NITER (loop_vinfo))
+ bound_epilog += vf - 1;
+ if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
+ bound_epilog += 1;
+ bool epilog_peeling = maybe_ne (bound_epilog, 0U);
+ poly_uint64 bound_scalar = bound_epilog;
if (!prolog_peeling && !epilog_peeling)
return NULL;
estimated_vf = 3;
prob_prolog = prob_epilog = profile_probability::guessed_always ()
.apply_scale (estimated_vf - 1, estimated_vf);
- poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
- struct loop *prolog, *epilog = NULL, *loop = LOOP_VINFO_LOOP (loop_vinfo);
- struct loop *first_loop = loop;
+ class loop *prolog, *epilog = NULL, *loop = LOOP_VINFO_LOOP (loop_vinfo);
+ class loop *first_loop = loop;
bool irred_flag = loop_preheader_edge (loop)->flags & EDGE_IRREDUCIBLE_LOOP;
create_lcssa_for_virtual_phi (loop);
update_ssa (TODO_update_ssa_only_virtuals);
}
initialize_original_copy_tables ();
+ /* Record the anchor bb at which the guard should be placed if the scalar
+ loop might be preferred. */
+ basic_block anchor = loop_preheader_edge (loop)->src;
+
+ /* Generate the number of iterations for the prolog loop. We do this here
+ so that we can also get the upper bound on the number of iterations. */
+ tree niters_prolog;
+ int bound_prolog = 0;
+ if (prolog_peeling)
+ niters_prolog = vect_gen_prolog_loop_niters (loop_vinfo, anchor,
+ &bound_prolog);
+ else
+ niters_prolog = build_int_cst (type, 0);
+
/* Prolog loop may be skipped. */
bool skip_prolog = (prolog_peeling != 0);
/* Skip to epilog if scalar loop may be preferred. It's only needed
when we peel for epilog loop and when it hasn't been checked with
loop versioning. */
- bool skip_vector = ((!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
- && !LOOP_REQUIRES_VERSIONING (loop_vinfo))
- || !vf.is_constant ());
+ bool skip_vector = (LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
+ ? maybe_lt (LOOP_VINFO_INT_NITERS (loop_vinfo),
+ bound_prolog + bound_epilog)
+ : !LOOP_REQUIRES_VERSIONING (loop_vinfo));
/* Epilog loop must be executed if the number of iterations for epilog
loop is known at compile time, otherwise we need to add a check at
the end of vector loop and skip to the end of epilog loop. */
if (LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo))
skip_epilog = false;
- /* Record the anchor bb at which guard should be placed if scalar loop
- may be preferred. */
- basic_block anchor = loop_preheader_edge (loop)->src;
if (skip_vector)
{
split_edge (loop_preheader_edge (loop));
}
}
- tree niters_prolog = build_int_cst (type, 0);
- source_location loop_loc = find_loop_location (loop);
- struct loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
+ dump_user_location_t loop_loc = find_loop_location (loop);
+ class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
if (prolog_peeling)
{
e = loop_preheader_edge (loop);
"slpeel_tree_duplicate_loop_to_edge_cfg failed.\n");
gcc_unreachable ();
}
+ prolog->force_vectorize = false;
slpeel_update_phi_nodes_for_loops (loop_vinfo, prolog, loop, true);
first_loop = prolog;
reset_original_copy_tables ();
- /* Generate and update the number of iterations for prolog loop. */
- niters_prolog = vect_gen_prolog_loop_niters (loop_vinfo, anchor,
- &bound_prolog);
+ /* Update the number of iterations for prolog loop. */
tree step_prolog = build_one_cst (TREE_TYPE (niters_prolog));
vect_set_loop_condition (prolog, NULL, niters_prolog,
step_prolog, NULL_TREE, false);
scale_loop_profile (prolog, prob_prolog, bound_prolog);
}
/* Update init address of DRs. */
- vect_update_inits_of_drs (loop_vinfo, niters_prolog);
+ vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR);
/* Update niters for vector loop. */
LOOP_VINFO_NITERS (loop_vinfo)
= fold_build2 (MINUS_EXPR, type, niters, niters_prolog);
"slpeel_tree_duplicate_loop_to_edge_cfg failed.\n");
gcc_unreachable ();
}
+ epilog->force_vectorize = false;
slpeel_update_phi_nodes_for_loops (loop_vinfo, loop, epilog, false);
/* Scalar version loop may be preferred. In this case, add guard
if (skip_vector)
{
/* Additional epilogue iteration is peeled if gap exists. */
- bool peel_for_gaps = LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo);
tree t = vect_gen_scalar_loop_niters (niters_prolog, prolog_peeling,
- bound_prolog,
- peel_for_gaps ? vf : vf - 1,
+ bound_prolog, bound_epilog,
th, &bound_scalar,
check_profitability);
/* Build guard against NITERSM1 since NITERS may overflow. */
else
slpeel_update_phi_nodes_for_lcssa (epilog);
- unsigned HOST_WIDE_INT bound1, bound2;
- if (vf.is_constant (&bound1) && bound_scalar.is_constant (&bound2))
+ unsigned HOST_WIDE_INT bound;
+ if (bound_scalar.is_constant (&bound))
{
- bound1 -= LOOP_VINFO_PEELING_FOR_GAPS (loop_vinfo) ? 1 : 2;
- if (bound2)
- /* We share epilog loop with scalar version loop. */
- bound1 = MAX (bound1, bound2 - 1);
- record_niter_bound (epilog, bound1, false, true);
+ gcc_assert (bound != 0);
+ /* -1 to convert loop iterations to latch iterations. */
+ record_niter_bound (epilog, bound - 1, false, true);
}
delete_update_ssa ();
/* Function vect_create_cond_for_niters_checks.
Create a conditional expression that represents the run-time checks for
- loop's niter. The loop is guaranteed to to terminate if the run-time
+ loop's niter. The loop is guaranteed to terminate if the run-time
checks hold.
Input:
tree *cond_expr,
gimple_seq *cond_expr_stmt_list)
{
- vec<gimple *> may_misalign_stmts
+ vec<stmt_vec_info> may_misalign_stmts
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
- gimple *ref_stmt;
+ stmt_vec_info stmt_info;
int mask = LOOP_VINFO_PTR_MASK (loop_vinfo);
tree mask_cst;
unsigned int i;
/* Create expression (mask & (dr_1 || ... || dr_n)) where dr_i is the address
of the first vector of the i'th data reference. */
- FOR_EACH_VEC_ELT (may_misalign_stmts, i, ref_stmt)
+ FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
{
gimple_seq new_stmt_list = NULL;
tree addr_base;
tree addr_tmp_name;
tree new_or_tmp_name;
gimple *addr_stmt, *or_stmt;
- stmt_vec_info stmt_vinfo = vinfo_for_stmt (ref_stmt);
- tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
+ tree vectype = STMT_VINFO_VECTYPE (stmt_info);
bool negative = tree_int_cst_compare
- (DR_STEP (STMT_VINFO_DATA_REF (stmt_vinfo)), size_zero_node) < 0;
+ (DR_STEP (STMT_VINFO_DATA_REF (stmt_info)), size_zero_node) < 0;
tree offset = negative
? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node;
/* create: addr_tmp = (int)(address_of_first_vector) */
addr_base =
- vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list,
+ vect_create_addr_base_for_vector_ref (stmt_info, &new_stmt_list,
offset);
if (new_stmt_list != NULL)
gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list);
}
}
+/* Create an expression that is true when all lower-bound conditions for
+ the vectorized loop are met. Chain this condition with *COND_EXPR. */
+
+static void
+vect_create_cond_for_lower_bounds (loop_vec_info loop_vinfo, tree *cond_expr)
+{
+ vec<vec_lower_bound> lower_bounds = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo);
+ for (unsigned int i = 0; i < lower_bounds.length (); ++i)
+ {
+ tree expr = lower_bounds[i].expr;
+ tree type = unsigned_type_for (TREE_TYPE (expr));
+ expr = fold_convert (type, expr);
+ poly_uint64 bound = lower_bounds[i].min_value;
+ if (!lower_bounds[i].unsigned_p)
+ {
+ expr = fold_build2 (PLUS_EXPR, type, expr,
+ build_int_cstu (type, bound - 1));
+ bound += bound - 1;
+ }
+ tree part_cond_expr = fold_build2 (GE_EXPR, boolean_type_node, expr,
+ build_int_cstu (type, bound));
+ chain_cond_expr (cond_expr, part_cond_expr);
+ }
+}
+
/* Function vect_create_cond_for_alias_checks.
Create a conditional expression that represents the run-time checks for
The versioning precondition(s) are placed in *COND_EXPR and
*COND_EXPR_STMT_LIST. */
-void
+class loop *
vect_loop_versioning (loop_vec_info loop_vinfo,
unsigned int th, bool check_profitability,
poly_uint64 versioning_threshold)
{
- struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *nloop;
- struct loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
+ class loop *loop = LOOP_VINFO_LOOP (loop_vinfo), *nloop;
+ class loop *scalar_loop = LOOP_VINFO_SCALAR_LOOP (loop_vinfo);
basic_block condition_bb;
gphi_iterator gsi;
gimple_stmt_iterator cond_exp_gsi;
bool version_align = LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo);
bool version_alias = LOOP_REQUIRES_VERSIONING_FOR_ALIAS (loop_vinfo);
bool version_niter = LOOP_REQUIRES_VERSIONING_FOR_NITERS (loop_vinfo);
+ tree version_simd_if_cond
+ = LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (loop_vinfo);
if (check_profitability)
cond_expr = fold_build2 (GE_EXPR, boolean_type_node, scalar_loop_iters,
vect_create_cond_for_niters_checks (loop_vinfo, &cond_expr);
if (cond_expr)
- cond_expr = force_gimple_operand_1 (cond_expr, &cond_expr_stmt_list,
+ cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
+ &cond_expr_stmt_list,
is_gimple_condexpr, NULL_TREE);
if (version_align)
if (version_alias)
{
vect_create_cond_for_unequal_addrs (loop_vinfo, &cond_expr);
+ vect_create_cond_for_lower_bounds (loop_vinfo, &cond_expr);
vect_create_cond_for_alias_checks (loop_vinfo, &cond_expr);
}
- cond_expr = force_gimple_operand_1 (cond_expr, &gimplify_stmt_list,
+ if (version_simd_if_cond)
+ {
+ gcc_assert (dom_info_available_p (CDI_DOMINATORS));
+ if (flag_checking)
+ if (basic_block bb
+ = gimple_bb (SSA_NAME_DEF_STMT (version_simd_if_cond)))
+ gcc_assert (bb != loop->header
+ && dominated_by_p (CDI_DOMINATORS, loop->header, bb)
+ && (scalar_loop == NULL
+ || (bb != scalar_loop->header
+ && dominated_by_p (CDI_DOMINATORS,
+ scalar_loop->header, bb))));
+ tree zero = build_zero_cst (TREE_TYPE (version_simd_if_cond));
+ tree c = fold_build2 (NE_EXPR, boolean_type_node,
+ version_simd_if_cond, zero);
+ if (cond_expr)
+ cond_expr = fold_build2 (TRUTH_AND_EXPR, boolean_type_node,
+ c, cond_expr);
+ else
+ cond_expr = c;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "created versioning for simd if condition check.\n");
+ }
+
+ cond_expr = force_gimple_operand_1 (unshare_expr (cond_expr),
+ &gimplify_stmt_list,
is_gimple_condexpr, NULL_TREE);
gimple_seq_add_seq (&cond_expr_stmt_list, gimplify_stmt_list);
- initialize_original_copy_tables ();
- if (scalar_loop)
+ /* Compute the outermost loop cond_expr and cond_expr_stmt_list are
+ invariant in. */
+ class loop *outermost = outermost_invariant_loop_for_expr (loop, cond_expr);
+ for (gimple_stmt_iterator gsi = gsi_start (cond_expr_stmt_list);
+ !gsi_end_p (gsi); gsi_next (&gsi))
{
- edge scalar_e;
- basic_block preheader, scalar_preheader;
+ gimple *stmt = gsi_stmt (gsi);
+ update_stmt (stmt);
+ ssa_op_iter iter;
+ use_operand_p use_p;
+ basic_block def_bb;
+ FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
+ if ((def_bb = gimple_bb (SSA_NAME_DEF_STMT (USE_FROM_PTR (use_p))))
+ && flow_bb_inside_loop_p (outermost, def_bb))
+ outermost = superloop_at_depth (loop, bb_loop_depth (def_bb) + 1);
+ }
- /* We don't want to scale SCALAR_LOOP's frequencies, we need to
- scale LOOP's frequencies instead. */
- nloop = loop_version (scalar_loop, cond_expr, &condition_bb,
- prob, prob.invert (), prob, prob.invert (), true);
- scale_loop_frequencies (loop, prob);
- /* CONDITION_BB was created above SCALAR_LOOP's preheader,
- while we need to move it above LOOP's preheader. */
- e = loop_preheader_edge (loop);
- scalar_e = loop_preheader_edge (scalar_loop);
- gcc_assert (empty_block_p (e->src)
- && single_pred_p (e->src));
- gcc_assert (empty_block_p (scalar_e->src)
- && single_pred_p (scalar_e->src));
- gcc_assert (single_pred_p (condition_bb));
- preheader = e->src;
- scalar_preheader = scalar_e->src;
- scalar_e = find_edge (condition_bb, scalar_preheader);
- e = single_pred_edge (preheader);
- redirect_edge_and_branch_force (single_pred_edge (condition_bb),
- scalar_preheader);
- redirect_edge_and_branch_force (scalar_e, preheader);
- redirect_edge_and_branch_force (e, condition_bb);
- set_immediate_dominator (CDI_DOMINATORS, condition_bb,
- single_pred (condition_bb));
- set_immediate_dominator (CDI_DOMINATORS, scalar_preheader,
- single_pred (scalar_preheader));
- set_immediate_dominator (CDI_DOMINATORS, preheader,
- condition_bb);
+ /* Search for the outermost loop we can version. Avoid versioning of
+ non-perfect nests but allow if-conversion versioned loops inside. */
+ class loop *loop_to_version = loop;
+ if (flow_loop_nested_p (outermost, loop))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "trying to apply versioning to outer loop %d\n",
+ outermost->num);
+ if (outermost->num == 0)
+ outermost = superloop_at_depth (loop, 1);
+ /* And avoid applying versioning on non-perfect nests. */
+ while (loop_to_version != outermost
+ && single_exit (loop_outer (loop_to_version))
+ && (!loop_outer (loop_to_version)->inner->next
+ || vect_loop_vectorized_call (loop_to_version))
+ && (!loop_outer (loop_to_version)->inner->next
+ || !loop_outer (loop_to_version)->inner->next->next))
+ loop_to_version = loop_outer (loop_to_version);
+ }
+
+ /* Apply versioning. If there is already a scalar version created by
+ if-conversion re-use that. Note we cannot re-use the copy of
+ an if-converted outer-loop when vectorizing the inner loop only. */
+ gcond *cond;
+ gimple *call;
+ if ((!loop_to_version->inner || loop == loop_to_version)
+ && (call = vect_loop_vectorized_call (loop_to_version, &cond)))
+ {
+ gcc_assert (scalar_loop);
+ condition_bb = gimple_bb (cond);
+ gimple_cond_set_condition_from_tree (cond, cond_expr);
+ update_stmt (cond);
+
+ if (cond_expr_stmt_list)
+ {
+ cond_exp_gsi = gsi_for_stmt (call);
+ gsi_insert_seq_before (&cond_exp_gsi, cond_expr_stmt_list,
+ GSI_SAME_STMT);
+ }
+
+ /* if-conversion uses profile_probability::always () for both paths,
+ reset the paths probabilities appropriately. */
+ edge te, fe;
+ extract_true_false_edges_from_block (condition_bb, &te, &fe);
+ te->probability = prob;
+ fe->probability = prob.invert ();
+ /* We can scale loops counts immediately but have to postpone
+ scaling the scalar loop because we re-use it during peeling. */
+ scale_loop_frequencies (loop_to_version, te->probability);
+ LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo) = fe->probability;
+
+ nloop = scalar_loop;
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "reusing %sloop version created by if conversion\n",
+ loop_to_version != loop ? "outer " : "");
}
else
- nloop = loop_version (loop, cond_expr, &condition_bb,
- prob, prob.invert (), prob, prob.invert (), true);
+ {
+ if (loop_to_version != loop
+ && dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "applying loop versioning to outer loop %d\n",
+ loop_to_version->num);
+
+ initialize_original_copy_tables ();
+ nloop = loop_version (loop_to_version, cond_expr, &condition_bb,
+ prob, prob.invert (), prob, prob.invert (), true);
+ gcc_assert (nloop);
+ nloop = get_loop_copy (loop);
+
+ /* Kill off IFN_LOOP_VECTORIZED_CALL in the copy, nobody will
+ reap those otherwise; they also refer to the original
+ loops. */
+ class loop *l = loop;
+ while (gimple *call = vect_loop_vectorized_call (l))
+ {
+ call = SSA_NAME_DEF_STMT (get_current_def (gimple_call_lhs (call)));
+ fold_loop_internal_call (call, boolean_false_node);
+ l = loop_outer (l);
+ }
+ free_original_copy_tables ();
+
+ if (cond_expr_stmt_list)
+ {
+ cond_exp_gsi = gsi_last_bb (condition_bb);
+ gsi_insert_seq_before (&cond_exp_gsi, cond_expr_stmt_list,
+ GSI_SAME_STMT);
+ }
+
+ /* Loop versioning violates an assumption we try to maintain during
+ vectorization - that the loop exit block has a single predecessor.
+ After versioning, the exit block of both loop versions is the same
+ basic block (i.e. it has two predecessors). Just in order to simplify
+ following transformations in the vectorizer, we fix this situation
+ here by adding a new (empty) block on the exit-edge of the loop,
+ with the proper loop-exit phis to maintain loop-closed-form.
+ If loop versioning wasn't done from loop, but scalar_loop instead,
+ merge_bb will have already just a single successor. */
+
+ merge_bb = single_exit (loop_to_version)->dest;
+ if (EDGE_COUNT (merge_bb->preds) >= 2)
+ {
+ gcc_assert (EDGE_COUNT (merge_bb->preds) >= 2);
+ new_exit_bb = split_edge (single_exit (loop_to_version));
+ new_exit_e = single_exit (loop_to_version);
+ e = EDGE_SUCC (new_exit_bb, 0);
+
+ for (gsi = gsi_start_phis (merge_bb); !gsi_end_p (gsi);
+ gsi_next (&gsi))
+ {
+ tree new_res;
+ orig_phi = gsi.phi ();
+ new_res = copy_ssa_name (PHI_RESULT (orig_phi));
+ new_phi = create_phi_node (new_res, new_exit_bb);
+ arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
+ add_phi_arg (new_phi, arg, new_exit_e,
+ gimple_phi_arg_location_from_edge (orig_phi, e));
+ adjust_phi_and_debug_stmts (orig_phi, e, PHI_RESULT (new_phi));
+ }
+ }
+
+ update_ssa (TODO_update_ssa);
+ }
if (version_niter)
{
loop_constraint_set (loop, LOOP_C_INFINITE);
}
- if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
+ if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
&& dump_enabled_p ())
{
if (version_alias)
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
+ vect_location,
"loop versioned for vectorization because of "
"possible aliasing\n");
if (version_align)
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS | MSG_PRIORITY_USER_FACING,
+ vect_location,
"loop versioned for vectorization to enhance "
"alignment\n");
}
- free_original_copy_tables ();
-
- /* Loop versioning violates an assumption we try to maintain during
- vectorization - that the loop exit block has a single predecessor.
- After versioning, the exit block of both loop versions is the same
- basic block (i.e. it has two predecessors). Just in order to simplify
- following transformations in the vectorizer, we fix this situation
- here by adding a new (empty) block on the exit-edge of the loop,
- with the proper loop-exit phis to maintain loop-closed-form.
- If loop versioning wasn't done from loop, but scalar_loop instead,
- merge_bb will have already just a single successor. */
-
- merge_bb = single_exit (loop)->dest;
- if (scalar_loop == NULL || EDGE_COUNT (merge_bb->preds) >= 2)
- {
- gcc_assert (EDGE_COUNT (merge_bb->preds) >= 2);
- new_exit_bb = split_edge (single_exit (loop));
- new_exit_e = single_exit (loop);
- e = EDGE_SUCC (new_exit_bb, 0);
-
- for (gsi = gsi_start_phis (merge_bb); !gsi_end_p (gsi); gsi_next (&gsi))
- {
- tree new_res;
- orig_phi = gsi.phi ();
- new_res = copy_ssa_name (PHI_RESULT (orig_phi));
- new_phi = create_phi_node (new_res, new_exit_bb);
- arg = PHI_ARG_DEF_FROM_EDGE (orig_phi, e);
- add_phi_arg (new_phi, arg, new_exit_e,
- gimple_phi_arg_location_from_edge (orig_phi, e));
- adjust_phi_and_debug_stmts (orig_phi, e, PHI_RESULT (new_phi));
- }
- }
-
- /* End loop-exit-fixes after versioning. */
- if (cond_expr_stmt_list)
- {
- cond_exp_gsi = gsi_last_bb (condition_bb);
- gsi_insert_seq_before (&cond_exp_gsi, cond_expr_stmt_list,
- GSI_SAME_STMT);
- }
- update_ssa (TODO_update_ssa);
+ return nloop;
}
projects / gcc.git / blobdiff