/* Vectorizer
- Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ Copyright (C) 2003-2019 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
This file is part of GCC.
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "dumpfile.h"
#include "backend.h"
-#include "predict.h"
#include "tree.h"
#include "gimple.h"
-#include "hard-reg-set.h"
+#include "predict.h"
+#include "tree-pass.h"
#include "ssa.h"
-#include "alias.h"
+#include "cgraph.h"
#include "fold-const.h"
#include "stor-layout.h"
-#include "tree-pretty-print.h"
-#include "internal-fn.h"
#include "gimple-iterator.h"
#include "gimple-walk.h"
-#include "cgraph.h"
#include "tree-ssa-loop-manip.h"
+#include "tree-ssa-loop-niter.h"
#include "tree-cfg.h"
#include "cfgloop.h"
#include "tree-vectorizer.h"
-#include "tree-pass.h"
#include "tree-ssa-propagate.h"
#include "dbgcnt.h"
-#include "gimple-fold.h"
#include "tree-scalar-evolution.h"
+#include "stringpool.h"
+#include "attribs.h"
+#include "gimple-pretty-print.h"
+#include "opt-problem.h"
+#include "internal-fn.h"
-/* Loop or bb location. */
-source_location vect_location;
+/* Loop or bb location, with hotness information. */
+dump_user_location_t vect_location;
-/* Vector mapping GIMPLE stmt to stmt_vec_info. */
-vec<stmt_vec_info> stmt_vec_info_vec;
+/* auto_purge_vect_location's dtor: reset the vect_location
+ global, to avoid stale location_t values that could reference
+ GC-ed blocks. */
+
+auto_purge_vect_location::~auto_purge_vect_location ()
+{
+ vect_location = dump_user_location_t ();
+}
+
+/* Dump a cost entry according to args to F. */
+
+void
+dump_stmt_cost (FILE *f, void *data, int count, enum vect_cost_for_stmt kind,
+ stmt_vec_info stmt_info, int misalign, unsigned cost,
+ enum vect_cost_model_location where)
+{
+ fprintf (f, "%p ", data);
+ if (stmt_info)
+ {
+ print_gimple_expr (f, STMT_VINFO_STMT (stmt_info), 0, TDF_SLIM);
+ fprintf (f, " ");
+ }
+ else
+ fprintf (f, "<unknown> ");
+ fprintf (f, "%d times ", count);
+ const char *ks = "unknown";
+ switch (kind)
+ {
+ case scalar_stmt:
+ ks = "scalar_stmt";
+ break;
+ case scalar_load:
+ ks = "scalar_load";
+ break;
+ case scalar_store:
+ ks = "scalar_store";
+ break;
+ case vector_stmt:
+ ks = "vector_stmt";
+ break;
+ case vector_load:
+ ks = "vector_load";
+ break;
+ case vector_gather_load:
+ ks = "vector_gather_load";
+ break;
+ case unaligned_load:
+ ks = "unaligned_load";
+ break;
+ case unaligned_store:
+ ks = "unaligned_store";
+ break;
+ case vector_store:
+ ks = "vector_store";
+ break;
+ case vector_scatter_store:
+ ks = "vector_scatter_store";
+ break;
+ case vec_to_scalar:
+ ks = "vec_to_scalar";
+ break;
+ case scalar_to_vec:
+ ks = "scalar_to_vec";
+ break;
+ case cond_branch_not_taken:
+ ks = "cond_branch_not_taken";
+ break;
+ case cond_branch_taken:
+ ks = "cond_branch_taken";
+ break;
+ case vec_perm:
+ ks = "vec_perm";
+ break;
+ case vec_promote_demote:
+ ks = "vec_promote_demote";
+ break;
+ case vec_construct:
+ ks = "vec_construct";
+ break;
+ }
+ fprintf (f, "%s ", ks);
+ if (kind == unaligned_load || kind == unaligned_store)
+ fprintf (f, "(misalign %d) ", misalign);
+ fprintf (f, "costs %u ", cost);
+ const char *ws = "unknown";
+ switch (where)
+ {
+ case vect_prologue:
+ ws = "prologue";
+ break;
+ case vect_body:
+ ws = "body";
+ break;
+ case vect_epilogue:
+ ws = "epilogue";
+ break;
+ }
+ fprintf (f, "in %s\n", ws);
+}
\f
/* For mapping simduid to vectorization factor. */
struct simduid_to_vf : free_ptr_hash<simduid_to_vf>
{
unsigned int simduid;
- int vf;
+ poly_uint64 vf;
/* hash_table support. */
static inline hashval_t hash (const simduid_to_vf *);
for (i = gsi_start_bb (bb); !gsi_end_p (i); )
{
- unsigned int vf = 1;
+ poly_uint64 vf = 1;
enum internal_fn ifn;
gimple *stmt = gsi_stmt (i);
tree t;
break;
case IFN_GOMP_SIMD_ORDERED_START:
case IFN_GOMP_SIMD_ORDERED_END:
+ if (integer_onep (gimple_call_arg (stmt, 0)))
+ {
+ enum built_in_function bcode
+ = (ifn == IFN_GOMP_SIMD_ORDERED_START
+ ? BUILT_IN_GOMP_ORDERED_START
+ : BUILT_IN_GOMP_ORDERED_END);
+ gimple *g
+ = gimple_build_call (builtin_decl_explicit (bcode), 0);
+ tree vdef = gimple_vdef (stmt);
+ gimple_set_vdef (g, vdef);
+ SSA_NAME_DEF_STMT (vdef) = g;
+ gimple_set_vuse (g, gimple_vuse (stmt));
+ gsi_replace (&i, g, true);
+ continue;
+ }
gsi_remove (&i, true);
unlink_stmt_vdef (stmt);
continue;
gcc_assert (TREE_CODE (arg) == SSA_NAME);
simduid_to_vf *p = NULL, data;
data.simduid = DECL_UID (SSA_NAME_VAR (arg));
+ /* Need to nullify loop safelen field since it's value is not
+ valid after transformation. */
+ if (bb->loop_father && bb->loop_father->safelen > 0)
+ bb->loop_father->safelen = 0;
if (htab)
{
p = htab->find (&data);
default:
gcc_unreachable ();
}
- update_call_from_tree (&i, t);
- gsi_next (&i);
+ tree lhs = gimple_call_lhs (stmt);
+ if (lhs)
+ replace_uses_by (lhs, t);
+ release_defs (stmt);
+ gsi_remove (&i, true);
}
}
}
if ((*iter)->simduid != -1U)
{
tree decl = (*iter)->decl;
- int vf = 1;
+ poly_uint64 vf = 1;
if (simduid_to_vf_htab)
{
simduid_to_vf *p = NULL, data;
delete simd_array_to_simduid_htab;
}
\f
-/* A helper function to free data refs. */
+/* Initialize the vec_info with kind KIND_IN and target cost data
+ TARGET_COST_DATA_IN. */
+
+vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in,
+ vec_info_shared *shared_)
+ : kind (kind_in),
+ shared (shared_),
+ target_cost_data (target_cost_data_in)
+{
+ stmt_vec_infos.create (50);
+}
+
+vec_info::~vec_info ()
+{
+ slp_instance instance;
+ unsigned int i;
+
+ FOR_EACH_VEC_ELT (slp_instances, i, instance)
+ vect_free_slp_instance (instance, true);
+
+ destroy_cost_data (target_cost_data);
+ free_stmt_vec_infos ();
+}
+
+vec_info_shared::vec_info_shared ()
+ : datarefs (vNULL),
+ datarefs_copy (vNULL),
+ ddrs (vNULL)
+{
+}
+
+vec_info_shared::~vec_info_shared ()
+{
+ free_data_refs (datarefs);
+ free_dependence_relations (ddrs);
+ datarefs_copy.release ();
+}
+
+void
+vec_info_shared::save_datarefs ()
+{
+ if (!flag_checking)
+ return;
+ datarefs_copy.reserve_exact (datarefs.length ());
+ for (unsigned i = 0; i < datarefs.length (); ++i)
+ datarefs_copy.quick_push (*datarefs[i]);
+}
+
+void
+vec_info_shared::check_datarefs ()
+{
+ if (!flag_checking)
+ return;
+ gcc_assert (datarefs.length () == datarefs_copy.length ());
+ for (unsigned i = 0; i < datarefs.length (); ++i)
+ if (memcmp (&datarefs_copy[i], datarefs[i], sizeof (data_reference)) != 0)
+ gcc_unreachable ();
+}
+
+/* Record that STMT belongs to the vectorizable region. Create and return
+ an associated stmt_vec_info. */
+
+stmt_vec_info
+vec_info::add_stmt (gimple *stmt)
+{
+ stmt_vec_info res = new_stmt_vec_info (stmt);
+ set_vinfo_for_stmt (stmt, res);
+ return res;
+}
+
+/* If STMT has an associated stmt_vec_info, return that vec_info, otherwise
+ return null. It is safe to call this function on any statement, even if
+ it might not be part of the vectorizable region. */
+
+stmt_vec_info
+vec_info::lookup_stmt (gimple *stmt)
+{
+ unsigned int uid = gimple_uid (stmt);
+ if (uid > 0 && uid - 1 < stmt_vec_infos.length ())
+ {
+ stmt_vec_info res = stmt_vec_infos[uid - 1];
+ if (res && res->stmt == stmt)
+ return res;
+ }
+ return NULL;
+}
+
+/* If NAME is an SSA_NAME and its definition has an associated stmt_vec_info,
+ return that stmt_vec_info, otherwise return null. It is safe to call
+ this on arbitrary operands. */
+
+stmt_vec_info
+vec_info::lookup_def (tree name)
+{
+ if (TREE_CODE (name) == SSA_NAME
+ && !SSA_NAME_IS_DEFAULT_DEF (name))
+ return lookup_stmt (SSA_NAME_DEF_STMT (name));
+ return NULL;
+}
+
+/* See whether there is a single non-debug statement that uses LHS and
+ whether that statement has an associated stmt_vec_info. Return the
+ stmt_vec_info if so, otherwise return null. */
+
+stmt_vec_info
+vec_info::lookup_single_use (tree lhs)
+{
+ use_operand_p dummy;
+ gimple *use_stmt;
+ if (single_imm_use (lhs, &dummy, &use_stmt))
+ return lookup_stmt (use_stmt);
+ return NULL;
+}
+
+/* Return vectorization information about DR. */
+
+dr_vec_info *
+vec_info::lookup_dr (data_reference *dr)
+{
+ stmt_vec_info stmt_info = lookup_stmt (DR_STMT (dr));
+ /* DR_STMT should never refer to a stmt in a pattern replacement. */
+ gcc_checking_assert (!is_pattern_stmt_p (stmt_info));
+ return STMT_VINFO_DR_INFO (stmt_info->dr_aux.stmt);
+}
+
+/* Record that NEW_STMT_INFO now implements the same data reference
+ as OLD_STMT_INFO. */
+
+void
+vec_info::move_dr (stmt_vec_info new_stmt_info, stmt_vec_info old_stmt_info)
+{
+ gcc_assert (!is_pattern_stmt_p (old_stmt_info));
+ STMT_VINFO_DR_INFO (old_stmt_info)->stmt = new_stmt_info;
+ new_stmt_info->dr_aux = old_stmt_info->dr_aux;
+ STMT_VINFO_DR_WRT_VEC_LOOP (new_stmt_info)
+ = STMT_VINFO_DR_WRT_VEC_LOOP (old_stmt_info);
+ STMT_VINFO_GATHER_SCATTER_P (new_stmt_info)
+ = STMT_VINFO_GATHER_SCATTER_P (old_stmt_info);
+}
+
+/* Permanently remove the statement described by STMT_INFO from the
+ function. */
void
-vect_destroy_datarefs (vec_info *vinfo)
+vec_info::remove_stmt (stmt_vec_info stmt_info)
+{
+ gcc_assert (!stmt_info->pattern_stmt_p);
+ set_vinfo_for_stmt (stmt_info->stmt, NULL);
+ gimple_stmt_iterator si = gsi_for_stmt (stmt_info->stmt);
+ unlink_stmt_vdef (stmt_info->stmt);
+ gsi_remove (&si, true);
+ release_defs (stmt_info->stmt);
+ free_stmt_vec_info (stmt_info);
+}
+
+/* Replace the statement at GSI by NEW_STMT, both the vectorization
+ information and the function itself. STMT_INFO describes the statement
+ at GSI. */
+
+void
+vec_info::replace_stmt (gimple_stmt_iterator *gsi, stmt_vec_info stmt_info,
+ gimple *new_stmt)
+{
+ gimple *old_stmt = stmt_info->stmt;
+ gcc_assert (!stmt_info->pattern_stmt_p && old_stmt == gsi_stmt (*gsi));
+ set_vinfo_for_stmt (old_stmt, NULL);
+ set_vinfo_for_stmt (new_stmt, stmt_info);
+ stmt_info->stmt = new_stmt;
+ gsi_replace (gsi, new_stmt, true);
+}
+
+/* Create and initialize a new stmt_vec_info struct for STMT. */
+
+stmt_vec_info
+vec_info::new_stmt_vec_info (gimple *stmt)
+{
+ stmt_vec_info res = XCNEW (struct _stmt_vec_info);
+ res->vinfo = this;
+ res->stmt = stmt;
+
+ STMT_VINFO_TYPE (res) = undef_vec_info_type;
+ STMT_VINFO_RELEVANT (res) = vect_unused_in_scope;
+ STMT_VINFO_VECTORIZABLE (res) = true;
+ STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION;
+ STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK;
+ STMT_VINFO_SLP_VECT_ONLY (res) = false;
+
+ if (gimple_code (stmt) == GIMPLE_PHI
+ && is_loop_header_bb_p (gimple_bb (stmt)))
+ STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type;
+ else
+ STMT_VINFO_DEF_TYPE (res) = vect_internal_def;
+
+ STMT_VINFO_SAME_ALIGN_REFS (res).create (0);
+ STMT_SLP_TYPE (res) = loop_vect;
+
+ /* This is really "uninitialized" until vect_compute_data_ref_alignment. */
+ res->dr_aux.misalignment = DR_MISALIGNMENT_UNINITIALIZED;
+
+ return res;
+}
+
+/* Associate STMT with INFO. */
+
+void
+vec_info::set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info)
+{
+ unsigned int uid = gimple_uid (stmt);
+ if (uid == 0)
+ {
+ gcc_checking_assert (info);
+ uid = stmt_vec_infos.length () + 1;
+ gimple_set_uid (stmt, uid);
+ stmt_vec_infos.safe_push (info);
+ }
+ else
+ {
+ gcc_checking_assert (info == NULL);
+ stmt_vec_infos[uid - 1] = info;
+ }
+}
+
+/* Free the contents of stmt_vec_infos. */
+
+void
+vec_info::free_stmt_vec_infos (void)
{
- struct data_reference *dr;
unsigned int i;
+ stmt_vec_info info;
+ FOR_EACH_VEC_ELT (stmt_vec_infos, i, info)
+ if (info != NULL)
+ free_stmt_vec_info (info);
+ stmt_vec_infos.release ();
+}
- FOR_EACH_VEC_ELT (vinfo->datarefs, i, dr)
- if (dr->aux)
- {
- free (dr->aux);
- dr->aux = NULL;
- }
+/* Free STMT_INFO. */
+
+void
+vec_info::free_stmt_vec_info (stmt_vec_info stmt_info)
+{
+ if (stmt_info->pattern_stmt_p)
+ {
+ gimple_set_bb (stmt_info->stmt, NULL);
+ tree lhs = gimple_get_lhs (stmt_info->stmt);
+ if (lhs && TREE_CODE (lhs) == SSA_NAME)
+ release_ssa_name (lhs);
+ }
- free_data_refs (vinfo->datarefs);
+ STMT_VINFO_SAME_ALIGN_REFS (stmt_info).release ();
+ STMT_VINFO_SIMD_CLONE_INFO (stmt_info).release ();
+ free (stmt_info);
}
+/* A helper function to free scev and LOOP niter information, as well as
+ clear loop constraint LOOP_C_FINITE. */
+
+void
+vect_free_loop_info_assumptions (struct loop *loop)
+{
+ scev_reset_htab ();
+ /* We need to explicitly reset upper bound information since they are
+ used even after free_numbers_of_iterations_estimates. */
+ loop->any_upper_bound = false;
+ loop->any_likely_upper_bound = false;
+ free_numbers_of_iterations_estimates (loop);
+ loop_constraint_clear (loop, LOOP_C_FINITE);
+}
/* If LOOP has been versioned during ifcvt, return the internal call
guarding it. */
if (!gsi_end_p (gsi))
{
g = gsi_stmt (gsi);
- if (is_gimple_call (g)
- && gimple_call_internal_p (g)
- && gimple_call_internal_fn (g) == IFN_LOOP_VECTORIZED
+ if (gimple_call_internal_p (g, IFN_LOOP_VECTORIZED)
&& (tree_to_shwi (gimple_call_arg (g, 0)) == loop->num
|| tree_to_shwi (gimple_call_arg (g, 1)) == loop->num))
return g;
return NULL;
}
-/* Fold LOOP_VECTORIZED internal call G to VALUE and
- update any immediate uses of it's LHS. */
+/* If LOOP has been versioned during loop distribution, return the gurading
+ internal call. */
-static void
-fold_loop_vectorized_call (gimple *g, tree value)
+static gimple *
+vect_loop_dist_alias_call (struct loop *loop)
{
- tree lhs = gimple_call_lhs (g);
- use_operand_p use_p;
- imm_use_iterator iter;
- gimple *use_stmt;
- gimple_stmt_iterator gsi = gsi_for_stmt (g);
+ basic_block bb;
+ basic_block entry;
+ struct loop *outer, *orig;
+ gimple_stmt_iterator gsi;
+ gimple *g;
+
+ if (loop->orig_loop_num == 0)
+ return NULL;
+
+ orig = get_loop (cfun, loop->orig_loop_num);
+ if (orig == NULL)
+ {
+ /* The original loop is somehow destroyed. Clear the information. */
+ loop->orig_loop_num = 0;
+ return NULL;
+ }
+
+ if (loop != orig)
+ bb = nearest_common_dominator (CDI_DOMINATORS, loop->header, orig->header);
+ else
+ bb = loop_preheader_edge (loop)->src;
- update_call_from_tree (&gsi, value);
- FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
+ outer = bb->loop_father;
+ entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
+
+ /* Look upward in dominance tree. */
+ for (; bb != entry && flow_bb_inside_loop_p (outer, bb);
+ bb = get_immediate_dominator (CDI_DOMINATORS, bb))
{
- FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
- SET_USE (use_p, value);
- update_stmt (use_stmt);
+ g = last_stmt (bb);
+ if (g == NULL || gimple_code (g) != GIMPLE_COND)
+ continue;
+
+ gsi = gsi_for_stmt (g);
+ gsi_prev (&gsi);
+ if (gsi_end_p (gsi))
+ continue;
+
+ g = gsi_stmt (gsi);
+ /* The guarding internal function call must have the same distribution
+ alias id. */
+ if (gimple_call_internal_p (g, IFN_LOOP_DIST_ALIAS)
+ && (tree_to_shwi (gimple_call_arg (g, 0)) == loop->orig_loop_num))
+ return g;
}
+ return NULL;
}
+
/* Set the uids of all the statements in basic blocks inside loop
represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
call guarding the loop which has been if converted. */
struct loop *scalar_loop = get_loop (cfun, tree_to_shwi (arg));
LOOP_VINFO_SCALAR_LOOP (loop_vinfo) = scalar_loop;
- gcc_checking_assert (vect_loop_vectorized_call
- (LOOP_VINFO_SCALAR_LOOP (loop_vinfo))
+ gcc_checking_assert (vect_loop_vectorized_call (scalar_loop)
== loop_vectorized_call);
+ /* If we are going to vectorize outer loop, prevent vectorization
+ of the inner loop in the scalar loop - either the scalar loop is
+ thrown away, so it is a wasted work, or is used only for
+ a few iterations. */
+ if (scalar_loop->inner)
+ {
+ gimple *g = vect_loop_vectorized_call (scalar_loop->inner);
+ if (g)
+ {
+ arg = gimple_call_arg (g, 0);
+ get_loop (cfun, tree_to_shwi (arg))->dont_vectorize = true;
+ fold_loop_internal_call (g, boolean_false_node);
+ }
+ }
bbs = get_loop_body (scalar_loop);
for (i = 0; i < scalar_loop->num_nodes; i++)
{
free (bbs);
}
+/* Try to vectorize LOOP. */
+
+static unsigned
+try_vectorize_loop_1 (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+ unsigned *num_vectorized_loops,
+ loop_p loop, loop_vec_info orig_loop_vinfo,
+ gimple *loop_vectorized_call,
+ gimple *loop_dist_alias_call)
+{
+ unsigned ret = 0;
+ vec_info_shared shared;
+ auto_purge_vect_location sentinel;
+ vect_location = find_loop_location (loop);
+ if (LOCATION_LOCUS (vect_location.get_location_t ()) != UNKNOWN_LOCATION
+ && dump_enabled_p ())
+ dump_printf (MSG_NOTE | MSG_PRIORITY_INTERNALS,
+ "\nAnalyzing loop at %s:%d\n",
+ LOCATION_FILE (vect_location.get_location_t ()),
+ LOCATION_LINE (vect_location.get_location_t ()));
+
+ /* Try to analyze the loop, retaining an opt_problem if dump_enabled_p. */
+ opt_loop_vec_info loop_vinfo
+ = vect_analyze_loop (loop, orig_loop_vinfo, &shared);
+ loop->aux = loop_vinfo;
+
+ if (!loop_vinfo)
+ if (dump_enabled_p ())
+ if (opt_problem *problem = loop_vinfo.get_problem ())
+ {
+ dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
+ "couldn't vectorize loop\n");
+ problem->emit_and_clear ();
+ }
+
+ if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
+ {
+ /* Free existing information if loop is analyzed with some
+ assumptions. */
+ if (loop_constraint_set_p (loop, LOOP_C_FINITE))
+ vect_free_loop_info_assumptions (loop);
+
+ /* If we applied if-conversion then try to vectorize the
+ BB of innermost loops.
+ ??? Ideally BB vectorization would learn to vectorize
+ control flow by applying if-conversion on-the-fly, the
+ following retains the if-converted loop body even when
+ only non-if-converted parts took part in BB vectorization. */
+ if (flag_tree_slp_vectorize != 0
+ && loop_vectorized_call
+ && ! loop->inner)
+ {
+ basic_block bb = loop->header;
+ bool require_loop_vectorize = false;
+ for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ gcall *call = dyn_cast <gcall *> (stmt);
+ if (call && gimple_call_internal_p (call))
+ {
+ internal_fn ifn = gimple_call_internal_fn (call);
+ if (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE
+ /* Don't keep the if-converted parts when the ifn with
+ specifc type is not supported by the backend. */
+ || (direct_internal_fn_p (ifn)
+ && !direct_internal_fn_supported_p
+ (call, OPTIMIZE_FOR_SPEED)))
+ {
+ require_loop_vectorize = true;
+ break;
+ }
+ }
+ gimple_set_uid (stmt, -1);
+ gimple_set_visited (stmt, false);
+ }
+ if (!require_loop_vectorize && vect_slp_bb (bb))
+ {
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location,
+ "basic block vectorized\n");
+ fold_loop_internal_call (loop_vectorized_call,
+ boolean_true_node);
+ loop_vectorized_call = NULL;
+ ret |= TODO_cleanup_cfg;
+ }
+ }
+ /* If outer loop vectorization fails for LOOP_VECTORIZED guarded
+ loop, don't vectorize its inner loop; we'll attempt to
+ vectorize LOOP_VECTORIZED guarded inner loop of the scalar
+ loop version. */
+ if (loop_vectorized_call && loop->inner)
+ loop->inner->dont_vectorize = true;
+ return ret;
+ }
+
+ if (!dbg_cnt (vect_loop))
+ {
+ /* Free existing information if loop is analyzed with some
+ assumptions. */
+ if (loop_constraint_set_p (loop, LOOP_C_FINITE))
+ vect_free_loop_info_assumptions (loop);
+ return ret;
+ }
+
+ if (loop_vectorized_call)
+ set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
+
+ unsigned HOST_WIDE_INT bytes;
+ if (dump_enabled_p ())
+ {
+ if (current_vector_size.is_constant (&bytes))
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ "loop vectorized using %wu byte vectors\n", bytes);
+ else
+ dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
+ "loop vectorized using variable length vectors\n");
+ }
+
+ loop_p new_loop = vect_transform_loop (loop_vinfo);
+ (*num_vectorized_loops)++;
+ /* Now that the loop has been vectorized, allow it to be unrolled
+ etc. */
+ loop->force_vectorize = false;
+
+ if (loop->simduid)
+ {
+ simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
+ if (!simduid_to_vf_htab)
+ simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
+ simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
+ simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
+ *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
+ = simduid_to_vf_data;
+ }
+
+ if (loop_vectorized_call)
+ {
+ fold_loop_internal_call (loop_vectorized_call, boolean_true_node);
+ loop_vectorized_call = NULL;
+ ret |= TODO_cleanup_cfg;
+ }
+ if (loop_dist_alias_call)
+ {
+ tree value = gimple_call_arg (loop_dist_alias_call, 1);
+ fold_loop_internal_call (loop_dist_alias_call, value);
+ loop_dist_alias_call = NULL;
+ ret |= TODO_cleanup_cfg;
+ }
+
+ /* Epilogue of vectorized loop must be vectorized too. */
+ if (new_loop)
+ ret |= try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
+ new_loop, loop_vinfo, NULL, NULL);
+
+ return ret;
+}
+
+/* Try to vectorize LOOP. */
+
+static unsigned
+try_vectorize_loop (hash_table<simduid_to_vf> *&simduid_to_vf_htab,
+ unsigned *num_vectorized_loops, loop_p loop)
+{
+ if (!((flag_tree_loop_vectorize
+ && optimize_loop_nest_for_speed_p (loop))
+ || loop->force_vectorize))
+ return 0;
+
+ return try_vectorize_loop_1 (simduid_to_vf_htab, num_vectorized_loops,
+ loop, NULL,
+ vect_loop_vectorized_call (loop),
+ vect_loop_dist_alias_call (loop));
+}
+
+
/* Function vectorize_loops.
Entry point to loop vectorization phase. */
if (cfun->has_simduid_loops)
note_simd_array_uses (&simd_array_to_simduid_htab);
- init_stmt_vec_info_vec ();
-
/* ----------- Analyze loops. ----------- */
/* If some loop was duplicated, it gets bigger number
only over initial loops skipping newly generated ones. */
FOR_EACH_LOOP (loop, 0)
if (loop->dont_vectorize)
- any_ifcvt_loops = true;
- else if ((flag_tree_loop_vectorize
- && optimize_loop_nest_for_speed_p (loop))
- || loop->force_vectorize)
{
- loop_vec_info loop_vinfo;
- vect_location = find_loop_location (loop);
- if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
- && dump_enabled_p ())
- dump_printf (MSG_NOTE, "\nAnalyzing loop at %s:%d\n",
- LOCATION_FILE (vect_location),
- LOCATION_LINE (vect_location));
-
- loop_vinfo = vect_analyze_loop (loop);
- loop->aux = loop_vinfo;
-
- if (!loop_vinfo || !LOOP_VINFO_VECTORIZABLE_P (loop_vinfo))
- continue;
-
- if (!dbg_cnt (vect_loop))
- break;
-
- gimple *loop_vectorized_call = vect_loop_vectorized_call (loop);
- if (loop_vectorized_call)
- set_uid_loop_bbs (loop_vinfo, loop_vectorized_call);
- if (LOCATION_LOCUS (vect_location) != UNKNOWN_LOCATION
- && dump_enabled_p ())
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
- "loop vectorized\n");
- vect_transform_loop (loop_vinfo);
- num_vectorized_loops++;
- /* Now that the loop has been vectorized, allow it to be unrolled
- etc. */
- loop->force_vectorize = false;
-
- if (loop->simduid)
- {
- simduid_to_vf *simduid_to_vf_data = XNEW (simduid_to_vf);
- if (!simduid_to_vf_htab)
- simduid_to_vf_htab = new hash_table<simduid_to_vf> (15);
- simduid_to_vf_data->simduid = DECL_UID (loop->simduid);
- simduid_to_vf_data->vf = loop_vinfo->vectorization_factor;
- *simduid_to_vf_htab->find_slot (simduid_to_vf_data, INSERT)
- = simduid_to_vf_data;
- }
-
- if (loop_vectorized_call)
+ any_ifcvt_loops = true;
+ /* If-conversion sometimes versions both the outer loop
+ (for the case when outer loop vectorization might be
+ desirable) as well as the inner loop in the scalar version
+ of the loop. So we have:
+ if (LOOP_VECTORIZED (1, 3))
+ {
+ loop1
+ loop2
+ }
+ else
+ loop3 (copy of loop1)
+ if (LOOP_VECTORIZED (4, 5))
+ loop4 (copy of loop2)
+ else
+ loop5 (copy of loop4)
+ If FOR_EACH_LOOP gives us loop3 first (which has
+ dont_vectorize set), make sure to process loop1 before loop4;
+ so that we can prevent vectorization of loop4 if loop1
+ is successfully vectorized. */
+ if (loop->inner)
{
- fold_loop_vectorized_call (loop_vectorized_call, boolean_true_node);
- ret |= TODO_cleanup_cfg;
+ gimple *loop_vectorized_call
+ = vect_loop_vectorized_call (loop);
+ if (loop_vectorized_call
+ && vect_loop_vectorized_call (loop->inner))
+ {
+ tree arg = gimple_call_arg (loop_vectorized_call, 0);
+ struct loop *vector_loop
+ = get_loop (cfun, tree_to_shwi (arg));
+ if (vector_loop && vector_loop != loop)
+ {
+ /* Make sure we don't vectorize it twice. */
+ vector_loop->dont_vectorize = true;
+ ret |= try_vectorize_loop (simduid_to_vf_htab,
+ &num_vectorized_loops,
+ vector_loop);
+ }
+ }
}
}
+ else
+ ret |= try_vectorize_loop (simduid_to_vf_htab, &num_vectorized_loops,
+ loop);
- vect_location = UNKNOWN_LOCATION;
+ vect_location = dump_user_location_t ();
statistics_counter_event (cfun, "Vectorized loops", num_vectorized_loops);
if (dump_enabled_p ()
/* ----------- Finalize. ----------- */
if (any_ifcvt_loops)
- for (i = 1; i < vect_loops_num; i++)
+ for (i = 1; i < number_of_loops (cfun); i++)
{
loop = get_loop (cfun, i);
if (loop && loop->dont_vectorize)
gimple *g = vect_loop_vectorized_call (loop);
if (g)
{
- fold_loop_vectorized_call (g, boolean_false_node);
+ fold_loop_internal_call (g, boolean_false_node);
+ ret |= TODO_cleanup_cfg;
+ g = NULL;
+ }
+ else
+ g = vect_loop_dist_alias_call (loop);
+
+ if (g)
+ {
+ fold_loop_internal_call (g, boolean_false_node);
ret |= TODO_cleanup_cfg;
}
}
}
- for (i = 1; i < vect_loops_num; i++)
+ for (i = 1; i < number_of_loops (cfun); i++)
{
loop_vec_info loop_vinfo;
+ bool has_mask_store;
loop = get_loop (cfun, i);
- if (!loop)
+ if (!loop || !loop->aux)
continue;
loop_vinfo = (loop_vec_info) loop->aux;
- destroy_loop_vec_info (loop_vinfo, true);
+ has_mask_store = LOOP_VINFO_HAS_MASK_STORE (loop_vinfo);
+ delete loop_vinfo;
+ if (has_mask_store
+ && targetm.vectorize.empty_mask_is_expensive (IFN_MASK_STORE))
+ optimize_mask_stores (loop);
loop->aux = NULL;
}
- free_stmt_vec_info_vec ();
-
/* Fold IFN_GOMP_SIMD_{VF,LANE,LAST_LANE,ORDERED_{START,END}} builtins. */
if (cfun->has_simduid_loops)
adjust_simduid_builtins (simduid_to_vf_htab);
unsigned int
pass_slp_vectorize::execute (function *fun)
{
+ auto_purge_vect_location sentinel;
basic_block bb;
bool in_loop_pipeline = scev_initialized_p ();
scev_initialize ();
}
- init_stmt_vec_info_vec ();
-
+ /* Mark all stmts as not belonging to the current region and unvisited. */
FOR_EACH_BB_FN (bb, fun)
{
- vect_location = find_bb_location (bb);
-
- if (vect_slp_analyze_bb (bb))
- {
- if (!dbg_cnt (vect_slp))
- break;
-
- vect_slp_transform_bb (bb);
- if (dump_enabled_p ())
- dump_printf_loc (MSG_OPTIMIZED_LOCATIONS, vect_location,
- "basic block vectorized\n");
- }
+ for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
+ gsi_next (&gsi))
+ {
+ gimple *stmt = gsi_stmt (gsi);
+ gimple_set_uid (stmt, -1);
+ gimple_set_visited (stmt, false);
+ }
}
- free_stmt_vec_info_vec ();
+ FOR_EACH_BB_FN (bb, fun)
+ {
+ if (vect_slp_bb (bb))
+ if (dump_enabled_p ())
+ dump_printf_loc (MSG_NOTE, vect_location, "basic block vectorized\n");
+ }
if (!in_loop_pipeline)
{
This should involve global alignment analysis and in the future also
array padding. */
+static unsigned get_vec_alignment_for_type (tree);
+static hash_map<tree, unsigned> *type_align_map;
+
+/* Return alignment of array's vector type corresponding to scalar type.
+ 0 if no vector type exists. */
+static unsigned
+get_vec_alignment_for_array_type (tree type)
+{
+ gcc_assert (TREE_CODE (type) == ARRAY_TYPE);
+ poly_uint64 array_size, vector_size;
+
+ tree vectype = get_vectype_for_scalar_type (strip_array_types (type));
+ if (!vectype
+ || !poly_int_tree_p (TYPE_SIZE (type), &array_size)
+ || !poly_int_tree_p (TYPE_SIZE (vectype), &vector_size)
+ || maybe_lt (array_size, vector_size))
+ return 0;
+
+ return TYPE_ALIGN (vectype);
+}
+
+/* Return alignment of field having maximum alignment of vector type
+ corresponding to it's scalar type. For now, we only consider fields whose
+ offset is a multiple of it's vector alignment.
+ 0 if no suitable field is found. */
+static unsigned
+get_vec_alignment_for_record_type (tree type)
+{
+ gcc_assert (TREE_CODE (type) == RECORD_TYPE);
+
+ unsigned max_align = 0, alignment;
+ HOST_WIDE_INT offset;
+ tree offset_tree;
+
+ if (TYPE_PACKED (type))
+ return 0;
+
+ unsigned *slot = type_align_map->get (type);
+ if (slot)
+ return *slot;
+
+ for (tree field = first_field (type);
+ field != NULL_TREE;
+ field = DECL_CHAIN (field))
+ {
+ /* Skip if not FIELD_DECL or if alignment is set by user. */
+ if (TREE_CODE (field) != FIELD_DECL
+ || DECL_USER_ALIGN (field)
+ || DECL_ARTIFICIAL (field))
+ continue;
+
+ /* We don't need to process the type further if offset is variable,
+ since the offsets of remaining members will also be variable. */
+ if (TREE_CODE (DECL_FIELD_OFFSET (field)) != INTEGER_CST
+ || TREE_CODE (DECL_FIELD_BIT_OFFSET (field)) != INTEGER_CST)
+ break;
+
+ /* Similarly stop processing the type if offset_tree
+ does not fit in unsigned HOST_WIDE_INT. */
+ offset_tree = bit_position (field);
+ if (!tree_fits_uhwi_p (offset_tree))
+ break;
+
+ offset = tree_to_uhwi (offset_tree);
+ alignment = get_vec_alignment_for_type (TREE_TYPE (field));
+
+ /* Get maximum alignment of vectorized field/array among those members
+ whose offset is multiple of the vector alignment. */
+ if (alignment
+ && (offset % alignment == 0)
+ && (alignment > max_align))
+ max_align = alignment;
+ }
+
+ type_align_map->put (type, max_align);
+ return max_align;
+}
+
+/* Return alignment of vector type corresponding to decl's scalar type
+ or 0 if it doesn't exist or the vector alignment is lesser than
+ decl's alignment. */
+static unsigned
+get_vec_alignment_for_type (tree type)
+{
+ if (type == NULL_TREE)
+ return 0;
+
+ gcc_assert (TYPE_P (type));
+
+ static unsigned alignment = 0;
+ switch (TREE_CODE (type))
+ {
+ case ARRAY_TYPE:
+ alignment = get_vec_alignment_for_array_type (type);
+ break;
+ case RECORD_TYPE:
+ alignment = get_vec_alignment_for_record_type (type);
+ break;
+ default:
+ alignment = 0;
+ break;
+ }
+
+ return (alignment > TYPE_ALIGN (type)) ? alignment : 0;
+}
+
+/* Entry point to increase_alignment pass. */
static unsigned int
increase_alignment (void)
{
varpool_node *vnode;
- vect_location = UNKNOWN_LOCATION;
+ vect_location = dump_user_location_t ();
+ type_align_map = new hash_map<tree, unsigned>;
/* Increase the alignment of all global arrays for vectorization. */
FOR_EACH_DEFINED_VARIABLE (vnode)
{
- tree vectype, decl = vnode->decl;
- tree t;
+ tree decl = vnode->decl;
unsigned int alignment;
- t = TREE_TYPE (decl);
- if (TREE_CODE (t) != ARRAY_TYPE)
- continue;
- vectype = get_vectype_for_scalar_type (strip_array_types (t));
- if (!vectype)
- continue;
- alignment = TYPE_ALIGN (vectype);
- if (DECL_ALIGN (decl) >= alignment)
- continue;
-
- if (vect_can_force_dr_alignment_p (decl, alignment))
+ if ((decl_in_symtab_p (decl)
+ && !symtab_node::get (decl)->can_increase_alignment_p ())
+ || DECL_USER_ALIGN (decl) || DECL_ARTIFICIAL (decl))
+ continue;
+
+ alignment = get_vec_alignment_for_type (TREE_TYPE (decl));
+ if (alignment && vect_can_force_dr_alignment_p (decl, alignment))
{
- vnode->increase_alignment (TYPE_ALIGN (vectype));
- dump_printf (MSG_NOTE, "Increasing alignment of decl: ");
- dump_generic_expr (MSG_NOTE, TDF_SLIM, decl);
- dump_printf (MSG_NOTE, "\n");
+ vnode->increase_alignment (alignment);
+ if (dump_enabled_p ())
+ dump_printf (MSG_NOTE, "Increasing alignment of decl: %T\n", decl);
}
}
+
+ delete type_align_map;
return 0;
}
projects / gcc.git / blobdiff