/* Vectorizer Specific Loop Manipulations
- Copyright (C) 2003-2015 Free Software Foundation, Inc.
+ Copyright (C) 2003-2016 Free Software Foundation, Inc.
Contributed by Dorit Naishlos <dorit@il.ibm.com>
and Ira Rosen <irar@il.ibm.com>
#include "config.h"
#include "system.h"
#include "coretypes.h"
-#include "dumpfile.h"
#include "backend.h"
-#include "cfghooks.h"
#include "tree.h"
#include "gimple.h"
-#include "hard-reg-set.h"
+#include "cfghooks.h"
+#include "tree-pass.h"
#include "ssa.h"
-#include "alias.h"
#include "fold-const.h"
#include "cfganal.h"
-#include "gimple-pretty-print.h"
-#include "internal-fn.h"
#include "gimplify.h"
#include "gimple-iterator.h"
#include "gimplify-me.h"
#include "tree-ssa-loop-manip.h"
#include "tree-into-ssa.h"
#include "tree-ssa.h"
-#include "tree-pass.h"
#include "cfgloop.h"
-#include "diagnostic-core.h"
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
-#include "langhooks.h"
/*************************************************************************
Simple Loop Peeling Utilities
}
-/* Renames the variables in basic block BB. */
+/* Renames the variables in basic block BB. Allow renaming of PHI argumnets
+ on edges incoming from outer-block header if RENAME_FROM_OUTER_LOOP is
+ true. */
static void
-rename_variables_in_bb (basic_block bb)
+rename_variables_in_bb (basic_block bb, bool rename_from_outer_loop)
{
- gimple stmt;
+ gimple *stmt;
use_operand_p use_p;
ssa_op_iter iter;
edge e;
edge_iterator ei;
struct loop *loop = bb->loop_father;
+ struct loop *outer_loop = NULL;
+
+ if (rename_from_outer_loop)
+ {
+ gcc_assert (loop);
+ outer_loop = loop_outer (loop);
+ }
for (gimple_stmt_iterator gsi = gsi_start_bb (bb); !gsi_end_p (gsi);
gsi_next (&gsi))
FOR_EACH_EDGE (e, ei, bb->preds)
{
- if (!flow_bb_inside_loop_p (loop, e->src))
+ if (!flow_bb_inside_loop_p (loop, e->src)
+ && (!rename_from_outer_loop || e->src != outer_loop->header))
continue;
for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
gsi_next (&gsi))
}
-typedef struct
+struct adjust_info
{
tree from, to;
basic_block bb;
-} adjust_info;
+};
/* A stack of values to be adjusted in debug stmts. We have to
process them LIFO, so that the closest substitution applies. If we
tree orig_def = ai->from;
tree new_def = ai->to;
imm_use_iterator imm_iter;
- gimple stmt;
+ gimple *stmt;
basic_block bbdef = gimple_bb (SSA_NAME_DEF_STMT (orig_def));
gcc_assert (dom_info_available_p (CDI_DOMINATORS));
transformations. */
static void
-adjust_phi_and_debug_stmts (gimple update_phi, edge e, tree new_def)
+adjust_phi_and_debug_stmts (gimple *update_phi, edge e, tree new_def)
{
tree orig_def = PHI_ARG_DEF_FROM_EDGE (update_phi, e);
set this earlier. Verify the PHI has the same value. */
if (new_name)
{
- gimple phi = SSA_NAME_DEF_STMT (new_name);
+ gimple *phi = SSA_NAME_DEF_STMT (new_name);
gcc_assert (gimple_code (phi) == GIMPLE_PHI
&& gimple_bb (phi) == *new_exit_bb
&& (PHI_ARG_DEF_FROM_EDGE (phi, single_exit (loop))
dump_printf (MSG_NOTE, "\nloop at %s:%d: ", LOCATION_FILE (loop_loc),
LOCATION_LINE (loop_loc));
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, cond_stmt, 0);
- dump_printf (MSG_NOTE, "\n");
}
loop->nb_iterations = niters;
}
for (gsi_from = gsi_start_phis (from->dest),
gsi_to = gsi_start_phis (to->dest);
- !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);
- gsi_next (&gsi_from), gsi_next (&gsi_to))
+ !gsi_end_p (gsi_from) && !gsi_end_p (gsi_to);)
{
- gimple from_phi = gsi_stmt (gsi_from);
- gimple to_phi = gsi_stmt (gsi_to);
+ gimple *from_phi = gsi_stmt (gsi_from);
+ gimple *to_phi = gsi_stmt (gsi_to);
tree from_arg = PHI_ARG_DEF_FROM_EDGE (from_phi, from);
+ if (TREE_CODE (from_arg) != SSA_NAME)
+ {
+ gsi_next (&gsi_from);
+ continue;
+ }
tree to_arg = PHI_ARG_DEF_FROM_EDGE (to_phi, to);
- if (TREE_CODE (from_arg) == SSA_NAME
- && TREE_CODE (to_arg) == SSA_NAME
- && get_current_def (to_arg) == NULL_TREE)
+ if (TREE_CODE (to_arg) != SSA_NAME)
+ {
+ gsi_next (&gsi_to);
+ continue;
+ }
+ if (get_current_def (to_arg) == NULL_TREE)
set_current_def (to_arg, get_current_def (from_arg));
+ gsi_next (&gsi_from);
+ gsi_next (&gsi_to);
}
}
bool was_imm_dom;
basic_block exit_dest;
edge exit, new_exit;
+ bool duplicate_outer_loop = false;
exit = single_exit (loop);
at_exit = (e == exit);
bbs = XNEWVEC (basic_block, scalar_loop->num_nodes + 1);
get_loop_body_with_size (scalar_loop, bbs, scalar_loop->num_nodes);
-
+ /* Allow duplication of outer loops. */
+ if (scalar_loop->inner)
+ duplicate_outer_loop = true;
/* Check whether duplication is possible. */
if (!can_copy_bbs_p (bbs, scalar_loop->num_nodes))
{
redirect_edge_and_branch_force (e, new_preheader);
flush_pending_stmts (e);
set_immediate_dominator (CDI_DOMINATORS, new_preheader, e->src);
- if (was_imm_dom)
+ if (was_imm_dom || duplicate_outer_loop)
set_immediate_dominator (CDI_DOMINATORS, exit_dest, new_exit->src);
/* And remove the non-necessary forwarder again. Keep the other
}
for (unsigned i = 0; i < scalar_loop->num_nodes + 1; i++)
- rename_variables_in_bb (new_bbs[i]);
+ rename_variables_in_bb (new_bbs[i], duplicate_outer_loop);
if (scalar_loop != loop)
{
free (new_bbs);
free (bbs);
-#ifdef ENABLE_CHECKING
- verify_dominators (CDI_DOMINATORS);
-#endif
+ checking_verify_dominators (CDI_DOMINATORS);
return new_loop;
}
/* This function verifies that the following restrictions apply to LOOP:
- (1) it is innermost
- (2) it consists of exactly 2 basic blocks - header, and an empty latch.
- (3) it is single entry, single exit
- (4) its exit condition is the last stmt in the header
- (5) E is the entry/exit edge of LOOP.
+ (1) it consists of exactly 2 basic blocks - header, and an empty latch
+ for innermost loop and 5 basic blocks for outer-loop.
+ (2) it is single entry, single exit
+ (3) its exit condition is the last stmt in the header
+ (4) E is the entry/exit edge of LOOP.
*/
bool
edge entry_e = loop_preheader_edge (loop);
gcond *orig_cond = get_loop_exit_condition (loop);
gimple_stmt_iterator loop_exit_gsi = gsi_last_bb (exit_e->src);
+ unsigned int num_bb = loop->inner? 5 : 2;
- if (loop->inner
/* All loops have an outer scope; the only case loop->outer is NULL is for
the function itself. */
- || !loop_outer (loop)
- || loop->num_nodes != 2
+ if (!loop_outer (loop)
+ || loop->num_nodes != num_bb
|| !empty_block_p (loop->latch)
|| !single_exit (loop)
/* Verify that new loop exit condition can be trivially modified. */
return true;
}
-#ifdef ENABLE_CHECKING
static void
-slpeel_verify_cfg_after_peeling (struct loop *first_loop,
- struct loop *second_loop)
+slpeel_checking_verify_cfg_after_peeling (struct loop *first_loop,
+ struct loop *second_loop)
{
+ if (!flag_checking)
+ return;
+
basic_block loop1_exit_bb = single_exit (first_loop)->dest;
basic_block loop2_entry_bb = loop_preheader_edge (second_loop)->src;
basic_block loop1_entry_bb = loop_preheader_edge (first_loop)->src;
second_loop. */
/* TODO */
}
-#endif
/* If the run time cost model check determines that vectorization is
not profitable and hence scalar loop should be generated then set
gphi *new_phi = create_phi_node (new_vop, exit_e->dest);
tree vop = PHI_ARG_DEF_FROM_EDGE (phi, EDGE_SUCC (loop->latch, 0));
imm_use_iterator imm_iter;
- gimple stmt;
+ gimple *stmt;
use_operand_p use_p;
add_phi_arg (new_phi, vop, exit_e, UNKNOWN_LOCATION);
gimple_phi_set_result (new_phi, new_vop);
FOR_EACH_IMM_USE_STMT (stmt, imm_iter, vop)
- if (stmt != new_phi && gimple_bb (stmt) != loop->header)
+ if (stmt != new_phi
+ && !flow_bb_inside_loop_p (loop, gimple_bb (stmt)))
FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
SET_USE (use_p, new_vop);
}
source_location
find_loop_location (struct loop *loop)
{
- gimple stmt = NULL;
+ gimple *stmt = NULL;
basic_block bb;
gimple_stmt_iterator si;
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
basic_block bb = loop->header;
- gimple phi;
+ gimple *phi;
gphi_iterator gsi;
/* Analyze phi functions of the loop header. */
{
dump_printf_loc (MSG_NOTE, vect_location, "Analyze phi: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- dump_printf (MSG_NOTE, "\n");
}
/* Skip virtual phi's. The data dependences that are associated with
dump_printf_loc (MSG_NOTE, vect_location,
"vect_update_ivs_after_vectorizer: phi: ");
dump_gimple_stmt (MSG_NOTE, TDF_SLIM, phi, 0);
- dump_printf (MSG_NOTE, "\n");
}
/* Skip virtual phi's. */
/* Skip reduction phis. */
stmt_info = vinfo_for_stmt (phi);
- if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def)
+ if (STMT_VINFO_DEF_TYPE (stmt_info) == vect_reduction_def
+ || STMT_VINFO_DEF_TYPE (stmt_info) == vect_double_reduction_def)
{
if (dump_enabled_p ())
dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
0, LOOP_VINFO_VECT_FACTOR (loop_vinfo));
gcc_assert (new_loop);
gcc_assert (loop_num == loop->num);
-#ifdef ENABLE_CHECKING
- slpeel_verify_cfg_after_peeling (loop, new_loop);
-#endif
+ slpeel_checking_verify_cfg_after_peeling (loop, new_loop);
/* A guard that controls whether the new_loop is to be executed or skipped
is placed in LOOP->exit. LOOP->exit therefore has two successors - one
tree iters, iters_name;
edge pe;
basic_block new_bb;
- gimple dr_stmt = DR_STMT (dr);
+ gimple *dr_stmt = DR_STMT (dr);
stmt_vec_info stmt_info = vinfo_for_stmt (dr_stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_info);
int vectype_align = TYPE_ALIGN (vectype) / BITS_PER_UNIT;
bound, 0);
gcc_assert (new_loop);
-#ifdef ENABLE_CHECKING
- slpeel_verify_cfg_after_peeling (new_loop, loop);
-#endif
+ slpeel_checking_verify_cfg_after_peeling (new_loop, loop);
/* For vectorization factor N, we need to copy at most N-1 values
for alignment and this means N-2 loopback edge executions. */
max_iter = LOOP_VINFO_VECT_FACTOR (loop_vinfo) - 2;
gimple_seq *cond_expr_stmt_list)
{
struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
- vec<gimple> may_misalign_stmts
+ vec<gimple *> may_misalign_stmts
= LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
- gimple ref_stmt;
+ gimple *ref_stmt;
int mask = LOOP_VINFO_PTR_MASK (loop_vinfo);
tree mask_cst;
unsigned int i;
char tmp_name[20];
tree or_tmp_name = NULL_TREE;
tree and_tmp_name;
- gimple and_stmt;
+ gimple *and_stmt;
tree ptrsize_zero;
tree part_cond_expr;
tree addr_base;
tree addr_tmp_name;
tree new_or_tmp_name;
- gimple addr_stmt, or_stmt;
+ gimple *addr_stmt, *or_stmt;
stmt_vec_info stmt_vinfo = vinfo_for_stmt (ref_stmt);
tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
bool negative = tree_int_cst_compare
const dr_with_seg_len& dr_b = comp_alias_ddrs[i].second;
tree segment_length_a = dr_a.seg_len;
tree segment_length_b = dr_b.seg_len;
+ tree addr_base_a = DR_BASE_ADDRESS (dr_a.dr);
+ tree addr_base_b = DR_BASE_ADDRESS (dr_b.dr);
+ tree offset_a = DR_OFFSET (dr_a.dr), offset_b = DR_OFFSET (dr_b.dr);
- tree addr_base_a
- = fold_build_pointer_plus (DR_BASE_ADDRESS (dr_a.dr), dr_a.offset);
- tree addr_base_b
- = fold_build_pointer_plus (DR_BASE_ADDRESS (dr_b.dr), dr_b.offset);
+ offset_a = fold_build2 (PLUS_EXPR, TREE_TYPE (offset_a),
+ offset_a, DR_INIT (dr_a.dr));
+ offset_b = fold_build2 (PLUS_EXPR, TREE_TYPE (offset_b),
+ offset_b, DR_INIT (dr_b.dr));
+ addr_base_a = fold_build_pointer_plus (addr_base_a, offset_a);
+ addr_base_b = fold_build_pointer_plus (addr_base_b, offset_b);
if (dump_enabled_p ())
{
dump_printf_loc (MSG_NOTE, vect_location,
"created %u versioning for alias checks.\n",
comp_alias_ddrs.length ());
-
- comp_alias_ddrs.release ();
}
projects / gcc.git / blobdiff