+2020-01-10 Andre Vieira <andre.simoesdiasvieira@arm.com>
+
+ * tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref): Use
+ get_dr_vinfo_offset
+ * tree-vect-loop.c (update_epilogue_loop_vinfo): Remove orig_drs_init
+ parameter and its use to reset DR_OFFSET's.
+ (vect_transform_loop): Remove orig_drs_init argument.
+ * tree-vect-loop-manip.c (vect_update_init_of_dr): Update the offset
+ member of dr_vec_info rather than the offset of the associated
+ data_reference's innermost_loop_behavior.
+ (vect_update_init_of_dr): Pass dr_vec_info instead of data_reference.
+ (vect_do_peeling): Remove orig_drs_init parameter and its construction.
+ * tree-vect-stmts.c (check_scan_store): Replace use of DR_OFFSET with
+ get_dr_vinfo_offset.
+ (vectorizable_store): Likewise.
+ (vectorizable_load): Likewise.
+
2020-01-10 Richard Biener <rguenther@suse.de>
* gimple-ssa-store-merging
innermost_loop_behavior *drb = vect_dr_behavior (dr_info);
tree data_ref_base = unshare_expr (drb->base_address);
- tree base_offset = unshare_expr (drb->offset);
+ tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true));
tree init = unshare_expr (drb->init);
if (loop_vinfo)
iterations before the scalar one (using masking to skip inactive
elements). This function updates the information recorded in DR to
account for the difference. Specifically, it updates the OFFSET
- field of DR. */
+ field of DR_INFO. */
static void
-vect_update_init_of_dr (struct data_reference *dr, tree niters, tree_code code)
+vect_update_init_of_dr (dr_vec_info *dr_info, tree niters, tree_code code)
{
- tree offset = DR_OFFSET (dr);
+ struct data_reference *dr = dr_info->dr;
+ tree offset = dr_info->offset;
+ if (!offset)
+ offset = build_zero_cst (sizetype);
niters = fold_build2 (MULT_EXPR, sizetype,
fold_convert (sizetype, niters),
fold_convert (sizetype, DR_STEP (dr)));
offset = fold_build2 (code, sizetype,
fold_convert (sizetype, offset), niters);
- DR_OFFSET (dr) = offset;
+ dr_info->offset = offset;
}
{
dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt))
- vect_update_init_of_dr (dr, niters, code);
+ vect_update_init_of_dr (dr_info, niters, code);
}
}
tree *niters_vector, tree *step_vector,
tree *niters_vector_mult_vf_var, int th,
bool check_profitability, bool niters_no_overflow,
- tree *advance, drs_init_vec &orig_drs_init)
+ tree *advance)
{
edge e, guard_e;
tree type = TREE_TYPE (niters), guard_cond;
scale_loop_profile (prolog, prob_prolog, bound_prolog);
}
- /* Save original inits for each data_reference before advancing them with
- NITERS_PROLOG. */
- unsigned int i;
- struct data_reference *dr;
- vec<data_reference_p> datarefs = loop_vinfo->shared->datarefs;
- FOR_EACH_VEC_ELT (datarefs, i, dr)
- orig_drs_init.safe_push (std::make_pair (dr, DR_OFFSET (dr)));
-
/* Update init address of DRs. */
vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR);
/* Update niters for vector loop. */
prologue of the main loop. */
static void
-update_epilogue_loop_vinfo (class loop *epilogue, tree advance,
- drs_init_vec &orig_drs_init)
+update_epilogue_loop_vinfo (class loop *epilogue, tree advance)
{
loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue);
auto_vec<gimple *> stmt_worklist;
gphi_iterator epilogue_phi_gsi;
stmt_vec_info stmt_vinfo = NULL, related_vinfo;
basic_block *epilogue_bbs = get_loop_body (epilogue);
+ unsigned i;
LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs;
- /* Restore original data_reference's offset, before the previous loop and its
- prologue. */
- std::pair<data_reference*, tree> *dr_init;
- unsigned i;
- for (i = 0; orig_drs_init.iterate (i, &dr_init); i++)
- DR_OFFSET (dr_init->first) = dr_init->second;
-
/* Advance data_reference's with the number of iterations of the previous
loop and its prologue. */
vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR);
epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector,
&step_vector, &niters_vector_mult_vf, th,
check_profitability, niters_no_overflow,
- &advance, orig_drs_init);
+ &advance);
if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo)
&& LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ())
if (epilogue)
{
- update_epilogue_loop_vinfo (epilogue, advance, orig_drs_init);
+ update_epilogue_loop_vinfo (epilogue, advance);
epilogue->simduid = loop->simduid;
epilogue->force_vectorize = loop->force_vectorize;
|| loop_vinfo == NULL
|| LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
|| STMT_VINFO_GROUPED_ACCESS (stmt_info)
- || !integer_zerop (DR_OFFSET (dr_info->dr))
+ || !integer_zerop (get_dr_vinfo_offset (dr_info))
|| !integer_zerop (DR_INIT (dr_info->dr))
|| !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)))
|| !alias_sets_conflict_p (get_alias_set (vectype),
tree running_off;
tree stride_base, stride_step, alias_off;
tree vec_oprnd;
+ tree dr_offset;
unsigned int g;
/* Checked by get_load_store_type. */
unsigned int const_nunits = nunits.to_constant ();
gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
gcc_assert (!nested_in_vect_loop_p (loop, stmt_info));
+ dr_offset = get_dr_vinfo_offset (first_dr_info);
stride_base
= fold_build_pointer_plus
(DR_BASE_ADDRESS (first_dr_info->dr),
size_binop (PLUS_EXPR,
- convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)),
+ convert_to_ptrofftype (dr_offset),
convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
&& !loop_masks
&& TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
&& VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
- && integer_zerop (DR_OFFSET (first_dr_info->dr))
+ && integer_zerop (get_dr_vinfo_offset (first_dr_info))
&& integer_zerop (DR_INIT (first_dr_info->dr))
&& alias_sets_conflict_p (get_alias_set (aggr_type),
get_alias_set (TREE_TYPE (ref_type))))
/* Checked by get_load_store_type. */
unsigned int const_nunits = nunits.to_constant ();
unsigned HOST_WIDE_INT cst_offset = 0;
+ tree dr_offset;
gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo));
gcc_assert (!nested_in_vect_loop);
ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr));
}
+ dr_offset = get_dr_vinfo_offset (first_dr_info);
stride_base
= fold_build_pointer_plus
(DR_BASE_ADDRESS (first_dr_info->dr),
size_binop (PLUS_EXPR,
- convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)),
+ convert_to_ptrofftype (dr_offset),
convert_to_ptrofftype (DR_INIT (first_dr_info->dr))));
stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr));
if (simd_lane_access_p
&& TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR
&& VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0))
- && integer_zerop (DR_OFFSET (first_dr_info->dr))
+ && integer_zerop (get_dr_vinfo_offset (first_dr_info))
&& integer_zerop (DR_INIT (first_dr_info->dr))
&& alias_sets_conflict_p (get_alias_set (aggr_type),
get_alias_set (TREE_TYPE (ref_type)))
/* If true the alignment of base_decl needs to be increased. */
bool base_misaligned;
tree base_decl;
+
+ /* Stores current vectorized loop's offset. To be added to the DR's
+ offset to calculate current offset of data reference. */
+ tree offset;
};
typedef struct data_reference *dr_p;
return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
}
+inline tree
+get_dr_vinfo_offset (dr_vec_info *dr_info, bool check_outer = false)
+{
+ innermost_loop_behavior *base;
+ if (check_outer)
+ base = vect_dr_behavior (dr_info);
+ else
+ base = &dr_info->dr->innermost;
+
+ tree offset = base->offset;
+
+ if (!dr_info->offset)
+ return offset;
+
+ offset = fold_convert (sizetype, offset);
+ return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
+ dr_info->offset);
+}
+
+
/* Return true if the vect cost model is unlimited. */
static inline bool
unlimited_cost_model (loop_p loop)
class loop *vect_loop_versioning (loop_vec_info);
extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
tree *, tree *, tree *, int, bool, bool,
- tree *, drs_init_vec &);
+ tree *);
extern void vect_prepare_for_masked_peels (loop_vec_info);
extern dump_user_location_t find_loop_location (class loop *);
extern bool vect_can_advance_ivs_p (loop_vec_info);
projects / gcc.git / commitdiff