From 67723321fb917e94acf5844c2524ca3d9655ff7b Mon Sep 17 00:00:00 2001 From: Andre Vieira Date: Fri, 10 Jan 2020 13:33:10 +0000 Subject: [PATCH] [vect] Keep track of DR_OFFSET advance in dr_vec_info rather than data_reference gcc/ChangeLog: 2020-01-10 Andre Vieira * tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref): Use get_dr_vinfo_offset * tree-vect-loop.c (update_epilogue_loop_vinfo): Remove orig_drs_init parameter and its use to reset DR_OFFSET's. (vect_transform_loop): Remove orig_drs_init argument. * tree-vect-loop-manip.c (vect_update_init_of_dr): Update the offset member of dr_vec_info rather than the offset of the associated data_reference's innermost_loop_behavior. (vect_update_init_of_dr): Pass dr_vec_info instead of data_reference. (vect_do_peeling): Remove orig_drs_init parameter and its construction. * tree-vect-stmts.c (check_scan_store): Replace use of DR_OFFSET with get_dr_vinfo_offset. (vectorizable_store): Likewise. (vectorizable_load): Likewise. From-SVN: r280107 --- gcc/ChangeLog | 17 +++++++++++++++++ gcc/tree-vect-data-refs.c | 2 +- gcc/tree-vect-loop-manip.c | 23 +++++++++-------------- gcc/tree-vect-loop.c | 15 ++++----------- gcc/tree-vect-stmts.c | 14 +++++++++----- gcc/tree-vectorizer.h | 26 +++++++++++++++++++++++++- 6 files changed, 65 insertions(+), 32 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 7f4b7fba724..e8ea7b79f4a 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,20 @@ +2020-01-10 Andre Vieira + + * tree-vect-data-refs.c (vect_create_addr_base_for_vector_ref): Use + get_dr_vinfo_offset + * tree-vect-loop.c (update_epilogue_loop_vinfo): Remove orig_drs_init + parameter and its use to reset DR_OFFSET's. + (vect_transform_loop): Remove orig_drs_init argument. + * tree-vect-loop-manip.c (vect_update_init_of_dr): Update the offset + member of dr_vec_info rather than the offset of the associated + data_reference's innermost_loop_behavior. + (vect_update_init_of_dr): Pass dr_vec_info instead of data_reference. + (vect_do_peeling): Remove orig_drs_init parameter and its construction. + * tree-vect-stmts.c (check_scan_store): Replace use of DR_OFFSET with + get_dr_vinfo_offset. + (vectorizable_store): Likewise. + (vectorizable_load): Likewise. + 2020-01-10 Richard Biener * gimple-ssa-store-merging diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 594d7e7d28f..554ef892254 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -4597,7 +4597,7 @@ vect_create_addr_base_for_vector_ref (stmt_vec_info stmt_info, innermost_loop_behavior *drb = vect_dr_behavior (dr_info); tree data_ref_base = unshare_expr (drb->base_address); - tree base_offset = unshare_expr (drb->offset); + tree base_offset = unshare_expr (get_dr_vinfo_offset (dr_info, true)); tree init = unshare_expr (drb->init); if (loop_vinfo) diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index ead8354176b..52ca4e3e56c 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -1716,19 +1716,22 @@ vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo, iterations before the scalar one (using masking to skip inactive elements). This function updates the information recorded in DR to account for the difference. Specifically, it updates the OFFSET - field of DR. */ + field of DR_INFO. */ static void -vect_update_init_of_dr (struct data_reference *dr, tree niters, tree_code code) +vect_update_init_of_dr (dr_vec_info *dr_info, tree niters, tree_code code) { - tree offset = DR_OFFSET (dr); + struct data_reference *dr = dr_info->dr; + tree offset = dr_info->offset; + if (!offset) + offset = build_zero_cst (sizetype); niters = fold_build2 (MULT_EXPR, sizetype, fold_convert (sizetype, niters), fold_convert (sizetype, DR_STEP (dr))); offset = fold_build2 (code, sizetype, fold_convert (sizetype, offset), niters); - DR_OFFSET (dr) = offset; + dr_info->offset = offset; } @@ -1758,7 +1761,7 @@ vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters, { dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr); if (!STMT_VINFO_GATHER_SCATTER_P (dr_info->stmt)) - vect_update_init_of_dr (dr, niters, code); + vect_update_init_of_dr (dr_info, niters, code); } } @@ -2446,7 +2449,7 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, tree *niters_vector, tree *step_vector, tree *niters_vector_mult_vf_var, int th, bool check_profitability, bool niters_no_overflow, - tree *advance, drs_init_vec &orig_drs_init) + tree *advance) { edge e, guard_e; tree type = TREE_TYPE (niters), guard_cond; @@ -2694,14 +2697,6 @@ vect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, scale_loop_profile (prolog, prob_prolog, bound_prolog); } - /* Save original inits for each data_reference before advancing them with - NITERS_PROLOG. */ - unsigned int i; - struct data_reference *dr; - vec datarefs = loop_vinfo->shared->datarefs; - FOR_EACH_VEC_ELT (datarefs, i, dr) - orig_drs_init.safe_push (std::make_pair (dr, DR_OFFSET (dr))); - /* Update init address of DRs. */ vect_update_inits_of_drs (loop_vinfo, niters_prolog, PLUS_EXPR); /* Update niters for vector loop. */ diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index cf9b7bcb9c4..596eafc2d3f 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -8333,8 +8333,7 @@ find_in_mapping (tree t, void *context) prologue of the main loop. */ static void -update_epilogue_loop_vinfo (class loop *epilogue, tree advance, - drs_init_vec &orig_drs_init) +update_epilogue_loop_vinfo (class loop *epilogue, tree advance) { loop_vec_info epilogue_vinfo = loop_vec_info_for_loop (epilogue); auto_vec stmt_worklist; @@ -8344,16 +8343,10 @@ update_epilogue_loop_vinfo (class loop *epilogue, tree advance, gphi_iterator epilogue_phi_gsi; stmt_vec_info stmt_vinfo = NULL, related_vinfo; basic_block *epilogue_bbs = get_loop_body (epilogue); + unsigned i; LOOP_VINFO_BBS (epilogue_vinfo) = epilogue_bbs; - /* Restore original data_reference's offset, before the previous loop and its - prologue. */ - std::pair *dr_init; - unsigned i; - for (i = 0; orig_drs_init.iterate (i, &dr_init); i++) - DR_OFFSET (dr_init->first) = dr_init->second; - /* Advance data_reference's with the number of iterations of the previous loop and its prologue. */ vect_update_inits_of_drs (epilogue_vinfo, advance, PLUS_EXPR); @@ -8569,7 +8562,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) epilogue = vect_do_peeling (loop_vinfo, niters, nitersm1, &niters_vector, &step_vector, &niters_vector_mult_vf, th, check_profitability, niters_no_overflow, - &advance, orig_drs_init); + &advance); if (LOOP_VINFO_SCALAR_LOOP (loop_vinfo) && LOOP_VINFO_SCALAR_LOOP_SCALING (loop_vinfo).initialized_p ()) @@ -8828,7 +8821,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) if (epilogue) { - update_epilogue_loop_vinfo (epilogue, advance, orig_drs_init); + update_epilogue_loop_vinfo (epilogue, advance); epilogue->simduid = loop->simduid; epilogue->force_vectorize = loop->force_vectorize; diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 0101d4c5bf8..df2899b4763 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -6629,7 +6629,7 @@ check_scan_store (stmt_vec_info stmt_info, tree vectype, || loop_vinfo == NULL || LOOP_VINFO_FULLY_MASKED_P (loop_vinfo) || STMT_VINFO_GROUPED_ACCESS (stmt_info) - || !integer_zerop (DR_OFFSET (dr_info->dr)) + || !integer_zerop (get_dr_vinfo_offset (dr_info)) || !integer_zerop (DR_INIT (dr_info->dr)) || !(ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr))) || !alias_sets_conflict_p (get_alias_set (vectype), @@ -7762,6 +7762,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, tree running_off; tree stride_base, stride_step, alias_off; tree vec_oprnd; + tree dr_offset; unsigned int g; /* Checked by get_load_store_type. */ unsigned int const_nunits = nunits.to_constant (); @@ -7769,11 +7770,12 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); gcc_assert (!nested_in_vect_loop_p (loop, stmt_info)); + dr_offset = get_dr_vinfo_offset (first_dr_info); stride_base = fold_build_pointer_plus (DR_BASE_ADDRESS (first_dr_info->dr), size_binop (PLUS_EXPR, - convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)), + convert_to_ptrofftype (dr_offset), convert_to_ptrofftype (DR_INIT (first_dr_info->dr)))); stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr)); @@ -8136,7 +8138,7 @@ vectorizable_store (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, && !loop_masks && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) - && integer_zerop (DR_OFFSET (first_dr_info->dr)) + && integer_zerop (get_dr_vinfo_offset (first_dr_info)) && integer_zerop (DR_INIT (first_dr_info->dr)) && alias_sets_conflict_p (get_alias_set (aggr_type), get_alias_set (TREE_TYPE (ref_type)))) @@ -8830,6 +8832,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, /* Checked by get_load_store_type. */ unsigned int const_nunits = nunits.to_constant (); unsigned HOST_WIDE_INT cst_offset = 0; + tree dr_offset; gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); gcc_assert (!nested_in_vect_loop); @@ -8860,11 +8863,12 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, ref_type = reference_alias_ptr_type (DR_REF (dr_info->dr)); } + dr_offset = get_dr_vinfo_offset (first_dr_info); stride_base = fold_build_pointer_plus (DR_BASE_ADDRESS (first_dr_info->dr), size_binop (PLUS_EXPR, - convert_to_ptrofftype (DR_OFFSET (first_dr_info->dr)), + convert_to_ptrofftype (dr_offset), convert_to_ptrofftype (DR_INIT (first_dr_info->dr)))); stride_step = fold_convert (sizetype, DR_STEP (first_dr_info->dr)); @@ -9329,7 +9333,7 @@ vectorizable_load (stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, if (simd_lane_access_p && TREE_CODE (DR_BASE_ADDRESS (first_dr_info->dr)) == ADDR_EXPR && VAR_P (TREE_OPERAND (DR_BASE_ADDRESS (first_dr_info->dr), 0)) - && integer_zerop (DR_OFFSET (first_dr_info->dr)) + && integer_zerop (get_dr_vinfo_offset (first_dr_info)) && integer_zerop (DR_INIT (first_dr_info->dr)) && alias_sets_conflict_p (get_alias_set (aggr_type), get_alias_set (TREE_TYPE (ref_type))) diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 68cf96f6766..58c12323fc9 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -910,6 +910,10 @@ public: /* If true the alignment of base_decl needs to be increased. */ bool base_misaligned; tree base_decl; + + /* Stores current vectorized loop's offset. To be added to the DR's + offset to calculate current offset of data reference. */ + tree offset; }; typedef struct data_reference *dr_p; @@ -1485,6 +1489,26 @@ vect_dr_behavior (dr_vec_info *dr_info) return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info); } +inline tree +get_dr_vinfo_offset (dr_vec_info *dr_info, bool check_outer = false) +{ + innermost_loop_behavior *base; + if (check_outer) + base = vect_dr_behavior (dr_info); + else + base = &dr_info->dr->innermost; + + tree offset = base->offset; + + if (!dr_info->offset) + return offset; + + offset = fold_convert (sizetype, offset); + return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset, + dr_info->offset); +} + + /* Return true if the vect cost model is unlimited. */ static inline bool unlimited_cost_model (loop_p loop) @@ -1655,7 +1679,7 @@ class loop *slpeel_tree_duplicate_loop_to_edge_cfg (class loop *, class loop *vect_loop_versioning (loop_vec_info); extern class loop *vect_do_peeling (loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, - tree *, drs_init_vec &); + tree *); extern void vect_prepare_for_masked_peels (loop_vec_info); extern dump_user_location_t find_loop_location (class loop *); extern bool vect_can_advance_ivs_p (loop_vec_info); -- 2.30.2