From: Richard Sandiford Date: Mon, 3 Jul 2017 13:36:13 +0000 (+0000) Subject: Use innermost_loop_behavior for outer loop vectorisation X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=3f5e8a7690d91daba3d8988318895fe51b7749ec;p=gcc.git Use innermost_loop_behavior for outer loop vectorisation This patch replaces the individual stmt_vinfo dr_* fields with an innermost_loop_behavior, so that the changes in later patches get picked up automatically. It also adds a helper function for getting the behavior of a data reference wrt the vectorised loop. 2017-07-03 Richard Sandiford gcc/ * tree-vectorizer.h (_stmt_vec_info): Replace individual dr_* fields with dr_wrt_vec_loop. (STMT_VINFO_DR_BASE_ADDRESS, STMT_VINFO_DR_INIT, STMT_VINFO_DR_OFFSET) (STMT_VINFO_DR_STEP, STMT_VINFO_DR_ALIGNED_TO): Update accordingly. (STMT_VINFO_DR_WRT_VEC_LOOP): New macro. (vect_dr_behavior): New function. (vect_create_addr_base_for_vector_ref): Remove loop parameter. * tree-vect-data-refs.c (vect_compute_data_ref_alignment): Use vect_dr_behavior. Use a step_preserves_misalignment_p boolean to track whether the step preserves the misalignment. (vect_create_addr_base_for_vector_ref): Remove loop parameter. Use vect_dr_behavior. (vect_setup_realignment): Update call accordingly. (vect_create_data_ref_ptr): Likewise. Use vect_dr_behavior. * tree-vect-loop-manip.c (vect_gen_prolog_loop_niters): Update call to vect_create_addr_base_for_vector_ref. (vect_create_cond_for_align_checks): Likewise. * tree-vect-patterns.c (vect_recog_bool_pattern): Copy STMT_VINFO_DR_WRT_VEC_LOOP as a block. (vect_recog_mask_conversion_pattern): Likewise. * tree-vect-stmts.c (compare_step_with_zero): Use vect_dr_behavior. (new_stmt_vec_info): Remove redundant zeroing. From-SVN: r249911 --- diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 05aaba91b68..a0db67d6890 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,28 @@ +2017-07-03 Richard Sandiford + + * tree-vectorizer.h (_stmt_vec_info): Replace individual dr_* + fields with dr_wrt_vec_loop. + (STMT_VINFO_DR_BASE_ADDRESS, STMT_VINFO_DR_INIT, STMT_VINFO_DR_OFFSET) + (STMT_VINFO_DR_STEP, STMT_VINFO_DR_ALIGNED_TO): Update accordingly. + (STMT_VINFO_DR_WRT_VEC_LOOP): New macro. + (vect_dr_behavior): New function. + (vect_create_addr_base_for_vector_ref): Remove loop parameter. + * tree-vect-data-refs.c (vect_compute_data_ref_alignment): Use + vect_dr_behavior. Use a step_preserves_misalignment_p boolean to + track whether the step preserves the misalignment. + (vect_create_addr_base_for_vector_ref): Remove loop parameter. + Use vect_dr_behavior. + (vect_setup_realignment): Update call accordingly. + (vect_create_data_ref_ptr): Likewise. Use vect_dr_behavior. + * tree-vect-loop-manip.c (vect_gen_prolog_loop_niters): Update + call to vect_create_addr_base_for_vector_ref. + (vect_create_cond_for_align_checks): Likewise. + * tree-vect-patterns.c (vect_recog_bool_pattern): Copy + STMT_VINFO_DR_WRT_VEC_LOOP as a block. + (vect_recog_mask_conversion_pattern): Likewise. + * tree-vect-stmts.c (compare_step_with_zero): Use vect_dr_behavior. + (new_stmt_vec_info): Remove redundant zeroing. + 2017-07-02 Richard Earnshaw * common/config/arm/arm-common.c (arm_be8_option): New function. diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 1595bb4247c..3b4d570cc6a 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -666,11 +666,8 @@ vect_compute_data_ref_alignment (struct data_reference *dr) loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); struct loop *loop = NULL; tree ref = DR_REF (dr); - tree vectype; - tree base, base_addr; - tree misalign = NULL_TREE; - tree aligned_to; - tree step; + tree vectype = STMT_VINFO_VECTYPE (stmt_info); + tree base; unsigned HOST_WIDE_INT alignment; if (dump_enabled_p ()) @@ -683,11 +680,15 @@ vect_compute_data_ref_alignment (struct data_reference *dr) /* Initialize misalignment to unknown. */ SET_DR_MISALIGNMENT (dr, DR_MISALIGNMENT_UNKNOWN); - if (tree_fits_shwi_p (DR_STEP (dr))) - misalign = DR_INIT (dr); - aligned_to = DR_ALIGNED_TO (dr); - base_addr = DR_BASE_ADDRESS (dr); - vectype = STMT_VINFO_VECTYPE (stmt_info); + innermost_loop_behavior *drb = vect_dr_behavior (dr); + bool step_preserves_misalignment_p; + + /* No step for BB vectorization. */ + if (!loop) + { + gcc_assert (integer_zerop (drb->step)); + step_preserves_misalignment_p = true; + } /* In case the dataref is in an inner-loop of the loop that is being vectorized (LOOP), we use the base and misalignment information @@ -695,26 +696,21 @@ vect_compute_data_ref_alignment (struct data_reference *dr) stays the same throughout the execution of the inner-loop, which is why we have to check that the stride of the dataref in the inner-loop evenly divides by the vector size. */ - if (loop && nested_in_vect_loop_p (loop, stmt)) + else if (nested_in_vect_loop_p (loop, stmt)) { tree step = DR_STEP (dr); + step_preserves_misalignment_p + = (tree_fits_shwi_p (step) + && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0); - if (tree_fits_shwi_p (step) - && tree_to_shwi (step) % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_NOTE, vect_location, - "inner step divides the vector-size.\n"); - misalign = STMT_VINFO_DR_INIT (stmt_info); - aligned_to = STMT_VINFO_DR_ALIGNED_TO (stmt_info); - base_addr = STMT_VINFO_DR_BASE_ADDRESS (stmt_info); - } - else + if (dump_enabled_p ()) { - if (dump_enabled_p ()) + if (step_preserves_misalignment_p) + dump_printf_loc (MSG_NOTE, vect_location, + "inner step divides the vector-size.\n"); + else dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "inner step doesn't divide the vector-size.\n"); - misalign = NULL_TREE; + "inner step doesn't divide the vector-size.\n"); } } @@ -725,18 +721,17 @@ vect_compute_data_ref_alignment (struct data_reference *dr) else { tree step = DR_STEP (dr); - unsigned vf = loop ? LOOP_VINFO_VECT_FACTOR (loop_vinfo) : 1; + unsigned vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo); + step_preserves_misalignment_p + = (tree_fits_shwi_p (step) + && ((tree_to_shwi (step) * vf) + % GET_MODE_SIZE (TYPE_MODE (vectype)) == 0)); - if (tree_fits_shwi_p (step) - && ((tree_to_shwi (step) * vf) - % GET_MODE_SIZE (TYPE_MODE (vectype)) != 0)) - { - if (dump_enabled_p ()) - dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, - "step doesn't divide the vector-size.\n"); - misalign = NULL_TREE; - } + if (!step_preserves_misalignment_p && dump_enabled_p ()) + dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location, + "step doesn't divide the vector-size.\n"); } + tree base_addr = drb->base_address; /* To look at alignment of the base we have to preserve an inner MEM_REF as that carries alignment information of the actual access. */ @@ -777,8 +772,8 @@ vect_compute_data_ref_alignment (struct data_reference *dr) alignment = TYPE_ALIGN_UNIT (vectype); - if ((compare_tree_int (aligned_to, alignment) < 0) - || !misalign) + if ((compare_tree_int (drb->aligned_to, alignment) < 0) + || !step_preserves_misalignment_p) { if (dump_enabled_p ()) { @@ -835,19 +830,16 @@ vect_compute_data_ref_alignment (struct data_reference *dr) DR_VECT_AUX (dr)->base_element_aligned = true; } - if (loop && nested_in_vect_loop_p (loop, stmt)) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (dr); /* If this is a backward running DR then first access in the larger vectype actually is N-1 elements before the address in the DR. Adjust misalign accordingly. */ - if (tree_int_cst_sgn (step) < 0) + tree misalign = drb->init; + if (tree_int_cst_sgn (drb->step) < 0) { tree offset = ssize_int (TYPE_VECTOR_SUBPARTS (vectype) - 1); /* DR_STEP(dr) is the same as -TYPE_SIZE of the scalar type, otherwise we wouldn't be here. */ - offset = fold_build2 (MULT_EXPR, ssizetype, offset, step); + offset = fold_build2 (MULT_EXPR, ssizetype, offset, drb->step); /* PLUS because STEP was negative. */ misalign = size_binop (PLUS_EXPR, misalign, offset); } @@ -3973,38 +3965,22 @@ tree vect_create_addr_base_for_vector_ref (gimple *stmt, gimple_seq *new_stmt_list, tree offset, - struct loop *loop, tree byte_offset) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); - tree data_ref_base; const char *base_name; tree addr_base; tree dest; gimple_seq seq = NULL; - tree base_offset; - tree init; tree vect_ptr_type; tree step = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dr))); loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + innermost_loop_behavior *drb = vect_dr_behavior (dr); - if (loop_vinfo && loop && loop != (gimple_bb (stmt))->loop_father) - { - struct loop *outer_loop = LOOP_VINFO_LOOP (loop_vinfo); - - gcc_assert (nested_in_vect_loop_p (outer_loop, stmt)); - - data_ref_base = unshare_expr (STMT_VINFO_DR_BASE_ADDRESS (stmt_info)); - base_offset = unshare_expr (STMT_VINFO_DR_OFFSET (stmt_info)); - init = unshare_expr (STMT_VINFO_DR_INIT (stmt_info)); - } - else - { - data_ref_base = unshare_expr (DR_BASE_ADDRESS (dr)); - base_offset = unshare_expr (DR_OFFSET (dr)); - init = unshare_expr (DR_INIT (dr)); - } + tree data_ref_base = unshare_expr (drb->base_address); + tree base_offset = unshare_expr (drb->offset); + tree init = unshare_expr (drb->init); if (loop_vinfo) base_name = get_name (data_ref_base); @@ -4169,11 +4145,7 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop, /* Check the step (evolution) of the load in LOOP, and record whether it's invariant. */ - if (nested_in_vect_loop) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); - + step = vect_dr_behavior (dr)->step; if (integer_zerop (step)) *inv_p = true; else @@ -4271,7 +4243,7 @@ vect_create_data_ref_ptr (gimple *stmt, tree aggr_type, struct loop *at_loop, /* Create: (&(base[init_val+offset]+byte_offset) in the loop preheader. */ new_temp = vect_create_addr_base_for_vector_ref (stmt, &new_stmt_list, - offset, loop, byte_offset); + offset, byte_offset); if (new_stmt_list) { if (pe) @@ -4985,7 +4957,7 @@ vect_setup_realignment (gimple *stmt, gimple_stmt_iterator *gsi, { /* Generate the INIT_ADDR computation outside LOOP. */ init_addr = vect_create_addr_base_for_vector_ref (stmt, &stmts, - NULL_TREE, loop); + NULL_TREE); if (loop) { pe = loop_preheader_edge (loop); diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index 95b7ea06d28..aa52c50f531 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -949,7 +949,6 @@ vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo, basic_block bb, int *bound) { struct data_reference *dr = LOOP_VINFO_UNALIGNED_DR (loop_vinfo); - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); tree var; tree niters_type = TREE_TYPE (LOOP_VINFO_NITERS (loop_vinfo)); gimple_seq stmts = NULL, new_stmts = NULL; @@ -977,7 +976,7 @@ vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo, tree offset = negative ? size_int (-TYPE_VECTOR_SUBPARTS (vectype) + 1) : size_zero_node; tree start_addr = vect_create_addr_base_for_vector_ref (dr_stmt, - &stmts, offset, loop); + &stmts, offset); tree type = unsigned_type_for (TREE_TYPE (start_addr)); tree vectype_align_minus_1 = build_int_cst (type, vectype_align - 1); HOST_WIDE_INT elem_size = @@ -1975,7 +1974,6 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo, tree *cond_expr, gimple_seq *cond_expr_stmt_list) { - struct loop *loop = LOOP_VINFO_LOOP (loop_vinfo); vec may_misalign_stmts = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo); gimple *ref_stmt; @@ -2016,7 +2014,7 @@ vect_create_cond_for_align_checks (loop_vec_info loop_vinfo, /* create: addr_tmp = (int)(address_of_first_vector) */ addr_base = vect_create_addr_base_for_vector_ref (ref_stmt, &new_stmt_list, - offset, loop); + offset); if (new_stmt_list != NULL) gimple_seq_add_seq (cond_expr_stmt_list, new_stmt_list); diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index 39f01337608..17d10835038 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -3789,14 +3789,8 @@ vect_recog_bool_pattern (vec *stmts, tree *type_in, set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); STMT_VINFO_DATA_REF (pattern_stmt_info) = STMT_VINFO_DATA_REF (stmt_vinfo); - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); - STMT_VINFO_DR_OFFSET (pattern_stmt_info) - = STMT_VINFO_DR_OFFSET (stmt_vinfo); - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; *type_out = vectype; *type_in = vectype; @@ -3930,14 +3924,8 @@ vect_recog_mask_conversion_pattern (vec *stmts, tree *type_in, set_vinfo_for_stmt (pattern_stmt, pattern_stmt_info); STMT_VINFO_DATA_REF (pattern_stmt_info) = STMT_VINFO_DATA_REF (stmt_vinfo); - STMT_VINFO_DR_BASE_ADDRESS (pattern_stmt_info) - = STMT_VINFO_DR_BASE_ADDRESS (stmt_vinfo); - STMT_VINFO_DR_INIT (pattern_stmt_info) = STMT_VINFO_DR_INIT (stmt_vinfo); - STMT_VINFO_DR_OFFSET (pattern_stmt_info) - = STMT_VINFO_DR_OFFSET (stmt_vinfo); - STMT_VINFO_DR_STEP (pattern_stmt_info) = STMT_VINFO_DR_STEP (stmt_vinfo); - STMT_VINFO_DR_ALIGNED_TO (pattern_stmt_info) - = STMT_VINFO_DR_ALIGNED_TO (stmt_vinfo); + STMT_VINFO_DR_WRT_VEC_LOOP (pattern_stmt_info) + = STMT_VINFO_DR_WRT_VEC_LOOP (stmt_vinfo); DR_STMT (STMT_VINFO_DATA_REF (stmt_vinfo)) = pattern_stmt; *type_out = vectype1; diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index 99c071557d4..1067421d575 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1692,13 +1692,9 @@ static int compare_step_with_zero (gimple *stmt) { stmt_vec_info stmt_info = vinfo_for_stmt (stmt); - loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); - tree step; - if (loop_vinfo && nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) - step = STMT_VINFO_DR_STEP (stmt_info); - else - step = DR_STEP (STMT_VINFO_DATA_REF (stmt_info)); - return tree_int_cst_compare (step, size_zero_node); + data_reference *dr = STMT_VINFO_DATA_REF (stmt_info); + return tree_int_cst_compare (vect_dr_behavior (dr)->step, + size_zero_node); } /* If the target supports a permute mask that reverses the elements in @@ -8845,12 +8841,6 @@ new_stmt_vec_info (gimple *stmt, vec_info *vinfo) STMT_VINFO_VEC_REDUCTION_TYPE (res) = TREE_CODE_REDUCTION; STMT_VINFO_VEC_CONST_COND_REDUC_CODE (res) = ERROR_MARK; - STMT_VINFO_DR_BASE_ADDRESS (res) = NULL; - STMT_VINFO_DR_OFFSET (res) = NULL; - STMT_VINFO_DR_INIT (res) = NULL; - STMT_VINFO_DR_STEP (res) = NULL; - STMT_VINFO_DR_ALIGNED_TO (res) = NULL; - if (gimple_code (stmt) == GIMPLE_PHI && is_loop_header_bb_p (gimple_bb (stmt))) STMT_VINFO_DEF_TYPE (res) = vect_unknown_def_type; diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index c4a84e8298d..4a1e302fe36 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -554,11 +554,7 @@ typedef struct _stmt_vec_info { /* Information about the data-ref relative to this loop nest (the loop that is being considered for vectorization). */ - tree dr_base_address; - tree dr_init; - tree dr_offset; - tree dr_step; - tree dr_aligned_to; + innermost_loop_behavior dr_wrt_vec_loop; /* For loop PHI nodes, the base and evolution part of it. This makes sure this information is still available in vect_update_ivs_after_vectorizer @@ -708,11 +704,12 @@ STMT_VINFO_BB_VINFO (stmt_vec_info stmt_vinfo) #define STMT_VINFO_VEC_REDUCTION_TYPE(S) (S)->v_reduc_type #define STMT_VINFO_VEC_CONST_COND_REDUC_CODE(S) (S)->const_cond_reduc_code -#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_base_address -#define STMT_VINFO_DR_INIT(S) (S)->dr_init -#define STMT_VINFO_DR_OFFSET(S) (S)->dr_offset -#define STMT_VINFO_DR_STEP(S) (S)->dr_step -#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_aligned_to +#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop +#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address +#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init +#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset +#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step +#define STMT_VINFO_DR_ALIGNED_TO(S) (S)->dr_wrt_vec_loop.aligned_to #define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt @@ -1014,6 +1011,22 @@ known_alignment_for_access_p (struct data_reference *data_ref_info) return (DR_MISALIGNMENT (data_ref_info) != DR_MISALIGNMENT_UNKNOWN); } +/* Return the behavior of DR with respect to the vectorization context + (which for outer loop vectorization might not be the behavior recorded + in DR itself). */ + +static inline innermost_loop_behavior * +vect_dr_behavior (data_reference *dr) +{ + gimple *stmt = DR_STMT (dr); + stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + loop_vec_info loop_vinfo = STMT_VINFO_LOOP_VINFO (stmt_info); + if (loop_vinfo == NULL + || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt)) + return &DR_INNERMOST (dr); + else + return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info); +} /* Return true if the vect cost model is unlimited. */ static inline bool @@ -1144,8 +1157,7 @@ extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *); extern tree vect_get_new_ssa_name (tree, enum vect_var_kind, const char * = NULL); extern tree vect_create_addr_base_for_vector_ref (gimple *, gimple_seq *, - tree, struct loop *, - tree = NULL_TREE); + tree, tree = NULL_TREE); /* In tree-vect-loop.c. */ /* FORNOW: Used in tree-parloops.c. */