From e14c1050008ecece4bff253a54eac0f15a6467bb Mon Sep 17 00:00:00 2001 From: Ira Rosen Date: Wed, 18 May 2011 11:09:44 +0000 Subject: [PATCH] tree-vect-loop-manip.c (vect_create_cond_for_alias_checks): Use new names for group elements access. * tree-vect-loop-manip.c (vect_create_cond_for_alias_checks): Use new names for group elements access. * tree-vectorizer.h (struct _stmt_vec_info): Use interleaving info for reduction chains as well. Remove data reference and interleaving related words from the fields names. * tree-vect-loop.c (vect_transform_loop): Use new names for group elements access. * tree-vect-data-refs.c (vect_get_place_in_interleaving_chain, vect_insert_into_interleaving_chain, vect_update_interleaving_chain, vect_update_interleaving_chain, vect_same_range_drs, vect_analyze_data_ref_dependence, vect_update_misalignment_for_peel, vect_verify_datarefs_alignment, vector_alignment_reachable_p, vect_peeling_hash_get_lowest_cost, vect_enhance_data_refs_alignment, vect_analyze_group_access, vect_analyze_data_ref_access, vect_create_data_ref_ptr, vect_transform_strided_load, vect_record_strided_load_vectors): Likewise. * tree-vect-stmts.c (vect_model_simple_cost, vect_model_store_cost, vect_model_load_cost, vectorizable_store, vectorizable_load, vect_remove_stores, new_stmt_vec_info): Likewise. * tree-vect-slp.c (vect_build_slp_tree, vect_supported_slp_permutation_p, vect_analyze_slp_instance): Likewise. From-SVN: r173855 --- gcc/ChangeLog | 24 ++++++ gcc/tree-vect-data-refs.c | 154 ++++++++++++++++++------------------- gcc/tree-vect-loop-manip.c | 4 +- gcc/tree-vect-loop.c | 2 +- gcc/tree-vect-slp.c | 26 +++---- gcc/tree-vect-stmts.c | 59 +++++++------- gcc/tree-vectorizer.h | 48 ++++++------ 7 files changed, 171 insertions(+), 146 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 9ebf4aad204..89575eac4d9 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,27 @@ +2011-05-18 Ira Rosen + + * tree-vect-loop-manip.c (vect_create_cond_for_alias_checks): Use new + names for group elements access. + * tree-vectorizer.h (struct _stmt_vec_info): Use interleaving info for + reduction chains as well. Remove data reference and interleaving + related words from the fields names. + * tree-vect-loop.c (vect_transform_loop): Use new names for group + elements access. + * tree-vect-data-refs.c (vect_get_place_in_interleaving_chain, + vect_insert_into_interleaving_chain, vect_update_interleaving_chain, + vect_update_interleaving_chain, vect_same_range_drs, + vect_analyze_data_ref_dependence, vect_update_misalignment_for_peel, + vect_verify_datarefs_alignment, vector_alignment_reachable_p, + vect_peeling_hash_get_lowest_cost, vect_enhance_data_refs_alignment, + vect_analyze_group_access, vect_analyze_data_ref_access, + vect_create_data_ref_ptr, vect_transform_strided_load, + vect_record_strided_load_vectors): Likewise. + * tree-vect-stmts.c (vect_model_simple_cost, vect_model_store_cost, + vect_model_load_cost, vectorizable_store, vectorizable_load, + vect_remove_stores, new_stmt_vec_info): Likewise. + * tree-vect-slp.c (vect_build_slp_tree, + vect_supported_slp_permutation_p, vect_analyze_slp_instance): Likewise. + 2011-05-18 Richard Guenther PR middle-end/48989 diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 21586db400a..8d3b2533dc4 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -135,13 +135,13 @@ vect_get_place_in_interleaving_chain (gimple stmt, gimple first_stmt) gimple next_stmt = first_stmt; int result = 0; - if (first_stmt != DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt))) + if (first_stmt != GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt))) return -1; while (next_stmt && next_stmt != stmt) { result++; - next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt)); + next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); } if (next_stmt) @@ -164,25 +164,25 @@ vect_insert_into_interleaving_chain (struct data_reference *dra, stmt_vec_info stmtinfo_a = vinfo_for_stmt (DR_STMT (dra)); stmt_vec_info stmtinfo_b = vinfo_for_stmt (DR_STMT (drb)); - prev = DR_GROUP_FIRST_DR (stmtinfo_b); - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)); + prev = GROUP_FIRST_ELEMENT (stmtinfo_b); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)); while (next) { next_init = DR_INIT (STMT_VINFO_DATA_REF (vinfo_for_stmt (next))); if (tree_int_cst_compare (next_init, DR_INIT (dra)) > 0) { /* Insert here. */ - DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)) = DR_STMT (dra); - DR_GROUP_NEXT_DR (stmtinfo_a) = next; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)) = DR_STMT (dra); + GROUP_NEXT_ELEMENT (stmtinfo_a) = next; return; } prev = next; - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)); } /* We got to the end of the list. Insert here. */ - DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)) = DR_STMT (dra); - DR_GROUP_NEXT_DR (stmtinfo_a) = NULL; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)) = DR_STMT (dra); + GROUP_NEXT_ELEMENT (stmtinfo_a) = NULL; } @@ -221,27 +221,27 @@ vect_update_interleaving_chain (struct data_reference *drb, gimple node, prev, next, first_stmt; /* 1. New stmts - both DRA and DRB are not a part of any chain. */ - if (!DR_GROUP_FIRST_DR (stmtinfo_a) && !DR_GROUP_FIRST_DR (stmtinfo_b)) + if (!GROUP_FIRST_ELEMENT (stmtinfo_a) && !GROUP_FIRST_ELEMENT (stmtinfo_b)) { - DR_GROUP_FIRST_DR (stmtinfo_a) = DR_STMT (drb); - DR_GROUP_FIRST_DR (stmtinfo_b) = DR_STMT (drb); - DR_GROUP_NEXT_DR (stmtinfo_b) = DR_STMT (dra); + GROUP_FIRST_ELEMENT (stmtinfo_a) = DR_STMT (drb); + GROUP_FIRST_ELEMENT (stmtinfo_b) = DR_STMT (drb); + GROUP_NEXT_ELEMENT (stmtinfo_b) = DR_STMT (dra); return; } /* 2. DRB is a part of a chain and DRA is not. */ - if (!DR_GROUP_FIRST_DR (stmtinfo_a) && DR_GROUP_FIRST_DR (stmtinfo_b)) + if (!GROUP_FIRST_ELEMENT (stmtinfo_a) && GROUP_FIRST_ELEMENT (stmtinfo_b)) { - DR_GROUP_FIRST_DR (stmtinfo_a) = DR_GROUP_FIRST_DR (stmtinfo_b); + GROUP_FIRST_ELEMENT (stmtinfo_a) = GROUP_FIRST_ELEMENT (stmtinfo_b); /* Insert DRA into the chain of DRB. */ vect_insert_into_interleaving_chain (dra, drb); return; } /* 3. DRA is a part of a chain and DRB is not. */ - if (DR_GROUP_FIRST_DR (stmtinfo_a) && !DR_GROUP_FIRST_DR (stmtinfo_b)) + if (GROUP_FIRST_ELEMENT (stmtinfo_a) && !GROUP_FIRST_ELEMENT (stmtinfo_b)) { - gimple old_first_stmt = DR_GROUP_FIRST_DR (stmtinfo_a); + gimple old_first_stmt = GROUP_FIRST_ELEMENT (stmtinfo_a); tree init_old = DR_INIT (STMT_VINFO_DATA_REF (vinfo_for_stmt ( old_first_stmt))); gimple tmp; @@ -251,29 +251,29 @@ vect_update_interleaving_chain (struct data_reference *drb, /* DRB's init is smaller than the init of the stmt previously marked as the first stmt of the interleaving chain of DRA. Therefore, we update FIRST_STMT and put DRB in the head of the list. */ - DR_GROUP_FIRST_DR (stmtinfo_b) = DR_STMT (drb); - DR_GROUP_NEXT_DR (stmtinfo_b) = old_first_stmt; + GROUP_FIRST_ELEMENT (stmtinfo_b) = DR_STMT (drb); + GROUP_NEXT_ELEMENT (stmtinfo_b) = old_first_stmt; /* Update all the stmts in the list to point to the new FIRST_STMT. */ tmp = old_first_stmt; while (tmp) { - DR_GROUP_FIRST_DR (vinfo_for_stmt (tmp)) = DR_STMT (drb); - tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (tmp)); + GROUP_FIRST_ELEMENT (vinfo_for_stmt (tmp)) = DR_STMT (drb); + tmp = GROUP_NEXT_ELEMENT (vinfo_for_stmt (tmp)); } } else { /* Insert DRB in the list of DRA. */ vect_insert_into_interleaving_chain (drb, dra); - DR_GROUP_FIRST_DR (stmtinfo_b) = DR_GROUP_FIRST_DR (stmtinfo_a); + GROUP_FIRST_ELEMENT (stmtinfo_b) = GROUP_FIRST_ELEMENT (stmtinfo_a); } return; } /* 4. both DRA and DRB are in some interleaving chains. */ - first_a = DR_GROUP_FIRST_DR (stmtinfo_a); - first_b = DR_GROUP_FIRST_DR (stmtinfo_b); + first_a = GROUP_FIRST_ELEMENT (stmtinfo_a); + first_b = GROUP_FIRST_ELEMENT (stmtinfo_b); if (first_a == first_b) return; init_dra_chain = DR_INIT (STMT_VINFO_DATA_REF (vinfo_for_stmt (first_a))); @@ -284,8 +284,8 @@ vect_update_interleaving_chain (struct data_reference *drb, /* Insert the nodes of DRA chain into the DRB chain. After inserting a node, continue from this node of the DRB chain (don't start from the beginning. */ - node = DR_GROUP_FIRST_DR (stmtinfo_a); - prev = DR_GROUP_FIRST_DR (stmtinfo_b); + node = GROUP_FIRST_ELEMENT (stmtinfo_a); + prev = GROUP_FIRST_ELEMENT (stmtinfo_b); first_stmt = first_b; } else @@ -293,38 +293,38 @@ vect_update_interleaving_chain (struct data_reference *drb, /* Insert the nodes of DRB chain into the DRA chain. After inserting a node, continue from this node of the DRA chain (don't start from the beginning. */ - node = DR_GROUP_FIRST_DR (stmtinfo_b); - prev = DR_GROUP_FIRST_DR (stmtinfo_a); + node = GROUP_FIRST_ELEMENT (stmtinfo_b); + prev = GROUP_FIRST_ELEMENT (stmtinfo_a); first_stmt = first_a; } while (node) { node_init = DR_INIT (STMT_VINFO_DATA_REF (vinfo_for_stmt (node))); - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)); while (next) { next_init = DR_INIT (STMT_VINFO_DATA_REF (vinfo_for_stmt (next))); if (tree_int_cst_compare (next_init, node_init) > 0) { /* Insert here. */ - DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)) = node; - DR_GROUP_NEXT_DR (vinfo_for_stmt (node)) = next; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)) = node; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (node)) = next; prev = node; break; } prev = next; - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)); } if (!next) { /* We got to the end of the list. Insert here. */ - DR_GROUP_NEXT_DR (vinfo_for_stmt (prev)) = node; - DR_GROUP_NEXT_DR (vinfo_for_stmt (node)) = NULL; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (prev)) = node; + GROUP_NEXT_ELEMENT (vinfo_for_stmt (node)) = NULL; prev = node; } - DR_GROUP_FIRST_DR (vinfo_for_stmt (node)) = first_stmt; - node = DR_GROUP_NEXT_DR (vinfo_for_stmt (node)); + GROUP_FIRST_ELEMENT (vinfo_for_stmt (node)) = first_stmt; + node = GROUP_NEXT_ELEMENT (vinfo_for_stmt (node)); } } @@ -482,10 +482,10 @@ vect_same_range_drs (data_reference_p dr_i, data_reference_p dr_j) gimple stmt_j = DR_STMT (dr_j); if (operand_equal_p (DR_REF (dr_i), DR_REF (dr_j), 0) - || (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i)) - && DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j)) - && (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_i)) - == DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_j))))) + || (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_i)) + && GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_j)) + && (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_i)) + == GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_j))))) return true; else return false; @@ -685,11 +685,11 @@ vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr, /* For interleaving, mark that there is a read-write dependency if necessary. We check before that one of the data-refs is store. */ if (DR_IS_READ (dra)) - DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_a) = true; + GROUP_READ_WRITE_DEPENDENCE (stmtinfo_a) = true; else { if (DR_IS_READ (drb)) - DR_GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true; + GROUP_READ_WRITE_DEPENDENCE (stmtinfo_b) = true; } continue; @@ -992,9 +992,9 @@ vect_update_misalignment_for_peel (struct data_reference *dr, /* For interleaved data accesses the step in the loop must be multiplied by the size of the interleaving group. */ if (STMT_VINFO_STRIDED_ACCESS (stmt_info)) - dr_size *= DR_GROUP_SIZE (vinfo_for_stmt (DR_GROUP_FIRST_DR (stmt_info))); + dr_size *= GROUP_SIZE (vinfo_for_stmt (GROUP_FIRST_ELEMENT (stmt_info))); if (STMT_VINFO_STRIDED_ACCESS (peel_stmt_info)) - dr_peel_size *= DR_GROUP_SIZE (peel_stmt_info); + dr_peel_size *= GROUP_SIZE (peel_stmt_info); /* It can be assumed that the data refs with the same alignment as dr_peel are aligned in the vector loop. */ @@ -1054,7 +1054,7 @@ vect_verify_datarefs_alignment (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo) /* For interleaving, only the alignment of the first access matters. Skip statements marked as not vectorizable. */ if ((STMT_VINFO_STRIDED_ACCESS (stmt_info) - && DR_GROUP_FIRST_DR (stmt_info) != stmt) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt) || !STMT_VINFO_VECTORIZABLE (stmt_info)) continue; @@ -1109,7 +1109,7 @@ vector_alignment_reachable_p (struct data_reference *dr) elem_size = GET_MODE_SIZE (TYPE_MODE (vectype)) / nelements; mis_in_elements = DR_MISALIGNMENT (dr) / elem_size; - if ((nelements - mis_in_elements) % DR_GROUP_SIZE (stmt_info)) + if ((nelements - mis_in_elements) % GROUP_SIZE (stmt_info)) return false; } @@ -1282,7 +1282,7 @@ vect_peeling_hash_get_lowest_cost (void **slot, void *data) /* For interleaving, only the alignment of the first access matters. */ if (STMT_VINFO_STRIDED_ACCESS (stmt_info) - && DR_GROUP_FIRST_DR (stmt_info) != stmt) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt) continue; save_misalignment = DR_MISALIGNMENT (dr); @@ -1494,7 +1494,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) /* For interleaving, only the alignment of the first access matters. */ if (STMT_VINFO_STRIDED_ACCESS (stmt_info) - && DR_GROUP_FIRST_DR (stmt_info) != stmt) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt) continue; supportable_dr_alignment = vect_supportable_dr_alignment (dr, true); @@ -1732,7 +1732,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) by the group size. */ stmt_info = vinfo_for_stmt (DR_STMT (dr0)); if (STMT_VINFO_STRIDED_ACCESS (stmt_info)) - npeel /= DR_GROUP_SIZE (stmt_info); + npeel /= GROUP_SIZE (stmt_info); if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "Try peeling by %d", npeel); @@ -1751,7 +1751,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) /* For interleaving, only the alignment of the first access matters. */ if (STMT_VINFO_STRIDED_ACCESS (stmt_info) - && DR_GROUP_FIRST_DR (stmt_info) != stmt) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt) continue; save_misalignment = DR_MISALIGNMENT (dr); @@ -1833,7 +1833,7 @@ vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo) matters. */ if (aligned_access_p (dr) || (STMT_VINFO_STRIDED_ACCESS (stmt_info) - && DR_GROUP_FIRST_DR (stmt_info) != stmt)) + && GROUP_FIRST_ELEMENT (stmt_info) != stmt)) continue; supportable_dr_alignment = vect_supportable_dr_alignment (dr, false); @@ -2051,7 +2051,7 @@ vect_analyze_group_access (struct data_reference *dr) stride = dr_step / type_size; /* Not consecutive access is possible only if it is a part of interleaving. */ - if (!DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt))) + if (!GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt))) { /* Check if it this DR is a part of interleaving, and is a single element of the group that is accessed in the loop. */ @@ -2063,8 +2063,8 @@ vect_analyze_group_access (struct data_reference *dr) && stride > 0 && exact_log2 (stride) != -1) { - DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) = stmt; - DR_GROUP_SIZE (vinfo_for_stmt (stmt)) = stride; + GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = stmt; + GROUP_SIZE (vinfo_for_stmt (stmt)) = stride; if (vect_print_dump_info (REPORT_DR_DETAILS)) { fprintf (vect_dump, "Detected single element interleaving "); @@ -2091,10 +2091,10 @@ vect_analyze_group_access (struct data_reference *dr) return false; } - if (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) == stmt) + if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) == stmt) { /* First stmt in the interleaving chain. Check the chain. */ - gimple next = DR_GROUP_NEXT_DR (vinfo_for_stmt (stmt)); + gimple next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (stmt)); struct data_reference *data_ref = dr; unsigned int count = 1; tree next_step; @@ -2121,8 +2121,8 @@ vect_analyze_group_access (struct data_reference *dr) /* Check that there is no load-store dependencies for this loads to prevent a case of load-store-load to the same location. */ - if (DR_GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (next)) - || DR_GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (prev))) + if (GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (next)) + || GROUP_READ_WRITE_DEPENDENCE (vinfo_for_stmt (prev))) { if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, @@ -2131,10 +2131,10 @@ vect_analyze_group_access (struct data_reference *dr) } /* For load use the same data-ref load. */ - DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next)) = prev; + GROUP_SAME_DR_STMT (vinfo_for_stmt (next)) = prev; prev = next; - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); continue; } prev = next; @@ -2168,11 +2168,11 @@ vect_analyze_group_access (struct data_reference *dr) } /* Store the gap from the previous member of the group. If there is no - gap in the access, DR_GROUP_GAP is always 1. */ - DR_GROUP_GAP (vinfo_for_stmt (next)) = diff; + gap in the access, GROUP_GAP is always 1. */ + GROUP_GAP (vinfo_for_stmt (next)) = diff; prev_init = DR_INIT (data_ref); - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); /* Count the number of data-refs in the chain. */ count++; } @@ -2203,7 +2203,7 @@ vect_analyze_group_access (struct data_reference *dr) /* There is a gap after the last load in the group. This gap is a difference between the stride and the number of elements. When there is no gap, this difference should be 0. */ - DR_GROUP_GAP (vinfo_for_stmt (stmt)) = stride - count; + GROUP_GAP (vinfo_for_stmt (stmt)) = stride - count; } else { @@ -2230,7 +2230,7 @@ vect_analyze_group_access (struct data_reference *dr) if (stride == 0) stride = count; - DR_GROUP_SIZE (vinfo_for_stmt (stmt)) = stride; + GROUP_SIZE (vinfo_for_stmt (stmt)) = stride; if (vect_print_dump_info (REPORT_DETAILS)) fprintf (vect_dump, "Detected interleaving of size %d", (int)stride); @@ -2284,7 +2284,7 @@ vect_analyze_data_ref_access (struct data_reference *dr) { /* Interleaved accesses are not yet supported within outer-loop vectorization for references in the inner-loop. */ - DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) = NULL; + GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL; /* For the rest of the analysis we use the outer-loop step. */ step = STMT_VINFO_DR_STEP (stmt_info); @@ -2307,7 +2307,7 @@ vect_analyze_data_ref_access (struct data_reference *dr) && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step))) { /* Mark that it is not interleaving. */ - DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) = NULL; + GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) = NULL; return true; } @@ -3081,9 +3081,9 @@ vect_create_data_ref_ptr (gimple stmt, tree aggr_type, struct loop *at_loop, } /* Likewise for any of the data references in the stmt group. */ - else if (STMT_VINFO_DR_GROUP_SIZE (stmt_info) > 1) + else if (STMT_VINFO_GROUP_SIZE (stmt_info) > 1) { - gimple orig_stmt = STMT_VINFO_DR_GROUP_FIRST_DR (stmt_info); + gimple orig_stmt = STMT_VINFO_GROUP_FIRST_ELEMENT (stmt_info); do { tree lhs = gimple_assign_lhs (orig_stmt); @@ -3099,7 +3099,7 @@ vect_create_data_ref_ptr (gimple stmt, tree aggr_type, struct loop *at_loop, break; } - orig_stmt = STMT_VINFO_DR_GROUP_NEXT_DR (vinfo_for_stmt (orig_stmt)); + orig_stmt = STMT_VINFO_GROUP_NEXT_ELEMENT (vinfo_for_stmt (orig_stmt)); } while (orig_stmt); } @@ -4048,7 +4048,7 @@ vect_transform_strided_load (gimple stmt, VEC(tree,heap) *dr_chain, int size, void vect_record_strided_load_vectors (gimple stmt, VEC(tree,heap) *result_chain) { - gimple first_stmt = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)); + gimple first_stmt = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)); gimple next_stmt, new_stmt; unsigned int i, gap_count; tree tmp_data_ref; @@ -4066,11 +4066,11 @@ vect_record_strided_load_vectors (gimple stmt, VEC(tree,heap) *result_chain) /* Skip the gaps. Loads created for the gaps will be removed by dead code elimination pass later. No need to check for the first stmt in the group, since it always exists. - DR_GROUP_GAP is the number of steps in elements from the previous - access (if there is no gap DR_GROUP_GAP is 1). We skip loads that + GROUP_GAP is the number of steps in elements from the previous + access (if there is no gap GROUP_GAP is 1). We skip loads that correspond to the gaps. */ if (next_stmt != first_stmt - && gap_count < DR_GROUP_GAP (vinfo_for_stmt (next_stmt))) + && gap_count < GROUP_GAP (vinfo_for_stmt (next_stmt))) { gap_count++; continue; @@ -4086,7 +4086,7 @@ vect_record_strided_load_vectors (gimple stmt, VEC(tree,heap) *result_chain) STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)) = new_stmt; else { - if (!DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) + if (!GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) { gimple prev_stmt = STMT_VINFO_VEC_STMT (vinfo_for_stmt (next_stmt)); @@ -4104,12 +4104,12 @@ vect_record_strided_load_vectors (gimple stmt, VEC(tree,heap) *result_chain) } } - next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt)); + next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); gap_count = 1; /* If NEXT_STMT accesses the same DR as the previous statement, put the same TMP_DATA_REF as its vectorized statement; otherwise get the next data-ref from RESULT_CHAIN. */ - if (!next_stmt || !DR_GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) + if (!next_stmt || !GROUP_SAME_DR_STMT (vinfo_for_stmt (next_stmt))) break; } } diff --git a/gcc/tree-vect-loop-manip.c b/gcc/tree-vect-loop-manip.c index b691cd2a0c0..7619d74b910 100644 --- a/gcc/tree-vect-loop-manip.c +++ b/gcc/tree-vect-loop-manip.c @@ -2437,7 +2437,7 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, dr_a = DDR_A (ddr); stmt_a = DR_STMT (DDR_A (ddr)); - dr_group_first_a = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_a)); + dr_group_first_a = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_a)); if (dr_group_first_a) { stmt_a = dr_group_first_a; @@ -2446,7 +2446,7 @@ vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, dr_b = DDR_B (ddr); stmt_b = DR_STMT (DDR_B (ddr)); - dr_group_first_b = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt_b)); + dr_group_first_b = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt_b)); if (dr_group_first_b) { stmt_b = dr_group_first_b; diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index 8f3b8e61c64..d786e679e76 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4924,7 +4924,7 @@ vect_transform_loop (loop_vec_info loop_vinfo) /* Interleaving. If IS_STORE is TRUE, the vectorization of the interleaving chain was completed - free all the stores in the chain. */ - vect_remove_stores (DR_GROUP_FIRST_DR (stmt_info)); + vect_remove_stores (GROUP_FIRST_ELEMENT (stmt_info)); gsi_remove (&si, true); continue; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 25eaf3e6f05..8dd07dc4da9 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -509,10 +509,10 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, { /* Load. */ /* FORNOW: Check that there is no gap between the loads. */ - if ((DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) == stmt - && DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 0) - || (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) != stmt - && DR_GROUP_GAP (vinfo_for_stmt (stmt)) != 1)) + if ((GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) == stmt + && GROUP_GAP (vinfo_for_stmt (stmt)) != 0) + || (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) != stmt + && GROUP_GAP (vinfo_for_stmt (stmt)) != 1)) { if (vect_print_dump_info (REPORT_SLP)) { @@ -526,7 +526,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, /* Check that the size of interleaved loads group is not greater than the SLP group size. */ - if (DR_GROUP_SIZE (vinfo_for_stmt (stmt)) > ncopies * group_size) + if (GROUP_SIZE (vinfo_for_stmt (stmt)) > ncopies * group_size) { if (vect_print_dump_info (REPORT_SLP)) { @@ -539,7 +539,7 @@ vect_build_slp_tree (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, return false; } - first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)); + first_load = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)); if (prev_first_load) { /* Check that there are no loads from different interleaving @@ -784,7 +784,7 @@ vect_supported_slp_permutation_p (slp_instance instance) { slp_tree node = VEC_index (slp_tree, SLP_INSTANCE_LOADS (instance), 0); gimple stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (node), 0); - gimple first_load = DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)); + gimple first_load = GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)); VEC (slp_tree, heap) *sorted_loads = NULL; int index; slp_tree *tmp_loads = NULL; @@ -803,7 +803,7 @@ vect_supported_slp_permutation_p (slp_instance instance) { gimple scalar_stmt = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (load), 0); /* Check that the loads are all in the same interleaving chain. */ - if (DR_GROUP_FIRST_DR (vinfo_for_stmt (scalar_stmt)) != first_load) + if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (scalar_stmt)) != first_load) { if (vect_print_dump_info (REPORT_DETAILS)) { @@ -933,7 +933,7 @@ vect_supported_load_permutation_p (slp_instance slp_instn, int group_size, first = stmt; else { - if (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) != first) + if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) != first) { if (complex_numbers != 2) return false; @@ -948,7 +948,7 @@ vect_supported_load_permutation_p (slp_instance slp_instn, int group_size, other_node_first = VEC_index (gimple, SLP_TREE_SCALAR_STMTS (other_complex_node), 0); - if (DR_GROUP_FIRST_DR (vinfo_for_stmt (stmt)) + if (GROUP_FIRST_ELEMENT (vinfo_for_stmt (stmt)) != other_node_first) return false; } @@ -1142,7 +1142,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, { slp_instance new_instance; slp_tree node = XNEW (struct _slp_tree); - unsigned int group_size = DR_GROUP_SIZE (vinfo_for_stmt (stmt)); + unsigned int group_size = GROUP_SIZE (vinfo_for_stmt (stmt)); unsigned int unrolling_factor = 1, nunits; tree vectype, scalar_type = NULL_TREE; gimple next; @@ -1157,7 +1157,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, { scalar_type = TREE_TYPE (DR_REF (dr)); vectype = get_vectype_for_scalar_type (scalar_type); - group_size = DR_GROUP_SIZE (vinfo_for_stmt (stmt)); + group_size = GROUP_SIZE (vinfo_for_stmt (stmt)); } else { @@ -1204,7 +1204,7 @@ vect_analyze_slp_instance (loop_vec_info loop_vinfo, bb_vec_info bb_vinfo, while (next) { VEC_safe_push (gimple, heap, SLP_TREE_SCALAR_STMTS (node), next); - next = DR_GROUP_NEXT_DR (vinfo_for_stmt (next)); + next = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); } } else diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index f9164efb084..33aab9bc2ea 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -708,10 +708,10 @@ vect_model_simple_cost (stmt_vec_info stmt_info, int ncopies, static int vect_cost_strided_group_size (stmt_vec_info stmt_info) { - gimple first_stmt = DR_GROUP_FIRST_DR (stmt_info); + gimple first_stmt = GROUP_FIRST_ELEMENT (stmt_info); if (first_stmt == STMT_VINFO_STMT (stmt_info)) - return DR_GROUP_SIZE (stmt_info); + return GROUP_SIZE (stmt_info); return 1; } @@ -740,7 +740,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, outside_cost = vect_get_stmt_cost (scalar_to_vec); /* Strided access? */ - if (DR_GROUP_FIRST_DR (stmt_info)) + if (STMT_VINFO_STRIDED_ACCESS (stmt_info)) { if (slp_node) { @@ -749,7 +749,7 @@ vect_model_store_cost (stmt_vec_info stmt_info, int ncopies, } else { - first_stmt = DR_GROUP_FIRST_DR (stmt_info); + first_stmt = GROUP_FIRST_ELEMENT (stmt_info); group_size = vect_cost_strided_group_size (stmt_info); } @@ -855,8 +855,8 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p, return; /* Strided accesses? */ - first_stmt = DR_GROUP_FIRST_DR (stmt_info); - if (first_stmt && !slp_node) + first_stmt = GROUP_FIRST_ELEMENT (stmt_info); + if (STMT_VINFO_STRIDED_ACCESS (stmt_info) && first_stmt && !slp_node) { group_size = vect_cost_strided_group_size (stmt_info); first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); @@ -885,7 +885,8 @@ vect_model_load_cost (stmt_vec_info stmt_info, int ncopies, bool load_lanes_p, /* The loads themselves. */ vect_get_load_cost (first_dr, ncopies, - ((!DR_GROUP_FIRST_DR (stmt_info)) || group_size > 1 || slp_node), + ((!STMT_VINFO_STRIDED_ACCESS (stmt_info)) || group_size > 1 + || slp_node), &inside_cost, &outside_cost); if (vect_print_dump_info (REPORT_COST)) @@ -3515,10 +3516,10 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (STMT_VINFO_STRIDED_ACCESS (stmt_info)) { strided_store = true; - first_stmt = DR_GROUP_FIRST_DR (stmt_info); + first_stmt = GROUP_FIRST_ELEMENT (stmt_info); if (!slp && !PURE_SLP_STMT (stmt_info)) { - group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); + group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); if (vect_store_lanes_supported (vectype, group_size)) store_lanes_p = true; else if (!vect_strided_store_supported (vectype, group_size)) @@ -3529,7 +3530,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, { /* STMT is the leader of the group. Check the operands of all the stmts of the group. */ - next_stmt = DR_GROUP_NEXT_DR (stmt_info); + next_stmt = GROUP_NEXT_ELEMENT (stmt_info); while (next_stmt) { gcc_assert (gimple_assign_single_p (next_stmt)); @@ -3541,7 +3542,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, fprintf (vect_dump, "use not simple."); return false; } - next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt)); + next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); } } } @@ -3558,17 +3559,17 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (strided_store) { first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); - group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); + group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); - DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++; + GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt))++; /* FORNOW */ gcc_assert (!loop || !nested_in_vect_loop_p (loop, stmt)); /* We vectorize all the stmts of the interleaving group when we reach the last stmt in the group. */ - if (DR_GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt)) - < DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)) + if (GROUP_STORE_COUNT (vinfo_for_stmt (first_stmt)) + < GROUP_SIZE (vinfo_for_stmt (first_stmt)) && !slp) { *vec_stmt = NULL; @@ -3695,7 +3696,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, NULL); VEC_quick_push(tree, dr_chain, vec_oprnd); VEC_quick_push(tree, oprnds, vec_oprnd); - next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt)); + next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); } } @@ -3808,7 +3809,7 @@ vectorizable_store (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (slp) continue; - next_stmt = DR_GROUP_NEXT_DR (vinfo_for_stmt (next_stmt)); + next_stmt = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next_stmt)); if (!next_stmt) break; } @@ -4052,10 +4053,10 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, /* FORNOW */ gcc_assert (! nested_in_vect_loop); - first_stmt = DR_GROUP_FIRST_DR (stmt_info); + first_stmt = GROUP_FIRST_ELEMENT (stmt_info); if (!slp && !PURE_SLP_STMT (stmt_info)) { - group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); + group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); if (vect_load_lanes_supported (vectype, group_size)) load_lanes_p = true; else if (!vect_strided_load_supported (vectype, group_size)) @@ -4096,7 +4097,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, if (strided_load) { - first_stmt = DR_GROUP_FIRST_DR (stmt_info); + first_stmt = GROUP_FIRST_ELEMENT (stmt_info); /* Check if the chain of loads is already vectorized. */ if (STMT_VINFO_VEC_STMT (vinfo_for_stmt (first_stmt))) { @@ -4104,7 +4105,7 @@ vectorizable_load (gimple stmt, gimple_stmt_iterator *gsi, gimple *vec_stmt, return true; } first_dr = STMT_VINFO_DATA_REF (vinfo_for_stmt (first_stmt)); - group_size = DR_GROUP_SIZE (vinfo_for_stmt (first_stmt)); + group_size = GROUP_SIZE (vinfo_for_stmt (first_stmt)); /* VEC_NUM is the number of vect stmts to be created for this group. */ if (slp) @@ -5138,7 +5139,7 @@ vect_remove_stores (gimple first_stmt) /* Free the attached stmt_vec_info and remove the stmt. */ next_si = gsi_for_stmt (next); gsi_remove (&next_si, true); - tmp = DR_GROUP_NEXT_DR (vinfo_for_stmt (next)); + tmp = GROUP_NEXT_ELEMENT (vinfo_for_stmt (next)); free_stmt_vec_info (next); next = tmp; } @@ -5185,13 +5186,13 @@ new_stmt_vec_info (gimple stmt, loop_vec_info loop_vinfo, STMT_VINFO_INSIDE_OF_LOOP_COST (res) = 0; STMT_VINFO_OUTSIDE_OF_LOOP_COST (res) = 0; STMT_SLP_TYPE (res) = loop_vect; - DR_GROUP_FIRST_DR (res) = NULL; - DR_GROUP_NEXT_DR (res) = NULL; - DR_GROUP_SIZE (res) = 0; - DR_GROUP_STORE_COUNT (res) = 0; - DR_GROUP_GAP (res) = 0; - DR_GROUP_SAME_DR_STMT (res) = NULL; - DR_GROUP_READ_WRITE_DEPENDENCE (res) = false; + GROUP_FIRST_ELEMENT (res) = NULL; + GROUP_NEXT_ELEMENT (res) = NULL; + GROUP_SIZE (res) = 0; + GROUP_STORE_COUNT (res) = 0; + GROUP_GAP (res) = 0; + GROUP_SAME_DR_STMT (res) = NULL; + GROUP_READ_WRITE_DEPENDENCE (res) = false; return res; } diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index e572511e88e..1f5e3efe705 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -468,15 +468,15 @@ typedef struct _stmt_vec_info { /* Whether the stmt is SLPed, loop-based vectorized, or both. */ enum slp_vect_type slp_type; - /* Interleaving info. */ - /* First data-ref in the interleaving group. */ - gimple first_dr; - /* Pointer to the next data-ref in the group. */ - gimple next_dr; - /* In case that two or more stmts share data-ref, this is the pointer to the - previously detected stmt with the same dr. */ + /* Interleaving and reduction chains info. */ + /* First element in the group. */ + gimple first_element; + /* Pointer to the next element in the group. */ + gimple next_element; + /* For data-refs, in case that two or more stmts share data-ref, this is the + pointer to the previously detected stmt with the same dr. */ gimple same_dr_stmt; - /* The size of the interleaving group. */ + /* The size of the group. */ unsigned int size; /* For stores, number of stores from this group seen. We vectorize the last one. */ @@ -527,22 +527,22 @@ typedef struct _stmt_vec_info { #define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt #define STMT_VINFO_SAME_ALIGN_REFS(S) (S)->same_align_refs #define STMT_VINFO_DEF_TYPE(S) (S)->def_type -#define STMT_VINFO_DR_GROUP_FIRST_DR(S) (S)->first_dr -#define STMT_VINFO_DR_GROUP_NEXT_DR(S) (S)->next_dr -#define STMT_VINFO_DR_GROUP_SIZE(S) (S)->size -#define STMT_VINFO_DR_GROUP_STORE_COUNT(S) (S)->store_count -#define STMT_VINFO_DR_GROUP_GAP(S) (S)->gap -#define STMT_VINFO_DR_GROUP_SAME_DR_STMT(S)(S)->same_dr_stmt -#define STMT_VINFO_DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep -#define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_dr != NULL) - -#define DR_GROUP_FIRST_DR(S) (S)->first_dr -#define DR_GROUP_NEXT_DR(S) (S)->next_dr -#define DR_GROUP_SIZE(S) (S)->size -#define DR_GROUP_STORE_COUNT(S) (S)->store_count -#define DR_GROUP_GAP(S) (S)->gap -#define DR_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt -#define DR_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep +#define STMT_VINFO_GROUP_FIRST_ELEMENT(S) (S)->first_element +#define STMT_VINFO_GROUP_NEXT_ELEMENT(S) (S)->next_element +#define STMT_VINFO_GROUP_SIZE(S) (S)->size +#define STMT_VINFO_GROUP_STORE_COUNT(S) (S)->store_count +#define STMT_VINFO_GROUP_GAP(S) (S)->gap +#define STMT_VINFO_GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt +#define STMT_VINFO_GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep +#define STMT_VINFO_STRIDED_ACCESS(S) ((S)->first_element != NULL && (S)->data_ref_info) + +#define GROUP_FIRST_ELEMENT(S) (S)->first_element +#define GROUP_NEXT_ELEMENT(S) (S)->next_element +#define GROUP_SIZE(S) (S)->size +#define GROUP_STORE_COUNT(S) (S)->store_count +#define GROUP_GAP(S) (S)->gap +#define GROUP_SAME_DR_STMT(S) (S)->same_dr_stmt +#define GROUP_READ_WRITE_DEPENDENCE(S) (S)->read_write_dep #define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope) #define STMT_VINFO_OUTSIDE_OF_LOOP_COST(S) (S)->cost.outside_of_loop -- 2.30.2