DOUBLE_REDUC is TRUE if double reduction phi nodes should be handled.
SLP_NODE is an SLP node containing a group of reduction statements. The
first one in this group is STMT.
- INDUCTION_INDEX is the index of the loop for condition reductions.
- Otherwise it is undefined.
This function:
1. Creates the reduction def-use cycles: sets the arguments for
int ncopies, enum tree_code reduc_code,
vec<gimple *> reduction_phis,
int reduc_index, bool double_reduc,
- slp_tree slp_node, tree induction_index)
+ slp_tree slp_node)
{
stmt_vec_info stmt_info = vinfo_for_stmt (stmt);
stmt_vec_info prev_phi_info;
tree bitsize;
tree adjustment_def = NULL;
tree vec_initial_def = NULL;
- tree reduction_op, expr, def, initial_def = NULL;
+ tree expr, def, initial_def = NULL;
tree orig_name, scalar_result;
imm_use_iterator imm_iter, phi_imm_iter;
use_operand_p use_p, phi_use_p;
bool slp_reduc = false;
tree new_phi_result;
gimple *inner_phi = NULL;
+ tree induction_index = NULL_TREE;
if (slp_node)
group_size = SLP_TREE_SCALAR_STMTS (slp_node).length ();
gcc_assert (!slp_node);
}
- reduction_op = get_reduction_op (stmt, reduc_index);
-
- vectype = get_vectype_for_scalar_type (TREE_TYPE (reduction_op));
+ vectype = STMT_VINFO_VECTYPE (stmt_info);
gcc_assert (vectype);
mode = TYPE_MODE (vectype);
{
/* Get at the scalar def before the loop, that defines the initial value
of the reduction variable. */
+ tree reduction_op = get_reduction_op (stmt, reduc_index);
gimple *def_stmt = SSA_NAME_DEF_STMT (reduction_op);
initial_def = PHI_ARG_DEF_FROM_EDGE (def_stmt,
loop_preheader_edge (loop));
}
}
+ /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
+ which is updated with the current index of the loop for every match of
+ the original loop's cond_expr (VEC_STMT). This results in a vector
+ containing the last time the condition passed for that vector lane.
+ The first match will be a 1 to allow 0 to be used for non-matching
+ indexes. If there are no matches at all then the vector will be all
+ zeroes. */
+ if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
+ {
+ tree indx_before_incr, indx_after_incr;
+ int nunits_out = TYPE_VECTOR_SUBPARTS (vectype);
+ int k;
+
+ gimple *vec_stmt = STMT_VINFO_VEC_STMT (stmt_info);
+ gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR);
+
+ int scalar_precision
+ = GET_MODE_PRECISION (TYPE_MODE (TREE_TYPE (vectype)));
+ tree cr_index_scalar_type = make_unsigned_type (scalar_precision);
+ tree cr_index_vector_type = build_vector_type
+ (cr_index_scalar_type, TYPE_VECTOR_SUBPARTS (vectype));
+
+ /* First we create a simple vector induction variable which starts
+ with the values {1,2,3,...} (SERIES_VECT) and increments by the
+ vector size (STEP). */
+
+ /* Create a {1,2,3,...} vector. */
+ tree *vtemp = XALLOCAVEC (tree, nunits_out);
+ for (k = 0; k < nunits_out; ++k)
+ vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
+ tree series_vect = build_vector (cr_index_vector_type, vtemp);
+
+ /* Create a vector of the step value. */
+ tree step = build_int_cst (cr_index_scalar_type, nunits_out);
+ tree vec_step = build_vector_from_val (cr_index_vector_type, step);
+
+ /* Create an induction variable. */
+ gimple_stmt_iterator incr_gsi;
+ bool insert_after;
+ standard_iv_increment_position (loop, &incr_gsi, &insert_after);
+ create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
+ insert_after, &indx_before_incr, &indx_after_incr);
+
+ /* Next create a new phi node vector (NEW_PHI_TREE) which starts
+ filled with zeros (VEC_ZERO). */
+
+ /* Create a vector of 0s. */
+ tree zero = build_zero_cst (cr_index_scalar_type);
+ tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
+
+ /* Create a vector phi node. */
+ tree new_phi_tree = make_ssa_name (cr_index_vector_type);
+ new_phi = create_phi_node (new_phi_tree, loop->header);
+ set_vinfo_for_stmt (new_phi,
+ new_stmt_vec_info (new_phi, loop_vinfo));
+ add_phi_arg (as_a <gphi *> (new_phi), vec_zero,
+ loop_preheader_edge (loop), UNKNOWN_LOCATION);
+
+ /* Now take the condition from the loops original cond_expr
+ (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
+ every match uses values from the induction variable
+ (INDEX_BEFORE_INCR) otherwise uses values from the phi node
+ (NEW_PHI_TREE).
+ Finally, we update the phi (NEW_PHI_TREE) to take the value of
+ the new cond_expr (INDEX_COND_EXPR). */
+
+ /* Duplicate the condition from vec_stmt. */
+ tree ccompare = unshare_expr (gimple_assign_rhs1 (vec_stmt));
+
+ /* Create a conditional, where the condition is taken from vec_stmt
+ (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
+ else is the phi (NEW_PHI_TREE). */
+ tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
+ ccompare, indx_before_incr,
+ new_phi_tree);
+ induction_index = make_ssa_name (cr_index_vector_type);
+ gimple *index_condition = gimple_build_assign (induction_index,
+ index_cond_expr);
+ gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
+ stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
+ loop_vinfo);
+ STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
+ set_vinfo_for_stmt (index_condition, index_vec_info);
+
+ /* Update the phi with the vec cond. */
+ add_phi_arg (as_a <gphi *> (new_phi), induction_index,
+ loop_latch_edge (loop), UNKNOWN_LOCATION);
+ }
+
/* 2. Create epilog code.
The reduction epilog code operates across the elements of the vector
of partial results computed by the vectorized loop.
prev_phi_info = vinfo_for_stmt (new_phi);
}
- tree indx_before_incr, indx_after_incr, cond_name = NULL;
-
/* Finalize the reduction-phi (set its arguments) and create the
epilog reduction code. */
if ((!single_defuse_cycle || code == COND_EXPR) && !slp_node)
- {
- new_temp = gimple_assign_lhs (*vec_stmt);
- vect_defs[0] = new_temp;
-
- /* For cond reductions we want to create a new vector (INDEX_COND_EXPR)
- which is updated with the current index of the loop for every match of
- the original loop's cond_expr (VEC_STMT). This results in a vector
- containing the last time the condition passed for that vector lane.
- The first match will be a 1 to allow 0 to be used for non-matching
- indexes. If there are no matches at all then the vector will be all
- zeroes. */
- if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == COND_REDUCTION)
- {
- int nunits_out = TYPE_VECTOR_SUBPARTS (vectype_out);
- int k;
-
- gcc_assert (gimple_assign_rhs_code (*vec_stmt) == VEC_COND_EXPR);
-
- /* First we create a simple vector induction variable which starts
- with the values {1,2,3,...} (SERIES_VECT) and increments by the
- vector size (STEP). */
-
- /* Create a {1,2,3,...} vector. */
- tree *vtemp = XALLOCAVEC (tree, nunits_out);
- for (k = 0; k < nunits_out; ++k)
- vtemp[k] = build_int_cst (cr_index_scalar_type, k + 1);
- tree series_vect = build_vector (cr_index_vector_type, vtemp);
-
- /* Create a vector of the step value. */
- tree step = build_int_cst (cr_index_scalar_type, nunits_out);
- tree vec_step = build_vector_from_val (cr_index_vector_type, step);
-
- /* Create an induction variable. */
- gimple_stmt_iterator incr_gsi;
- bool insert_after;
- standard_iv_increment_position (loop, &incr_gsi, &insert_after);
- create_iv (series_vect, vec_step, NULL_TREE, loop, &incr_gsi,
- insert_after, &indx_before_incr, &indx_after_incr);
-
- /* Next create a new phi node vector (NEW_PHI_TREE) which starts
- filled with zeros (VEC_ZERO). */
-
- /* Create a vector of 0s. */
- tree zero = build_zero_cst (cr_index_scalar_type);
- tree vec_zero = build_vector_from_val (cr_index_vector_type, zero);
-
- /* Create a vector phi node. */
- tree new_phi_tree = make_ssa_name (cr_index_vector_type);
- new_phi = create_phi_node (new_phi_tree, loop->header);
- set_vinfo_for_stmt (new_phi,
- new_stmt_vec_info (new_phi, loop_vinfo));
- add_phi_arg (new_phi, vec_zero, loop_preheader_edge (loop),
- UNKNOWN_LOCATION);
-
- /* Now take the condition from the loops original cond_expr
- (VEC_STMT) and produce a new cond_expr (INDEX_COND_EXPR) which for
- every match uses values from the induction variable
- (INDEX_BEFORE_INCR) otherwise uses values from the phi node
- (NEW_PHI_TREE).
- Finally, we update the phi (NEW_PHI_TREE) to take the value of
- the new cond_expr (INDEX_COND_EXPR). */
-
- /* Duplicate the condition from vec_stmt. */
- tree ccompare = unshare_expr (gimple_assign_rhs1 (*vec_stmt));
-
- /* Create a conditional, where the condition is taken from vec_stmt
- (CCOMPARE), then is the induction index (INDEX_BEFORE_INCR) and
- else is the phi (NEW_PHI_TREE). */
- tree index_cond_expr = build3 (VEC_COND_EXPR, cr_index_vector_type,
- ccompare, indx_before_incr,
- new_phi_tree);
- cond_name = make_ssa_name (cr_index_vector_type);
- gimple *index_condition = gimple_build_assign (cond_name,
- index_cond_expr);
- gsi_insert_before (&incr_gsi, index_condition, GSI_SAME_STMT);
- stmt_vec_info index_vec_info = new_stmt_vec_info (index_condition,
- loop_vinfo);
- STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type;
- set_vinfo_for_stmt (index_condition, index_vec_info);
-
- /* Update the phi with the vec cond. */
- add_phi_arg (new_phi, cond_name, loop_latch_edge (loop),
- UNKNOWN_LOCATION);
- }
- }
+ vect_defs[0] = gimple_assign_lhs (*vec_stmt);
vect_create_epilog_for_reduction (vect_defs, stmt, epilog_copies,
epilog_reduc_code, phis, reduc_index,
- double_reduc, slp_node, cond_name);
+ double_reduc, slp_node);
return true;
}
projects / gcc.git / commitdiff