From 16edaeb8a6b1cd72b2eedfe8cac684ac3f4785c2 Mon Sep 17 00:00:00 2001 From: Richard Sandiford Date: Tue, 31 Jul 2018 14:22:17 +0000 Subject: [PATCH] [15/46] Make SLP_TREE_VEC_STMTS a vec This patch changes SLP_TREE_VEC_STMTS from a vec to a vec. This involved making the same change to the phis vector in vectorizable_reduction, since SLP_TREE_VEC_STMTS is spliced into it here: phis.splice (SLP_TREE_VEC_STMTS (slp_node_instance->reduc_phis)); 2018-07-31 Richard Sandiford gcc/ * tree-vectorizer.h (_slp_tree::vec_stmts): Change from a vec to a vec. * tree-vect-loop.c (vect_create_epilog_for_reduction): Change the reduction_phis argument from a vec to a vec. (vectorizable_reduction): Likewise the phis local variable that is passed to vect_create_epilog_for_reduction. Update for new type of SLP_TREE_VEC_STMTS. (vectorizable_induction): Update for new type of SLP_TREE_VEC_STMTS. (vectorizable_live_operation): Likewise. * tree-vect-slp.c (vect_get_slp_vect_defs): Likewise. (vect_transform_slp_perm_load, vect_schedule_slp_instance): Likewise. From-SVN: r263130 --- gcc/ChangeLog | 15 +++++++++++ gcc/tree-vect-loop.c | 63 ++++++++++++++++++++++--------------------- gcc/tree-vect-slp.c | 44 ++++++++++++++++-------------- gcc/tree-vectorizer.h | 2 +- 4 files changed, 73 insertions(+), 51 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index d89186b0f0f..cff993ffa1e 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,18 @@ +2018-07-31 Richard Sandiford + + * tree-vectorizer.h (_slp_tree::vec_stmts): Change from a + vec to a vec. + * tree-vect-loop.c (vect_create_epilog_for_reduction): Change + the reduction_phis argument from a vec to a + vec. + (vectorizable_reduction): Likewise the phis local variable that + is passed to vect_create_epilog_for_reduction. Update for new type + of SLP_TREE_VEC_STMTS. + (vectorizable_induction): Update for new type of SLP_TREE_VEC_STMTS. + (vectorizable_live_operation): Likewise. + * tree-vect-slp.c (vect_get_slp_vect_defs): Likewise. + (vect_transform_slp_perm_load, vect_schedule_slp_instance): Likewise. + 2018-07-31 Richard Sandiford * tree-vectorizer.h (_stmt_vec_info::vectorized_stmt): Change from diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index a115fd15c01..7f335cf6351 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4412,7 +4412,7 @@ static void vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, gimple *reduc_def_stmt, int ncopies, internal_fn reduc_fn, - vec reduction_phis, + vec reduction_phis, bool double_reduc, slp_tree slp_node, slp_instance slp_node_instance, @@ -4429,6 +4429,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree scalar_dest; tree scalar_type; gimple *new_phi = NULL, *phi; + stmt_vec_info phi_info; gimple_stmt_iterator exit_gsi; tree vec_dest; tree new_temp = NULL_TREE, new_dest, new_name, new_scalar_dest; @@ -4442,7 +4443,8 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree orig_name, scalar_result; imm_use_iterator imm_iter, phi_imm_iter; use_operand_p use_p, phi_use_p; - gimple *use_stmt, *reduction_phi = NULL; + gimple *use_stmt; + stmt_vec_info reduction_phi_info = NULL; bool nested_in_vect_loop = false; auto_vec new_phis; auto_vec inner_phis; @@ -4540,7 +4542,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, } /* Set phi nodes arguments. */ - FOR_EACH_VEC_ELT (reduction_phis, i, phi) + FOR_EACH_VEC_ELT (reduction_phis, i, phi_info) { tree vec_init_def = vec_initial_defs[i]; tree def = vect_defs[i]; @@ -4548,7 +4550,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, { if (j != 0) { - phi = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (phi)); + phi_info = STMT_VINFO_RELATED_STMT (phi_info); if (nested_in_vect_loop) vec_init_def = vect_get_vec_def_for_stmt_copy (initial_def_dt, @@ -4557,6 +4559,7 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, /* Set the loop-entry arg of the reduction-phi. */ + gphi *phi = as_a (phi_info->stmt); if (STMT_VINFO_VEC_REDUCTION_TYPE (stmt_info) == INTEGER_INDUC_COND_REDUCTION) { @@ -4569,19 +4572,18 @@ vect_create_epilog_for_reduction (vec vect_defs, gimple *stmt, tree induc_val_vec = build_vector_from_val (vec_init_def_type, induc_val); - add_phi_arg (as_a (phi), induc_val_vec, - loop_preheader_edge (loop), UNKNOWN_LOCATION); + add_phi_arg (phi, induc_val_vec, loop_preheader_edge (loop), + UNKNOWN_LOCATION); } else - add_phi_arg (as_a (phi), vec_init_def, - loop_preheader_edge (loop), UNKNOWN_LOCATION); + add_phi_arg (phi, vec_init_def, loop_preheader_edge (loop), + UNKNOWN_LOCATION); /* Set the loop-latch arg for the reduction-phi. */ if (j > 0) def = vect_get_vec_def_for_stmt_copy (vect_unknown_def_type, def); - add_phi_arg (as_a (phi), def, loop_latch_edge (loop), - UNKNOWN_LOCATION); + add_phi_arg (phi, def, loop_latch_edge (loop), UNKNOWN_LOCATION); if (dump_enabled_p ()) { @@ -5599,7 +5601,7 @@ vect_finalize_reduction: if (k % ratio == 0) { epilog_stmt = new_phis[k / ratio]; - reduction_phi = reduction_phis[k / ratio]; + reduction_phi_info = reduction_phis[k / ratio]; if (double_reduc) inner_phi = inner_phis[k / ratio]; } @@ -5672,7 +5674,6 @@ vect_finalize_reduction: stmt_vec_info use_stmt_vinfo; tree vect_phi_init, preheader_arg, vect_phi_res; basic_block bb = gimple_bb (use_stmt); - gimple *use; /* Check that USE_STMT is really double reduction phi node. */ @@ -5722,13 +5723,14 @@ vect_finalize_reduction: /* Replace the use, i.e., set the correct vs1 in the regular reduction phi node. FORNOW, NCOPIES is always 1, so the loop is redundant. */ - use = reduction_phi; - for (j = 0; j < ncopies; j++) - { - edge pr_edge = loop_preheader_edge (loop); - SET_PHI_ARG_DEF (use, pr_edge->dest_idx, vect_phi_res); - use = STMT_VINFO_RELATED_STMT (vinfo_for_stmt (use)); - } + stmt_vec_info use_info = reduction_phi_info; + for (j = 0; j < ncopies; j++) + { + edge pr_edge = loop_preheader_edge (loop); + SET_PHI_ARG_DEF (as_a (use_info->stmt), + pr_edge->dest_idx, vect_phi_res); + use_info = STMT_VINFO_RELATED_STMT (use_info); + } } } } @@ -6112,7 +6114,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, auto_vec vec_oprnds1; auto_vec vec_oprnds2; auto_vec vect_defs; - auto_vec phis; + auto_vec phis; int vec_num; tree def0, tem; tree cr_index_scalar_type = NULL_TREE, cr_index_vector_type = NULL_TREE; @@ -6218,7 +6220,7 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); else { if (j == 0) @@ -7075,9 +7077,9 @@ vectorizable_reduction (gimple *stmt, gimple_stmt_iterator *gsi, if (code == COND_EXPR) { gcc_assert (!slp_node); - vectorizable_condition (stmt, gsi, vec_stmt, - PHI_RESULT (phis[0]), - reduc_index, NULL, NULL); + vectorizable_condition (stmt, gsi, vec_stmt, + PHI_RESULT (phis[0]->stmt), + reduc_index, NULL, NULL); /* Multiple types are not supported for condition. */ break; } @@ -7501,7 +7503,8 @@ vectorizable_induction (gimple *phi, /* Create the induction-phi that defines the induction-operand. */ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); induction_phi = create_phi_node (vec_dest, iv_loop->header); - loop_vinfo->add_stmt (induction_phi); + stmt_vec_info induction_phi_info + = loop_vinfo->add_stmt (induction_phi); induc_def = PHI_RESULT (induction_phi); /* Create the iv update inside the loop */ @@ -7515,7 +7518,7 @@ vectorizable_induction (gimple *phi, add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), UNKNOWN_LOCATION); - SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi); + SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi_info); } /* Re-use IVs when we can. */ @@ -7540,7 +7543,7 @@ vectorizable_induction (gimple *phi, vec_step = vect_init_vector (phi, new_vec, vectype, NULL); for (; ivn < nvects; ++ivn) { - gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; + gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt; tree def; if (gimple_code (iv) == GIMPLE_PHI) def = gimple_phi_result (iv); @@ -7556,8 +7559,8 @@ vectorizable_induction (gimple *phi, gimple_stmt_iterator tgsi = gsi_for_stmt (iv); gsi_insert_after (&tgsi, new_stmt, GSI_CONTINUE_LINKING); } - loop_vinfo->add_stmt (new_stmt); - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + SLP_TREE_VEC_STMTS (slp_node).quick_push + (loop_vinfo->add_stmt (new_stmt)); } } @@ -7943,7 +7946,7 @@ vectorizable_live_operation (gimple *stmt, gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); /* Get the correct slp vectorized stmt. */ - gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]; + gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]->stmt; if (gphi *phi = dyn_cast (vec_stmt)) vec_lhs = gimple_phi_result (phi); else diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index c7af18b9b9a..c3d72ecaab4 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -3557,18 +3557,18 @@ static void vect_get_slp_vect_defs (slp_tree slp_node, vec *vec_oprnds) { tree vec_oprnd; - gimple *vec_def_stmt; + stmt_vec_info vec_def_stmt_info; unsigned int i; gcc_assert (SLP_TREE_VEC_STMTS (slp_node).exists ()); - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt) + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), i, vec_def_stmt_info) { - gcc_assert (vec_def_stmt); - if (gimple_code (vec_def_stmt) == GIMPLE_PHI) - vec_oprnd = gimple_phi_result (vec_def_stmt); + gcc_assert (vec_def_stmt_info); + if (gphi *vec_def_phi = dyn_cast (vec_def_stmt_info->stmt)) + vec_oprnd = gimple_phi_result (vec_def_phi); else - vec_oprnd = gimple_get_lhs (vec_def_stmt); + vec_oprnd = gimple_get_lhs (vec_def_stmt_info->stmt); vec_oprnds->quick_push (vec_oprnd); } } @@ -3687,6 +3687,7 @@ vect_transform_slp_perm_load (slp_tree node, vec dr_chain, { gimple *stmt = SLP_TREE_SCALAR_STMTS (node)[0]; stmt_vec_info stmt_info = vinfo_for_stmt (stmt); + vec_info *vinfo = stmt_info->vinfo; tree mask_element_type = NULL_TREE, mask_type; int vec_index = 0; tree vectype = STMT_VINFO_VECTYPE (stmt_info); @@ -3827,26 +3828,28 @@ vect_transform_slp_perm_load (slp_tree node, vec dr_chain, /* Generate the permute statement if necessary. */ tree first_vec = dr_chain[first_vec_index]; tree second_vec = dr_chain[second_vec_index]; - gimple *perm_stmt; + stmt_vec_info perm_stmt_info; if (! noop_p) { tree perm_dest = vect_create_destination_var (gimple_assign_lhs (stmt), vectype); perm_dest = make_ssa_name (perm_dest); - perm_stmt = gimple_build_assign (perm_dest, - VEC_PERM_EXPR, - first_vec, second_vec, - mask_vec); - vect_finish_stmt_generation (stmt, perm_stmt, gsi); + gassign *perm_stmt + = gimple_build_assign (perm_dest, VEC_PERM_EXPR, + first_vec, second_vec, + mask_vec); + perm_stmt_info + = vect_finish_stmt_generation (stmt, perm_stmt, gsi); } else /* If mask was NULL_TREE generate the requested identity transform. */ - perm_stmt = SSA_NAME_DEF_STMT (first_vec); + perm_stmt_info = vinfo->lookup_def (first_vec); /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt; + SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] + = perm_stmt_info; } index = 0; @@ -3948,8 +3951,8 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance, mask.quick_push (0); if (ocode != ERROR_MARK) { - vec v0; - vec v1; + vec v0; + vec v1; unsigned j; tree tmask = NULL_TREE; vect_transform_stmt (stmt, &si, &grouped_store, node, instance); @@ -3990,10 +3993,11 @@ vect_schedule_slp_instance (slp_tree node, slp_instance instance, gimple *vstmt; vstmt = gimple_build_assign (make_ssa_name (vectype), VEC_PERM_EXPR, - gimple_assign_lhs (v0[j]), - gimple_assign_lhs (v1[j]), tmask); - vect_finish_stmt_generation (stmt, vstmt, &si); - SLP_TREE_VEC_STMTS (node).quick_push (vstmt); + gimple_assign_lhs (v0[j]->stmt), + gimple_assign_lhs (v1[j]->stmt), + tmask); + SLP_TREE_VEC_STMTS (node).quick_push + (vect_finish_stmt_generation (stmt, vstmt, &si)); } v0.release (); v1.release (); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index 1fb2513ed75..f4c2672e97b 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -143,7 +143,7 @@ struct _slp_tree { permutation. */ vec load_permutation; /* Vectorized stmt/s. */ - vec vec_stmts; + vec vec_stmts; /* Number of vector stmts that are created to replace the group of scalar stmts. It is calculated during the transformation phase as the number of scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF -- 2.30.2