From: Richard Biener Date: Wed, 10 Jun 2020 11:47:12 +0000 (+0200) Subject: Make {SLP_TREE,STMT_VINFO}_VEC_STMTS a vector of gimple * X-Git-Url: https://git.libre-soc.org/?a=commitdiff_plain;h=f25161bdcc2b098a4678a42bb2c581ac47625b17;p=gcc.git Make {SLP_TREE,STMT_VINFO}_VEC_STMTS a vector of gimple * This makes {SLP_TREE,STMT_VINFO}_VEC_STMTS a vector of gimple * and not allocate a stmt_vec_info for vectorizer generated stmts since this is now possible after removing the only use which was chaining of vector stmts via STMT_VINFO_RELATED_STMT. This also removes all stmt_vec_info allocations done for vector stmts, the remaining ones are for stmts in the scalar IL and for patterns which are not part of the IL. Thus after this the stmt UIDs inside a basic-block are suitable for dominance checking if you ignore (or lazy-fill) UIDs of zero of the vector stmts inserted during transform. This property is ensured by a new flag set when pattern analysis is complete. 2020-06-10 Richard Biener * tree-vectorizer.h (_slp_tree::vec_stmts): Make it a vector of gimple * stmts. (_stmt_vec_info::vec_stmts): Likewise. (vec_info::stmt_vec_info_ro): New flag. (vect_finish_replace_stmt): Adjust declaration. (vect_finish_stmt_generation): Likewise. (vectorizable_induction): Likewise. (vect_transform_reduction): Likewise. (vectorizable_lc_phi): Likewise. * tree-vect-data-refs.c (vect_create_data_ref_ptr): Do not allocate stmt infos for increments. (vect_record_grouped_load_vectors): Adjust. * tree-vect-loop.c (vect_create_epilog_for_reduction): Likewise. (vectorize_fold_left_reduction): Likewise. (vect_transform_reduction): Likewise. (vect_transform_cycle_phi): Likewise. (vectorizable_lc_phi): Likewise. (vectorizable_induction): Likewise. (vectorizable_live_operation): Likewise. (vect_transform_loop): Likewise. * tree-vect-patterns.c (vect_pattern_recog): Set stmt_vec_info_ro. * tree-vect-slp.c (vect_get_slp_vect_def): Adjust. (vect_get_slp_defs): Likewise. (vect_transform_slp_perm_load): Likewise. (vect_schedule_slp_instance): Likewise. (vectorize_slp_instance_root_stmt): Likewise. * tree-vect-stmts.c (vect_get_vec_defs_for_operand): Likewise. (vect_finish_stmt_generation_1): Do not allocate a stmt info. (vect_finish_replace_stmt): Do not return anything. (vect_finish_stmt_generation): Likewise. (vect_build_gather_load_calls): Adjust. (vectorizable_bswap): Likewise. (vectorizable_call): Likewise. (vectorizable_simd_clone_call): Likewise. (vect_create_vectorized_demotion_stmts): Likewise. (vectorizable_conversion): Likewise. (vectorizable_assignment): Likewise. (vectorizable_shift): Likewise. (vectorizable_operation): Likewise. (vectorizable_scan_store): Likewise. (vectorizable_store): Likewise. (vectorizable_load): Likewise. (vectorizable_condition): Likewise. (vectorizable_comparison): Likewise. (vect_transform_stmt): Likewise. * tree-vectorizer.c (vec_info::vec_info): Initialize stmt_vec_info_ro. (vec_info::replace_stmt): Copy over stmt UID rather than unsetting/setting a stmt info allocating a new UID. (vec_info::set_vinfo_for_stmt): Assert !stmt_vec_info_ro. --- diff --git a/gcc/tree-vect-data-refs.c b/gcc/tree-vect-data-refs.c index 7edd9ebe3b6..39d5a1b554c 100644 --- a/gcc/tree-vect-data-refs.c +++ b/gcc/tree-vect-data-refs.c @@ -4896,7 +4896,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, aggr_ptr, loop, &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); /* Copy the points-to information if it exists. */ if (DR_PTR_INFO (dr)) @@ -4926,7 +4925,6 @@ vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info, containing_loop, &incr_gsi, insert_after, &indx_before_incr, &indx_after_incr); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); /* Copy the points-to information if it exists. */ if (DR_PTR_INFO (dr)) @@ -6407,7 +6405,7 @@ vect_transform_grouped_load (vec_info *vinfo, stmt_vec_info stmt_info, for each vector to the associated scalar statement. */ void -vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info, +vect_record_grouped_load_vectors (vec_info *, stmt_vec_info stmt_info, vec result_chain) { stmt_vec_info first_stmt_info = DR_GROUP_FIRST_ELEMENT (stmt_info); @@ -6441,10 +6439,10 @@ vect_record_grouped_load_vectors (vec_info *vinfo, stmt_vec_info stmt_info, DR_GROUP_SAME_DR_STMT. */ if (next_stmt_info) { - stmt_vec_info new_stmt_info = vinfo->lookup_def (tmp_data_ref); + gimple *new_stmt = SSA_NAME_DEF_STMT (tmp_data_ref); /* We assume that if VEC_STMT is not NULL, this is a case of multiple copies, and we put the new vector statement last. */ - STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (next_stmt_info).safe_push (new_stmt); next_stmt_info = DR_GROUP_NEXT_ELEMENT (next_stmt_info); gap_count = 1; diff --git a/gcc/tree-vect-loop.c b/gcc/tree-vect-loop.c index cc3d391d7c2..f0b33258ac5 100644 --- a/gcc/tree-vect-loop.c +++ b/gcc/tree-vect-loop.c @@ -4585,7 +4585,7 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, { if (gimple_assign_rhs_code (cond_info->stmt) == COND_EXPR) { - gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]->stmt; + gimple *vec_stmt = STMT_VINFO_VEC_STMTS (cond_info)[0]; gcc_assert (gimple_assign_rhs_code (vec_stmt) == VEC_COND_EXPR); ccompares.safe_push (std::make_pair (unshare_expr (gimple_assign_rhs1 (vec_stmt)), @@ -4636,7 +4636,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, /* Create a vector phi node. */ tree new_phi_tree = make_ssa_name (cr_index_vector_type); new_phi = create_phi_node (new_phi_tree, loop->header); - loop_vinfo->add_stmt (new_phi); add_phi_arg (as_a (new_phi), vec_zero, loop_preheader_edge (loop), UNKNOWN_LOCATION); @@ -4663,9 +4662,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, new_phi_tree, indx_before_incr); } gsi_insert_seq_before (&incr_gsi, stmts, GSI_SAME_STMT); - stmt_vec_info index_vec_info - = loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (new_phi_tree)); - STMT_VINFO_VECTYPE (index_vec_info) = cr_index_vector_type; /* Update the phi with the vec cond. */ induction_index = new_phi_tree; @@ -4712,17 +4708,16 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, if (slp_node) def = vect_get_slp_vect_def (slp_node, i); else - def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]->stmt); + def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[0]); for (j = 0; j < ncopies; j++) { tree new_def = copy_ssa_name (def); phi = create_phi_node (new_def, exit_bb); - loop_vinfo->add_stmt (phi); if (j == 0) new_phis.quick_push (phi); else { - def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]->stmt); + def = gimple_get_lhs (STMT_VINFO_VEC_STMTS (rdef_info)[j]); new_phis.quick_push (phi); } @@ -5446,10 +5441,6 @@ vect_create_epilog_for_reduction (loop_vec_info loop_vinfo, gsi_insert_seq_before (&exit_gsi, stmts, GSI_SAME_STMT); if (nested_in_vect_loop) { - stmt_vec_info epilog_stmt_info = loop_vinfo->add_stmt (epilog_stmt); - STMT_VINFO_RELATED_STMT (epilog_stmt_info) - = STMT_VINFO_RELATED_STMT (loop_vinfo->lookup_stmt (new_phi)); - if (!double_reduc) scalar_results.quick_push (new_temp); else @@ -5666,7 +5657,7 @@ static bool vectorize_fold_left_reduction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, gimple *reduc_def_stmt, tree_code code, internal_fn reduc_fn, tree ops[3], tree vectype_in, @@ -5674,7 +5665,6 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo, { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); - stmt_vec_info new_stmt_info = NULL; internal_fn mask_reduc_fn = get_masked_reduction_fn (reduc_fn, vectype_in); int ncopies; @@ -5785,23 +5775,22 @@ vectorize_fold_left_reduction (loop_vec_info loop_vinfo, if (i == vec_num - 1) { gimple_set_lhs (new_stmt, scalar_dest); - new_stmt_info = vect_finish_replace_stmt (loop_vinfo, - scalar_dest_def_info, - new_stmt); + vect_finish_replace_stmt (loop_vinfo, + scalar_dest_def_info, + new_stmt); } else - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, - scalar_dest_def_info, - new_stmt, gsi); + vect_finish_stmt_generation (loop_vinfo, + scalar_dest_def_info, + new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); - } - - if (!slp_node) - { - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); - *vec_stmt = new_stmt_info; + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); + else + { + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); + *vec_stmt = new_stmt; + } } return true; @@ -6822,7 +6811,7 @@ vectorizable_reduction (loop_vec_info loop_vinfo, bool vect_transform_reduction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node) + gimple **vec_stmt, slp_tree slp_node) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -6883,7 +6872,6 @@ vect_transform_reduction (loop_vec_info loop_vinfo, bool mask_by_cond_expr = use_mask_by_cond_expr_p (code, cond_fn, vectype_in); /* Transform. */ - stmt_vec_info new_stmt_info = NULL; tree new_temp = NULL_TREE; auto_vec vec_oprnds0; auto_vec vec_oprnds1; @@ -6938,6 +6926,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, FOR_EACH_VEC_ELT (vec_oprnds0, i, def0) { + gimple *new_stmt; tree vop[3] = { def0, vec_oprnds1[i], NULL_TREE }; if (masked_loop_p && !mask_by_cond_expr) { @@ -6954,8 +6943,8 @@ vect_transform_reduction (loop_vec_info loop_vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, - stmt_info, call, gsi); + vect_finish_stmt_generation (loop_vinfo, stmt_info, call, gsi); + new_stmt = call; } else { @@ -6969,28 +6958,27 @@ vect_transform_reduction (loop_vec_info loop_vinfo, build_vect_cond_expr (code, vop, mask, gsi); } - gassign *new_stmt = gimple_build_assign (vec_dest, code, - vop[0], vop[1], vop[2]); + new_stmt = gimple_build_assign (vec_dest, code, + vop[0], vop[1], vop[2]); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info = vect_finish_stmt_generation (loop_vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (loop_vinfo, stmt_info, new_stmt, gsi); } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else if (single_defuse_cycle && i < ncopies - 1) { if (reduc_index == 0) - vec_oprnds0.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds0.safe_push (gimple_get_lhs (new_stmt)); else if (reduc_index == 1) - vec_oprnds1.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds1.safe_push (gimple_get_lhs (new_stmt)); else if (reduc_index == 2) - vec_oprnds2.safe_push (gimple_get_lhs (new_stmt_info->stmt)); + vec_oprnds2.safe_push (gimple_get_lhs (new_stmt)); } else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -7003,7 +6991,7 @@ vect_transform_reduction (loop_vec_info loop_vinfo, bool vect_transform_cycle_phi (loop_vec_info loop_vinfo, - stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, + stmt_vec_info stmt_info, gimple **vec_stmt, slp_tree slp_node, slp_instance slp_node_instance) { tree vectype_out = STMT_VINFO_VECTYPE (stmt_info); @@ -7138,7 +7126,6 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, /* Create the reduction-phi that defines the reduction operand. */ gphi *new_phi = create_phi_node (vec_dest, loop->header); - stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); /* Set the loop-entry arg of the reduction-phi. */ if (j != 0 && nested_cycle) @@ -7149,12 +7136,12 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, /* The loop-latch arg is set in epilogue processing. */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); else { if (j == 0) - *vec_stmt = new_phi_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info); + *vec_stmt = new_phi; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi); } } } @@ -7166,7 +7153,7 @@ vect_transform_cycle_phi (loop_vec_info loop_vinfo, bool vectorizable_lc_phi (loop_vec_info loop_vinfo, - stmt_vec_info stmt_info, stmt_vec_info *vec_stmt, + stmt_vec_info stmt_info, gimple **vec_stmt, slp_tree slp_node) { if (!loop_vinfo @@ -7198,11 +7185,10 @@ vectorizable_lc_phi (loop_vec_info loop_vinfo, /* Create the vectorized LC PHI node. */ gphi *new_phi = create_phi_node (vec_dest, bb); add_phi_arg (new_phi, vec_oprnds[i], e, UNKNOWN_LOCATION); - stmt_vec_info new_phi_info = loop_vinfo->add_stmt (new_phi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_phi); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_phi); } if (!slp_node) *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; @@ -7262,7 +7248,7 @@ bool vectorizable_induction (loop_vec_info loop_vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi ATTRIBUTE_UNUSED, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { class loop *loop = LOOP_VINFO_LOOP (loop_vinfo); @@ -7495,8 +7481,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Create the induction-phi that defines the induction-operand. */ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); induction_phi = create_phi_node (vec_dest, iv_loop->header); - stmt_vec_info induction_phi_info - = loop_vinfo->add_stmt (induction_phi); induc_def = PHI_RESULT (induction_phi); /* Create the iv update inside the loop */ @@ -7505,7 +7489,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, vec_def = gimple_build (&stmts, PLUS_EXPR, step_vectype, vec_def, vec_step); vec_def = gimple_convert (&stmts, vectype, vec_def); - loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (vec_def)); gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); /* Set the arguments of the phi node: */ @@ -7513,7 +7496,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), UNKNOWN_LOCATION); - SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (induction_phi); } /* Fill up to the number of vectors we need for the whole group. */ nivs = least_common_multiple (group_size, @@ -7545,7 +7528,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, step_vectype, NULL); for (; ivn < nvects; ++ivn) { - gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]->stmt; + gimple *iv = SLP_TREE_VEC_STMTS (slp_node)[ivn - nivs]; tree def; if (gimple_code (iv) == GIMPLE_PHI) def = gimple_phi_result (iv); @@ -7563,8 +7546,8 @@ vectorizable_induction (loop_vec_info loop_vinfo, gimple_stmt_iterator tgsi = gsi_for_stmt (iv); gsi_insert_seq_after (&tgsi, stmts, GSI_CONTINUE_LINKING); } - SLP_TREE_VEC_STMTS (slp_node).quick_push - (loop_vinfo->add_stmt (SSA_NAME_DEF_STMT (def))); + SLP_TREE_VEC_STMTS (slp_node) + .quick_push (SSA_NAME_DEF_STMT (def)); } } @@ -7595,7 +7578,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, new_bb = gsi_insert_on_edge_immediate (loop_preheader_edge (iv_loop), new_stmt); gcc_assert (!new_bb); - loop_vinfo->add_stmt (new_stmt); } } else @@ -7701,7 +7683,6 @@ vectorizable_induction (loop_vec_info loop_vinfo, /* Create the induction-phi that defines the induction-operand. */ vec_dest = vect_get_new_vect_var (vectype, vect_simple_var, "vec_iv_"); induction_phi = create_phi_node (vec_dest, iv_loop->header); - stmt_vec_info induction_phi_info = loop_vinfo->add_stmt (induction_phi); induc_def = PHI_RESULT (induction_phi); /* Create the iv update inside the loop */ @@ -7711,15 +7692,14 @@ vectorizable_induction (loop_vec_info loop_vinfo, vec_def = gimple_convert (&stmts, vectype, vec_def); gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); new_stmt = SSA_NAME_DEF_STMT (vec_def); - stmt_vec_info new_stmt_info = loop_vinfo->add_stmt (new_stmt); /* Set the arguments of the phi node: */ add_phi_arg (induction_phi, vec_init, pe, UNKNOWN_LOCATION); add_phi_arg (induction_phi, vec_def, loop_latch_edge (iv_loop), UNKNOWN_LOCATION); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi_info); - *vec_stmt = induction_phi_info; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (induction_phi); + *vec_stmt = induction_phi; /* In case that vectorization factor (VF) is bigger than the number of elements that we can fit in a vectype (nunits), we have to generate @@ -7768,8 +7748,7 @@ vectorizable_induction (loop_vec_info loop_vinfo, gsi_insert_seq_before (&si, stmts, GSI_SAME_STMT); new_stmt = SSA_NAME_DEF_STMT (vec_def); - new_stmt_info = loop_vinfo->add_stmt (new_stmt); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -7944,7 +7923,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo, gcc_assert (!LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)); /* Get the correct slp vectorized stmt. */ - gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]->stmt; + gimple *vec_stmt = SLP_TREE_VEC_STMTS (slp_node)[vec_entry]; if (gphi *phi = dyn_cast (vec_stmt)) vec_lhs = gimple_phi_result (phi); else @@ -7957,7 +7936,7 @@ vectorizable_live_operation (loop_vec_info loop_vinfo, else { /* For multiple copies, get the last copy. */ - vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()->stmt); + vec_lhs = gimple_get_lhs (STMT_VINFO_VEC_STMTS (stmt_info).last ()); /* Get the last lane in the vector. */ bitstart = int_const_binop (MINUS_EXPR, vec_bitsize, bitsize); @@ -8683,6 +8662,7 @@ vect_transform_loop (loop_vec_info loop_vinfo, gimple *loop_vectorized_call) } else { + /* Ignore vector stmts created in the outer loop. */ stmt_info = loop_vinfo->lookup_stmt (stmt); /* vector stmts created in the outer-loop during vectorization of diff --git a/gcc/tree-vect-patterns.c b/gcc/tree-vect-patterns.c index a1fd67341d7..930f47e0742 100644 --- a/gcc/tree-vect-patterns.c +++ b/gcc/tree-vect-patterns.c @@ -5503,4 +5503,7 @@ vect_pattern_recog (vec_info *vinfo) &vect_vect_recog_func_ptrs[j], stmt_info); } } + + /* After this no more add_stmt calls are allowed. */ + vinfo->stmt_vec_info_ro = true; } diff --git a/gcc/tree-vect-slp.c b/gcc/tree-vect-slp.c index 44944bf8964..0217a524f05 100644 --- a/gcc/tree-vect-slp.c +++ b/gcc/tree-vect-slp.c @@ -3666,7 +3666,7 @@ tree vect_get_slp_vect_def (slp_tree slp_node, unsigned i) { if (SLP_TREE_VEC_STMTS (slp_node).exists ()) - return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]->stmt); + return gimple_get_lhs (SLP_TREE_VEC_STMTS (slp_node)[i]); else return SLP_TREE_VEC_DEFS (slp_node)[i]; } @@ -3680,9 +3680,9 @@ vect_get_slp_defs (slp_tree slp_node, vec *vec_defs) if (SLP_TREE_DEF_TYPE (slp_node) == vect_internal_def) { unsigned j; - stmt_vec_info vec_def_stmt_info; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt_info) - vec_defs->quick_push (gimple_get_lhs (vec_def_stmt_info->stmt)); + gimple *vec_def_stmt; + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (slp_node), j, vec_def_stmt) + vec_defs->quick_push (gimple_get_lhs (vec_def_stmt)); } else vec_defs->splice (SLP_TREE_VEC_DEFS (slp_node)); @@ -3880,7 +3880,7 @@ vect_transform_slp_perm_load (vec_info *vinfo, /* Generate the permute statement if necessary. */ tree first_vec = dr_chain[first_vec_index + ri]; tree second_vec = dr_chain[second_vec_index + ri]; - stmt_vec_info perm_stmt_info; + gimple *perm_stmt; if (! noop_p) { gassign *stmt = as_a (stmt_info->stmt); @@ -3888,23 +3888,20 @@ vect_transform_slp_perm_load (vec_info *vinfo, = vect_create_destination_var (gimple_assign_lhs (stmt), vectype); perm_dest = make_ssa_name (perm_dest); - gassign *perm_stmt + perm_stmt = gimple_build_assign (perm_dest, VEC_PERM_EXPR, first_vec, second_vec, mask_vec); - perm_stmt_info - = vect_finish_stmt_generation (vinfo, - stmt_info, perm_stmt, - gsi); + vect_finish_stmt_generation (vinfo, stmt_info, perm_stmt, + gsi); } else /* If mask was NULL_TREE generate the requested identity transform. */ - perm_stmt_info = vinfo->lookup_def (first_vec); + perm_stmt = SSA_NAME_DEF_STMT (first_vec); /* Store the vector statement in NODE. */ - SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] - = perm_stmt_info; + SLP_TREE_VEC_STMTS (node)[vect_stmts_counter++] = perm_stmt; } } @@ -3995,8 +3992,8 @@ vect_schedule_slp_instance (vec_info *vinfo, } if (ocode != ERROR_MARK) { - vec v0; - vec v1; + vec v0; + vec v1; unsigned j; tree tmask = NULL_TREE; vect_transform_stmt (vinfo, stmt_info, &si, node, instance); @@ -4037,11 +4034,11 @@ vect_schedule_slp_instance (vec_info *vinfo, gimple *vstmt; vstmt = gimple_build_assign (make_ssa_name (vectype), VEC_PERM_EXPR, - gimple_assign_lhs (v0[j]->stmt), - gimple_assign_lhs (v1[j]->stmt), + gimple_assign_lhs (v0[j]), + gimple_assign_lhs (v1[j]), tmask); - SLP_TREE_VEC_STMTS (node).quick_push - (vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si)); + vect_finish_stmt_generation (vinfo, stmt_info, vstmt, &si); + SLP_TREE_VEC_STMTS (node).quick_push (vstmt); } v0.release (); v1.release (); @@ -4109,12 +4106,12 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) == 1) { - stmt_vec_info child_stmt_info; + gimple *child_stmt; int j; - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info) + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt) { - tree vect_lhs = gimple_get_lhs (child_stmt_info->stmt); + tree vect_lhs = gimple_get_lhs (child_stmt); tree root_lhs = gimple_get_lhs (instance->root_stmt->stmt); if (!useless_type_conversion_p (TREE_TYPE (root_lhs), TREE_TYPE (vect_lhs))) @@ -4127,16 +4124,16 @@ vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance) else if (SLP_TREE_NUMBER_OF_VEC_STMTS (node) > 1) { int nelts = SLP_TREE_NUMBER_OF_VEC_STMTS (node); - stmt_vec_info child_stmt_info; + gimple *child_stmt; int j; vec *v; vec_alloc (v, nelts); - FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt_info) + FOR_EACH_VEC_ELT (SLP_TREE_VEC_STMTS (node), j, child_stmt) { CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, - gimple_get_lhs (child_stmt_info->stmt)); + gimple_get_lhs (child_stmt)); } tree lhs = gimple_get_lhs (instance->root_stmt->stmt); tree rtype = TREE_TYPE (gimple_assign_rhs1 (instance->root_stmt->stmt)); diff --git a/gcc/tree-vect-stmts.c b/gcc/tree-vect-stmts.c index d9db2f079ad..686eb389f0c 100644 --- a/gcc/tree-vect-stmts.c +++ b/gcc/tree-vect-stmts.c @@ -1478,7 +1478,7 @@ vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info stmt_vinfo, gcc_assert (STMT_VINFO_VEC_STMTS (def_stmt_info).length () == ncopies); for (unsigned i = 0; i < ncopies; ++i) vec_oprnds->quick_push (gimple_get_lhs - (STMT_VINFO_VEC_STMTS (def_stmt_info)[i]->stmt)); + (STMT_VINFO_VEC_STMTS (def_stmt_info)[i])); } } @@ -1540,13 +1540,10 @@ vect_get_vec_defs (vec_info *vinfo, stmt_vec_info stmt_info, slp_tree slp_node, vect_finish_stmt_generation. Set the location of the new statement and create and return a stmt_vec_info for it. */ -static stmt_vec_info -vect_finish_stmt_generation_1 (vec_info *vinfo, +static void +vect_finish_stmt_generation_1 (vec_info *, stmt_vec_info stmt_info, gimple *vec_stmt) { - stmt_vec_info vec_stmt_info = vinfo->add_stmt (vec_stmt); - vec_stmt_info->vector_stmt = 1; - if (dump_enabled_p ()) dump_printf_loc (MSG_NOTE, vect_location, "add new stmt: %G", vec_stmt); @@ -1563,15 +1560,13 @@ vect_finish_stmt_generation_1 (vec_info *vinfo, } else gcc_assert (!stmt_could_throw_p (cfun, vec_stmt)); - - return vec_stmt_info; } /* Replace the scalar statement STMT_INFO with a new vector statement VEC_STMT, which sets the same scalar result as STMT_INFO did. Create and return a stmt_vec_info for VEC_STMT. */ -stmt_vec_info +void vect_finish_replace_stmt (vec_info *vinfo, stmt_vec_info stmt_info, gimple *vec_stmt) { @@ -1581,13 +1576,13 @@ vect_finish_replace_stmt (vec_info *vinfo, gimple_stmt_iterator gsi = gsi_for_stmt (scalar_stmt); gsi_replace (&gsi, vec_stmt, true); - return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); + vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); } /* Add VEC_STMT to the vectorized implementation of STMT_INFO and insert it before *GSI. Create and return a stmt_vec_info for VEC_STMT. */ -stmt_vec_info +void vect_finish_stmt_generation (vec_info *vinfo, stmt_vec_info stmt_info, gimple *vec_stmt, gimple_stmt_iterator *gsi) @@ -1621,7 +1616,7 @@ vect_finish_stmt_generation (vec_info *vinfo, } } gsi_insert_before (gsi, vec_stmt, GSI_SAME_STMT); - return vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); + vect_finish_stmt_generation_1 (vinfo, stmt_info, vec_stmt); } /* We want to vectorize a call to combined function CFN with function @@ -2564,7 +2559,7 @@ vect_build_zero_merge_argument (vec_info *vinfo, static void vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, gather_scatter_info *gs_info, tree mask) { @@ -2661,7 +2656,6 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, tree src_op = NULL_TREE; tree mask_op = NULL_TREE; tree prev_res = NULL_TREE; - stmt_vec_info prev_stmt_info = NULL; if (!mask) { @@ -2765,29 +2759,26 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, } src_op = build_zero_cst (srctype); } - gcall *new_call = gimple_build_call (gs_info->decl, 5, src_op, ptr, op, - mask_arg, scale); + gimple *new_stmt = gimple_build_call (gs_info->decl, 5, src_op, ptr, op, + mask_arg, scale); - stmt_vec_info new_stmt_info; if (!useless_type_conversion_p (vectype, rettype)) { gcc_assert (known_eq (TYPE_VECTOR_SUBPARTS (vectype), TYPE_VECTOR_SUBPARTS (rettype))); op = vect_get_new_ssa_name (rettype, vect_simple_var); - gimple_call_set_lhs (new_call, op); - vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + gimple_call_set_lhs (new_stmt, op); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); var = make_ssa_name (vec_dest); op = build1 (VIEW_CONVERT_EXPR, vectype, op); - gassign *new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (var, VIEW_CONVERT_EXPR, op); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { - var = make_ssa_name (vec_dest, new_call); - gimple_call_set_lhs (new_call, var); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + var = make_ssa_name (vec_dest, new_stmt); + gimple_call_set_lhs (new_stmt, var); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (modifier == NARROW) @@ -2799,14 +2790,12 @@ vect_build_gather_load_calls (vec_info *vinfo, stmt_vec_info stmt_info, } var = permute_vec_elements (vinfo, prev_res, var, perm_mask, stmt_info, gsi); - new_stmt_info = loop_vinfo->lookup_def (var); + new_stmt = SSA_NAME_DEF_STMT (var); } - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); - if (prev_stmt_info == NULL) - *vec_stmt = new_stmt_info; - prev_stmt_info = new_stmt_info; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } + *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } /* Prepare the base and offset in GS_INFO for vectorization. @@ -2905,7 +2894,7 @@ vect_get_data_ptr_increment (vec_info *vinfo, static bool vectorizable_bswap (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, slp_tree *slp_op, tree vectype_in, stmt_vector_for_cost *cost_vec) { @@ -2993,12 +2982,11 @@ vectorizable_bswap (vec_info *vinfo, tem = make_ssa_name (vectype); new_stmt = gimple_build_assign (tem, build1 (VIEW_CONVERT_EXPR, vectype, tem2)); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -3043,7 +3031,7 @@ simple_integer_narrowing (tree vectype_out, tree vectype_in, static bool vectorizable_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { gcall *stmt; @@ -3333,7 +3321,6 @@ vectorizable_call (vec_info *vinfo, bool masked_loop_p = loop_vinfo && LOOP_VINFO_FULLY_MASKED_P (loop_vinfo); - stmt_vec_info new_stmt_info = NULL; if (modifier == NONE || ifn != IFN_LAST) { tree prev_res = NULL_TREE; @@ -3359,6 +3346,7 @@ vectorizable_call (vec_info *vinfo, vec vec_oprndsk = vec_defs[k]; vargs[k] = vec_oprndsk[i]; } + gimple *new_stmt; if (modifier == NARROW) { /* We don't define any narrowing conditional functions @@ -3376,12 +3364,10 @@ vectorizable_call (vec_info *vinfo, continue; } new_temp = make_ssa_name (vec_dest); - gimple *new_stmt - = gimple_build_assign (new_temp, convert_code, - prev_res, half_res); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, convert_code, + prev_res, half_res); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } else { @@ -3404,11 +3390,10 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); } continue; } @@ -3434,6 +3419,7 @@ vectorizable_call (vec_info *vinfo, vargs[mask_opno], gsi); } + gimple *new_stmt; if (cfn == CFN_GOMP_SIMD_LANE) { tree cst = build_index_vector (vectype_out, j * nunits_out, 1); @@ -3442,9 +3428,8 @@ vectorizable_call (vec_info *vinfo, gimple *init_stmt = gimple_build_assign (new_var, cst); vect_init_vector_1 (vinfo, stmt_info, init_stmt, NULL); new_temp = make_ssa_name (vec_dest); - gimple *new_stmt = gimple_build_assign (new_temp, new_var); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, new_var); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else if (modifier == NARROW) { @@ -3462,10 +3447,9 @@ vectorizable_call (vec_info *vinfo, continue; } new_temp = make_ssa_name (vec_dest); - gassign *new_stmt = gimple_build_assign (new_temp, convert_code, - prev_res, half_res); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, convert_code, + prev_res, half_res); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { @@ -3477,13 +3461,13 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } if (j == (modifier == NARROW ? 1 : 0)) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } for (i = 0; i < nargs; i++) { @@ -3530,9 +3514,8 @@ vectorizable_call (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + SLP_TREE_VEC_STMTS (slp_node).quick_push (call); } continue; } @@ -3556,10 +3539,9 @@ vectorizable_call (vec_info *vinfo, gcall *new_stmt = gimple_build_call_vec (fndecl, vargs); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_call_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -3695,7 +3677,7 @@ simd_clone_subparts (tree vectype) static bool vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *) { tree vec_dest; @@ -4154,7 +4136,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, } tree phi_res = copy_ssa_name (op); gphi *new_phi = create_phi_node (phi_res, loop->header); - loop_vinfo->add_stmt (new_phi); add_phi_arg (new_phi, arginfo[i].op, loop_preheader_edge (loop), UNKNOWN_LOCATION); enum tree_code code @@ -4171,7 +4152,6 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, = gimple_build_assign (phi_arg, code, phi_res, tcst); gimple_stmt_iterator si = gsi_after_labels (loop->header); gsi_insert_after (&si, new_stmt, GSI_NEW_STMT); - loop_vinfo->add_stmt (new_stmt); add_phi_arg (new_phi, phi_arg, loop_latch_edge (loop), UNKNOWN_LOCATION); arginfo[i].op = phi_res; @@ -4220,8 +4200,8 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, new_temp = make_ssa_name (rtype, new_call); gimple_call_set_lhs (new_call, new_temp); } - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_call, gsi); + gimple *new_stmt = new_call; if (vec_dest) { @@ -4244,15 +4224,12 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, else t = build3 (BIT_FIELD_REF, vectype, new_temp, bitsize_int (prec), bitsize_int (l * prec)); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (vectype), t); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vectype), t); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (j == 0 && l == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (ratype) @@ -4273,11 +4250,10 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, { tree tem = build4 (ARRAY_REF, rtype, new_temp, size_int (m), NULL_TREE, NULL_TREE); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (rtype), tem); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (rtype), + tem); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); CONSTRUCTOR_APPEND_ELT (ret_ctor_elts, NULL_TREE, gimple_assign_lhs (new_stmt)); } @@ -4288,14 +4264,13 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, if ((j & (k - 1)) != k - 1) continue; vec_oprnd0 = build_constructor (vectype, ret_ctor_elts); - gimple *new_stmt + new_stmt = gimple_build_assign (make_ssa_name (vec_dest), vec_oprnd0); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if ((unsigned) j == k - 1) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); continue; } else if (ratype) @@ -4303,17 +4278,15 @@ vectorizable_simd_clone_call (vec_info *vinfo, stmt_vec_info stmt_info, tree t = build_fold_addr_expr (new_temp); t = build2 (MEM_REF, vectype, t, build_int_cst (TREE_TYPE (t), 0)); - gimple *new_stmt - = gimple_build_assign (make_ssa_name (vec_dest), t); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vec_dest), t); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); vect_clobber_variable (vinfo, stmt_info, gsi, new_temp); } } if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } for (i = 0; i < nargs; ++i) @@ -4403,8 +4376,7 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec *vec_oprnds, gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1); new_tmp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_tmp); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (multi_step_cvt) /* Store the resulting vector for next recursive call. */ @@ -4415,9 +4387,9 @@ vect_create_vectorized_demotion_stmts (vec_info *vinfo, vec *vec_oprnds, vectors in SLP_NODE or in vector info of the scalar statement (or in STMT_VINFO_RELATED_STMT chain). */ if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -4503,7 +4475,7 @@ vect_create_vectorized_promotion_stmts (vec_info *vinfo, static bool vectorizable_conversion (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -4887,19 +4859,17 @@ vectorizable_conversion (vec_info *vinfo, op0, &vec_oprnds0); FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { - stmt_vec_info new_stmt_info; /* Arguments are ready, create the new vector stmt. */ gcc_assert (TREE_CODE_LENGTH (code1) == unary_op); gassign *new_stmt = gimple_build_assign (vec_dest, code1, vop0); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } break; @@ -4936,23 +4906,21 @@ vectorizable_conversion (vec_info *vinfo, FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (cvt_type) { gcc_assert (TREE_CODE_LENGTH (codecvt1) == unary_op); new_temp = make_ssa_name (vec_dest); - gassign *new_stmt - = gimple_build_assign (new_temp, codecvt1, vop0); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, codecvt1, vop0); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else - new_stmt_info = vinfo->lookup_def (vop0); + new_stmt = SSA_NAME_DEF_STMT (vop0); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } break; @@ -5025,7 +4993,7 @@ vect_nop_conversion_p (stmt_vec_info stmt_info) static bool vectorizable_assignment (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -5155,7 +5123,6 @@ vectorizable_assignment (vec_info *vinfo, vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op, &vec_oprnds); /* Arguments are ready. create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds, i, vop) { if (CONVERT_EXPR_CODE_P (code) @@ -5164,12 +5131,11 @@ vectorizable_assignment (vec_info *vinfo, gassign *new_stmt = gimple_build_assign (vec_dest, vop); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; @@ -5225,7 +5191,7 @@ vect_supportable_shift (vec_info *vinfo, enum tree_code code, tree scalar_type) static bool vectorizable_shift (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -5606,19 +5572,17 @@ vectorizable_shift (vec_info *vinfo, vec_oprnd1 ? NULL_TREE : op1, &vec_oprnds1); /* Arguments are ready. Create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { vop1 = vec_oprnds1[i]; gassign *new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -5642,7 +5606,7 @@ vectorizable_shift (vec_info *vinfo, static bool vectorizable_operation (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree vec_dest; @@ -6015,9 +5979,9 @@ vectorizable_operation (vec_info *vinfo, vect_get_vec_defs (vinfo, stmt_info, slp_node, ncopies, op0, &vec_oprnds0, op1, &vec_oprnds1, op2, &vec_oprnds2); /* Arguments are ready. Create the new vector stmt. */ - stmt_vec_info new_stmt_info = NULL; FOR_EACH_VEC_ELT (vec_oprnds0, i, vop0) { + gimple *new_stmt = NULL; vop1 = ((op_type == binary_op || op_type == ternary_op) ? vec_oprnds1[i] : NULL_TREE); vop2 = ((op_type == ternary_op) ? vec_oprnds2[i] : NULL_TREE); @@ -6034,33 +5998,30 @@ vectorizable_operation (vec_info *vinfo, new_temp = make_ssa_name (vec_dest, call); gimple_call_set_lhs (call, new_temp); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } else { - gassign *new_stmt = gimple_build_assign (vec_dest, code, - vop0, vop1, vop2); + new_stmt = gimple_build_assign (vec_dest, code, vop0, vop1, vop2); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (vec_cvt_dest) { new_temp = build1 (VIEW_CONVERT_EXPR, vectype_out, new_temp); - gassign *new_stmt - = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR, - new_temp); + new_stmt = gimple_build_assign (vec_cvt_dest, VIEW_CONVERT_EXPR, + new_temp); new_temp = make_ssa_name (vec_cvt_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, + new_stmt, gsi); } } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -6723,7 +6684,7 @@ check_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype, static bool vectorizable_scan_store (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, int ncopies) + gimple **vec_stmt, int ncopies) { loop_vec_info loop_vinfo = dyn_cast (vinfo); dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info); @@ -6843,7 +6804,6 @@ vectorizable_scan_store (vec_info *vinfo, rhs2, &vec_oprnds3); for (int j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; vec_oprnd1 = vec_oprnds1[j]; if (ldataref_ptr == NULL) vec_oprnd2 = vec_oprnds2[j]; @@ -6861,8 +6821,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (load1_dr_info->dr)); gimple *g = gimple_build_assign (vec_oprnd2, data_ref); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; } @@ -6876,8 +6836,8 @@ vectorizable_scan_store (vec_info *vinfo, != scan_store_kind_perm)) ? zero_vec : vec_oprnd1, v, perms[i]); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; if (zero_vec && use_whole_vector[i] == scan_store_kind_lshift_cond) @@ -6893,9 +6853,9 @@ vectorizable_scan_store (vec_info *vinfo, tree new_temp2 = make_ssa_name (vectype); g = gimple_build_assign (new_temp2, VEC_COND_EXPR, vb.build (), new_temp, vec_oprnd1); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); new_temp = new_temp2; } @@ -6912,16 +6872,16 @@ vectorizable_scan_store (vec_info *vinfo, tree new_temp2 = make_ssa_name (vectype); g = gimple_build_assign (new_temp2, code, v, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); v = new_temp2; } tree new_temp = make_ssa_name (vectype); gimple *g = gimple_build_assign (new_temp, code, orig, v); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); tree last_perm_arg = new_temp; /* For exclusive scan, new_temp computed above is the exclusive scan @@ -6931,15 +6891,15 @@ vectorizable_scan_store (vec_info *vinfo, { last_perm_arg = make_ssa_name (vectype); g = gimple_build_assign (last_perm_arg, code, new_temp, vec_oprnd2); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } orig = make_ssa_name (vectype); g = gimple_build_assign (orig, VEC_PERM_EXPR, last_perm_arg, last_perm_arg, perms[units_log2]); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); if (!inscan_var_store) { @@ -6948,8 +6908,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); g = gimple_build_assign (data_ref, new_temp); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } } @@ -6964,9 +6924,8 @@ vectorizable_scan_store (vec_info *vinfo, dataref_offset); vect_copy_ref_info (data_ref, DR_REF (dr_info->dr)); gimple *g = gimple_build_assign (data_ref, orig); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + vect_finish_stmt_generation (vinfo, stmt_info, g, gsi); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (g); } return true; } @@ -6983,7 +6942,7 @@ vectorizable_scan_store (vec_info *vinfo, static bool vectorizable_store (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree data_ref; @@ -7399,10 +7358,9 @@ vectorizable_store (vec_info *vinfo, gcall *new_stmt = gimple_build_call (gs_info.decl, 5, ptr, mask_arg, op, src, scale); - stmt_vec_info new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; return true; @@ -7580,7 +7538,6 @@ vectorizable_store (vec_info *vinfo, loop, &incr_gsi, insert_after, &offvar, NULL); incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); @@ -7643,9 +7600,7 @@ vectorizable_store (vec_info *vinfo, /* And store it to *running_off. */ assign = gimple_build_assign (newref, elem); - stmt_vec_info assign_info - = vect_finish_stmt_generation (vinfo, stmt_info, - assign, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, assign, gsi); group_el += lnel; if (! slp @@ -7663,8 +7618,8 @@ vectorizable_store (vec_info *vinfo, && !slp) { if (j == 0 && i == 0) - *vec_stmt = assign_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign_info); + *vec_stmt = assign; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (assign); } } } @@ -7777,7 +7732,7 @@ vectorizable_store (vec_info *vinfo, gvec_oprnds.safe_grow_cleared (group_size); for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (j == 0) { if (slp) @@ -7924,15 +7879,15 @@ vectorizable_store (vec_info *vinfo, gimple_call_set_lhs (call, data_ref); } gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; /* Record that VEC_ARRAY is now dead. */ vect_clobber_variable (vinfo, stmt_info, gsi, vec_array); } else { - new_stmt_info = NULL; + new_stmt = NULL; if (grouped_store) { if (j == 0) @@ -7970,8 +7925,8 @@ vectorizable_store (vec_info *vinfo, (IFN_SCATTER_STORE, 4, dataref_ptr, vec_offset, scale, vec_oprnd); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; break; } @@ -8029,8 +7984,8 @@ vectorizable_store (vec_info *vinfo, dataref_ptr, ptr, final_mask, vec_oprnd); gimple_call_set_nothrow (call, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; } else { @@ -8050,10 +8005,8 @@ vectorizable_store (vec_info *vinfo, = build_aligned_type (TREE_TYPE (data_ref), TYPE_ALIGN (elem_type)); vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - gassign *new_stmt - = gimple_build_assign (data_ref, vec_oprnd); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (data_ref, vec_oprnd); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (slp) @@ -8067,8 +8020,8 @@ vectorizable_store (vec_info *vinfo, if (!slp) { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } @@ -8205,7 +8158,7 @@ hoist_defs_of_uses (stmt_vec_info stmt_info, class loop *loop) static bool vectorizable_load (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, slp_tree slp_node, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest; @@ -8535,27 +8488,20 @@ vectorizable_load (vec_info *vinfo, gsi_next (&gsi2); for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info; if (hoist_p) - { - new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, - vectype, NULL); - gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); - new_stmt_info = vinfo->add_stmt (new_stmt); - } + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, NULL); else - { - new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, - vectype, &gsi2); - new_stmt_info = vinfo->lookup_def (new_temp); - } + new_temp = vect_init_vector (vinfo, stmt_info, scalar_dest, + vectype, &gsi2); + gimple *new_stmt = SSA_NAME_DEF_STMT (new_temp); if (slp) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } return true; @@ -8566,7 +8512,6 @@ vectorizable_load (vec_info *vinfo, { gimple_stmt_iterator incr_gsi; bool insert_after; - gimple *incr; tree offvar; tree ivstep; tree running_off; @@ -8641,8 +8586,6 @@ vectorizable_load (vec_info *vinfo, create_iv (stride_base, ivstep, NULL, loop, &incr_gsi, insert_after, &offvar, NULL); - incr = gsi_stmt (incr_gsi); - loop_vinfo->add_stmt (incr); stride_step = cse_and_gimplify_to_preheader (loop_vinfo, stride_step); @@ -8712,17 +8655,15 @@ vectorizable_load (vec_info *vinfo, { if (nloads > 1) vec_alloc (v, nloads); - stmt_vec_info new_stmt_info = NULL; + gimple *new_stmt = NULL; for (i = 0; i < nloads; i++) { tree this_off = build_int_cst (TREE_TYPE (alias_off), group_el * elsz + cst_offset); tree data_ref = build2 (MEM_REF, ltype, running_off, this_off); vect_copy_ref_info (data_ref, DR_REF (first_dr_info->dr)); - gassign *new_stmt - = gimple_build_assign (make_ssa_name (ltype), data_ref); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (ltype), data_ref); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (nloads > 1) CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, gimple_assign_lhs (new_stmt)); @@ -8745,32 +8686,29 @@ vectorizable_load (vec_info *vinfo, tree vec_inv = build_constructor (lvectype, v); new_temp = vect_init_vector (vinfo, stmt_info, vec_inv, lvectype, gsi); - new_stmt_info = vinfo->lookup_def (new_temp); + new_stmt = SSA_NAME_DEF_STMT (new_temp); if (lvectype != vectype) { - gassign *new_stmt - = gimple_build_assign (make_ssa_name (vectype), - VIEW_CONVERT_EXPR, - build1 (VIEW_CONVERT_EXPR, - vectype, new_temp)); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + new_stmt = gimple_build_assign (make_ssa_name (vectype), + VIEW_CONVERT_EXPR, + build1 (VIEW_CONVERT_EXPR, + vectype, new_temp)); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } if (slp) { if (slp_perm) - dr_chain.quick_push (gimple_assign_lhs (new_stmt_info->stmt)); + dr_chain.quick_push (gimple_assign_lhs (new_stmt)); else - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); } else { if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + *vec_stmt = new_stmt; + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } if (slp_perm) @@ -9052,7 +8990,6 @@ vectorizable_load (vec_info *vinfo, poly_uint64 group_elt = 0; for (j = 0; j < ncopies; j++) { - stmt_vec_info new_stmt_info = NULL; /* 1. Create the vector or array pointer update chain. */ if (j == 0) { @@ -9131,6 +9068,7 @@ vectorizable_load (vec_info *vinfo, if (grouped_load || slp_perm) dr_chain.create (vec_num); + gimple *new_stmt = NULL; if (memory_access_type == VMAT_LOAD_STORE_LANES) { tree vec_array; @@ -9166,8 +9104,8 @@ vectorizable_load (vec_info *vinfo, } gimple_call_set_lhs (call, vec_array); gimple_call_set_nothrow (call, true); - new_stmt_info = vect_finish_stmt_generation (vinfo, stmt_info, - call, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, call, gsi); + new_stmt = call; /* Extract each vector into an SSA_NAME. */ for (i = 0; i < vec_num; i++) @@ -9202,7 +9140,6 @@ vectorizable_load (vec_info *vinfo, gsi, stmt_info, bump); /* 2. Create the vector-load in the loop. */ - gimple *new_stmt = NULL; switch (alignment_support_scheme) { case dr_aligned: @@ -9448,9 +9385,7 @@ vectorizable_load (vec_info *vinfo, } new_temp = make_ssa_name (vec_dest, new_stmt); gimple_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); /* 3. Handle explicit realignment if necessary/supported. Create in loop: @@ -9466,9 +9401,7 @@ vectorizable_load (vec_info *vinfo, msq, lsq, realignment_token); new_temp = make_ssa_name (vec_dest, new_stmt); gimple_assign_set_lhs (new_stmt, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, - new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (alignment_support_scheme == dr_explicit_realign_optimized) { @@ -9486,7 +9419,7 @@ vectorizable_load (vec_info *vinfo, tree perm_mask = perm_mask_for_reverse (vectype); new_temp = permute_vec_elements (vinfo, new_temp, new_temp, perm_mask, stmt_info, gsi); - new_stmt_info = vinfo->lookup_def (new_temp); + new_stmt = SSA_NAME_DEF_STMT (new_temp); } /* Collect vector loads and later create their permutation in @@ -9496,7 +9429,7 @@ vectorizable_load (vec_info *vinfo, /* Store vector loads in the corresponding SLP_NODE. */ if (slp && !slp_perm) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); /* With SLP permutation we load the gaps as well, without we need to skip the gaps after we manage to fully load @@ -9549,13 +9482,13 @@ vectorizable_load (vec_info *vinfo, } else { - if (j == 0) - *vec_stmt = new_stmt_info; - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } } dr_chain.release (); } + if (!slp) + *vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info)[0]; return true; } @@ -9667,7 +9600,7 @@ vect_is_simple_cond (tree cond, vec_info *vinfo, stmt_vec_info stmt_info, static bool vectorizable_condition (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree scalar_dest = NULL_TREE; @@ -10090,19 +10023,18 @@ vectorizable_condition (vec_info *vinfo, } } - stmt_vec_info new_stmt_info; + gimple *new_stmt; if (reduction_type == EXTRACT_LAST_REDUCTION) { gimple *old_stmt = vect_orig_stmt (stmt_info)->stmt; tree lhs = gimple_get_lhs (old_stmt); - gcall *new_stmt = gimple_build_call_internal + new_stmt = gimple_build_call_internal (IFN_FOLD_EXTRACT_LAST, 3, else_clause, vec_compare, vec_then_clause); gimple_call_set_lhs (new_stmt, lhs); SSA_NAME_DEF_STMT (lhs) = new_stmt; if (old_stmt == gsi_stmt (*gsi)) - new_stmt_info = vect_finish_replace_stmt (vinfo, - stmt_info, new_stmt); + vect_finish_replace_stmt (vinfo, stmt_info, new_stmt); else { /* In this case we're moving the definition to later in the @@ -10110,23 +10042,20 @@ vectorizable_condition (vec_info *vinfo, lhs are in phi statements. */ gimple_stmt_iterator old_gsi = gsi_for_stmt (old_stmt); gsi_remove (&old_gsi, true); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } else { new_temp = make_ssa_name (vec_dest); - gassign *new_stmt - = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare, - vec_then_clause, vec_else_clause); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, VEC_COND_EXPR, vec_compare, + vec_then_clause, vec_else_clause); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -10151,7 +10080,7 @@ vectorizable_condition (vec_info *vinfo, static bool vectorizable_comparison (vec_info *vinfo, stmt_vec_info stmt_info, gimple_stmt_iterator *gsi, - stmt_vec_info *vec_stmt, + gimple **vec_stmt, slp_tree slp_node, stmt_vector_for_cost *cost_vec) { tree lhs, rhs1, rhs2; @@ -10344,27 +10273,24 @@ vectorizable_comparison (vec_info *vinfo, /* Arguments are ready. Create the new vector stmt. */ FOR_EACH_VEC_ELT (vec_oprnds0, i, vec_rhs1) { - stmt_vec_info new_stmt_info; + gimple *new_stmt; vec_rhs2 = vec_oprnds1[i]; new_temp = make_ssa_name (mask); if (bitop1 == NOP_EXPR) { - gassign *new_stmt = gimple_build_assign (new_temp, code, - vec_rhs1, vec_rhs2); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + new_stmt = gimple_build_assign (new_temp, code, + vec_rhs1, vec_rhs2); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } else { - gassign *new_stmt; if (bitop1 == BIT_NOT_EXPR) new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs2); else new_stmt = gimple_build_assign (new_temp, bitop1, vec_rhs1, vec_rhs2); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); if (bitop2 != NOP_EXPR) { tree res = make_ssa_name (mask); @@ -10373,14 +10299,13 @@ vectorizable_comparison (vec_info *vinfo, else new_stmt = gimple_build_assign (res, bitop2, vec_rhs1, new_temp); - new_stmt_info - = vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); + vect_finish_stmt_generation (vinfo, stmt_info, new_stmt, gsi); } } if (slp_node) - SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt_info); + SLP_TREE_VEC_STMTS (slp_node).quick_push (new_stmt); else - STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt_info); + STMT_VINFO_VEC_STMTS (stmt_info).safe_push (new_stmt); } if (!slp_node) @@ -10666,7 +10591,7 @@ vect_transform_stmt (vec_info *vinfo, slp_tree slp_node, slp_instance slp_node_instance) { bool is_store = false; - stmt_vec_info vec_stmt = NULL; + gimple *vec_stmt = NULL; bool done; gcc_assert (slp_node || !PURE_SLP_STMT (stmt_info)); @@ -10808,14 +10733,14 @@ vect_transform_stmt (vec_info *vinfo, && (PHI_ARG_DEF_FROM_EDGE (phi, e) == gimple_get_lhs (orig_stmt_info->stmt))) { - vec &phi_info + vec &phi_info = STMT_VINFO_VEC_STMTS (STMT_VINFO_REDUC_DEF (orig_stmt_info)); - vec &vec_stmt + vec &vec_stmt = STMT_VINFO_VEC_STMTS (stmt_info); gcc_assert (phi_info.length () == vec_stmt.length ()); for (unsigned i = 0; i < phi_info.length (); ++i) - add_phi_arg (as_a (phi_info[i]->stmt), - gimple_get_lhs (vec_stmt[i]->stmt), e, + add_phi_arg (as_a (phi_info[i]), + gimple_get_lhs (vec_stmt[i]), e, gimple_phi_arg_location (phi, e->dest_idx)); } else if (slp_node @@ -10827,7 +10752,7 @@ vect_transform_stmt (vec_info *vinfo, gcc_assert (SLP_TREE_VEC_STMTS (phi_node).length () == SLP_TREE_VEC_STMTS (slp_node).length ()); for (unsigned i = 0; i < SLP_TREE_VEC_STMTS (phi_node).length (); ++i) - add_phi_arg (as_a (SLP_TREE_VEC_STMTS (phi_node)[i]->stmt), + add_phi_arg (as_a (SLP_TREE_VEC_STMTS (phi_node)[i]), vect_get_slp_vect_def (slp_node, i), e, gimple_phi_arg_location (phi, e->dest_idx)); } diff --git a/gcc/tree-vectorizer.c b/gcc/tree-vectorizer.c index e9fe3840e1e..76cfba5d497 100644 --- a/gcc/tree-vectorizer.c +++ b/gcc/tree-vectorizer.c @@ -461,6 +461,7 @@ vec_info::vec_info (vec_info::vec_kind kind_in, void *target_cost_data_in, vec_info_shared *shared_) : kind (kind_in), shared (shared_), + stmt_vec_info_ro (false), target_cost_data (target_cost_data_in) { stmt_vec_infos.create (50); @@ -619,8 +620,7 @@ vec_info::replace_stmt (gimple_stmt_iterator *gsi, stmt_vec_info stmt_info, { gimple *old_stmt = stmt_info->stmt; gcc_assert (!stmt_info->pattern_stmt_p && old_stmt == gsi_stmt (*gsi)); - set_vinfo_for_stmt (old_stmt, NULL); - set_vinfo_for_stmt (new_stmt, stmt_info); + gimple_set_uid (new_stmt, gimple_uid (old_stmt)); stmt_info->stmt = new_stmt; gsi_replace (gsi, new_stmt, true); } @@ -666,6 +666,7 @@ vec_info::set_vinfo_for_stmt (gimple *stmt, stmt_vec_info info) unsigned int uid = gimple_uid (stmt); if (uid == 0) { + gcc_assert (!stmt_vec_info_ro); gcc_checking_assert (info); uid = stmt_vec_infos.length () + 1; gimple_set_uid (stmt, uid); diff --git a/gcc/tree-vectorizer.h b/gcc/tree-vectorizer.h index df056381531..14f68f7c4f6 100644 --- a/gcc/tree-vectorizer.h +++ b/gcc/tree-vectorizer.h @@ -138,7 +138,7 @@ struct _slp_tree { tree vectype; /* Vectorized stmt/s. */ - vec vec_stmts; + vec vec_stmts; vec vec_defs; /* Number of vector stmts that are created to replace the group of scalar stmts. It is calculated during the transformation phase as the number of @@ -338,6 +338,8 @@ public: /* The mapping of GIMPLE UID to stmt_vec_info. */ vec stmt_vec_infos; + /* Whether the above mapping is complete. */ + bool stmt_vec_info_ro; /* The SLP graph. */ auto_vec slp_instances; @@ -965,7 +967,7 @@ public: tree vectype; /* The vectorized stmts. */ - vec vec_stmts; + vec vec_stmts; /* The following is relevant only for stmts that contain a non-scalar data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have @@ -1126,8 +1128,6 @@ public: /* True if this is only suitable for SLP vectorization. */ bool slp_vect_only_p; - - bool vector_stmt; }; /* Information about a gather/scatter call. */ @@ -1169,7 +1169,7 @@ struct gather_scatter_info { #define STMT_VINFO_RELEVANT(S) (S)->relevant #define STMT_VINFO_LIVE_P(S) (S)->live #define STMT_VINFO_VECTYPE(S) (S)->vectype -#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts +#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts #define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable #define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0) #define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p @@ -1737,11 +1737,9 @@ record_stmt_cost (stmt_vector_for_cost *body_cost_vec, int count, STMT_VINFO_VECTYPE (stmt_info), misalign, where); } -extern stmt_vec_info vect_finish_replace_stmt (vec_info *, - stmt_vec_info, gimple *); -extern stmt_vec_info vect_finish_stmt_generation (vec_info *, - stmt_vec_info, gimple *, - gimple_stmt_iterator *); +extern void vect_finish_replace_stmt (vec_info *, stmt_vec_info, gimple *); +extern void vect_finish_stmt_generation (vec_info *, stmt_vec_info, gimple *, + gimple_stmt_iterator *); extern opt_result vect_mark_stmts_to_be_vectorized (loop_vec_info, bool *); extern tree vect_get_store_rhs (stmt_vec_info); void vect_get_vec_defs_for_operand (vec_info *vinfo, stmt_vec_info, unsigned, @@ -1872,16 +1870,16 @@ extern bool vectorizable_reduction (loop_vec_info, stmt_vec_info, stmt_vector_for_cost *); extern bool vectorizable_induction (loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, slp_tree, + gimple **, slp_tree, stmt_vector_for_cost *); extern bool vect_transform_reduction (loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, - stmt_vec_info *, slp_tree); + gimple **, slp_tree); extern bool vect_transform_cycle_phi (loop_vec_info, stmt_vec_info, - stmt_vec_info *, + gimple **, slp_tree, slp_instance); extern bool vectorizable_lc_phi (loop_vec_info, stmt_vec_info, - stmt_vec_info *, slp_tree); + gimple **, slp_tree); extern bool vect_worthwhile_without_simd_p (vec_info *, tree_code); extern int vect_get_known_peeling_cost (loop_vec_info, int, int *, stmt_vector_for_cost *,