From 47b4c53f035502f1140ae6e286c4576faa57c7ea Mon Sep 17 00:00:00 2001 From: Bin Cheng Date: Fri, 14 Dec 2018 05:40:07 +0000 Subject: [PATCH] auto-profile.c (AFDO_EINFO): New macro. * auto-profile.c (AFDO_EINFO): New macro. (class edge_info): New class. (is_edge_annotated, set_edge_annotated): Delete. (afdo_propagate_edge, afdo_propagate_circuit, afdo_propagate): Remove parameter. Adjust edge count computation and annotation using class edge_info. (afdo_calculate_branch_prob, afdo_annotate_cfg): Likewise. From-SVN: r267119 --- gcc/ChangeLog | 10 +++ gcc/auto-profile.c | 190 +++++++++++++++++++++++---------------------- 2 files changed, 107 insertions(+), 93 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index e3baaa6026b..b59574863b3 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,13 @@ +2018-12-14 Bin Cheng + + * auto-profile.c (AFDO_EINFO): New macro. + (class edge_info): New class. + (is_edge_annotated, set_edge_annotated): Delete. + (afdo_propagate_edge, afdo_propagate_circuit, afdo_propagate): Remove + parameter. Adjust edge count computation and annotation using class + edge_info. + (afdo_calculate_branch_prob, afdo_annotate_cfg): Likewise. + 2018-12-13 Michael Ploujnikov * ipa-cp.c (print_all_lattices): Skip cp clones. diff --git a/gcc/auto-profile.c b/gcc/auto-profile.c index cde4f41c1d9..7e0020bc232 100644 --- a/gcc/auto-profile.c +++ b/gcc/auto-profile.c @@ -101,6 +101,23 @@ along with GCC; see the file COPYING3. If not see namespace autofdo { +/* Intermediate edge info used when propagating AutoFDO profile information. + We can't edge->count() directly since it's computed from edge's probability + while probability is yet not decided during propagation. */ +#define AFDO_EINFO(e) ((struct edge_info *) e->aux) +class edge_info +{ +public: + edge_info () : count_ (profile_count::zero ().afdo ()), annotated_ (false) {} + bool is_annotated () const { return annotated_; } + void set_annotated () { annotated_ = true; } + profile_count get_count () const { return count_; } + void set_count (profile_count count) { count_ = count; } +private: + profile_count count_; + bool annotated_; +}; + /* Represent a source location: (function_decl, lineno). */ typedef std::pair decl_lineno; @@ -1067,18 +1084,6 @@ set_bb_annotated (basic_block bb, bb_set *annotated) annotated->insert (bb); } -static bool -is_edge_annotated (const edge e, const edge_set &annotated) -{ - return annotated.find (e) != annotated.end (); -} - -static void -set_edge_annotated (edge e, edge_set *annotated) -{ - annotated->insert (e); -} - /* For a given BB, set its execution count. Attach value profile if a stmt is not in PROMOTED, because we only want to promote an indirect call once. Return TRUE if BB is annotated. */ @@ -1188,12 +1193,11 @@ afdo_find_equiv_class (bb_set *annotated_bb) edges' counts are known, then the basic block's unknown count can also be calculated. IS_SUCC is true if out edges of a basic blocks are examined. - Update ANNOTATED_BB and ANNOTATED_EDGE accordingly. + Update ANNOTATED_BB accordingly. Return TRUE if any basic block/edge count is changed. */ static bool -afdo_propagate_edge (bool is_succ, bb_set *annotated_bb, - edge_set *annotated_edge) +afdo_propagate_edge (bool is_succ, bb_set *annotated_bb) { basic_block bb; bool changed = false; @@ -1206,30 +1210,30 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb, profile_count total_known_count = profile_count::zero ().afdo (); FOR_EACH_EDGE (e, ei, is_succ ? bb->succs : bb->preds) - if (!is_edge_annotated (e, *annotated_edge)) - num_unknown_edge++, unknown_edge = e; - else - total_known_count += e->count (); + { + gcc_assert (AFDO_EINFO (e) != NULL); + if (! AFDO_EINFO (e)->is_annotated ()) + num_unknown_edge++, unknown_edge = e; + else + total_known_count += AFDO_EINFO (e)->get_count (); + } - if (num_unknown_edge == 0) + /* Be careful not to annotate block with no successor in special cases. */ + if (num_unknown_edge == 0 && total_known_count > bb->count) { - if (total_known_count > bb->count) - { - bb->count = total_known_count; - changed = true; - } - if (!is_bb_annotated (bb, *annotated_bb)) - { - set_bb_annotated (bb, annotated_bb); - changed = true; - } + bb->count = total_known_count; + if (!is_bb_annotated (bb, *annotated_bb)) + set_bb_annotated (bb, annotated_bb); + changed = true; } else if (num_unknown_edge == 1 && is_bb_annotated (bb, *annotated_bb)) { - unknown_edge->probability - = total_known_count.probability_in (bb->count); - set_edge_annotated (unknown_edge, annotated_edge); - changed = true; + if (bb->count > total_known_count) + AFDO_EINFO (unknown_edge)->set_count (bb->count - total_known_count); + else + AFDO_EINFO (unknown_edge)->set_count (profile_count::zero().afdo ()); + AFDO_EINFO (unknown_edge)->set_annotated (); + changed = true; } } return changed; @@ -1265,11 +1269,10 @@ afdo_propagate_edge (bool is_succ, bb_set *annotated_bb, goto BB3 In this case, we need to propagate through PHI to determine the edge - count of BB1->BB.t1, BB.t1->BB.t2. - Update ANNOTATED_EDGE accordingly. */ + count of BB1->BB.t1, BB.t1->BB.t2. */ static void -afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge) +afdo_propagate_circuit (const bb_set &annotated_bb) { basic_block bb; FOR_ALL_BB_FN (bb, cfun) @@ -1308,7 +1311,7 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge) bool check_value_one = (((integer_onep (cmp_rhs)) ^ (gimple_cond_code (cmp_stmt) == EQ_EXPR)) ^ ((e->flags & EDGE_TRUE_VALUE) != 0)); - if (!is_edge_annotated (e, *annotated_edge)) + if (! AFDO_EINFO (e)->is_annotated ()) continue; for (i = 0; i < gimple_phi_num_args (phi_stmt); i++) { @@ -1322,18 +1325,18 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge) continue; total++; only_one = ep; - if (!e->probability.initialized_p () - && !is_edge_annotated (ep, *annotated_edge)) - { - ep->probability = profile_probability::never ().afdo (); - set_edge_annotated (ep, annotated_edge); - } - } - if (total == 1 && !is_edge_annotated (only_one, *annotated_edge)) - { - only_one->probability = e->probability; - set_edge_annotated (only_one, annotated_edge); - } + if (! (AFDO_EINFO (e)->get_count ()).nonzero_p () + && ! AFDO_EINFO (ep)->is_annotated ()) + { + AFDO_EINFO (ep)->set_count (profile_count::zero ().afdo ()); + AFDO_EINFO (ep)->set_annotated (); + } + } + if (total == 1 && ! AFDO_EINFO (only_one)->is_annotated ()) + { + AFDO_EINFO (only_one)->set_count (AFDO_EINFO (e)->get_count ()); + AFDO_EINFO (only_one)->set_annotated (); + } } } } @@ -1342,7 +1345,7 @@ afdo_propagate_circuit (const bb_set &annotated_bb, edge_set *annotated_edge) graph. We do the propagation iteratively until stablize. */ static void -afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge) +afdo_propagate (bb_set *annotated_bb) { basic_block bb; bool changed = true; @@ -1359,11 +1362,11 @@ afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge) { changed = false; - if (afdo_propagate_edge (true, annotated_bb, annotated_edge)) + if (afdo_propagate_edge (true, annotated_bb)) changed = true; - if (afdo_propagate_edge (false, annotated_bb, annotated_edge)) + if (afdo_propagate_edge (false, annotated_bb)) changed = true; - afdo_propagate_circuit (*annotated_bb, annotated_edge); + afdo_propagate_circuit (*annotated_bb); } } @@ -1371,52 +1374,59 @@ afdo_propagate (bb_set *annotated_bb, edge_set *annotated_edge) probabilities. */ static void -afdo_calculate_branch_prob (bb_set *annotated_bb, edge_set *annotated_edge) +afdo_calculate_branch_prob (bb_set *annotated_bb) { + edge e; + edge_iterator ei; basic_block bb; - bool has_sample = false; - - FOR_EACH_BB_FN (bb, cfun) - { - if (bb->count > profile_count::zero ()) - { - has_sample = true; - break; - } - } - - if (!has_sample) - return; calculate_dominance_info (CDI_POST_DOMINATORS); calculate_dominance_info (CDI_DOMINATORS); loop_optimizer_init (0); + FOR_ALL_BB_FN (bb, cfun) + { + gcc_assert (bb->aux == NULL); + FOR_EACH_EDGE (e, ei, bb->succs) + { + gcc_assert (e->aux == NULL); + e->aux = new edge_info (); + } + } + afdo_find_equiv_class (annotated_bb); - afdo_propagate (annotated_bb, annotated_edge); + afdo_propagate (annotated_bb); FOR_EACH_BB_FN (bb, cfun) { - edge e; - edge_iterator ei; int num_unknown_succ = 0; profile_count total_count = profile_count::zero ().afdo (); FOR_EACH_EDGE (e, ei, bb->succs) { - if (!is_edge_annotated (e, *annotated_edge)) + gcc_assert (AFDO_EINFO (e) != NULL); + if (! AFDO_EINFO (e)->is_annotated ()) num_unknown_succ++; else - total_count += e->count (); + total_count += AFDO_EINFO (e)->get_count (); } if (num_unknown_succ == 0 && total_count > profile_count::zero ()) { - FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = e->count ().probability_in (total_count); + FOR_EACH_EDGE (e, ei, bb->succs) + e->probability + = AFDO_EINFO (e)->get_count ().probability_in (total_count); } } FOR_ALL_BB_FN (bb, cfun) - bb->aux = NULL; + { + bb->aux = NULL; + FOR_EACH_EDGE (e, ei, bb->succs) + if (AFDO_EINFO (e) != NULL) + { + delete AFDO_EINFO (e); + e->aux = NULL; + } + } loop_optimizer_finalize (); free_dominance_info (CDI_DOMINATORS); @@ -1496,7 +1506,6 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts) { basic_block bb; bb_set annotated_bb; - edge_set annotated_edge; const function_instance *s = afdo_source_profile->get_function_instance_by_decl ( current_function_decl); @@ -1511,21 +1520,15 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts) profile_count max_count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count; FOR_EACH_BB_FN (bb, cfun) - { - edge e; - edge_iterator ei; - - /* As autoFDO uses sampling approach, we have to assume that all - counters are zero when not seen by autoFDO. */ - bb->count = profile_count::zero ().afdo (); - FOR_EACH_EDGE (e, ei, bb->succs) - e->probability = profile_probability::uninitialized (); - - if (afdo_set_bb_count (bb, promoted_stmts)) - set_bb_annotated (bb, &annotated_bb); - if (bb->count > max_count) - max_count = bb->count; - } + { + /* As autoFDO uses sampling approach, we have to assume that all + counters are zero when not seen by autoFDO. */ + bb->count = profile_count::zero ().afdo (); + if (afdo_set_bb_count (bb, promoted_stmts)) + set_bb_annotated (bb, &annotated_bb); + if (bb->count > max_count) + max_count = bb->count; + } if (ENTRY_BLOCK_PTR_FOR_FN (cfun)->count > ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb->count) { @@ -1546,7 +1549,8 @@ afdo_annotate_cfg (const stmt_set &promoted_stmts) afdo_source_profile->mark_annotated (cfun->function_end_locus); if (max_count > profile_count::zero ()) { - afdo_calculate_branch_prob (&annotated_bb, &annotated_edge); + /* Calculate, propagate count and probability information on CFG. */ + afdo_calculate_branch_prob (&annotated_bb); update_max_bb_count (); profile_status_for_fn (cfun) = PROFILE_READ; } -- 2.30.2