From 1bad9c1806ac51abc2bf1bdf18b96924b1104ebb Mon Sep 17 00:00:00 2001 From: Jan Hubicka Date: Fri, 10 Nov 2017 21:14:52 +0100 Subject: [PATCH] auto-profile.c (afdo_indirect_call): Drop frequency. * auto-profile.c (afdo_indirect_call): Drop frequency. * cgraph.c (symbol_table::create_edge): Drop frequency argument. (cgraph_node::create_edge): Drop frequency argument. (cgraph_node::create_indirect_edge): Drop frequency argument. (cgraph_edge::make_speculative): Drop frequency arguments. (cgraph_edge::resolve_speculation): Do not update frequencies (cgraph_edge::dump_edge_flags): Do not dump frequency. (cgraph_node::dump): Check consistency in IPA mode. (cgraph_edge::maybe_hot_p): Use IPA counter. (cgraph_edge::verify_count_and_frequency): Rename to ... (cgraph_edge::verify_count): ... this one; drop frequency checking. (cgraph_node::verify_node): Update. * cgraph.h (struct cgraph_edge): Drop frequency. (cgraph_edge::frequency): New function. * cgraphbuild.c (pass_build_cgraph_edges::execute): Donot pass frequencies. (cgraph_edge::rebuild_edges): Likewise. * cgraphclones.c (cgraph_edge::clone): Scale only counts. (duplicate_thunk_for_node): Do not pass frequency. (cgraph_node::create_clone): Scale only counts. (cgraph_node::create_virtual_clone): Do not pass frequency. (cgraph_node::create_edge_including_clones): Do not pass frequency. (cgraph_node::create_version_clone): Do not pass frequency. * cgraphunit.c (cgraph_node::analyze): Do not pass frequency. (cgraph_node::expand_thunk): Do not pass frequency. (cgraph_node::create_wrapper): Do not pass frequency. * gimple-iterator.c (update_call_edge_frequencies): Do not pass frequency. * gimple-streamer-in.c (input_bb): Scale only IPA counts. * ipa-chkp.c (chkp_produce_thunks): Do not pass frequency. * ipa-cp.c (ipcp_lattice::print): Use frequency function. (gather_caller_stats): Use frequency function. (ipcp_cloning_candidate_p): Use frequency function. (ipcp_propagate_stage): Use frequency function. (get_info_about_necessary_edges): Use frequency function. (update_profiling_info): Update only IPA profile. (update_specialized_profile): Use frequency functoin. (perhaps_add_new_callers): Update only IPA profile. * ipa-devirt.c (ipa_devirt): Use IPA profile. * ipa-fnsummary.c (redirect_to_unreachable): Do not set frequrency. (dump_ipa_call_summary): Use frequency function. (estimate_edge_size_and_time): Use frequency function. (ipa_merge_fn_summary_after_inlining): Use frequency function. * ipa-inline-analysis.c (do_estimate_edge_time): Use IPA profile. * ipa-inline-transform.c (update_noncloned_frequencies): Rename to .. (update_noncloned_counts): ... ths one; scale counts only. (clone_inlined_nodes): Do not scale frequency. (inline_call): Do not pass frequency. * ipa-inline.c (compute_uninlined_call_time): Use IPA profile. (compute_inlined_call_time): Use IPA profile. (want_inline_small_function_p): Use IPA profile. (want_inline_self_recursive_call_p): Use IPA profile. (edge_badness): Use IPA profile. (lookup_recursive_calls): Use IPA profile. (recursive_inlining): Do not pass frequency. (resolve_noninline_speculation): Do not update frequency. (inline_small_functions): Collect max of IPA profile. (dump_overall_stats): Dump IPA porfile. (dump_inline_stats): Dump IPA porfile. (ipa_inline): Collect IPA stats. * ipa-inline.h (clone_inlined_nodes): Update prototype. * ipa-profile.c (ipa_propagate_frequency_1): Use frequency function. (ipa_propagate_frequency): Use frequency function. (ipa_profile): Cleanup. * ipa-prop.c (ipa_make_edge_direct_to_target): Do not pass frequency * ipa-utils.c (ipa_merge_profiles): Merge all profiles. * lto-cgraph.c (lto_output_edge): Do not stream frequency. (input_node): Do not stream frequency. (input_edge): Do not stream frequency. (merge_profile_summaries): Scale only IPA profiles. * omp-simd-clone.c (simd_clone_adjust): Do not pass frequency. * predict.c (drop_profile): Do not recompute frequency. * trans-mem.c (ipa_tm_insert_irr_call): Do not pass frequency. (ipa_tm_insert_gettmclone_call): Do not pass frequency. * tree-cfg.c (execute_fixup_cfg): Drop profile to global0 if needed. * tree-chkp.c (chkp_copy_bounds_for_assign): Do not pass frequency. * tree-emutls.c (gen_emutls_addr): Do not pass frequency. * tree-inline.c (copy_bb): Do not scale frequency. (expand_call_inline): Do not scale frequency. (tree_function_versioning): Do not scale frequency. * ubsan.c (ubsan_create_edge): Do not pass frequency. lto/ChangeLog: 2017-11-10 Jan Hubicka * lto-partition.c (lto_balanced_map): Use frequency accessor. From-SVN: r254636 --- gcc/ChangeLog | 84 +++++++++++++++ gcc/auto-profile.c | 2 +- gcc/cgraph.c | 149 +++++++++------------------ gcc/cgraph.h | 37 ++++--- gcc/cgraphbuild.c | 14 +-- gcc/cgraphclones.c | 72 +++++++------ gcc/cgraphunit.c | 6 +- gcc/gimple-iterator.c | 9 +- gcc/gimple-streamer-in.c | 7 +- gcc/ipa-chkp.c | 2 +- gcc/ipa-cp.c | 28 ++--- gcc/ipa-devirt.c | 5 +- gcc/ipa-fnsummary.c | 11 +- gcc/ipa-inline-analysis.c | 8 +- gcc/ipa-inline-transform.c | 39 ++----- gcc/ipa-inline.c | 205 +++++++++++++++++++------------------ gcc/ipa-inline.h | 3 +- gcc/ipa-profile.c | 12 +-- gcc/ipa-prop.c | 2 +- gcc/ipa-utils.c | 38 ++----- gcc/lto-cgraph.c | 18 ++-- gcc/lto/ChangeLog | 4 + gcc/lto/lto-partition.c | 4 +- gcc/omp-simd-clone.c | 4 +- gcc/predict.c | 9 +- gcc/trans-mem.c | 8 +- gcc/tree-cfg.c | 17 +-- gcc/tree-chkp.c | 8 +- gcc/tree-emutls.c | 2 +- gcc/tree-inline.c | 81 ++++----------- gcc/ubsan.c | 4 +- 31 files changed, 428 insertions(+), 464 deletions(-) diff --git a/gcc/ChangeLog b/gcc/ChangeLog index 103dd26a0bc..ba6d46b0d79 100644 --- a/gcc/ChangeLog +++ b/gcc/ChangeLog @@ -1,3 +1,87 @@ +2017-11-10 Jan Hubicka + + * auto-profile.c (afdo_indirect_call): Drop frequency. + * cgraph.c (symbol_table::create_edge): Drop frequency argument. + (cgraph_node::create_edge): Drop frequency argument. + (cgraph_node::create_indirect_edge): Drop frequency argument. + (cgraph_edge::make_speculative): Drop frequency arguments. + (cgraph_edge::resolve_speculation): Do not update frequencies + (cgraph_edge::dump_edge_flags): Do not dump frequency. + (cgraph_node::dump): Check consistency in IPA mode. + (cgraph_edge::maybe_hot_p): Use IPA counter. + (cgraph_edge::verify_count_and_frequency): Rename to ... + (cgraph_edge::verify_count): ... this one; drop frequency checking. + (cgraph_node::verify_node): Update. + * cgraph.h (struct cgraph_edge): Drop frequency. + (cgraph_edge::frequency): New function. + * cgraphbuild.c (pass_build_cgraph_edges::execute): Donot pass + frequencies. + (cgraph_edge::rebuild_edges): Likewise. + * cgraphclones.c (cgraph_edge::clone): Scale only counts. + (duplicate_thunk_for_node): Do not pass frequency. + (cgraph_node::create_clone): Scale only counts. + (cgraph_node::create_virtual_clone): Do not pass frequency. + (cgraph_node::create_edge_including_clones): Do not pass frequency. + (cgraph_node::create_version_clone): Do not pass frequency. + * cgraphunit.c (cgraph_node::analyze): Do not pass frequency. + (cgraph_node::expand_thunk): Do not pass frequency. + (cgraph_node::create_wrapper): Do not pass frequency. + * gimple-iterator.c (update_call_edge_frequencies): Do not pass + frequency. + * gimple-streamer-in.c (input_bb): Scale only IPA counts. + * ipa-chkp.c (chkp_produce_thunks): Do not pass frequency. + * ipa-cp.c (ipcp_lattice::print): Use frequency function. + (gather_caller_stats): Use frequency function. + (ipcp_cloning_candidate_p): Use frequency function. + (ipcp_propagate_stage): Use frequency function. + (get_info_about_necessary_edges): Use frequency function. + (update_profiling_info): Update only IPA profile. + (update_specialized_profile): Use frequency functoin. + (perhaps_add_new_callers): Update only IPA profile. + * ipa-devirt.c (ipa_devirt): Use IPA profile. + * ipa-fnsummary.c (redirect_to_unreachable): Do not set frequrency. + (dump_ipa_call_summary): Use frequency function. + (estimate_edge_size_and_time): Use frequency function. + (ipa_merge_fn_summary_after_inlining): Use frequency function. + * ipa-inline-analysis.c (do_estimate_edge_time): Use IPA profile. + * ipa-inline-transform.c (update_noncloned_frequencies): Rename to .. + (update_noncloned_counts): ... ths one; scale counts only. + (clone_inlined_nodes): Do not scale frequency. + (inline_call): Do not pass frequency. + * ipa-inline.c (compute_uninlined_call_time): Use IPA profile. + (compute_inlined_call_time): Use IPA profile. + (want_inline_small_function_p): Use IPA profile. + (want_inline_self_recursive_call_p): Use IPA profile. + (edge_badness): Use IPA profile. + (lookup_recursive_calls): Use IPA profile. + (recursive_inlining): Do not pass frequency. + (resolve_noninline_speculation): Do not update frequency. + (inline_small_functions): Collect max of IPA profile. + (dump_overall_stats): Dump IPA porfile. + (dump_inline_stats): Dump IPA porfile. + (ipa_inline): Collect IPA stats. + * ipa-inline.h (clone_inlined_nodes): Update prototype. + * ipa-profile.c (ipa_propagate_frequency_1): Use frequency function. + (ipa_propagate_frequency): Use frequency function. + (ipa_profile): Cleanup. + * ipa-prop.c (ipa_make_edge_direct_to_target): Do not pass frequency + * ipa-utils.c (ipa_merge_profiles): Merge all profiles. + * lto-cgraph.c (lto_output_edge): Do not stream frequency. + (input_node): Do not stream frequency. + (input_edge): Do not stream frequency. + (merge_profile_summaries): Scale only IPA profiles. + * omp-simd-clone.c (simd_clone_adjust): Do not pass frequency. + * predict.c (drop_profile): Do not recompute frequency. + * trans-mem.c (ipa_tm_insert_irr_call): Do not pass frequency. + (ipa_tm_insert_gettmclone_call): Do not pass frequency. + * tree-cfg.c (execute_fixup_cfg): Drop profile to global0 if needed. + * tree-chkp.c (chkp_copy_bounds_for_assign): Do not pass frequency. + * tree-emutls.c (gen_emutls_addr): Do not pass frequency. + * tree-inline.c (copy_bb): Do not scale frequency. + (expand_call_inline): Do not scale frequency. + (tree_function_versioning): Do not scale frequency. + * ubsan.c (ubsan_create_edge): Do not pass frequency. + 2017-11-10 Julia Koval * config/i386/gfniintrin.h (_mm_gf2p8affine_epi64_epi8, diff --git a/gcc/auto-profile.c b/gcc/auto-profile.c index 130d8df5b1e..f8c8871cf5e 100644 --- a/gcc/auto-profile.c +++ b/gcc/auto-profile.c @@ -1061,7 +1061,7 @@ afdo_indirect_call (gimple_stmt_iterator *gsi, const icall_target_map &map, /* FIXME: Count should be initialized. */ struct cgraph_edge *new_edge = indirect_edge->make_speculative (direct_call, - profile_count::uninitialized (), 0); + profile_count::uninitialized ()); new_edge->redirect_call_stmt_to_callee (); gimple_remove_histogram_value (cfun, stmt, hist); inline_call (new_edge, true, NULL, NULL, false); diff --git a/gcc/cgraph.c b/gcc/cgraph.c index 7c3507c6ece..83e496b4239 100644 --- a/gcc/cgraph.c +++ b/gcc/cgraph.c @@ -822,7 +822,7 @@ cgraph_edge::set_call_stmt (gcall *new_stmt, bool update_speculative) cgraph_edge * symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee, - gcall *call_stmt, profile_count count, int freq, + gcall *call_stmt, profile_count count, bool indir_unknown_callee) { cgraph_edge *edge; @@ -862,10 +862,7 @@ symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee, edge->next_callee = NULL; edge->lto_stmt_uid = 0; - edge->count = count.ipa (); - edge->frequency = freq; - gcc_checking_assert (freq >= 0); - gcc_checking_assert (freq <= CGRAPH_FREQ_MAX); + edge->count = count; edge->call_stmt = call_stmt; push_cfun (DECL_STRUCT_FUNCTION (caller->decl)); @@ -907,10 +904,10 @@ symbol_table::create_edge (cgraph_node *caller, cgraph_node *callee, cgraph_edge * cgraph_node::create_edge (cgraph_node *callee, - gcall *call_stmt, profile_count count, int freq) + gcall *call_stmt, profile_count count) { cgraph_edge *edge = symtab->create_edge (this, callee, call_stmt, count, - freq, false); + false); initialize_inline_failed (edge); @@ -944,11 +941,11 @@ cgraph_allocate_init_indirect_info (void) cgraph_edge * cgraph_node::create_indirect_edge (gcall *call_stmt, int ecf_flags, - profile_count count, int freq, + profile_count count, bool compute_indirect_info) { cgraph_edge *edge = symtab->create_edge (this, NULL, call_stmt, - count, freq, true); + count, true); tree target; initialize_inline_failed (edge); @@ -1060,8 +1057,7 @@ cgraph_edge::remove (void) Return direct edge created. */ cgraph_edge * -cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count, - int direct_frequency) +cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count) { cgraph_node *n = caller; ipa_ref *ref = NULL; @@ -1071,7 +1067,7 @@ cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count, fprintf (dump_file, "Indirect call -> speculative call %s => %s\n", n->dump_name (), n2->dump_name ()); speculative = true; - e2 = n->create_edge (n2, call_stmt, direct_count, direct_frequency); + e2 = n->create_edge (n2, call_stmt, direct_count); initialize_inline_failed (e2); e2->speculative = true; if (TREE_NOTHROW (n2->decl)) @@ -1081,7 +1077,6 @@ cgraph_edge::make_speculative (cgraph_node *n2, profile_count direct_count, e2->lto_stmt_uid = lto_stmt_uid; e2->in_polymorphic_cdtor = in_polymorphic_cdtor; count -= e2->count; - frequency -= e2->frequency; symtab->call_edge_duplication_hooks (this, e2); ref = n->create_reference (n2, IPA_REF_ADDR, call_stmt); ref->lto_stmt_uid = lto_stmt_uid; @@ -1198,9 +1193,6 @@ cgraph_edge::resolve_speculation (tree callee_decl) in the functions inlined through it. */ } edge->count += e2->count; - edge->frequency += e2->frequency; - if (edge->frequency > CGRAPH_FREQ_MAX) - edge->frequency = CGRAPH_FREQ_MAX; edge->speculative = false; e2->speculative = false; ref->remove_reference (); @@ -1308,9 +1300,7 @@ cgraph_edge::redirect_call_stmt_to_callee (void) /* We are producing the final function body and will throw away the callgraph edges really soon. Reset the counts/frequencies to keep verifier happy in the case of roundoff errors. */ - e->count = gimple_bb (e->call_stmt)->count.ipa (); - e->frequency = compute_call_stmt_bb_frequency - (e->caller->decl, gimple_bb (e->call_stmt)); + e->count = gimple_bb (e->call_stmt)->count; } /* Expand speculation into GIMPLE code. */ else @@ -1329,12 +1319,7 @@ cgraph_edge::redirect_call_stmt_to_callee (void) profile_probability prob = e->count.probability_in (e->count + e2->count); - if (prob.initialized_p ()) - ; - else if (e->frequency || e2->frequency) - prob = profile_probability::probability_in_gcov_type - (e->frequency, e->frequency + e2->frequency).guessed (); - else + if (!prob.initialized_p ()) prob = profile_probability::even (); new_stmt = gimple_ic (e->call_stmt, dyn_cast (ref->referred), @@ -1355,24 +1340,11 @@ cgraph_edge::redirect_call_stmt_to_callee (void) gcall *ibndret = chkp_retbnd_call_by_val (iresult); struct cgraph_edge *iedge = e2->caller->cgraph_node::get_edge (ibndret); - struct cgraph_edge *dedge; if (dbndret) - { - dedge = iedge->caller->create_edge (iedge->callee, - dbndret, e->count, - e->frequency); - dedge->frequency = compute_call_stmt_bb_frequency - (dedge->caller->decl, gimple_bb (dedge->call_stmt)); - } - iedge->frequency = compute_call_stmt_bb_frequency - (iedge->caller->decl, gimple_bb (iedge->call_stmt)); + iedge->caller->create_edge (iedge->callee, dbndret, e->count); } - e->frequency = compute_call_stmt_bb_frequency - (e->caller->decl, gimple_bb (e->call_stmt)); - e2->frequency = compute_call_stmt_bb_frequency - (e2->caller->decl, gimple_bb (e2->call_stmt)); e2->speculative = false; ref->speculative = false; ref->stmt = NULL; @@ -1610,7 +1582,6 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node, cgraph_edge *e = node->get_edge (old_stmt); cgraph_edge *ne = NULL; profile_count count; - int frequency; if (e) { @@ -1644,8 +1615,7 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node, /* Otherwise remove edge and create new one; we can't simply redirect since function has changed, so inline plan and other information attached to edge is invalid. */ - count = e->count.ipa (); - frequency = e->frequency; + count = e->count; if (e->indirect_unknown_callee || e->inline_failed) e->remove (); else @@ -1655,16 +1625,13 @@ cgraph_update_edges_for_call_stmt_node (cgraph_node *node, { /* We are seeing new direct call; compute profile info based on BB. */ basic_block bb = gimple_bb (new_stmt); - count = bb->count.ipa (); - frequency = compute_call_stmt_bb_frequency (current_function_decl, - bb); + count = bb->count; } if (new_call) { ne = node->create_edge (cgraph_node::get_create (new_call), - as_a (new_stmt), count, - frequency); + as_a (new_stmt), count); gcc_assert (ne->inline_failed); } } @@ -2056,10 +2023,9 @@ cgraph_edge::dump_edge_flags (FILE *f) { fprintf (f, "("); count.dump (f); - fprintf (f, ")"); + fprintf (f, ","); + fprintf (f, "%.2f per call) ", frequency () / (double)CGRAPH_FREQ_BASE); } - if (frequency) - fprintf (f, "(%.2f per call) ", frequency / (double)CGRAPH_FREQ_BASE); if (can_throw_external) fprintf (f, "(can throw external) "); } @@ -2205,7 +2171,7 @@ cgraph_node::dump (FILE *f) } fprintf (f, "\n"); - if (count.initialized_p ()) + if (count.ipa ().initialized_p ()) { bool ok = true; bool min = false; @@ -2213,14 +2179,14 @@ cgraph_node::dump (FILE *f) FOR_EACH_ALIAS (this, ref) if (dyn_cast (ref->referring)->count.initialized_p ()) - sum += dyn_cast (ref->referring)->count; + sum += dyn_cast (ref->referring)->count.ipa (); if (global.inlined_to || (symtab->state < EXPANSION && ultimate_alias_target () == this && only_called_directly_p ())) ok = !count.differs_from_p (sum); - else if (count > profile_count::from_gcov_type (100) - && count < sum.apply_scale (99, 100)) + else if (count.ipa () > profile_count::from_gcov_type (100) + && count.ipa () < sum.apply_scale (99, 100)) ok = false, min = true; if (!ok) { @@ -2826,7 +2792,7 @@ cgraph_edge::cannot_lead_to_return_p (void) bool cgraph_edge::maybe_hot_p (void) { - if (!maybe_hot_count_p (NULL, count)) + if (!maybe_hot_count_p (NULL, count.ipa ())) return false; if (caller->frequency == NODE_FREQUENCY_UNLIKELY_EXECUTED || (callee @@ -2845,12 +2811,12 @@ cgraph_edge::maybe_hot_p (void) if (symtab->state < IPA_SSA) return true; if (caller->frequency == NODE_FREQUENCY_EXECUTED_ONCE - && frequency < CGRAPH_FREQ_BASE * 3 / 2) + && frequency () < CGRAPH_FREQ_BASE * 3 / 2) return false; if (opt_for_fn (caller->decl, flag_guess_branch_prob)) { if (PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION) == 0 - || frequency <= (CGRAPH_FREQ_BASE + || frequency () <= (CGRAPH_FREQ_BASE / PARAM_VALUE (HOT_BB_FREQUENCY_FRACTION))) return false; } @@ -3079,7 +3045,7 @@ clone_of_p (cgraph_node *node, cgraph_node *node2) /* Verify edge count and frequency. */ bool -cgraph_edge::verify_count_and_frequency () +cgraph_edge::verify_count () { bool error_found = false; if (!count.verify ()) @@ -3087,21 +3053,6 @@ cgraph_edge::verify_count_and_frequency () error ("caller edge count invalid"); error_found = true; } - if (count.initialized_p () && !(count.ipa () == count)) - { - error ("caller edge count is local"); - error_found = true; - } - if (frequency < 0) - { - error ("caller edge frequency is negative"); - error_found = true; - } - if (frequency > CGRAPH_FREQ_MAX) - { - error ("caller edge frequency is too large"); - error_found = true; - } return error_found; } @@ -3193,11 +3144,6 @@ cgraph_node::verify_node (void) error ("cgraph count invalid"); error_found = true; } - if (count.initialized_p () && !(count.ipa () == count)) - { - error ("cgraph count is local"); - error_found = true; - } if (global.inlined_to && same_comdat_group) { error ("inline clone in same comdat group list"); @@ -3244,7 +3190,7 @@ cgraph_node::verify_node (void) bool check_comdat = comdat_local_p (); for (e = callers; e; e = e->next_caller) { - if (e->verify_count_and_frequency ()) + if (e->verify_count ()) error_found = true; if (check_comdat && !in_same_comdat_group_p (e->caller)) @@ -3277,46 +3223,49 @@ cgraph_node::verify_node (void) } for (e = callees; e; e = e->next_callee) { - if (e->verify_count_and_frequency ()) + if (e->verify_count ()) error_found = true; - /* FIXME: re-enable once cgraph is converted to counts. */ if (gimple_has_body_p (e->caller->decl) - && 0 && !e->caller->global.inlined_to && !e->speculative /* Optimized out calls are redirected to __builtin_unreachable. */ - && (e->frequency + && (e->count.nonzero_p () || ! e->callee->decl || DECL_BUILT_IN_CLASS (e->callee->decl) != BUILT_IN_NORMAL || DECL_FUNCTION_CODE (e->callee->decl) != BUILT_IN_UNREACHABLE) - && (e->frequency - != compute_call_stmt_bb_frequency (e->caller->decl, - gimple_bb (e->call_stmt)))) + && count + == ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (decl))->count + && (!e->count.ipa_p () + && e->count.differs_from_p (gimple_bb (e->call_stmt)->count))) { - error ("caller edge frequency %i does not match BB frequency %i", - e->frequency, - compute_call_stmt_bb_frequency (e->caller->decl, - gimple_bb (e->call_stmt))); + error ("caller edge count does not match BB count"); + fprintf (stderr, "edge count: "); + e->count.dump (stderr); + fprintf (stderr, "\n bb count: "); + gimple_bb (e->call_stmt)->count.dump (stderr); + fprintf (stderr, "\n"); error_found = true; } } for (e = indirect_calls; e; e = e->next_callee) { - if (e->verify_count_and_frequency ()) + if (e->verify_count ()) error_found = true; - /* FIXME: re-enable once cgraph is converted to counts. */ if (gimple_has_body_p (e->caller->decl) && !e->caller->global.inlined_to && !e->speculative - && 0 - && (e->frequency - != compute_call_stmt_bb_frequency (e->caller->decl, - gimple_bb (e->call_stmt)))) + && e->count.ipa_p () + && count + == ENTRY_BLOCK_PTR_FOR_FN (DECL_STRUCT_FUNCTION (decl))->count + && (!e->count.ipa_p () + && e->count.differs_from_p (gimple_bb (e->call_stmt)->count))) { - error ("indirect call frequency %i does not match BB frequency %i", - e->frequency, - compute_call_stmt_bb_frequency (e->caller->decl, - gimple_bb (e->call_stmt))); + error ("indirect call count does not match BB count"); + fprintf (stderr, "edge count: "); + e->count.dump (stderr); + fprintf (stderr, "\n bb count: "); + gimple_bb (e->call_stmt)->count.dump (stderr); + fprintf (stderr, "\n"); error_found = true; } } diff --git a/gcc/cgraph.h b/gcc/cgraph.h index 84824e9f814..8da8f605b6f 100644 --- a/gcc/cgraph.h +++ b/gcc/cgraph.h @@ -942,7 +942,7 @@ public: All hooks will see this in node's global.inlined_to, when invoked. Can be NULL if the node is not inlined. SUFFIX is string that is appended to the original name. */ - cgraph_node *create_clone (tree decl, profile_count count, int freq, + cgraph_node *create_clone (tree decl, profile_count count, bool update_original, vec redirect_callers, bool call_duplication_hook, @@ -1110,14 +1110,13 @@ public: /* Create edge from a given function to CALLEE in the cgraph. */ cgraph_edge *create_edge (cgraph_node *callee, - gcall *call_stmt, profile_count count, - int freq); + gcall *call_stmt, profile_count count); /* Create an indirect edge with a yet-undetermined callee where the call statement destination is a formal parameter of the caller with index PARAM_INDEX. */ cgraph_edge *create_indirect_edge (gcall *call_stmt, int ecf_flags, - profile_count count, int freq, + profile_count count, bool compute_indirect_info = true); /* Like cgraph_create_edge walk the clone tree and update all clones sharing @@ -1126,7 +1125,6 @@ public: void create_edge_including_clones (cgraph_node *callee, gimple *old_stmt, gcall *stmt, profile_count count, - int freq, cgraph_inline_failed_t reason); /* Return the callgraph edge representing the GIMPLE_CALL statement @@ -1665,8 +1663,7 @@ struct GTY((chain_next ("%h.next_caller"), chain_prev ("%h.prev_caller"), /* Turn edge into speculative call calling N2. Update the profile so the direct call is taken COUNT times with FREQUENCY. */ - cgraph_edge *make_speculative (cgraph_node *n2, profile_count direct_count, - int direct_frequency); + cgraph_edge *make_speculative (cgraph_node *n2, profile_count direct_count); /* Given speculative call edge, return all three components. */ void speculative_call_info (cgraph_edge *&direct, cgraph_edge *&indirect, @@ -1684,11 +1681,11 @@ struct GTY((chain_next ("%h.next_caller"), chain_prev ("%h.prev_caller"), /* Create clone of edge in the node N represented by CALL_EXPR the callgraph. */ cgraph_edge * clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, - profile_count num, profile_count den, int freq_scale, + profile_count num, profile_count den, bool update_original); /* Verify edge count and frequency. */ - bool verify_count_and_frequency (); + bool verify_count (); /* Return true when call of edge can not lead to return from caller and thus it is safe to ignore its side effects for IPA analysis @@ -1728,10 +1725,6 @@ struct GTY((chain_next ("%h.next_caller"), chain_prev ("%h.prev_caller"), /* The stmt_uid of call_stmt. This is used by LTO to recover the call_stmt when the function is serialized in. */ unsigned int lto_stmt_uid; - /* Expected frequency of executions within the function. - When set to CGRAPH_FREQ_BASE, the edge is expected to be called once - per function call. The range is 0 to CGRAPH_FREQ_MAX. */ - int frequency; /* Unique id of the edge. */ int uid; /* Whether this edge was made direct by indirect inlining. */ @@ -1769,6 +1762,10 @@ struct GTY((chain_next ("%h.next_caller"), chain_prev ("%h.prev_caller"), /* Return true if call must bind to current definition. */ bool binds_to_current_def_p (); + /* Expected frequency of executions within the function. + When set to CGRAPH_FREQ_BASE, the edge is expected to be called once + per function call. The range is 0 to CGRAPH_FREQ_MAX. */ + int frequency (); private: /* Remove the edge from the list of the callers of the callee. */ void remove_caller (void); @@ -2287,7 +2284,7 @@ private: parameters of which only CALLEE can be NULL (when creating an indirect call edge). */ cgraph_edge *create_edge (cgraph_node *caller, cgraph_node *callee, - gcall *call_stmt, profile_count count, int freq, + gcall *call_stmt, profile_count count, bool indir_unknown_callee); /* Put the edge onto the free list. */ @@ -3111,6 +3108,18 @@ cgraph_edge::binds_to_current_def_p () return false; } +/* Expected frequency of executions within the function. + When set to CGRAPH_FREQ_BASE, the edge is expected to be called once + per function call. The range is 0 to CGRAPH_FREQ_MAX. */ + +inline int +cgraph_edge::frequency () +{ + return count.to_cgraph_frequency (caller->global.inlined_to + ? caller->global.inlined_to->count + : caller->count); +} + /* Return true if the TM_CLONE bit is set for a given FNDECL. */ static inline bool decl_is_tm_clone (const_tree fndecl) diff --git a/gcc/cgraphbuild.c b/gcc/cgraphbuild.c index dd4bf9a7fa3..efb333cbae0 100644 --- a/gcc/cgraphbuild.c +++ b/gcc/cgraphbuild.c @@ -317,17 +317,15 @@ pass_build_cgraph_edges::execute (function *fun) if (gcall *call_stmt = dyn_cast (stmt)) { - int freq = compute_call_stmt_bb_frequency (current_function_decl, - bb); decl = gimple_call_fndecl (call_stmt); if (decl) - node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count, freq); + node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count); else if (gimple_call_internal_p (call_stmt)) ; else node->create_indirect_edge (call_stmt, gimple_call_flags (call_stmt), - bb->count, freq); + bb->count); } node->record_stmt_references (stmt); if (gomp_parallel *omp_par_stmt = dyn_cast (stmt)) @@ -402,7 +400,7 @@ cgraph_edge::rebuild_edges (void) node->remove_callees (); node->remove_all_references (); - node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa (); + node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count; FOR_EACH_BB_FN (bb, cfun) { @@ -413,18 +411,16 @@ cgraph_edge::rebuild_edges (void) if (gcall *call_stmt = dyn_cast (stmt)) { - int freq = compute_call_stmt_bb_frequency (current_function_decl, - bb); decl = gimple_call_fndecl (call_stmt); if (decl) node->create_edge (cgraph_node::get_create (decl), call_stmt, - bb->count, freq); + bb->count); else if (gimple_call_internal_p (call_stmt)) ; else node->create_indirect_edge (call_stmt, gimple_call_flags (call_stmt), - bb->count, freq); + bb->count); } node->record_stmt_references (stmt); } diff --git a/gcc/cgraphclones.c b/gcc/cgraphclones.c index 6513aa768be..a575a34b0c6 100644 --- a/gcc/cgraphclones.c +++ b/gcc/cgraphclones.c @@ -87,20 +87,11 @@ along with GCC; see the file COPYING3. If not see cgraph_edge * cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, profile_count num, profile_count den, - int freq_scale, bool update_original) + bool update_original) { cgraph_edge *new_edge; - profile_count gcov_count - = (num == profile_count::zero () || den > 0) - ? count.apply_scale (num, den) : count; - gcov_type freq; - - /* We do not want to ignore loop nest after frequency drops to 0. */ - if (!freq_scale) - freq_scale = 1; - freq = frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; - if (freq > CGRAPH_FREQ_MAX) - freq = CGRAPH_FREQ_MAX; + profile_count::adjust_for_ipa_scaling (&num, &den); + profile_count gcov_count = count.apply_scale (num, den); if (indirect_unknown_callee) { @@ -113,19 +104,19 @@ cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, { cgraph_node *callee = cgraph_node::get (decl); gcc_checking_assert (callee); - new_edge = n->create_edge (callee, call_stmt, gcov_count, freq); + new_edge = n->create_edge (callee, call_stmt, gcov_count); } else { new_edge = n->create_indirect_edge (call_stmt, indirect_info->ecf_flags, - gcov_count, freq, false); + gcov_count, false); *new_edge->indirect_info = *indirect_info; } } else { - new_edge = n->create_edge (callee, call_stmt, gcov_count, freq); + new_edge = n->create_edge (callee, call_stmt, gcov_count); if (indirect_info) { new_edge->indirect_info @@ -142,10 +133,14 @@ cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p; new_edge->speculative = speculative; new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor; - if (update_original) - { - count -= new_edge->count; - } + + /* Update IPA profile. Local profiles need no updating in original. */ + if (update_original + && count.ipa () == count && new_edge->count.ipa () == new_edge->count) + count -= new_edge->count; + else if (caller->count.global0 () == caller->count + && !(count == profile_count::zero ())) + count = count.global0 (); symtab->call_edge_duplication_hooks (this, new_edge); return new_edge; } @@ -337,8 +332,7 @@ duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node) new_thunk->clone.args_to_skip = node->clone.args_to_skip; new_thunk->clone.combined_args_to_skip = node->clone.combined_args_to_skip; - cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count, - CGRAPH_FREQ_BASE); + cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count); symtab->call_edge_duplication_hooks (thunk->callees, e); symtab->call_cgraph_duplication_hooks (thunk, new_thunk); return new_thunk; @@ -422,7 +416,7 @@ dump_callgraph_transformation (const cgraph_node *original, node is not inlined. */ cgraph_node * -cgraph_node::create_clone (tree new_decl, profile_count prof_count, int freq, +cgraph_node::create_clone (tree new_decl, profile_count prof_count, bool update_original, vec redirect_callers, bool call_duplication_hook, @@ -432,11 +426,27 @@ cgraph_node::create_clone (tree new_decl, profile_count prof_count, int freq, cgraph_node *new_node = symtab->create_empty (); cgraph_edge *e; unsigned i; + profile_count old_count = count; if (new_inlined_to) dump_callgraph_transformation (this, new_inlined_to, "inlining to"); + if (prof_count == profile_count::zero () + && !(count == profile_count::zero ())) + prof_count = count.global0 (); + new_node->count = prof_count; + + /* Update IPA profile. Local profiles need no updating in original. */ + if (update_original && !(count == profile_count::zero ()) + && count.ipa () == count && prof_count.ipa () == prof_count) + { + if (count.nonzero_p () + && !(count - prof_count).nonzero_p ()) + count = count.global0 (); + else + count -= prof_count; + } new_node->decl = new_decl; new_node->register_symbol (); new_node->origin = origin; @@ -489,12 +499,12 @@ cgraph_node::create_clone (tree new_decl, profile_count prof_count, int freq, new_node->expand_all_artificial_thunks (); for (e = callees;e; e=e->next_callee) - e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, count, - freq, update_original); + e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count, + update_original); for (e = indirect_calls; e; e = e->next_callee) e->clone (new_node, e->call_stmt, e->lto_stmt_uid, - new_node->count, count, freq, update_original); + new_node->count, old_count, update_original); new_node->clone_references (this); new_node->next_sibling_clone = clones; @@ -503,9 +513,6 @@ cgraph_node::create_clone (tree new_decl, profile_count prof_count, int freq, clones = new_node; new_node->clone_of = this; - if (update_original) - count -= prof_count; - if (call_duplication_hook) symtab->call_cgraph_duplication_hooks (this, new_node); @@ -591,7 +598,7 @@ cgraph_node::create_virtual_clone (vec redirect_callers, SET_DECL_ASSEMBLER_NAME (new_decl, clone_function_name (old_decl, suffix)); SET_DECL_RTL (new_decl, NULL); - new_node = create_clone (new_decl, count, CGRAPH_FREQ_BASE, false, + new_node = create_clone (new_decl, count, false, redirect_callers, false, NULL, args_to_skip, suffix); /* Update the properties. @@ -773,7 +780,6 @@ void cgraph_node::create_edge_including_clones (cgraph_node *callee, gimple *old_stmt, gcall *stmt, profile_count count, - int freq, cgraph_inline_failed_t reason) { cgraph_node *node; @@ -781,7 +787,7 @@ cgraph_node::create_edge_including_clones (cgraph_node *callee, if (!get_edge (stmt)) { - edge = create_edge (callee, stmt, count, freq); + edge = create_edge (callee, stmt, count); edge->inline_failed = reason; } @@ -801,7 +807,7 @@ cgraph_node::create_edge_including_clones (cgraph_node *callee, edge->set_call_stmt (stmt); else if (! node->get_edge (stmt)) { - edge = node->create_edge (callee, stmt, count, freq); + edge = node->create_edge (callee, stmt, count); edge->inline_failed = reason; } @@ -904,14 +910,12 @@ cgraph_node::create_version_clone (tree new_decl, || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) e->clone (new_version, e->call_stmt, e->lto_stmt_uid, count, count, - CGRAPH_FREQ_BASE, true); for (e = indirect_calls; e; e=e->next_callee) if (!bbs_to_copy || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) e->clone (new_version, e->call_stmt, e->lto_stmt_uid, count, count, - CGRAPH_FREQ_BASE, true); FOR_EACH_VEC_ELT (redirect_callers, i, e) { diff --git a/gcc/cgraphunit.c b/gcc/cgraphunit.c index c5183a02058..3b68d646ffa 100644 --- a/gcc/cgraphunit.c +++ b/gcc/cgraphunit.c @@ -620,7 +620,7 @@ cgraph_node::analyze (void) { cgraph_node *t = cgraph_node::get (thunk.alias); - create_edge (t, NULL, t->count, CGRAPH_FREQ_BASE); + create_edge (t, NULL, t->count); callees->can_throw_external = !TREE_NOTHROW (t->decl); /* Target code in expand_thunk may need the thunk's target to be analyzed, so recurse here. */ @@ -1950,7 +1950,7 @@ cgraph_node::expand_thunk (bool output_asm_thunks, bool force_gimple_thunk) resbnd = chkp_insert_retbnd_call (NULL, restmp, &bsi); create_edge (get_create (gimple_call_fndecl (gsi_stmt (bsi))), as_a (gsi_stmt (bsi)), - callees->count, callees->frequency); + callees->count); } if (restmp && !this_adjusting @@ -2759,7 +2759,7 @@ cgraph_node::create_wrapper (cgraph_node *target) memset (&thunk, 0, sizeof (cgraph_thunk_info)); thunk.thunk_p = true; - create_edge (target, NULL, count, CGRAPH_FREQ_BASE); + create_edge (target, NULL, count); callees->can_throw_external = !TREE_NOTHROW (target->decl); tree arguments = DECL_ARGUMENTS (decl); diff --git a/gcc/gimple-iterator.c b/gcc/gimple-iterator.c index 3b74cc540f6..d9d02d305ef 100644 --- a/gcc/gimple-iterator.c +++ b/gcc/gimple-iterator.c @@ -83,7 +83,6 @@ static void update_call_edge_frequencies (gimple_seq_node first, basic_block bb) { struct cgraph_node *cfun_node = NULL; - int bb_freq = 0; gimple_seq_node n; for (n = first; n ; n = n->next) @@ -94,15 +93,11 @@ update_call_edge_frequencies (gimple_seq_node first, basic_block bb) /* These function calls are expensive enough that we want to avoid calling them if we never see any calls. */ if (cfun_node == NULL) - { - cfun_node = cgraph_node::get (current_function_decl); - bb_freq = (compute_call_stmt_bb_frequency - (current_function_decl, bb)); - } + cfun_node = cgraph_node::get (current_function_decl); e = cfun_node->get_edge (n); if (e != NULL) - e->frequency = bb_freq; + e->count = bb->count; } } diff --git a/gcc/gimple-streamer-in.c b/gcc/gimple-streamer-in.c index 0dabe1adcf6..56f748a23ae 100644 --- a/gcc/gimple-streamer-in.c +++ b/gcc/gimple-streamer-in.c @@ -264,8 +264,11 @@ input_bb (struct lto_input_block *ib, enum LTO_tags tag, index = streamer_read_uhwi (ib); bb = BASIC_BLOCK_FOR_FN (fn, index); - bb->count = profile_count::stream_in (ib).apply_scale - (count_materialization_scale, REG_BR_PROB_BASE); + bb->count = profile_count::stream_in (ib); + if (count_materialization_scale != REG_BR_PROB_BASE + && bb->count.ipa ().nonzero_p ()) + bb->count + = bb->count.apply_scale (count_materialization_scale, REG_BR_PROB_BASE); bb->flags = streamer_read_hwi (ib); /* LTO_bb1 has statements. LTO_bb0 does not. */ diff --git a/gcc/ipa-chkp.c b/gcc/ipa-chkp.c index 704ef6e4550..30684baa3f8 100644 --- a/gcc/ipa-chkp.c +++ b/gcc/ipa-chkp.c @@ -717,7 +717,7 @@ chkp_produce_thunks (bool early) node->thunk.thunk_p = true; node->thunk.add_pointer_bounds_args = true; node->create_edge (node->instrumented_version, NULL, - node->count, CGRAPH_FREQ_BASE); + node->count); node->create_reference (node->instrumented_version, IPA_REF_CHKP, NULL); /* Thunk shouldn't be a cdtor. */ diff --git a/gcc/ipa-cp.c b/gcc/ipa-cp.c index 24d2be79103..bc1e3ae799d 100644 --- a/gcc/ipa-cp.c +++ b/gcc/ipa-cp.c @@ -498,7 +498,7 @@ ipcp_lattice::print (FILE * f, bool dump_sources, bool dump_benefits) fprintf (f, " [from:"); for (s = val->sources; s; s = s->next) fprintf (f, " %i(%i)", s->cs->caller->order, - s->cs->frequency); + s->cs->frequency ()); fprintf (f, "]"); } @@ -677,9 +677,9 @@ gather_caller_stats (struct cgraph_node *node, void *data) for (cs = node->callers; cs; cs = cs->next_caller) if (!cs->caller->thunk.thunk_p) { - if (cs->count.initialized_p ()) - stats->count_sum += cs->count; - stats->freq_sum += cs->frequency; + if (cs->count.ipa ().initialized_p ()) + stats->count_sum += cs->count.ipa (); + stats->freq_sum += cs->frequency (); stats->n_calls++; if (cs->maybe_hot_p ()) stats->n_hot_calls ++; @@ -731,7 +731,7 @@ ipcp_cloning_candidate_p (struct cgraph_node *node) significantly. */ if (max_count > profile_count::zero ()) { - if (stats.count_sum > node->count.apply_scale (90, 100)) + if (stats.count_sum > node->count.ipa ().apply_scale (90, 100)) { if (dump_file) fprintf (dump_file, "Considering %s for cloning; " @@ -3272,7 +3272,7 @@ ipcp_propagate_stage (struct ipa_topo_info *topo) } if (node->definition && !node->alias) overall_size += ipa_fn_summaries->get (node)->self_size; - max_count = max_count.max (node->count); + max_count = max_count.max (node->count.ipa ()); } max_new_size = overall_size; @@ -3550,9 +3550,9 @@ get_info_about_necessary_edges (ipcp_value *val, cgraph_node *dest, if (cgraph_edge_brings_value_p (cs, src, dest)) { count++; - freq += cs->frequency; - if (cs->count.initialized_p ()) - cnt += cs->count; + freq += cs->frequency (); + if (cs->count.ipa ().initialized_p ()) + cnt += cs->count.ipa (); hot |= cs->maybe_hot_p (); } cs = get_next_cgraph_edge_clone (cs); @@ -3662,7 +3662,7 @@ update_profiling_info (struct cgraph_node *orig_node, profile_count new_sum, orig_sum; profile_count remainder, orig_node_count = orig_node->count; - if (!(orig_node_count > profile_count::zero ())) + if (!(orig_node_count.ipa () > profile_count::zero ())) return; init_caller_stats (&stats); @@ -3701,7 +3701,7 @@ update_profiling_info (struct cgraph_node *orig_node, for (cs = new_node->callees; cs; cs = cs->next_callee) /* FIXME: why we care about non-zero frequency here? */ - if (cs->frequency) + if (cs->frequency ()) cs->count = cs->count.apply_scale (new_sum, orig_node_count); else cs->count = profile_count::zero (); @@ -3741,7 +3741,7 @@ update_specialized_profile (struct cgraph_node *new_node, orig_node->count -= redirected_sum; for (cs = new_node->callees; cs; cs = cs->next_callee) - if (cs->frequency) + if (cs->frequency ()) cs->count += cs->count.apply_scale (redirected_sum, new_node_count); else cs->count = profile_count::zero (); @@ -4463,8 +4463,8 @@ perhaps_add_new_callers (cgraph_node *node, ipcp_value *val) cs->redirect_callee_duplicating_thunks (val->spec_node); val->spec_node->expand_all_artificial_thunks (); - if (cs->count.initialized_p ()) - redirected_sum = redirected_sum + cs->count; + if (cs->count.ipa ().initialized_p ()) + redirected_sum = redirected_sum + cs->count.ipa (); } cs = get_next_cgraph_edge_clone (cs); } diff --git a/gcc/ipa-devirt.c b/gcc/ipa-devirt.c index f03c7f099f7..540f038a2d5 100644 --- a/gcc/ipa-devirt.c +++ b/gcc/ipa-devirt.c @@ -3566,7 +3566,7 @@ ipa_devirt (void) bool final; if (final_warning_records) - final_warning_records->dyn_count = e->count; + final_warning_records->dyn_count = e->count.ipa (); vec targets = possible_polymorphic_call_targets @@ -3727,8 +3727,7 @@ ipa_devirt (void) nconverted++; update = true; e->make_speculative - (likely_target, e->count.apply_scale (8, 10), - e->frequency * 8 / 10); + (likely_target, e->count.apply_scale (8, 10)); } } if (update) diff --git a/gcc/ipa-fnsummary.c b/gcc/ipa-fnsummary.c index 5168886fe14..6c0e4370066 100644 --- a/gcc/ipa-fnsummary.c +++ b/gcc/ipa-fnsummary.c @@ -244,7 +244,6 @@ redirect_to_unreachable (struct cgraph_edge *e) e->redirect_callee (target); struct ipa_call_summary *es = ipa_call_summaries->get (e); e->inline_failed = CIF_UNREACHABLE; - e->frequency = 0; e->count = profile_count::zero (); es->call_stmt_size = 0; es->call_stmt_time = 0; @@ -823,7 +822,7 @@ dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node, indent, "", callee->name (), callee->order, !edge->inline_failed ? "inlined" : cgraph_inline_failed_string (edge-> inline_failed), - indent, "", es->loop_depth, edge->frequency, + indent, "", es->loop_depth, edge->frequency (), es->call_stmt_size, es->call_stmt_time, (int) ipa_fn_summaries->get (callee)->size / ipa_fn_summary::size_scale, (int) ipa_fn_summaries->get (callee)->estimated_stack_size); @@ -865,7 +864,7 @@ dump_ipa_call_summary (FILE *f, int indent, struct cgraph_node *node, " time: %2i", indent, "", es->loop_depth, - edge->frequency, es->call_stmt_size, es->call_stmt_time); + edge->frequency (), es->call_stmt_size, es->call_stmt_time); if (es->predicate) { fprintf (f, "predicate: "); @@ -2579,9 +2578,9 @@ estimate_edge_size_and_time (struct cgraph_edge *e, int *size, int *min_size, if (min_size) *min_size += cur_size; if (prob == REG_BR_PROB_BASE) - *time += ((sreal)(call_time * e->frequency)) / CGRAPH_FREQ_BASE; + *time += ((sreal)(call_time * e->frequency ())) / CGRAPH_FREQ_BASE; else - *time += ((sreal)call_time) * (prob * e->frequency) + *time += ((sreal)call_time) * (prob * e->frequency ()) / (CGRAPH_FREQ_BASE * REG_BR_PROB_BASE); } @@ -3059,7 +3058,7 @@ ipa_merge_fn_summary_after_inlining (struct cgraph_edge *edge) toplev_predicate); if (p != false && nonconstp != false) { - sreal add_time = ((sreal)e->time * edge->frequency) / CGRAPH_FREQ_BASE; + sreal add_time = ((sreal)e->time * edge->frequency ()) / CGRAPH_FREQ_BASE; int prob = e->nonconst_predicate.probability (callee_info->conds, clause, es->param); add_time = add_time * prob / REG_BR_PROB_BASE; diff --git a/gcc/ipa-inline-analysis.c b/gcc/ipa-inline-analysis.c index 38f3d2e762a..54a441f142d 100644 --- a/gcc/ipa-inline-analysis.c +++ b/gcc/ipa-inline-analysis.c @@ -175,11 +175,11 @@ do_estimate_edge_time (struct cgraph_edge *edge) edges and for those we disable size limits. Don't do that when probability that caller will call the callee is low however, since it may hurt optimization of the caller's hot path. */ - if (edge->count.initialized_p () && edge->maybe_hot_p () - && (edge->count.apply_scale (2, 1) + if (edge->count.ipa ().initialized_p () && edge->maybe_hot_p () + && (edge->count.ipa ().apply_scale (2, 1) > (edge->caller->global.inlined_to - ? edge->caller->global.inlined_to->count - : edge->caller->count))) + ? edge->caller->global.inlined_to->count.ipa () + : edge->caller->count.ipa ()))) hints |= INLINE_HINT_known_hot; known_vals.release (); diff --git a/gcc/ipa-inline-transform.c b/gcc/ipa-inline-transform.c index b2363e23043..8e66483016a 100644 --- a/gcc/ipa-inline-transform.c +++ b/gcc/ipa-inline-transform.c @@ -51,36 +51,24 @@ along with GCC; see the file COPYING3. If not see int ncalls_inlined; int nfunctions_inlined; -/* Scale frequency of NODE edges by FREQ_SCALE. */ +/* Scale counts of NODE edges by NUM/DEN. */ static void -update_noncloned_frequencies (struct cgraph_node *node, - int freq_scale, profile_count num, - profile_count den) +update_noncloned_counts (struct cgraph_node *node, + profile_count num, profile_count den) { struct cgraph_edge *e; profile_count::adjust_for_ipa_scaling (&num, &den); - /* We do not want to ignore high loop nest after freq drops to 0. */ - if (!freq_scale) - freq_scale = 1; for (e = node->callees; e; e = e->next_callee) { - e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; - if (e->frequency > CGRAPH_FREQ_MAX) - e->frequency = CGRAPH_FREQ_MAX; if (!e->inline_failed) - update_noncloned_frequencies (e->callee, freq_scale, num, den); + update_noncloned_counts (e->callee, num, den); e->count = e->count.apply_scale (num, den); } for (e = node->indirect_calls; e; e = e->next_callee) - { - e->frequency = e->frequency * (gcov_type) freq_scale / CGRAPH_FREQ_BASE; - if (e->frequency > CGRAPH_FREQ_MAX) - e->frequency = CGRAPH_FREQ_MAX; - e->count = e->count.apply_scale (num, den); - } + e->count = e->count.apply_scale (num, den); node->count = node->count.apply_scale (num, den); } @@ -169,12 +157,11 @@ master_clone_with_noninline_clones_p (struct cgraph_node *node) By default the offline copy is removed, when it appears dead after inlining. UPDATE_ORIGINAL prevents this transformation. If OVERALL_SIZE is non-NULL, the size is updated to reflect the - transformation. - FREQ_SCALE specify the scaling of frequencies of call sites. */ + transformation. */ void clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, - bool update_original, int *overall_size, int freq_scale) + bool update_original, int *overall_size) { struct cgraph_node *inlining_into; struct cgraph_edge *next; @@ -218,8 +205,7 @@ clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, } duplicate = false; e->callee->externally_visible = false; - update_noncloned_frequencies (e->callee, e->frequency, - e->count, e->callee->count); + update_noncloned_counts (e->callee, e->count, e->callee->count); dump_callgraph_transformation (e->callee, inlining_into, "inlining to"); @@ -228,11 +214,8 @@ clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, { struct cgraph_node *n; - if (freq_scale == -1) - freq_scale = e->frequency; n = e->callee->create_clone (e->callee->decl, - MIN (e->count, e->callee->count), - freq_scale, + e->count, update_original, vNULL, true, inlining_into, NULL); @@ -250,7 +233,7 @@ clone_inlined_nodes (struct cgraph_edge *e, bool duplicate, { next = e->next_callee; if (!e->inline_failed) - clone_inlined_nodes (e, duplicate, update_original, overall_size, freq_scale); + clone_inlined_nodes (e, duplicate, update_original, overall_size); } } @@ -457,7 +440,7 @@ inline_call (struct cgraph_edge *e, bool update_original, } } - clone_inlined_nodes (e, true, update_original, overall_size, e->frequency); + clone_inlined_nodes (e, true, update_original, overall_size); gcc_assert (curr->callee->global.inlined_to == to); diff --git a/gcc/ipa-inline.c b/gcc/ipa-inline.c index 687996876ce..3128f320bdd 100644 --- a/gcc/ipa-inline.c +++ b/gcc/ipa-inline.c @@ -640,12 +640,12 @@ compute_uninlined_call_time (struct cgraph_edge *edge, ? edge->caller->global.inlined_to : edge->caller); - if (edge->count.nonzero_p () - && caller->count.nonzero_p ()) - uninlined_call_time *= (sreal)edge->count.to_gcov_type () - / caller->count.to_gcov_type (); - if (edge->frequency) - uninlined_call_time *= cgraph_freq_base_rec * edge->frequency; + if (edge->count.ipa ().nonzero_p () + && caller->count.ipa ().nonzero_p ()) + uninlined_call_time *= (sreal)edge->count.ipa ().to_gcov_type () + / caller->count.ipa ().to_gcov_type (); + if (edge->frequency ()) + uninlined_call_time *= cgraph_freq_base_rec * edge->frequency (); else uninlined_call_time = uninlined_call_time >> 11; @@ -665,17 +665,17 @@ compute_inlined_call_time (struct cgraph_edge *edge, : edge->caller); sreal caller_time = ipa_fn_summaries->get (caller)->time; - if (edge->count.nonzero_p () - && caller->count.nonzero_p ()) + if (edge->count.ipa ().nonzero_p () + && caller->count.ipa ().nonzero_p ()) time *= (sreal)edge->count.to_gcov_type () / caller->count.to_gcov_type (); - if (edge->frequency) - time *= cgraph_freq_base_rec * edge->frequency; + if (edge->frequency ()) + time *= cgraph_freq_base_rec * edge->frequency (); else time = time >> 11; /* This calculation should match one in ipa-inline-analysis.c (estimate_edge_size_and_time). */ - time -= (sreal) edge->frequency + time -= (sreal) edge->frequency () * ipa_call_summaries->get (edge)->call_stmt_time / CGRAPH_FREQ_BASE; time += caller_time; if (time <= 0) @@ -724,7 +724,7 @@ want_inline_small_function_p (struct cgraph_edge *e, bool report) promote non-inline functions to inline and we increase MAX_INLINE_INSNS_SINGLE 16-fold for inline functions. */ else if ((!DECL_DECLARED_INLINE_P (callee->decl) - && (!e->count.initialized_p () || !e->maybe_hot_p ())) + && (!e->count.ipa ().initialized_p () || !e->maybe_hot_p ())) && ipa_fn_summaries->get (callee)->min_size - ipa_call_summaries->get (e)->call_stmt_size > MAX (MAX_INLINE_INSNS_SINGLE, MAX_INLINE_INSNS_AUTO)) @@ -733,7 +733,7 @@ want_inline_small_function_p (struct cgraph_edge *e, bool report) want_inline = false; } else if ((DECL_DECLARED_INLINE_P (callee->decl) - || e->count.nonzero_p ()) + || e->count.ipa ().nonzero_p ()) && ipa_fn_summaries->get (callee)->min_size - ipa_call_summaries->get (e)->call_stmt_size > 16 * MAX_INLINE_INSNS_SINGLE) @@ -843,7 +843,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge, reason = "recursive call is cold"; want_inline = false; } - else if (!outer_node->count.nonzero_p ()) + else if (!outer_node->count.ipa ().nonzero_p ()) { reason = "not executed in profile"; want_inline = false; @@ -855,7 +855,7 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge, } if (outer_node->global.inlined_to) - caller_freq = outer_node->callers->frequency; + caller_freq = outer_node->callers->frequency (); if (!caller_freq) { @@ -881,16 +881,16 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge, int i; for (i = 1; i < depth; i++) max_prob = max_prob * max_prob / CGRAPH_FREQ_BASE; - if (max_count.nonzero_p () && edge->count.nonzero_p () - && (edge->count.to_gcov_type () * CGRAPH_FREQ_BASE - / outer_node->count.to_gcov_type () + if (max_count.nonzero_p () && edge->count.ipa ().nonzero_p () + && (edge->count.ipa ().to_gcov_type () * CGRAPH_FREQ_BASE + / outer_node->count.ipa ().to_gcov_type () >= max_prob)) { reason = "profile of recursive call is too large"; want_inline = false; } if (!max_count.nonzero_p () - && (edge->frequency * CGRAPH_FREQ_BASE / caller_freq + && (edge->frequency () * CGRAPH_FREQ_BASE / caller_freq >= max_prob)) { reason = "frequency of recursive call is too large"; @@ -915,17 +915,17 @@ want_inline_self_recursive_call_p (struct cgraph_edge *edge, methods. */ else { - if (max_count.nonzero_p () && edge->count.initialized_p () - && (edge->count.to_gcov_type () * 100 - / outer_node->count.to_gcov_type () + if (max_count.nonzero_p () && edge->count.ipa ().initialized_p () + && (edge->count.ipa ().to_gcov_type () * 100 + / outer_node->count.ipa ().to_gcov_type () <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY))) { reason = "profile of recursive call is too small"; want_inline = false; } else if ((!max_count.nonzero_p () - || !edge->count.initialized_p ()) - && (edge->frequency * 100 / caller_freq + || !edge->count.ipa ().initialized_p ()) + && (edge->frequency () * 100 / caller_freq <= PARAM_VALUE (PARAM_MIN_INLINE_RECURSIVE_PROBABILITY))) { reason = "frequency of recursive call is too small"; @@ -1023,8 +1023,11 @@ edge_badness (struct cgraph_edge *edge, bool dump) edge_time = estimate_edge_time (edge, &unspec_edge_time); hints = estimate_edge_hints (edge); gcc_checking_assert (edge_time >= 0); - /* Check that inlined time is better, but tolerate some roundoff issues. */ - gcc_checking_assert ((edge_time - callee_info->time).to_int () <= 0); + /* Check that inlined time is better, but tolerate some roundoff issues. + FIXME: When callee profile drops to 0 we account calls more. This + should be fixed by never doing that. */ + gcc_checking_assert ((edge_time - callee_info->time).to_int () <= 0 + || callee->count.ipa ().initialized_p ()); gcc_checking_assert (growth <= callee_info->size); if (dump) @@ -1070,7 +1073,7 @@ edge_badness (struct cgraph_edge *edge, bool dump) then calls without. */ else if (opt_for_fn (caller->decl, flag_guess_branch_prob) - || caller->count.nonzero_p ()) + || caller->count.ipa ().nonzero_p ()) { sreal numerator, denominator; int overall_growth; @@ -1080,9 +1083,9 @@ edge_badness (struct cgraph_edge *edge, bool dump) - inlined_time); if (numerator == 0) numerator = ((sreal) 1 >> 8); - if (caller->count.nonzero_p ()) - numerator *= caller->count.to_gcov_type (); - else if (caller->count.initialized_p ()) + if (caller->count.ipa ().nonzero_p ()) + numerator *= caller->count.ipa ().to_gcov_type (); + else if (caller->count.ipa ().initialized_p ()) numerator = numerator >> 11; denominator = growth; @@ -1108,14 +1111,14 @@ edge_badness (struct cgraph_edge *edge, bool dump) && callee_info->single_caller && !edge->caller->global.inlined_to /* ... and edges executed only conditionally ... */ - && edge->frequency < CGRAPH_FREQ_BASE + && edge->frequency () < CGRAPH_FREQ_BASE /* ... consider case where callee is not inline but caller is ... */ && ((!DECL_DECLARED_INLINE_P (edge->callee->decl) && DECL_DECLARED_INLINE_P (caller->decl)) /* ... or when early optimizers decided to split and edge frequency still indicates splitting is a win ... */ || (callee->split_part && !caller->split_part - && edge->frequency + && edge->frequency () < CGRAPH_FREQ_BASE * PARAM_VALUE (PARAM_PARTIAL_INLINING_ENTRY_PROBABILITY) / 100 @@ -1166,9 +1169,9 @@ edge_badness (struct cgraph_edge *edge, bool dump) " overall growth %i (current) %i (original)" " %i (compensated)\n", badness.to_double (), - (double)edge->frequency / CGRAPH_FREQ_BASE, - edge->count.initialized_p () ? edge->count.to_gcov_type () : -1, - caller->count.initialized_p () ? caller->count.to_gcov_type () : -1, + (double)edge->frequency () / CGRAPH_FREQ_BASE, + edge->count.ipa ().initialized_p () ? edge->count.ipa ().to_gcov_type () : -1, + caller->count.ipa ().initialized_p () ? caller->count.ipa ().to_gcov_type () : -1, compute_uninlined_call_time (edge, unspec_edge_time).to_double (), compute_inlined_call_time (edge, edge_time).to_double (), @@ -1430,8 +1433,8 @@ lookup_recursive_calls (struct cgraph_node *node, struct cgraph_node *where, { /* When profile feedback is available, prioritize by expected number of calls. */ - heap->insert (!(max_count > 0) || !e->count.initialized_p () ? -e->frequency - : -(e->count.to_gcov_type () + heap->insert (!(max_count > 0) || !e->count.ipa ().initialized_p () ? -e->frequency () + : -(e->count.ipa ().to_gcov_type () / ((max_count.to_gcov_type () + (1<<24) - 1) / (1<<24))), e); @@ -1533,11 +1536,10 @@ recursive_inlining (struct cgraph_edge *edge, { /* We need original clone to copy around. */ master_clone = node->create_clone (node->decl, node->count, - CGRAPH_FREQ_BASE, false, vNULL, - true, NULL, NULL); + false, vNULL, true, NULL, NULL); for (e = master_clone->callees; e; e = e->next_callee) if (!e->inline_failed) - clone_inlined_nodes (e, true, false, NULL, CGRAPH_FREQ_BASE); + clone_inlined_nodes (e, true, false, NULL); curr->redirect_callee (master_clone); reset_edge_growth_cache (curr); } @@ -1684,8 +1686,8 @@ resolve_noninline_speculation (edge_heap_t *edge_heap, struct cgraph_edge *edge) ? node->global.inlined_to : node; auto_bitmap updated_nodes; - if (edge->count.initialized_p ()) - spec_rem += edge->count; + if (edge->count.ipa ().initialized_p ()) + spec_rem += edge->count.ipa (); edge->resolve_speculation (); reset_edge_caches (where); ipa_update_overall_fn_summary (where); @@ -1790,7 +1792,7 @@ inline_small_functions (void) } for (edge = node->callers; edge; edge = edge->next_caller) - max_count = max_count.max (edge->count); + max_count = max_count.max (edge->count.ipa ()); } ipa_free_postorder_info (); initialize_growth_caches (); @@ -1874,35 +1876,40 @@ inline_small_functions (void) continue; #if CHECKING_P - /* Be sure that caches are maintained consistent. */ - sreal cached_badness = edge_badness (edge, false); - - int old_size_est = estimate_edge_size (edge); - sreal old_time_est = estimate_edge_time (edge); - int old_hints_est = estimate_edge_hints (edge); - - reset_edge_growth_cache (edge); - gcc_assert (old_size_est == estimate_edge_size (edge)); - gcc_assert (old_time_est == estimate_edge_time (edge)); - /* FIXME: - - gcc_assert (old_hints_est == estimate_edge_hints (edge)); - - fails with profile feedback because some hints depends on - maybe_hot_edge_p predicate and because callee gets inlined to other - calls, the edge may become cold. - This ought to be fixed by computing relative probabilities - for given invocation but that will be better done once whole - code is converted to sreals. Disable for now and revert to "wrong" - value so enable/disable checking paths agree. */ - edge_growth_cache[edge->uid].hints = old_hints_est + 1; - - /* When updating the edge costs, we only decrease badness in the keys. - Increases of badness are handled lazilly; when we see key with out - of date value on it, we re-insert it now. */ - current_badness = edge_badness (edge, false); - gcc_assert (cached_badness == current_badness); - gcc_assert (current_badness >= badness); + /* Be sure that caches are maintained consistent. + This check is affected by scaling roundoff errors when compiling for + IPA this we skip it in that case. */ + if (!edge->callee->count.ipa_p ()) + { + sreal cached_badness = edge_badness (edge, false); + + int old_size_est = estimate_edge_size (edge); + sreal old_time_est = estimate_edge_time (edge); + int old_hints_est = estimate_edge_hints (edge); + + reset_edge_growth_cache (edge); + gcc_assert (old_size_est == estimate_edge_size (edge)); + gcc_assert (old_time_est == estimate_edge_time (edge)); + /* FIXME: + + gcc_assert (old_hints_est == estimate_edge_hints (edge)); + + fails with profile feedback because some hints depends on + maybe_hot_edge_p predicate and because callee gets inlined to other + calls, the edge may become cold. + This ought to be fixed by computing relative probabilities + for given invocation but that will be better done once whole + code is converted to sreals. Disable for now and revert to "wrong" + value so enable/disable checking paths agree. */ + edge_growth_cache[edge->uid].hints = old_hints_est + 1; + + /* When updating the edge costs, we only decrease badness in the keys. + Increases of badness are handled lazilly; when we see key with out + of date value on it, we re-insert it now. */ + current_badness = edge_badness (edge, false); + gcc_assert (cached_badness == current_badness); + gcc_assert (current_badness >= badness); + } #else current_badness = edge_badness (edge, false); #endif @@ -1945,11 +1952,11 @@ inline_small_functions (void) ? gimple_lineno ((const gimple *) edge->call_stmt) : -1, badness.to_double (), - edge->frequency / (double)CGRAPH_FREQ_BASE); - if (edge->count.initialized_p ()) + edge->frequency () / (double)CGRAPH_FREQ_BASE); + if (edge->count.ipa ().initialized_p ()) { fprintf (dump_file, " Called "); - edge->count.dump (dump_file); + edge->count.ipa ().dump (dump_file); fprintf (dump_file, "times\n"); } if (dump_flags & TDF_DETAILS) @@ -2255,8 +2262,8 @@ dump_overall_stats (void) { sreal time = ipa_fn_summaries->get (node)->time; sum += time; - if (node->count.initialized_p ()) - sum_weighted += time * node->count.to_gcov_type (); + if (node->count.ipa ().initialized_p ()) + sum_weighted += time * node->count.ipa ().to_gcov_type (); } fprintf (dump_file, "Overall time estimate: " "%f weighted by profile: " @@ -2286,57 +2293,57 @@ dump_inline_stats (void) { if (e->inline_failed) { - if (e->count.initialized_p ()) - reason[(int) e->inline_failed][0] += e->count.to_gcov_type (); - reason[(int) e->inline_failed][1] += e->frequency; + if (e->count.ipa ().initialized_p ()) + reason[(int) e->inline_failed][0] += e->count.ipa ().to_gcov_type (); + reason[(int) e->inline_failed][1] += e->frequency (); reason[(int) e->inline_failed][2] ++; if (DECL_VIRTUAL_P (e->callee->decl) - && e->count.initialized_p ()) + && e->count.ipa ().initialized_p ()) { if (e->indirect_inlining_edge) - noninlined_virt_indir_cnt += e->count.to_gcov_type (); + noninlined_virt_indir_cnt += e->count.ipa ().to_gcov_type (); else - noninlined_virt_cnt += e->count.to_gcov_type (); + noninlined_virt_cnt += e->count.ipa ().to_gcov_type (); } - else if (e->count.initialized_p ()) + else if (e->count.ipa ().initialized_p ()) { if (e->indirect_inlining_edge) - noninlined_indir_cnt += e->count.to_gcov_type (); + noninlined_indir_cnt += e->count.ipa ().to_gcov_type (); else - noninlined_cnt += e->count.to_gcov_type (); + noninlined_cnt += e->count.ipa ().to_gcov_type (); } } - else if (e->count.initialized_p ()) + else if (e->count.ipa ().initialized_p ()) { if (e->speculative) { if (DECL_VIRTUAL_P (e->callee->decl)) - inlined_speculative_ply += e->count.to_gcov_type (); + inlined_speculative_ply += e->count.ipa ().to_gcov_type (); else - inlined_speculative += e->count.to_gcov_type (); + inlined_speculative += e->count.ipa ().to_gcov_type (); } else if (DECL_VIRTUAL_P (e->callee->decl)) { if (e->indirect_inlining_edge) - inlined_virt_indir_cnt += e->count.to_gcov_type (); + inlined_virt_indir_cnt += e->count.ipa ().to_gcov_type (); else - inlined_virt_cnt += e->count.to_gcov_type (); + inlined_virt_cnt += e->count.ipa ().to_gcov_type (); } else { if (e->indirect_inlining_edge) - inlined_indir_cnt += e->count.to_gcov_type (); + inlined_indir_cnt += e->count.ipa ().to_gcov_type (); else - inlined_cnt += e->count.to_gcov_type (); + inlined_cnt += e->count.ipa ().to_gcov_type (); } } } for (e = node->indirect_calls; e; e = e->next_callee) if (e->indirect_info->polymorphic - & e->count.initialized_p ()) - indirect_poly_cnt += e->count.to_gcov_type (); - else if (e->count.initialized_p ()) - indirect_cnt += e->count.to_gcov_type (); + & e->count.ipa ().initialized_p ()) + indirect_poly_cnt += e->count.ipa ().to_gcov_type (); + else if (e->count.ipa ().initialized_p ()) + indirect_cnt += e->count.ipa ().to_gcov_type (); } if (max_count.initialized_p ()) { @@ -2488,8 +2495,8 @@ ipa_inline (void) next = edge->next_callee; if (edge->speculative && !speculation_useful_p (edge, false)) { - if (edge->count.initialized_p ()) - spec_rem += edge->count; + if (edge->count.ipa ().initialized_p ()) + spec_rem += edge->count.ipa (); edge->resolve_speculation (); update = true; remove_functions = true; diff --git a/gcc/ipa-inline.h b/gcc/ipa-inline.h index 4b9a1c2191a..ebf5d9718fb 100644 --- a/gcc/ipa-inline.h +++ b/gcc/ipa-inline.h @@ -59,8 +59,7 @@ bool inline_account_function_p (struct cgraph_node *node); bool inline_call (struct cgraph_edge *, bool, vec *, int *, bool, bool *callee_removed = NULL); unsigned int inline_transform (struct cgraph_node *); -void clone_inlined_nodes (struct cgraph_edge *e, bool, bool, int *, - int freq_scale); +void clone_inlined_nodes (struct cgraph_edge *e, bool, bool, int *); extern int ncalls_inlined; extern int nfunctions_inlined; diff --git a/gcc/ipa-profile.c b/gcc/ipa-profile.c index 8eb03dd7c24..cdcd0505f03 100644 --- a/gcc/ipa-profile.c +++ b/gcc/ipa-profile.c @@ -340,7 +340,7 @@ ipa_propagate_frequency_1 (struct cgraph_node *node, void *data) && edge->caller->global.inlined_to->frequency != NODE_FREQUENCY_UNLIKELY_EXECUTED))) d->maybe_unlikely_executed = false; - if (!edge->frequency) + if (!edge->frequency ()) continue; switch (edge->caller->frequency) { @@ -431,11 +431,11 @@ ipa_propagate_frequency (struct cgraph_node *node) } /* With profile we can decide on hot/normal based on count. */ - if (node->count.initialized_p ()) + if (node->count. ipa().initialized_p ()) { bool hot = false; - if (!(node->count == profile_count::zero ()) - && node->count >= get_hot_bb_threshold ()) + if (!(node->count. ipa() == profile_count::zero ()) + && node->count. ipa() >= get_hot_bb_threshold ()) hot = true; if (!hot) hot |= contains_hot_call_p (node); @@ -667,9 +667,7 @@ ipa_profile (void) e->make_speculative (n2, e->count.apply_probability - (e->indirect_info->common_target_probability), - apply_scale (e->frequency, - e->indirect_info->common_target_probability)); + (e->indirect_info->common_target_probability)); update = true; } } diff --git a/gcc/ipa-prop.c b/gcc/ipa-prop.c index 729251714de..4b3c2361418 100644 --- a/gcc/ipa-prop.c +++ b/gcc/ipa-prop.c @@ -2982,7 +2982,7 @@ ipa_make_edge_direct_to_target (struct cgraph_edge *ie, tree target, } /* make_speculative will update ie's cost to direct call cost. */ ie = ie->make_speculative - (callee, ie->count.apply_scale (8, 10), ie->frequency * 8 / 10); + (callee, ie->count.apply_scale (8, 10)); } return ie; diff --git a/gcc/ipa-utils.c b/gcc/ipa-utils.c index e9ab78cdabb..808db436b6a 100644 --- a/gcc/ipa-utils.c +++ b/gcc/ipa-utils.c @@ -404,17 +404,18 @@ ipa_merge_profiles (struct cgraph_node *dst, /* FIXME when we merge in unknown profile, we ought to set counts as unsafe. */ - if (!src->count.initialized_p ()) + if (!src->count.initialized_p () + || !(src->count.ipa () == src->count)) return; if (symtab->dump_file) { fprintf (symtab->dump_file, "Merging profiles of %s to %s\n", src->dump_name (), dst->dump_name ()); } - if (dst->count.initialized_p ()) - dst->count += src->count; - else - dst->count = src->count; + if (dst->count.initialized_p () && dst->count.ipa () == dst->count) + dst->count += src->count.ipa (); + else + dst->count = src->count.ipa (); /* This is ugly. We need to get both function bodies into memory. If declaration is merged, we need to duplicate it to be able @@ -564,18 +565,12 @@ ipa_merge_profiles (struct cgraph_node *dst, { if (e->speculative) continue; - e->count = gimple_bb (e->call_stmt)->count.ipa (); - e->frequency = compute_call_stmt_bb_frequency - (dst->decl, - gimple_bb (e->call_stmt)); + e->count = gimple_bb (e->call_stmt)->count; } for (e = dst->indirect_calls, e2 = src->indirect_calls; e; e2 = (e2 ? e2->next_callee : NULL), e = e->next_callee) { profile_count count = gimple_bb (e->call_stmt)->count; - int freq = compute_call_stmt_bb_frequency - (dst->decl, - gimple_bb (e->call_stmt)); /* When call is speculative, we need to re-distribute probabilities the same way as they was. This is not really correct because in the other copy the speculation may differ; but probably it @@ -624,12 +619,6 @@ ipa_merge_profiles (struct cgraph_node *dst, indirect->count += indirect2->count; } } - int prob = direct->count.probability_in (direct->count - + indirect->count). - to_reg_br_prob_base (); - direct->frequency = RDIV (freq * prob, REG_BR_PROB_BASE); - indirect->frequency = RDIV (freq * (REG_BR_PROB_BASE - prob), - REG_BR_PROB_BASE); } else /* At the moment we should have only profile feedback based @@ -642,18 +631,11 @@ ipa_merge_profiles (struct cgraph_node *dst, ipa_ref *ref; e2->speculative_call_info (direct, indirect, ref); - e->count = count.ipa (); - e->frequency = freq; - int prob = direct->count.probability_in (e->count) - .to_reg_br_prob_base (); - e->make_speculative (direct->callee, direct->count, - RDIV (freq * prob, REG_BR_PROB_BASE)); + e->count = count; + e->make_speculative (direct->callee, direct->count); } else - { - e->count = count.ipa (); - e->frequency = freq; - } + e->count = count; } if (!preserve_body) src->release_body (); diff --git a/gcc/lto-cgraph.c b/gcc/lto-cgraph.c index 15f0eaadf20..a19f8a13dfb 100644 --- a/gcc/lto-cgraph.c +++ b/gcc/lto-cgraph.c @@ -266,7 +266,6 @@ lto_output_edge (struct lto_simple_output_block *ob, struct cgraph_edge *edge, bp_pack_enum (&bp, cgraph_inline_failed_t, CIF_N_REASONS, edge->inline_failed); bp_pack_var_len_unsigned (&bp, uid); - bp_pack_var_len_unsigned (&bp, edge->frequency); bp_pack_value (&bp, edge->indirect_inlining_edge, 1); bp_pack_value (&bp, edge->speculative, 1); bp_pack_value (&bp, edge->call_stmt_cannot_inline_p, 1); @@ -1248,7 +1247,7 @@ input_node (struct lto_file_decl_data *file_data, if (clone_ref != LCC_NOT_FOUND) { node = dyn_cast (nodes[clone_ref])->create_clone (fn_decl, - profile_count::uninitialized (), CGRAPH_FREQ_BASE, false, + profile_count::uninitialized (), false, vNULL, false, NULL, NULL); } else @@ -1464,7 +1463,6 @@ input_edge (struct lto_input_block *ib, vec nodes, struct cgraph_edge *edge; unsigned int stmt_id; profile_count count; - int freq; cgraph_inline_failed_t inline_failed; struct bitpack_d bp; int ecf_flags = 0; @@ -1487,12 +1485,11 @@ input_edge (struct lto_input_block *ib, vec nodes, bp = streamer_read_bitpack (ib); inline_failed = bp_unpack_enum (&bp, cgraph_inline_failed_t, CIF_N_REASONS); stmt_id = bp_unpack_var_len_unsigned (&bp); - freq = (int) bp_unpack_var_len_unsigned (&bp); if (indirect) - edge = caller->create_indirect_edge (NULL, 0, count, freq); + edge = caller->create_indirect_edge (NULL, 0, count); else - edge = caller->create_edge (callee, NULL, count, freq); + edge = caller->create_edge (callee, NULL, count); edge->indirect_inlining_edge = bp_unpack_value (&bp, 1); edge->speculative = bp_unpack_value (&bp, 1); @@ -1823,8 +1820,13 @@ merge_profile_summaries (struct lto_file_decl_data **file_data_vec) if (scale == REG_BR_PROB_BASE) continue; for (edge = node->callees; edge; edge = edge->next_callee) - edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE); - node->count = node->count.apply_scale (scale, REG_BR_PROB_BASE); + if (edge->count.ipa ().nonzero_p ()) + edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE); + for (edge = node->indirect_calls; edge; edge = edge->next_callee) + if (edge->count.ipa ().nonzero_p ()) + edge->count = edge->count.apply_scale (scale, REG_BR_PROB_BASE); + if (node->count.ipa ().nonzero_p ()) + node->count = node->count.apply_scale (scale, REG_BR_PROB_BASE); } } diff --git a/gcc/lto/ChangeLog b/gcc/lto/ChangeLog index 173cde67369..4edb153ed76 100644 --- a/gcc/lto/ChangeLog +++ b/gcc/lto/ChangeLog @@ -1,3 +1,7 @@ +2017-11-10 Jan Hubicka + + * lto-partition.c (lto_balanced_map): Use frequency accessor. + 2017-10-13 Jan Hubicka * lto-lang.c (lto_post_options): Clean shlib flag when not doing PIC. diff --git a/gcc/lto/lto-partition.c b/gcc/lto/lto-partition.c index 3141ecec07c..2d0663eb93c 100644 --- a/gcc/lto/lto-partition.c +++ b/gcc/lto/lto-partition.c @@ -587,7 +587,7 @@ lto_balanced_map (int n_lto_partitions, int max_partition_size) for (edge = node->callees; edge; edge = edge->next_callee) if (edge->callee->definition) { - int edge_cost = edge->frequency; + int edge_cost = edge->frequency (); int index; if (!edge_cost) @@ -603,7 +603,7 @@ lto_balanced_map (int n_lto_partitions, int max_partition_size) } for (edge = node->callers; edge; edge = edge->next_caller) { - int edge_cost = edge->frequency; + int edge_cost = edge->frequency (); int index; gcc_assert (edge->caller->definition); diff --git a/gcc/omp-simd-clone.c b/gcc/omp-simd-clone.c index b8386037dfd..5cd85642546 100644 --- a/gcc/omp-simd-clone.c +++ b/gcc/omp-simd-clone.c @@ -1389,10 +1389,8 @@ simd_clone_adjust (struct cgraph_node *node) (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); - int freq = compute_call_stmt_bb_frequency (current_function_decl, - entry_bb); node->create_edge (cgraph_node::get_create (fn), - call, entry_bb->count, freq); + call, entry_bb->count); imm_use_iterator iter; use_operand_p use_p; diff --git a/gcc/predict.c b/gcc/predict.c index 0fd2b72e135..07b4bf7c398 100644 --- a/gcc/predict.c +++ b/gcc/predict.c @@ -3222,11 +3222,10 @@ drop_profile (struct cgraph_node *node, profile_count call_count) pop_cfun (); struct cgraph_edge *e; - for (e = node->callees; e; e = e->next_caller) - { - e->frequency = compute_call_stmt_bb_frequency (e->caller->decl, - gimple_bb (e->call_stmt)); - } + for (e = node->callees; e; e = e->next_callee) + e->count = gimple_bb (e->call_stmt)->count; + for (e = node->indirect_calls; e; e = e->next_callee) + e->count = gimple_bb (e->call_stmt)->count; profile_status_for_fn (fn) = (flag_guess_branch_prob ? PROFILE_GUESSED : PROFILE_ABSENT); diff --git a/gcc/trans-mem.c b/gcc/trans-mem.c index ef5655aa61a..63cac31829e 100644 --- a/gcc/trans-mem.c +++ b/gcc/trans-mem.c @@ -5065,9 +5065,7 @@ ipa_tm_insert_irr_call (struct cgraph_node *node, struct tm_region *region, node->create_edge (cgraph_node::get_create (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE)), - g, gimple_bb (g)->count, - compute_call_stmt_bb_frequency (node->decl, - gimple_bb (g))); + g, gimple_bb (g)->count); } /* Construct a call to TM_GETTMCLONE and insert it before GSI. */ @@ -5116,9 +5114,7 @@ ipa_tm_insert_gettmclone_call (struct cgraph_node *node, gsi_insert_before (gsi, g, GSI_SAME_STMT); - node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count, - compute_call_stmt_bb_frequency (node->decl, - gimple_bb (g))); + node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count); /* Cast return value from tm_gettmclone* into appropriate function pointer. */ diff --git a/gcc/tree-cfg.c b/gcc/tree-cfg.c index def3f0b8d9b..9a2fa1d98ca 100644 --- a/gcc/tree-cfg.c +++ b/gcc/tree-cfg.c @@ -9225,7 +9225,7 @@ execute_fixup_cfg (void) gimple_stmt_iterator gsi; int todo = 0; cgraph_node *node = cgraph_node::get (current_function_decl); - profile_count num = node->count; + profile_count num = node->count.ipa (); profile_count den = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count; bool scale = num.initialized_p () && den.ipa_p () && (den.nonzero_p () || num == profile_count::zero ()) @@ -9241,7 +9241,15 @@ execute_fixup_cfg (void) FOR_EACH_BB_FN (bb, cfun) { if (scale) - bb->count = bb->count.apply_scale (num, den); + { + if (num == profile_count::zero ()) + { + if (!(bb->count == profile_count::zero ())) + bb->count = bb->count.global0 (); + } + else + bb->count = bb->count.apply_scale (num, den); + } for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi);) { gimple *stmt = gsi_stmt (gsi); @@ -9331,11 +9339,8 @@ execute_fixup_cfg (void) if (!cfun->after_inlining) { gcall *call_stmt = dyn_cast (stmt); - int freq - = compute_call_stmt_bb_frequency (current_function_decl, - bb); node->create_edge (cgraph_node::get_create (fndecl), - call_stmt, bb->count, freq); + call_stmt, bb->count); } } } diff --git a/gcc/tree-chkp.c b/gcc/tree-chkp.c index 025155aca72..6d24355aba6 100644 --- a/gcc/tree-chkp.c +++ b/gcc/tree-chkp.c @@ -4231,18 +4231,12 @@ chkp_copy_bounds_for_assign (gimple *assign, struct cgraph_edge *edge) { tree fndecl = gimple_call_fndecl (stmt); struct cgraph_node *callee = cgraph_node::get_create (fndecl); - struct cgraph_edge *new_edge; gcc_assert (chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDSTX) || chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDLDX) || chkp_gimple_call_builtin_p (stmt, BUILT_IN_CHKP_BNDRET)); - new_edge = edge->caller->create_edge (callee, - as_a (stmt), - edge->count, - edge->frequency); - new_edge->frequency = compute_call_stmt_bb_frequency - (edge->caller->decl, gimple_bb (stmt)); + edge->caller->create_edge (callee, as_a (stmt), edge->count); } gsi_prev (&iter); } diff --git a/gcc/tree-emutls.c b/gcc/tree-emutls.c index 951e7d3f513..9136a0b2856 100644 --- a/gcc/tree-emutls.c +++ b/gcc/tree-emutls.c @@ -417,7 +417,7 @@ gen_emutls_addr (tree decl, struct lower_emutls_data *d) gimple_seq_add_stmt (&d->seq, x); - d->cfun_node->create_edge (d->builtin_node, x, d->bb->count, d->bb_freq); + d->cfun_node->create_edge (d->builtin_node, x, d->bb->count); /* We may be adding a new reference to a new variable to the function. This means we have to play with the ipa-reference web. */ diff --git a/gcc/tree-inline.c b/gcc/tree-inline.c index fc5d2c703ab..5c49214663f 100644 --- a/gcc/tree-inline.c +++ b/gcc/tree-inline.c @@ -2001,23 +2001,16 @@ copy_bb (copy_body_data *id, basic_block bb, edge = id->src_node->get_edge (orig_stmt); if (edge) { - int edge_freq = edge->frequency; - int new_freq; struct cgraph_edge *old_edge = edge; + profile_count old_cnt = edge->count; edge = edge->clone (id->dst_node, call_stmt, gimple_uid (stmt), - profile_count::one (), - profile_count::one (), - CGRAPH_FREQ_BASE, + num, den, true); - /* We could also just rescale the frequency, but - doing so would introduce roundoff errors and make - verifier unhappy. */ - new_freq = compute_call_stmt_bb_frequency (id->dst_node->decl, - copy_basic_block); - - /* Speculative calls consist of two edges - direct and indirect. - Duplicate the whole thing and distribute frequencies accordingly. */ + + /* Speculative calls consist of two edges - direct and + indirect. Duplicate the whole thing and distribute + frequencies accordingly. */ if (edge->speculative) { struct cgraph_edge *direct, *indirect; @@ -2025,42 +2018,22 @@ copy_bb (copy_body_data *id, basic_block bb, gcc_assert (!edge->indirect_unknown_callee); old_edge->speculative_call_info (direct, indirect, ref); + + profile_count indir_cnt = indirect->count; indirect = indirect->clone (id->dst_node, call_stmt, gimple_uid (stmt), - profile_count::one (), - profile_count::one (), - CGRAPH_FREQ_BASE, + num, den, true); - if (old_edge->frequency + indirect->frequency) - { - edge->frequency = MIN (RDIV ((gcov_type)new_freq * old_edge->frequency, - (old_edge->frequency + indirect->frequency)), - CGRAPH_FREQ_MAX); - indirect->frequency = MIN (RDIV ((gcov_type)new_freq * indirect->frequency, - (old_edge->frequency + indirect->frequency)), - CGRAPH_FREQ_MAX); - } + + profile_probability prob + = indir_cnt.probability_in (old_cnt + indir_cnt); + indirect->count + = copy_basic_block->count.apply_probability (prob); + edge->count = copy_basic_block->count - indirect->count; id->dst_node->clone_reference (ref, stmt); } else - { - edge->frequency = new_freq; - if (dump_file - && profile_status_for_fn (cfun) != PROFILE_ABSENT - && (edge_freq > edge->frequency + 10 - || edge_freq < edge->frequency - 10)) - { - fprintf (dump_file, "Edge frequency estimated by " - "cgraph %i diverge from inliner's estimate %i\n", - edge_freq, - edge->frequency); - fprintf (dump_file, - "Orig bb: %i, orig bb freq %i, new bb freq %i\n", - bb->index, - bb->count.to_frequency (cfun), - copy_basic_block->count.to_frequency (cfun)); - } - } + edge->count = copy_basic_block->count; } break; @@ -2103,15 +2076,10 @@ copy_bb (copy_body_data *id, basic_block bb, if (id->transform_call_graph_edges == CB_CGE_MOVE_CLONES) id->dst_node->create_edge_including_clones (dest, orig_stmt, call_stmt, bb->count, - compute_call_stmt_bb_frequency (id->dst_node->decl, - copy_basic_block), CIF_ORIGINALLY_INDIRECT_CALL); else id->dst_node->create_edge (dest, call_stmt, - bb->count, - compute_call_stmt_bb_frequency - (id->dst_node->decl, - copy_basic_block))->inline_failed + bb->count)->inline_failed = CIF_ORIGINALLY_INDIRECT_CALL; if (dump_file) { @@ -4475,7 +4443,6 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id) { cgraph_edge *edge; tree virtual_offset = NULL; - int freq = cg_edge->frequency; profile_count count = cg_edge->count; tree op; gimple_stmt_iterator iter = gsi_for_stmt (stmt); @@ -4485,9 +4452,7 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id) gimple_uid (stmt), profile_count::one (), profile_count::one (), - CGRAPH_FREQ_BASE, true); - edge->frequency = freq; edge->count = count; if (id->src_node->thunk.virtual_offset_p) virtual_offset = size_int (id->src_node->thunk.virtual_value); @@ -4709,7 +4674,7 @@ expand_call_inline (basic_block bb, gimple *stmt, copy_body_data *id) print_generic_expr (dump_file, id->src_fn); fprintf (dump_file, " to "); print_generic_expr (dump_file, id->dst_fn); - fprintf (dump_file, " with frequency %i\n", cg_edge->frequency); + fprintf (dump_file, " with frequency %i\n", cg_edge->frequency ()); } /* This is it. Duplicate the callee body. Assume callee is @@ -6089,20 +6054,16 @@ tree_function_versioning (tree old_decl, tree new_decl, struct cgraph_edge *e; rebuild_frequencies (); - new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count.ipa (); + new_version_node->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count; for (e = new_version_node->callees; e; e = e->next_callee) { basic_block bb = gimple_bb (e->call_stmt); - e->frequency = compute_call_stmt_bb_frequency (current_function_decl, - bb); - e->count = bb->count.ipa (); + e->count = bb->count; } for (e = new_version_node->indirect_calls; e; e = e->next_callee) { basic_block bb = gimple_bb (e->call_stmt); - e->frequency = compute_call_stmt_bb_frequency (current_function_decl, - bb); - e->count = bb->count.ipa (); + e->count = bb->count; } } diff --git a/gcc/ubsan.c b/gcc/ubsan.c index 52dacd001f8..560d4654850 100644 --- a/gcc/ubsan.c +++ b/gcc/ubsan.c @@ -675,12 +675,10 @@ ubsan_create_edge (gimple *stmt) { gcall *call_stmt = dyn_cast (stmt); basic_block bb = gimple_bb (stmt); - int freq = compute_call_stmt_bb_frequency (current_function_decl, bb); cgraph_node *node = cgraph_node::get (current_function_decl); tree decl = gimple_call_fndecl (call_stmt); if (decl) - node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count, - freq); + node->create_edge (cgraph_node::get_create (decl), call_stmt, bb->count); } /* Expand the UBSAN_BOUNDS special builtin function. */ -- 2.30.2