From 71dbabccbfb295c87d91719fe72d9d60511c0b44 Mon Sep 17 00:00:00 2001 From: Jan Hubicka Date: Mon, 12 Oct 2020 16:17:10 +0200 Subject: [PATCH] Cleanup ipa-modref this is largely mechanical patch fixing some suboptimal datastructure decision in modref. It records three different things 1) optimization_summaries that are used by tree-ssa-alias to disambiguate (computed by local passes or ipa execute) 2) summaries produced by local analysis and used by the ipa execute 3) summaries_lto produced by analysis when streaming is expected, streamed, used by ipa execute All three items are stored in "summaries" datastructure where 1 dn 2 are mixed and differentiated by "finished" flags. This use extra memory and also makes it impossible to use modref while producing other IPA summaries (by ipa-prop and ipa-devirt). This patch separates the summaries into three special purpose datastructures. There is one fix to propagation in ipa_merge_modref_summary_after_inlining where check to ignore stores was placed incorrectly. This seems to lead to increased clobber disambiguations: Alias oracle query stats: refs_may_alias_p: 64267006 disambiguations, 74475486 queries ref_maybe_used_by_call_p: 142119 disambiguations, 65169365 queries call_may_clobber_ref_p: 22975 disambiguations, 28762 queries nonoverlapping_component_refs_p: 0 disambiguations, 36803 queries nonoverlapping_refs_since_match_p: 19401 disambiguations, 55550 must overlaps, 75722 queries aliasing_component_refs_p: 54714 disambiguations, 759027 queries TBAA oracle: 23636760 disambiguations 56001742 queries 16112157 are in alias set 0 10614737 queries asked about the same object 125 queries asked about the same alias set 0 access volatile 3994423 are dependent in the DAG 1643540 are aritificially in conflict with void * Modref stats: modref use: 11667 disambiguations, 40207 queries modref clobber: 1508990 disambiguations, 1829697 queries 3916688 tbaa queries (2.140621 per modref query) 623504 base compares (0.340769 per modref query) PTA query stats: pt_solution_includes: 967354 disambiguations, 13605701 queries pt_solutions_intersect: 1032982 disambiguations, 13121107 queries Bootstrapped/regtested x86_64-linux. I plan to commit it tomorrow if there are no complains. gcc/ChangeLog: 2020-10-11 Jan Hubicka * ipa-modref.c (modref_summaries): Remove field IPA. (class modref_summary_lto): New global variable. (class modref_summaries_lto): New. (modref_summary::modref_summary): Remove loads_lto and stores_lto. (modref_summary::~modref_summary): Remove loads_lto and stores_lto. (modref_summary::useful_p): Do not use lto_useful. (modref_records_lto): New typedef. (struct modref_summary_lto): New type. (modref_summary_lto::modref_summary_lto): New member function. (modref_summary_lto::~modref_summary_lto): New member function. (modref_summary_lto::useful_p): New member function. (modref_summary::dump): Do not handle lto. (modref_summary_lto::dump): New member function. (get_modref_function_summary): Use optimization_summary. (merge_call_side_effects): Use optimization_summary. (analyze_call): Use optimization_summary. (struct summary_ptrs): New struture. (analyze_load): Update to handle separate lto and non-lto summaries. (analyze_store): Likewise. (analyze_stmt): Likewise. (remove_summary): Break out from ... (analyze_function): ... here; update to handle seprated summaries. (modref_summaries::insert): Do not handle lto summary. (modref_summaries_lto::insert): New member function. (modref_summaries::duplicate): Do not handle lto summary. (modref_summaries_lto::duplicate): New member function. (read_modref_records): Expect nolto_ret or lto_ret to be NULL> (modref_write): Write lto summary. (read_section): Handle separated summaries. (modref_read): Initialize separated summaries. (modref_transform): Handle separated summaries. (pass_modref::execute): Turn summary to optimization_summary; handle separate summaries. (ignore_edge): Handle separate summaries. (ipa_merge_modref_summary_after_inlining): Likewise. (collapse_loads): Likewise. (modref_propagate_in_scc): Likewise. (pass_ipa_modref::execute): Likewise. (ipa_modref_c_finalize): Likewise. * ipa-modref.h (modref_records_lto): Remove typedef. (struct modref_summary): Remove stores_lto, loads_lto and finished fields; remove lto_useful_p member function. --- gcc/ipa-modref.c | 776 ++++++++++++++++++++++++++++++++--------------- gcc/ipa-modref.h | 9 - 2 files changed, 528 insertions(+), 257 deletions(-) diff --git a/gcc/ipa-modref.c b/gcc/ipa-modref.c index b815eb8cc88..1d4eaf8d7ad 100644 --- a/gcc/ipa-modref.c +++ b/gcc/ipa-modref.c @@ -73,11 +73,6 @@ public: cgraph_node *dst_node, modref_summary *src_data, modref_summary *dst_data); - /* This flag controls whether newly inserted functions should be analyzed - in IPA or normal mode. Functions inserted between IPA analysis and - ipa-modref pass execution needs to be analyzed in IPA mode while all - other insertions leads to normal analysis. */ - bool ipa; static modref_summaries *create_ggc (symbol_table *symtab) { return new (ggc_alloc_no_dtor ()) @@ -85,14 +80,49 @@ public: } }; -/* Global variable holding all modref summaries. */ -static GTY(()) fast_function_summary *summaries; +class modref_summary_lto; + +/* Class (from which there is one global instance) that holds modref summaries + for all analyzed functions. */ +class GTY((user)) modref_summaries_lto + : public fast_function_summary +{ +public: + modref_summaries_lto (symbol_table *symtab) + : fast_function_summary (symtab), + propagated (false) {} + virtual void insert (cgraph_node *, modref_summary_lto *state); + virtual void duplicate (cgraph_node *src_node, + cgraph_node *dst_node, + modref_summary_lto *src_data, + modref_summary_lto *dst_data); + static modref_summaries_lto *create_ggc (symbol_table *symtab) + { + return new (ggc_alloc_no_dtor ()) + modref_summaries_lto (symtab); + } + bool propagated; +}; + +/* Global variable holding all modref summaries + (from analysis to IPA propagation time). */ +static GTY(()) fast_function_summary + *summaries; + +/* Global variable holding all modref optimizaiton summaries + (from IPA propagation time or used by local optimization pass). */ +static GTY(()) fast_function_summary + *optimization_summaries; + +/* LTO summaries hold info from analysis to LTO streaming or from LTO + stream-in through propagation to LTO stream-out. */ +static GTY(()) fast_function_summary + *summaries_lto; /* Summary for a single function which this pass produces. */ modref_summary::modref_summary () - : loads (NULL), stores (NULL), loads_lto (NULL), - stores_lto (NULL), finished (0) + : loads (NULL), stores (NULL) { } @@ -102,35 +132,63 @@ modref_summary::~modref_summary () ggc_delete (loads); if (stores) ggc_delete (stores); - if (loads_lto) - ggc_delete (loads_lto); - if (stores_lto) - ggc_delete (stores_lto); } -/* Return true if lto summary is potentially useful for optimization. */ +/* Return true if summary is potentially useful for optimization. */ bool -modref_summary::lto_useful_p (int ecf_flags) +modref_summary::useful_p (int ecf_flags) { if (ecf_flags & (ECF_CONST | ECF_NOVOPS)) return false; - if (loads_lto && !loads_lto->every_base) + if (loads && !loads->every_base) return true; if (ecf_flags & ECF_PURE) return false; - return stores_lto && !stores_lto->every_base; + return stores && !stores->every_base; } -/* Return true if summary is potentially useful for optimization. */ +/* Single function summary used for LTO. */ + +typedef modref_tree modref_records_lto; +struct GTY(()) modref_summary_lto +{ + /* Load and stores in functions using types rather then alias sets. + + This is necessary to make the information streamable for LTO but is also + more verbose and thus more likely to hit the limits. */ + modref_records_lto *loads; + modref_records_lto *stores; + + modref_summary_lto (); + ~modref_summary_lto (); + void dump (FILE *); + bool useful_p (int ecf_flags); +}; + +/* Summary for a single function which this pass produces. */ + +modref_summary_lto::modref_summary_lto () + : loads (NULL), stores (NULL) +{ +} + +modref_summary_lto::~modref_summary_lto () +{ + if (loads) + ggc_delete (loads); + if (stores) + ggc_delete (stores); +} + + +/* Return true if lto summary is potentially useful for optimization. */ bool -modref_summary::useful_p (int ecf_flags) +modref_summary_lto::useful_p (int ecf_flags) { if (ecf_flags & (ECF_CONST | ECF_NOVOPS)) return false; - if (lto_useful_p (ecf_flags)) - return true; if (loads && !loads->every_base) return true; if (ecf_flags & ECF_PURE) @@ -266,26 +324,32 @@ modref_summary::dump (FILE *out) fprintf (out, " stores:\n"); dump_records (stores, out); } - if (loads_lto) +} + +/* Dump summary. */ + +void +modref_summary_lto::dump (FILE *out) +{ + if (loads) { - fprintf (out, " LTO loads:\n"); - dump_lto_records (loads_lto, out); + fprintf (out, " loads:\n"); + dump_lto_records (loads, out); } - if (stores_lto) + if (stores) { - fprintf (out, " LTO stores:\n"); - dump_lto_records (stores_lto, out); + fprintf (out, " stores:\n"); + dump_lto_records (stores, out); } } - /* Get function summary for FUNC if it exists, return NULL otherwise. */ modref_summary * get_modref_function_summary (cgraph_node *func) { /* Avoid creation of the summary too early (e.g. when front-end calls us). */ - if (!summaries) + if (!optimization_summaries) return NULL; /* A single function body may be represented by multiple symbols with @@ -298,13 +362,8 @@ get_modref_function_summary (cgraph_node *func) if (avail <= AVAIL_INTERPOSABLE) return NULL; - /* Attempt to get summary for FUNC. If analysis of FUNC hasn't finished yet, - don't return anything. */ - modref_summary *r = summaries->get (func); - if (r && r->finished) - return r; - - return NULL; + modref_summary *r = optimization_summaries->get (func); + return r; } /* Construct modref_access_node from REF. */ @@ -509,17 +568,11 @@ merge_call_side_effects (modref_summary *cur_summary, /* Merge with callee's summary. */ if (cur_summary->loads) changed |= cur_summary->loads->merge (callee_summary->loads, &parm_map); - if (cur_summary->loads_lto) - changed |= cur_summary->loads_lto->merge (callee_summary->loads_lto, - &parm_map); if (!ignore_stores) { if (cur_summary->stores) changed |= cur_summary->stores->merge (callee_summary->stores, &parm_map); - if (cur_summary->stores_lto) - changed |= cur_summary->stores_lto->merge (callee_summary->stores_lto, - &parm_map); } return changed; } @@ -562,10 +615,7 @@ analyze_call (modref_summary *cur_summary, if (dump_file) fprintf (dump_file, " - Indirect call which does not write memory, " "discarding loads.\n"); - if (cur_summary->loads) - cur_summary->loads->collapse (); - if (cur_summary->loads_lto) - cur_summary->loads_lto->collapse (); + cur_summary->loads->collapse (); return true; } if (dump_file) @@ -583,10 +633,7 @@ analyze_call (modref_summary *cur_summary, { if (dump_file) fprintf (dump_file, " - May be interposed: collapsing loads.\n"); - if (cur_summary->loads) - cur_summary->loads->collapse (); - if (cur_summary->loads_lto) - cur_summary->loads_lto->collapse (); + cur_summary->loads->collapse (); } /* If this is a recursive call, the target summary is the same as ours, so @@ -610,10 +657,7 @@ analyze_call (modref_summary *cur_summary, symbols. */ if (ignore_stores) { - if (cur_summary->loads) - cur_summary->loads->collapse (); - if (cur_summary->loads_lto) - cur_summary->loads_lto->collapse (); + cur_summary->loads->collapse (); return true; } if (dump_file) @@ -623,15 +667,13 @@ analyze_call (modref_summary *cur_summary, /* Get callee's modref summary. As above, if there's no summary, we either have to give up or, if stores are ignored, we can just purge loads. */ - modref_summary *callee_summary = summaries->get (callee_node); + modref_summary *callee_summary = optimization_summaries->get (callee_node); if (!callee_summary) { if (ignore_stores) { if (cur_summary->loads) cur_summary->loads->collapse (); - if (cur_summary->loads_lto) - cur_summary->loads_lto->collapse (); return true; } if (dump_file) @@ -644,12 +686,21 @@ analyze_call (modref_summary *cur_summary, return true; } +/* Support analyzis in non-lto and lto mode in parallel. */ + +struct summary_ptrs +{ + struct modref_summary *nolto; + struct modref_summary_lto *lto; +}; + /* Helper for analyze_stmt. */ static bool analyze_load (gimple *, tree, tree op, void *data) { - modref_summary *summary = (modref_summary *)data; + modref_summary *summary = ((summary_ptrs *)data)->nolto; + modref_summary_lto *summary_lto = ((summary_ptrs *)data)->lto; if (dump_file) { @@ -664,10 +715,10 @@ analyze_load (gimple *, tree, tree op, void *data) ao_ref r; ao_ref_init (&r, op); - if (summary->loads) + if (summary) record_access (summary->loads, &r); - if (summary->loads_lto) - record_access_lto (summary->loads_lto, &r); + if (summary_lto) + record_access_lto (summary_lto->loads, &r); return false; } @@ -676,7 +727,8 @@ analyze_load (gimple *, tree, tree op, void *data) static bool analyze_store (gimple *, tree, tree op, void *data) { - modref_summary *summary = (modref_summary *)data; + modref_summary *summary = ((summary_ptrs *)data)->nolto; + modref_summary_lto *summary_lto = ((summary_ptrs *)data)->lto; if (dump_file) { @@ -691,10 +743,10 @@ analyze_store (gimple *, tree, tree op, void *data) ao_ref r; ao_ref_init (&r, op); - if (summary->stores) - record_access (((modref_summary *)data)->stores, &r); - if (summary->stores_lto) - record_access_lto (((modref_summary *)data)->stores_lto, &r); + if (summary) + record_access (summary->stores, &r); + if (summary_lto) + record_access_lto (summary_lto->stores, &r); return false; } @@ -702,8 +754,8 @@ analyze_store (gimple *, tree, tree op, void *data) If IPA is true do not merge in side effects of calls. */ static bool -analyze_stmt (modref_summary *summary, gimple *stmt, bool ipa, - vec *recursive_calls) +analyze_stmt (modref_summary *summary, modref_summary_lto *summary_lto, + gimple *stmt, bool ipa, vec *recursive_calls) { /* In general we can not ignore clobbers because they are barries for code motion, however after inlining it is safe to do becuase local optimization @@ -712,8 +764,10 @@ analyze_stmt (modref_summary *summary, gimple *stmt, bool ipa, if ((ipa || cfun->after_inlining) && gimple_clobber_p (stmt)) return true; + struct summary_ptrs sums = {summary, summary_lto}; + /* Analyze all loads and stores in STMT. */ - walk_stmt_load_store_ops (stmt, summary, + walk_stmt_load_store_ops (stmt, &sums, analyze_load, analyze_store); switch (gimple_code (stmt)) @@ -737,8 +791,30 @@ analyze_stmt (modref_summary *summary, gimple *stmt, bool ipa, } } -/* Analyze function F. IPA indicates whether we're running in local mode (false) - or the IPA mode (true). */ +/* Remove summary of current function because during the function body + scan we determined it is not useful. LTO, NOLTO and IPA determines the + mode of scan. */ + +static void +remove_summary (bool lto, bool nolto, bool ipa) +{ + cgraph_node *fnode = cgraph_node::get (current_function_decl); + if (!ipa) + optimization_summaries->remove (fnode); + else + { + if (nolto) + summaries->remove (fnode); + if (lto) + summaries_lto->remove (fnode); + } + if (dump_file) + fprintf (dump_file, + " - modref done with result: not tracked.\n"); +} + +/* Analyze function F. IPA indicates whether we're running in local mode + (false) or the IPA mode (true). */ static void analyze_function (function *f, bool ipa) @@ -753,32 +829,62 @@ analyze_function (function *f, bool ipa) if (!flag_ipa_modref) return; - /* Initialize the summary. */ - if (!summaries) - summaries = modref_summaries::create_ggc (symtab); - else /* Remove existing summary if we are re-running the pass. */ - { - if (dump_file && summaries->get (cgraph_node::get (f->decl))) - { - fprintf (dump_file, "Past summary:\n"); - summaries->get (cgraph_node::get (f->decl))->dump (dump_file); - } - summaries->remove (cgraph_node::get (f->decl)); - } - - ((modref_summaries *)summaries)->ipa = ipa; - - modref_summary *summary = summaries->get_create (cgraph_node::get (f->decl)); - /* Compute no-LTO summaries when local optimization is going to happen. */ bool nolto = (!ipa || ((!flag_lto || flag_fat_lto_objects) && !in_lto_p) || (in_lto_p && !flag_wpa && flag_incremental_link != INCREMENTAL_LINK_LTO)); - /* Compute LTO when LTO streaming is going to happen. */ bool lto = ipa && ((flag_lto && !in_lto_p) || flag_wpa || flag_incremental_link == INCREMENTAL_LINK_LTO); + cgraph_node *fnode = cgraph_node::get (current_function_decl); + + modref_summary *summary = NULL; + modref_summary_lto *summary_lto = NULL; + + /* Initialize the summary. + If we run in local mode there is possibly pre-existing summary from + IPA pass. Dump it so it is easy to compare if mod-ref info has + improved. */ + if (!ipa) + { + if (!optimization_summaries) + optimization_summaries = modref_summaries::create_ggc (symtab); + else /* Remove existing summary if we are re-running the pass. */ + { + if (dump_file + && optimization_summaries->get (cgraph_node::get (f->decl))) + { + fprintf (dump_file, "Past summary:\n"); + optimization_summaries->get + (cgraph_node::get (f->decl))->dump (dump_file); + } + optimization_summaries->remove (cgraph_node::get (f->decl)); + } + summary = optimization_summaries->get_create (cgraph_node::get (f->decl)); + gcc_checking_assert (nolto && !lto); + } + /* In IPA mode we analyze every function precisely once. Asser that. */ + else + { + if (nolto) + { + if (!summaries) + summaries = modref_summaries::create_ggc (symtab); + else + summaries->remove (cgraph_node::get (f->decl)); + summary = summaries->get_create (cgraph_node::get (f->decl)); + } + if (lto) + { + if (!summaries_lto) + summaries_lto = modref_summaries_lto::create_ggc (symtab); + else + summaries_lto->remove (cgraph_node::get (f->decl)); + summary_lto = summaries_lto->get_create (cgraph_node::get (f->decl)); + } + } + /* Create and initialize summary for F. Note that summaries may be already allocated from previous @@ -796,18 +902,17 @@ analyze_function (function *f, bool ipa) } if (lto) { - gcc_assert (!summary->loads_lto); - summary->loads_lto = modref_records_lto::create_ggc + gcc_assert (!summary_lto->loads); + summary_lto->loads = modref_records_lto::create_ggc (param_modref_max_bases, param_modref_max_refs, param_modref_max_accesses); - gcc_assert (!summary->stores_lto); - summary->stores_lto = modref_records_lto::create_ggc + gcc_assert (!summary_lto->stores); + summary_lto->stores = modref_records_lto::create_ggc (param_modref_max_bases, param_modref_max_refs, param_modref_max_accesses); } - summary->finished = false; int ecf_flags = flags_from_decl_or_type (current_function_decl); auto_vec recursive_calls; @@ -820,14 +925,12 @@ analyze_function (function *f, bool ipa) gimple_stmt_iterator si; for (si = gsi_after_labels (bb); !gsi_end_p (si); gsi_next (&si)) { - if (!analyze_stmt (summary, gsi_stmt (si), ipa, &recursive_calls) - || !summary->useful_p (ecf_flags)) + if (!analyze_stmt (summary, summary_lto, + gsi_stmt (si), ipa, &recursive_calls) + || ((!summary || !summary->useful_p (ecf_flags)) + && (!summary_lto || !summary_lto->useful_p (ecf_flags)))) { - cgraph_node *fnode = cgraph_node::get (current_function_decl); - summaries->remove (fnode); - if (dump_file) - fprintf (dump_file, - " - modref done with result: not tracked.\n"); + remove_summary (lto, nolto, ipa); return; } } @@ -850,24 +953,33 @@ analyze_function (function *f, bool ipa) (recursive_calls[i]))); if (!summary->useful_p (ecf_flags)) { - cgraph_node *fnode = cgraph_node::get (current_function_decl); - summaries->remove (fnode); - if (dump_file) - fprintf (dump_file, - " - modref done with result: not tracked.\n"); + remove_summary (lto, nolto, ipa); return; } } } } - - if (!ipa) - summary->finished = true; + if (summary && !summary->useful_p (ecf_flags)) + { + if (!ipa) + optimization_summaries->remove (fnode); + else + summaries->remove (fnode); + summary = NULL; + } + if (summary_lto && !summary_lto->useful_p (ecf_flags)) + { + summaries_lto->remove (fnode); + summary_lto = NULL; + } if (dump_file) { fprintf (dump_file, " - modref done with result: tracked.\n"); - summary->dump (dump_file); + if (summary) + summary->dump (dump_file); + if (summary_lto) + summary_lto->dump (dump_file); } } @@ -894,9 +1006,36 @@ void modref_summaries::insert (struct cgraph_node *node, modref_summary *) { if (!DECL_STRUCT_FUNCTION (node->decl)) - return; + { + optimization_summaries->remove (node); + summaries->remove (node); + } + push_cfun (DECL_STRUCT_FUNCTION (node->decl)); + /* This is not very pretty: We do not know if we insert into optimization + summary or summary. Do both but check for duplicated effort. */ + if (optimization_summaries && !optimization_summaries->get (node)->loads) + analyze_function (DECL_STRUCT_FUNCTION (node->decl), false); + if (summaries && !summaries->get (node)->loads) + analyze_function (DECL_STRUCT_FUNCTION (node->decl), true); + pop_cfun (); +} + +/* Called when a new function is inserted to callgraph late. */ + +void +modref_summaries_lto::insert (struct cgraph_node *node, modref_summary_lto *) +{ + /* We do not support adding new function when IPA information is already + propagated. This is done only by SIMD cloning that is not very + critical. */ + if (!DECL_STRUCT_FUNCTION (node->decl) + || propagated) + { + summaries_lto->remove (node); + return; + } push_cfun (DECL_STRUCT_FUNCTION (node->decl)); - analyze_function (DECL_STRUCT_FUNCTION (node->decl), ipa); + analyze_function (DECL_STRUCT_FUNCTION (node->decl), true); pop_cfun (); } @@ -907,7 +1046,6 @@ modref_summaries::duplicate (cgraph_node *, cgraph_node *, modref_summary *src_data, modref_summary *dst_data) { - dst_data->finished = src_data->finished; if (src_data->stores) { dst_data->stores = modref_records::create_ggc @@ -924,21 +1062,30 @@ modref_summaries::duplicate (cgraph_node *, cgraph_node *, src_data->loads->max_accesses); dst_data->loads->copy_from (src_data->loads); } - if (src_data->stores_lto) +} + +/* Called when new clone is inserted to callgraph late. */ + +void +modref_summaries_lto::duplicate (cgraph_node *, cgraph_node *, + modref_summary_lto *src_data, + modref_summary_lto *dst_data) +{ + if (src_data->stores) { - dst_data->stores_lto = modref_records_lto::create_ggc - (src_data->stores_lto->max_bases, - src_data->stores_lto->max_refs, - src_data->stores_lto->max_accesses); - dst_data->stores_lto->copy_from (src_data->stores_lto); + dst_data->stores = modref_records_lto::create_ggc + (src_data->stores->max_bases, + src_data->stores->max_refs, + src_data->stores->max_accesses); + dst_data->stores->copy_from (src_data->stores); } - if (src_data->loads_lto) + if (src_data->loads) { - dst_data->loads_lto = modref_records_lto::create_ggc - (src_data->loads_lto->max_bases, - src_data->loads_lto->max_refs, - src_data->stores_lto->max_accesses); - dst_data->loads_lto->copy_from (src_data->loads_lto); + dst_data->loads = modref_records_lto::create_ggc + (src_data->loads->max_bases, + src_data->loads->max_refs, + src_data->loads->max_accesses); + dst_data->loads->copy_from (src_data->loads); } } @@ -1038,14 +1185,13 @@ read_modref_records (lto_input_block *ib, struct data_in *data_in, size_t max_refs = streamer_read_uhwi (ib); size_t max_accesses = streamer_read_uhwi (ib); - /* Decide whether we want to turn LTO data types to non-LTO (i.e. when - LTO re-streaming is not going to happen). */ - if (flag_wpa || flag_incremental_link == INCREMENTAL_LINK_LTO) + if (lto_ret) *lto_ret = modref_records_lto::create_ggc (max_bases, max_refs, max_accesses); - else + if (nolto_ret) *nolto_ret = modref_records::create_ggc (max_bases, max_refs, max_accesses); + gcc_checking_assert (lto_ret || nolto_ret); size_t every_base = streamer_read_uhwi (ib); size_t nbase = streamer_read_uhwi (ib); @@ -1053,9 +1199,9 @@ read_modref_records (lto_input_block *ib, struct data_in *data_in, gcc_assert (!every_base || nbase == 0); if (every_base) { - if (*nolto_ret) + if (nolto_ret) (*nolto_ret)->collapse (); - if (*lto_ret) + if (lto_ret) (*lto_ret)->collapse (); } for (size_t i = 0; i < nbase; i++) @@ -1079,11 +1225,11 @@ read_modref_records (lto_input_block *ib, struct data_in *data_in, base_tree = NULL; } - if (*nolto_ret) + if (nolto_ret) nolto_base_node = (*nolto_ret)->insert_base (base_tree ? get_alias_set (base_tree) : 0); - if (*lto_ret) + if (lto_ret) lto_base_node = (*lto_ret)->insert_base (base_tree); size_t every_ref = streamer_read_uhwi (ib); size_t nref = streamer_read_uhwi (ib); @@ -1159,9 +1305,9 @@ read_modref_records (lto_input_block *ib, struct data_in *data_in, } } } - if (*lto_ret) + if (lto_ret) (*lto_ret)->cleanup (); - if (*nolto_ret) + if (nolto_ret) (*nolto_ret)->cleanup (); } @@ -1175,7 +1321,7 @@ modref_write () unsigned int count = 0; int i; - if (!summaries) + if (!summaries_lto) { streamer_write_uhwi (ob, 0); streamer_write_char_stream (ob->main_stream, 0); @@ -1188,11 +1334,11 @@ modref_write () { symtab_node *snode = lto_symtab_encoder_deref (encoder, i); cgraph_node *cnode = dyn_cast (snode); - modref_summary *r; + modref_summary_lto *r; if (cnode && cnode->definition && !cnode->alias - && (r = summaries->get (cnode)) - && r->lto_useful_p (flags_from_decl_or_type (cnode->decl))) + && (r = summaries_lto->get (cnode)) + && r->useful_p (flags_from_decl_or_type (cnode->decl))) count++; } streamer_write_uhwi (ob, count); @@ -1205,19 +1351,19 @@ modref_write () if (cnode && cnode->definition && !cnode->alias) { - modref_summary *r = summaries->get (cnode); + modref_summary_lto *r = summaries_lto->get (cnode); - if (!r || !r->lto_useful_p (flags_from_decl_or_type (cnode->decl))) + if (!r || !r->useful_p (flags_from_decl_or_type (cnode->decl))) continue; streamer_write_uhwi (ob, lto_symtab_encoder_encode (encoder, cnode)); - streamer_write_uhwi (ob, r->loads_lto ? 1 : 0); - streamer_write_uhwi (ob, r->stores_lto ? 1 : 0); - if (r->loads_lto) - write_modref_records (r->loads_lto, ob); - if (r->stores_lto) - write_modref_records (r->stores_lto, ob); + streamer_write_uhwi (ob, r->loads ? 1 : 0); + streamer_write_uhwi (ob, r->stores ? 1 : 0); + if (r->loads) + write_modref_records (r->loads, ob); + if (r->stores) + write_modref_records (r->stores, ob); } } streamer_write_char_stream (ob->main_stream, 0); @@ -1255,30 +1401,38 @@ read_section (struct lto_file_decl_data *file_data, const char *data, node = dyn_cast (lto_symtab_encoder_deref (encoder, index)); - modref_summary *modref_sum = summaries->get_create (node); - modref_sum->finished = false; + modref_summary *modref_sum = summaries + ? summaries->get_create (node) : NULL; + modref_summary_lto *modref_sum_lto = summaries_lto + ? summaries_lto->get_create (node) + : NULL; int have_loads = streamer_read_uhwi (&ib); int have_stores = streamer_read_uhwi (&ib); - gcc_assert (!modref_sum->loads_lto - && !modref_sum->stores_lto - && !modref_sum->loads - && !modref_sum->stores); + + if (optimization_summaries) + modref_sum = optimization_summaries->get_create (node); + + gcc_assert (!modref_sum || (!modref_sum->loads + && !modref_sum->stores)); + gcc_assert (!modref_sum_lto || (!modref_sum_lto->loads + && !modref_sum_lto->stores)); if (have_loads) read_modref_records (&ib, data_in, - &modref_sum->loads, - &modref_sum->loads_lto); + modref_sum ? &modref_sum->loads : NULL, + modref_sum_lto ? &modref_sum_lto->loads : NULL); if (have_stores) read_modref_records (&ib, data_in, - &modref_sum->stores, - &modref_sum->stores_lto); + modref_sum ? &modref_sum->stores : NULL, + modref_sum_lto ? &modref_sum_lto->stores : NULL); if (dump_file) { fprintf (dump_file, "Read modref for %s\n", node->dump_name ()); - modref_sum->dump (dump_file); + if (modref_sum) + modref_sum->dump (dump_file); + if (modref_sum_lto) + modref_sum_lto->dump (dump_file); } - if (flag_ltrans) - modref_sum->finished = true; } lto_free_section_data (file_data, LTO_section_ipa_modref, NULL, data, @@ -1295,9 +1449,18 @@ modref_read (void) struct lto_file_decl_data *file_data; unsigned int j = 0; - if (!summaries) - summaries = modref_summaries::create_ggc (symtab); - ((modref_summaries *)summaries)->ipa = !flag_ltrans; + gcc_checking_assert (!optimization_summaries && !summaries && !summaries_lto); + if (flag_ltrans) + optimization_summaries = modref_summaries::create_ggc (symtab); + else + { + if (flag_wpa || flag_incremental_link == INCREMENTAL_LINK_LTO) + summaries_lto = modref_summaries_lto::create_ggc (symtab); + if (!flag_wpa + || (flag_incremental_link == INCREMENTAL_LINK_LTO + && flag_fat_lto_objects)) + summaries = modref_summaries::create_ggc (symtab); + } while ((file_data = file_data_vec[j++])) { @@ -1348,9 +1511,9 @@ remap_arguments (vec *map, modref_records *tt) static unsigned int modref_transform (struct cgraph_node *node) { - if (!node->clone.param_adjustments || !summaries) + if (!node->clone.param_adjustments || !optimization_summaries) return 0; - modref_summary *r = summaries->get (node); + modref_summary *r = optimization_summaries->get (node); if (!r) return 0; if (dump_file) @@ -1436,7 +1599,7 @@ public: unsigned int pass_modref::execute (function *f) { /* If new function is being added during IPA, we can skip analysis. */ - if (summaries && ((modref_summaries *)summaries)->ipa) + if (!optimization_summaries) return 0; analyze_function (f, false); return 0; @@ -1465,7 +1628,8 @@ ignore_edge (struct cgraph_edge *e) (&avail, e->caller); return (avail <= AVAIL_INTERPOSABLE - || !summaries->get (callee) + || ((!summaries || !summaries->get (callee)) + && (!summaries_lto || !summaries_lto->get (callee))) || flags_from_decl_or_type (e->callee->decl) & (ECF_CONST | ECF_NOVOPS)); } @@ -1553,36 +1717,51 @@ compute_parm_map (cgraph_edge *callee_edge, vec *parm_map) void ipa_merge_modref_summary_after_inlining (cgraph_edge *edge) { - if (!summaries) + if (!summaries && !summaries_lto) return; struct cgraph_node *to = (edge->caller->inlined_to ? edge->caller->inlined_to : edge->caller); - class modref_summary *to_info = summaries->get (to); + class modref_summary *to_info = summaries ? summaries->get (to) : NULL; + class modref_summary_lto *to_info_lto = summaries_lto + ? summaries_lto->get (to) : NULL; - if (!to_info) - return; + if (!to_info && !to_info_lto) + { + if (summaries) + summaries->remove (edge->callee); + if (summaries_lto) + summaries_lto->remove (edge->callee); + return; + } - class modref_summary *callee_info = summaries->get (edge->callee); + class modref_summary *callee_info = summaries ? summaries->get (edge->callee) + : NULL; + class modref_summary_lto *callee_info_lto + = summaries_lto ? summaries_lto->get (edge->callee) : NULL; int flags = flags_from_decl_or_type (edge->callee->decl); - if (!callee_info) + if (!callee_info && to_info) { if (ignore_stores_p (edge->callee->decl, flags)) + to_info->loads->collapse (); + else { - if (to_info->loads) - to_info->loads->collapse (); - if (to_info->loads_lto) - to_info->loads_lto->collapse (); + summaries->remove (to); + to_info = NULL; } + } + if (!callee_info_lto && to_info_lto) + { + if (ignore_stores_p (edge->callee->decl, flags)) + to_info_lto->loads->collapse (); else { - summaries->remove (to); - summaries->remove (edge->callee); - return; + summaries_lto->remove (to); + to_info_lto = NULL; } } - else + if (callee_info || callee_info_lto) { auto_vec parm_map; @@ -1590,38 +1769,50 @@ ipa_merge_modref_summary_after_inlining (cgraph_edge *edge) if (!ignore_stores_p (edge->callee->decl, flags)) { - if (to_info->loads) - to_info->loads->merge (callee_info->loads, &parm_map); - if (to_info->stores) + if (to_info && callee_info) to_info->stores->merge (callee_info->stores, &parm_map); + if (to_info_lto && callee_info_lto) + to_info_lto->stores->merge (callee_info_lto->stores, &parm_map); } - if (to_info->loads_lto) - to_info->loads_lto->merge (callee_info->loads_lto, &parm_map); - if (to_info->stores_lto) - to_info->stores_lto->merge (callee_info->stores_lto, &parm_map); - } - if (!to_info->useful_p (flags)) - summaries->remove (to); - summaries->remove (edge->callee); + if (to_info && callee_info) + to_info->loads->merge (callee_info->loads, &parm_map); + if (to_info_lto && callee_info_lto) + to_info_lto->loads->merge (callee_info_lto->loads, &parm_map); + } + if (summaries) + { + if (to_info && !to_info->useful_p (flags)) + summaries->remove (to); + if (callee_info) + summaries->remove (edge->callee); + } + if (summaries_lto) + { + if (to_info_lto && !to_info_lto->useful_p (flags)) + summaries_lto->remove (to); + if (callee_info_lto) + summaries_lto->remove (edge->callee); + } return; } /* Collapse loads and return true if something changed. */ bool -collapse_loads (modref_summary *cur_summary) +collapse_loads (modref_summary *cur_summary, + modref_summary_lto *cur_summary_lto) { bool changed = false; - if (cur_summary->loads && !cur_summary->loads->every_base) + if (cur_summary && !cur_summary->loads->every_base) { cur_summary->loads->collapse (); changed = true; } - if (cur_summary->loads_lto - && !cur_summary->loads_lto->every_base) + if (cur_summary_lto + && !cur_summary_lto->loads->every_base) { - cur_summary->loads_lto->collapse (); + cur_summary_lto->loads->collapse (); changed = true; } return changed; @@ -1642,9 +1833,14 @@ modref_propagate_in_scc (cgraph_node *component_node) cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle) { cgraph_node *node = cur->inlined_to ? cur->inlined_to : cur; - modref_summary *cur_summary = summaries->get (node); - - if (!cur_summary) + modref_summary *cur_summary = optimization_summaries + ? optimization_summaries->get (node) + : NULL; + modref_summary_lto *cur_summary_lto = summaries_lto + ? summaries_lto->get (node) + : NULL; + + if (!cur_summary && !cur_summary_lto) continue; if (dump_file) @@ -1662,27 +1858,32 @@ modref_propagate_in_scc (cgraph_node *component_node) if (dump_file) fprintf (dump_file, " Indirect call: " "collapsing loads\n"); - changed |= collapse_loads (cur_summary); + changed |= collapse_loads (cur_summary, cur_summary_lto); } else { if (dump_file) fprintf (dump_file, " Indirect call: giving up\n"); - summaries->remove (node); + if (optimization_summaries) + optimization_summaries->remove (node); + if (summaries_lto) + summaries_lto->remove (node); changed = true; cur_summary = NULL; + cur_summary_lto = NULL; break; } } - if (!cur_summary) + if (!cur_summary && !cur_summary_lto) continue; for (cgraph_edge *callee_edge = cur->callees; callee_edge; callee_edge = callee_edge->next_callee) { int flags = flags_from_decl_or_type (callee_edge->callee->decl); - modref_summary *callee_summary; + modref_summary *callee_summary = NULL; + modref_summary_lto *callee_summary_lto = NULL; struct cgraph_node *callee; if (flags & (ECF_CONST | ECF_NOVOPS) @@ -1708,35 +1909,83 @@ modref_propagate_in_scc (cgraph_node *component_node) bool ignore_stores = ignore_stores_p (cur->decl, flags); - /* We don't know anything about CALLEE, hence we cannot tell - anything about the entire component. */ - - if (avail <= AVAIL_INTERPOSABLE - || !(callee_summary = summaries->get (callee))) + if (avail <= AVAIL_INTERPOSABLE) { if (!ignore_stores) { - if (dump_file && avail <= AVAIL_INTERPOSABLE) + if (dump_file) fprintf (dump_file, " Call target interposable" " or not available\n"); - else if (dump_file) - fprintf (dump_file, " No call target summary\n"); - summaries->remove (node); + if (optimization_summaries) + optimization_summaries->remove (node); + if (summaries_lto) + summaries_lto->remove (node); changed = true; break; } else { - if (dump_file && avail <= AVAIL_INTERPOSABLE) + if (dump_file) fprintf (dump_file, " Call target interposable" " or not available; collapsing loads\n"); - else if (dump_file) + + changed |= collapse_loads (cur_summary, cur_summary_lto); + continue; + } + } + + /* We don't know anything about CALLEE, hence we cannot tell + anything about the entire component. */ + + if (cur_summary + && !(callee_summary = optimization_summaries->get (callee))) + { + if (!ignore_stores) + { + if (dump_file) + fprintf (dump_file, " No call target summary\n"); + + optimization_summaries->remove (node); + cur_summary = NULL; + changed = true; + } + else + { + if (dump_file) fprintf (dump_file, " No call target summary;" " collapsing loads\n"); - changed |= collapse_loads (cur_summary); - continue; + if (!cur_summary->loads->every_base) + { + cur_summary->loads->collapse (); + changed = true; + } + } + } + if (cur_summary_lto + && !(callee_summary_lto = summaries_lto->get (callee))) + { + if (!ignore_stores) + { + if (dump_file) + fprintf (dump_file, " No call target summary\n"); + + summaries_lto->remove (node); + cur_summary_lto = NULL; + changed = true; + } + else + { + if (dump_file) + fprintf (dump_file, " No call target summary;" + " collapsing loads\n"); + + if (!cur_summary_lto->loads->every_base) + { + cur_summary_lto->loads->collapse (); + changed = true; + } } } @@ -1746,7 +1995,7 @@ modref_propagate_in_scc (cgraph_node *component_node) the interposed variant. */ if (!callee_edge->binds_to_current_def_p ()) { - changed |= collapse_loads (cur_summary); + changed |= collapse_loads (cur_summary, cur_summary_lto); if (dump_file) fprintf (dump_file, " May not bind local;" " collapsing loads\n"); @@ -1758,31 +2007,35 @@ modref_propagate_in_scc (cgraph_node *component_node) compute_parm_map (callee_edge, &parm_map); /* Merge in callee's information. */ - if (callee_summary->loads) - changed |= cur_summary->loads->merge - (callee_summary->loads, &parm_map); - if (callee_summary->stores) - changed |= cur_summary->stores->merge - (callee_summary->stores, &parm_map); - if (callee_summary->loads_lto) - changed |= cur_summary->loads_lto->merge - (callee_summary->loads_lto, &parm_map); - if (callee_summary->stores_lto) - changed |= cur_summary->stores_lto->merge - (callee_summary->stores_lto, &parm_map); + if (callee_summary) + { + if (callee_summary->loads) + changed |= cur_summary->loads->merge + (callee_summary->loads, &parm_map); + if (callee_summary->stores) + changed |= cur_summary->stores->merge + (callee_summary->stores, &parm_map); + } + if (callee_summary_lto) + { + if (callee_summary_lto->loads) + changed |= cur_summary_lto->loads->merge + (callee_summary_lto->loads, &parm_map); + if (callee_summary_lto->stores) + changed |= cur_summary_lto->stores->merge + (callee_summary_lto->stores, &parm_map); + } if (dump_file && changed) - cur_summary->dump (dump_file); + { + if (cur_summary) + cur_summary->dump (dump_file); + if (cur_summary_lto) + cur_summary_lto->dump (dump_file); + } } } iteration++; } - for (struct cgraph_node *cur = component_node; cur; - cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle) - { - modref_summary *cur_summary = summaries->get (cur); - if (cur_summary) - cur_summary->finished = true; - } if (dump_file) { fprintf (dump_file, @@ -1791,16 +2044,31 @@ modref_propagate_in_scc (cgraph_node *component_node) cur = ((struct ipa_dfs_info *) cur->aux)->next_cycle) if (!cur->inlined_to) { - modref_summary *cur_summary = summaries->get (cur); + modref_summary *cur_summary = optimization_summaries + ? optimization_summaries->get (cur) + : NULL; + modref_summary_lto *cur_summary_lto = summaries_lto + ? summaries_lto->get (cur) + : NULL; fprintf (dump_file, "Propagated modref for %s%s%s\n", cur->dump_name (), TREE_READONLY (cur->decl) ? " (const)" : "", DECL_PURE_P (cur->decl) ? " (pure)" : ""); - if (cur_summary) - cur_summary->dump (dump_file); - else - fprintf (dump_file, " Not tracked\n"); + if (optimization_summaries) + { + if (cur_summary) + cur_summary->dump (dump_file); + else + fprintf (dump_file, " Not tracked\n"); + } + if (summaries_lto) + { + if (cur_summary_lto) + cur_summary_lto->dump (dump_file); + else + fprintf (dump_file, " Not tracked (lto)\n"); + } } } } @@ -1813,9 +2081,14 @@ modref_propagate_in_scc (cgraph_node *component_node) unsigned int pass_ipa_modref::execute (function *) { - if (!summaries) + if (!summaries && !summaries_lto) return 0; + if (optimization_summaries) + ggc_delete (optimization_summaries); + optimization_summaries = summaries; + summaries = NULL; + struct cgraph_node **order = XCNEWVEC (struct cgraph_node *, symtab->cgraph_count); int order_pos; @@ -1834,7 +2107,8 @@ pass_ipa_modref::execute (function *) modref_propagate_in_scc (component_node); } - ((modref_summaries *)summaries)->ipa = false; + if (summaries_lto) + ((modref_summaries_lto *)summaries_lto)->propagated = true; ipa_free_postorder_info (); free (order); return 0; @@ -1845,9 +2119,15 @@ pass_ipa_modref::execute (function *) void ipa_modref_c_finalize () { - if (summaries) - ggc_delete (summaries); - summaries = NULL; + if (optimization_summaries) + ggc_delete (optimization_summaries); + optimization_summaries = NULL; + gcc_checking_assert (!summaries); + if (summaries_lto) + { + ggc_delete (summaries_lto); + summaries_lto = NULL; + } } #include "gt-ipa-modref.h" diff --git a/gcc/ipa-modref.h b/gcc/ipa-modref.h index b6621b498f0..88281044951 100644 --- a/gcc/ipa-modref.h +++ b/gcc/ipa-modref.h @@ -21,7 +21,6 @@ along with GCC; see the file COPYING3. If not see #define IPA_MODREF_H typedef modref_tree modref_records; -typedef modref_tree modref_records_lto; /* Single function summary. */ @@ -31,18 +30,10 @@ struct GTY(()) modref_summary modref_records *loads; modref_records *stores; - /* The same but using tree types rather than alias sets. This is necessary - to make the information streamable for LTO but is also more verbose - and thus more likely to hit the limits. */ - modref_records_lto *loads_lto; - modref_records_lto *stores_lto; - bool finished; - modref_summary (); ~modref_summary (); void dump (FILE *); bool useful_p (int ecf_flags); - bool lto_useful_p (int ecf_flags); }; modref_summary *get_modref_function_summary (cgraph_node *func); -- 2.30.2