X-Git-Url: https://git.libre-soc.org/?a=blobdiff_plain;f=gcc%2Fvalue-prof.c;h=67bc2c8ea8e86423ed877d435280d724c3fa418a;hb=6c84d5762f1e1a60b1d4cee0caf8b4e7e0ba5f6f;hp=12f1ecc5962b7e9c9c006c507031edd3c9af95cf;hpb=eb99f777b117a9f9007d88f0a45ea62fcf4b22b5;p=gcc.git diff --git a/gcc/value-prof.c b/gcc/value-prof.c index 12f1ecc5962..67bc2c8ea8e 100644 --- a/gcc/value-prof.c +++ b/gcc/value-prof.c @@ -1,6 +1,5 @@ /* Transformations based on profile information for values. - Copyright (C) 2003, 2004, 2005, 2006, 2007, 2008, 2009 Free Software - Foundation, Inc. + Copyright (C) 2003-2013 Free Software Foundation, Inc. This file is part of GCC. @@ -27,7 +26,6 @@ along with GCC; see the file COPYING3. If not see #include "hard-reg-set.h" #include "basic-block.h" #include "value-prof.h" -#include "output.h" #include "flags.h" #include "insn-config.h" #include "recog.h" @@ -37,16 +35,16 @@ along with GCC; see the file COPYING3. If not see #include "tree-flow.h" #include "tree-flow-inline.h" #include "diagnostic.h" +#include "gimple-pretty-print.h" #include "coverage.h" #include "tree.h" #include "gcov-io.h" #include "cgraph.h" #include "timevar.h" -#include "tree-pass.h" -#include "toplev.h" +#include "dumpfile.h" #include "pointer-set.h" - -static struct value_prof_hooks *value_prof_hooks; +#include "profile.h" +#include "data-streamer.h" /* In this file value profile based optimizations are placed. Currently the following optimizations are implemented (for more detailed descriptions @@ -55,28 +53,63 @@ static struct value_prof_hooks *value_prof_hooks; 1) Division/modulo specialization. Provided that we can determine that the operands of the division have some special properties, we may use it to produce more effective code. - 2) Speculative prefetching. If we are able to determine that the difference - between addresses accessed by a memory reference is usually constant, we - may add the prefetch instructions. - FIXME: This transformation was removed together with RTL based value - profiling. - 3) Indirect/virtual call specialization. If we can determine most + 2) Indirect/virtual call specialization. If we can determine most common function callee in indirect/virtual call. We can use this information to improve code effectiveness (especially info for - inliner). + the inliner). - Every such optimization should add its requirements for profiled values to - insn_values_to_profile function. This function is called from branch_prob - in profile.c and the requested values are instrumented by it in the first - compilation with -fprofile-arcs. The optimization may then read the - gathered data in the second compilation with -fbranch-probabilities. + 3) Speculative prefetching. If we are able to determine that the difference + between addresses accessed by a memory reference is usually constant, we + may add the prefetch instructions. + FIXME: This transformation was removed together with RTL based value + profiling. - The measured data is pointed to from the histograms - field of the statement annotation of the instrumented insns. It is - kept as a linked list of struct histogram_value_t's, which contain the - same information as above. */ + Value profiling internals + ========================== + + Every value profiling transformation starts with defining what values + to profile. There are different histogram types (see HIST_TYPE_* in + value-prof.h) and each transformation can request one or more histogram + types per GIMPLE statement. The function gimple_find_values_to_profile() + collects the values to profile in a vec, and adds the number of counters + required for the different histogram types. + + For a -fprofile-generate run, the statements for which values should be + recorded, are instrumented in instrument_values(). The instrumentation + is done by helper functions that can be found in tree-profile.c, where + new types of histograms can be added if necessary. + + After a -fprofile-use, the value profiling data is read back in by + compute_value_histograms() that translates the collected data to + histograms and attaches them to the profiled statements via + gimple_add_histogram_value(). Histograms are stored in a hash table + that is attached to every intrumented function, see VALUE_HISTOGRAMS + in function.h. + + The value-profile transformations driver is the function + gimple_value_profile_transformations(). It traverses all statements in + the to-be-transformed function, and looks for statements with one or + more histograms attached to it. If a statement has histograms, the + transformation functions are called on the statement. + + Limitations / FIXME / TODO: + * Only one histogram of each type can be associated with a statement. + * Currently, HIST_TYPE_CONST_DELTA is not implemented. + (This type of histogram was originally used to implement a form of + stride profiling based speculative prefetching to improve SPEC2000 + scores for memory-bound benchmarks, mcf and equake. However, this + was an RTL value-profiling transformation, and those have all been + removed.) + * Some value profile transformations are done in builtins.c (?!) + * Updating of histograms needs some TLC. + * The value profiling code could be used to record analysis results + from non-profiling (e.g. VRP). + * Adding new profilers should be simplified, starting with a cleanup + of what-happens-where andwith making gimple_find_values_to_profile + and gimple_value_profile_transformations table-driven, perhaps... +*/ static tree gimple_divmod_fixed_value (gimple, tree, int, gcov_type, gcov_type); static tree gimple_mod_pow2 (gimple, int, gcov_type, gcov_type); @@ -86,7 +119,7 @@ static bool gimple_divmod_fixed_value_transform (gimple_stmt_iterator *); static bool gimple_mod_pow2_value_transform (gimple_stmt_iterator *); static bool gimple_mod_subtract_transform (gimple_stmt_iterator *); static bool gimple_stringops_transform (gimple_stmt_iterator *); -static bool gimple_ic_transform (gimple); +static bool gimple_ic_transform (gimple_stmt_iterator *); /* Allocate histogram value. */ @@ -109,7 +142,7 @@ histogram_hash (const void *x) return htab_hash_pointer (((const_histogram_value)x)->hvalue.stmt); } -/* Return nonzero if decl_id of die_struct X is the same as UID of decl *Y. */ +/* Return nonzero if statement for histogram_value X is Y. */ static int histogram_eq (const void *x, const void *y) @@ -301,9 +334,97 @@ dump_histogram_value (FILE *dump_file, histogram_value hist) } fprintf (dump_file, ".\n"); break; + case HIST_TYPE_MAX: + gcc_unreachable (); } } +/* Dump information about HIST to DUMP_FILE. */ + +void +stream_out_histogram_value (struct output_block *ob, histogram_value hist) +{ + struct bitpack_d bp; + unsigned int i; + + bp = bitpack_create (ob->main_stream); + bp_pack_enum (&bp, hist_type, HIST_TYPE_MAX, hist->type); + bp_pack_value (&bp, hist->hvalue.next != NULL, 1); + streamer_write_bitpack (&bp); + switch (hist->type) + { + case HIST_TYPE_INTERVAL: + streamer_write_hwi (ob, hist->hdata.intvl.int_start); + streamer_write_uhwi (ob, hist->hdata.intvl.steps); + break; + default: + break; + } + for (i = 0; i < hist->n_counters; i++) + streamer_write_gcov_count (ob, hist->hvalue.counters[i]); + if (hist->hvalue.next) + stream_out_histogram_value (ob, hist->hvalue.next); +} +/* Dump information about HIST to DUMP_FILE. */ + +void +stream_in_histogram_value (struct lto_input_block *ib, gimple stmt) +{ + enum hist_type type; + unsigned int ncounters = 0; + struct bitpack_d bp; + unsigned int i; + histogram_value new_val; + bool next; + histogram_value *next_p = NULL; + + do + { + bp = streamer_read_bitpack (ib); + type = bp_unpack_enum (&bp, hist_type, HIST_TYPE_MAX); + next = bp_unpack_value (&bp, 1); + new_val = gimple_alloc_histogram_value (cfun, type, stmt, NULL); + switch (type) + { + case HIST_TYPE_INTERVAL: + new_val->hdata.intvl.int_start = streamer_read_hwi (ib); + new_val->hdata.intvl.steps = streamer_read_uhwi (ib); + ncounters = new_val->hdata.intvl.steps + 2; + break; + + case HIST_TYPE_POW2: + case HIST_TYPE_AVERAGE: + ncounters = 2; + break; + + case HIST_TYPE_SINGLE_VALUE: + case HIST_TYPE_INDIR_CALL: + ncounters = 3; + break; + + case HIST_TYPE_CONST_DELTA: + ncounters = 4; + break; + + case HIST_TYPE_IOR: + ncounters = 1; + break; + case HIST_TYPE_MAX: + gcc_unreachable (); + } + new_val->hvalue.counters = XNEWVAR (gcov_type, sizeof (*new_val->hvalue.counters) * ncounters); + new_val->n_counters = ncounters; + for (i = 0; i < ncounters; i++) + new_val->hvalue.counters[i] = streamer_read_gcov_count (ib); + if (!next_p) + gimple_add_histogram_value (cfun, stmt, new_val); + else + *next_p = new_val; + next_p = &new_val->hvalue.next; + } + while (next); +} + /* Dump all histograms attached to STMT to DUMP_FILE. */ void @@ -311,7 +432,7 @@ dump_histograms_for_stmt (struct function *fun, FILE *dump_file, gimple stmt) { histogram_value hist; for (hist = gimple_histogram_value (fun, stmt); hist; hist = hist->hvalue.next) - dump_histogram_value (dump_file, hist); + dump_histogram_value (dump_file, hist); } /* Remove all histograms associated with STMT. */ @@ -373,7 +494,7 @@ visit_hist (void **slot, void *data) histogram_value hist = *(histogram_value *) slot; if (!pointer_set_contains (visited, hist)) { - error ("Dead histogram"); + error ("dead histogram"); dump_histogram_value (stderr, hist); debug_gimple_stmt (hist->hvalue.stmt); error_found = true; @@ -384,7 +505,7 @@ visit_hist (void **slot, void *data) /* Verify sanity of the histograms. */ -void +DEBUG_FUNCTION void verify_histograms (void) { basic_block bb; @@ -464,7 +585,7 @@ check_counter (gimple stmt, const char * name, : DECL_SOURCE_LOCATION (current_function_decl); if (flag_profile_correction) { - inform (locus, "Correcting inconsistent value profile: " + inform (locus, "correcting inconsistent value profile: " "%s profiler overall count (%d) does not match BB count " "(%d)", name, (int)*all, (int)bb_count); *all = bb_count; @@ -474,9 +595,13 @@ check_counter (gimple stmt, const char * name, } else { - error_at (locus, "Corrupted value profile: %s " - "profiler overall count (%d) does not match BB count (%d)", - name, (int)*all, (int)bb_count); + error_at (locus, "corrupted value profile: %s " + "profile counter (%d out of %d) inconsistent with " + "basic-block count (%d)", + name, + (int) *count, + (int) *all, + (int) bb_count); return true; } } @@ -487,7 +612,7 @@ check_counter (gimple stmt, const char * name, /* GIMPLE based transformations. */ -static bool +bool gimple_value_profile_transformations (void) { basic_block bb; @@ -517,12 +642,11 @@ gimple_value_profile_transformations (void) will be added before the current statement, and that the current statement remain valid (although possibly modified) upon return. */ - if (flag_value_profile_transformations - && (gimple_mod_subtract_transform (&gsi) - || gimple_divmod_fixed_value_transform (&gsi) - || gimple_mod_pow2_value_transform (&gsi) - || gimple_stringops_transform (&gsi) - || gimple_ic_transform (stmt))) + if (gimple_mod_subtract_transform (&gsi) + || gimple_divmod_fixed_value_transform (&gsi) + || gimple_mod_pow2_value_transform (&gsi) + || gimple_stringops_transform (&gsi) + || gimple_ic_transform (&gsi)) { stmt = gsi_stmt (gsi); changed = true; @@ -556,7 +680,7 @@ gimple_divmod_fixed_value (gimple stmt, tree value, int prob, gcov_type count, gcov_type all) { gimple stmt1, stmt2, stmt3; - tree tmp1, tmp2, tmpv; + tree tmp0, tmp1, tmp2; gimple bb1end, bb2end, bb3end; basic_block bb, bb2, bb3, bb4; tree optype, op1, op2; @@ -574,19 +698,19 @@ gimple_divmod_fixed_value (gimple stmt, tree value, int prob, gcov_type count, bb = gimple_bb (stmt); gsi = gsi_for_stmt (stmt); - tmpv = create_tmp_var (optype, "PROF"); - tmp1 = create_tmp_var (optype, "PROF"); - stmt1 = gimple_build_assign (tmpv, fold_convert (optype, value)); + tmp0 = make_temp_ssa_name (optype, NULL, "PROF"); + tmp1 = make_temp_ssa_name (optype, NULL, "PROF"); + stmt1 = gimple_build_assign (tmp0, fold_convert (optype, value)); stmt2 = gimple_build_assign (tmp1, op2); - stmt3 = gimple_build_cond (NE_EXPR, tmp1, tmpv, NULL_TREE, NULL_TREE); + stmt3 = gimple_build_cond (NE_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE); gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); gsi_insert_before (&gsi, stmt2, GSI_SAME_STMT); gsi_insert_before (&gsi, stmt3, GSI_SAME_STMT); bb1end = stmt3; - tmp2 = create_tmp_var (optype, "PROF"); + tmp2 = create_tmp_reg (optype, "PROF"); stmt1 = gimple_build_assign_with_ops (gimple_assign_rhs_code (stmt), tmp2, - op1, tmpv); + op1, tmp0); gsi_insert_before (&gsi, stmt1, GSI_SAME_STMT); bb2end = stmt1; @@ -617,7 +741,7 @@ gimple_divmod_fixed_value (gimple stmt, tree value, int prob, gcov_type count, e13->count = all - count; remove_edge (e23); - + e24 = make_edge (bb2, bb4, EDGE_FALLTHRU); e24->probability = REG_BR_PROB_BASE; e24->count = count; @@ -649,7 +773,7 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) return false; code = gimple_assign_rhs_code (stmt); - + if (code != TRUNC_DIV_EXPR && code != TRUNC_MOD_EXPR) return false; @@ -677,7 +801,7 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) /* Compute probability of taking the optimal path. */ if (all > 0) - prob = (count * REG_BR_PROB_BASE + all / 2) / all; + prob = GCOV_COMPUTE_SCALE (count, all); else prob = 0; @@ -697,13 +821,14 @@ gimple_divmod_fixed_value_transform (gimple_stmt_iterator *si) } gimple_assign_set_rhs_from_tree (si, result); + update_stmt (gsi_stmt (*si)); return true; } /* Generate code for transformation 2 (with parent gimple assign STMT and probability of taking the optimal path PROB, which is equivalent to COUNT/ALL - within roundoff error). This generates the result into a temp and returns + within roundoff error). This generates the result into a temp and returns the temp; it does not replace or alter the original STMT. */ static tree gimple_mod_pow2 (gimple stmt, int prob, gcov_type count, gcov_type all) @@ -727,9 +852,9 @@ gimple_mod_pow2 (gimple stmt, int prob, gcov_type count, gcov_type all) bb = gimple_bb (stmt); gsi = gsi_for_stmt (stmt); - result = create_tmp_var (optype, "PROF"); - tmp2 = create_tmp_var (optype, "PROF"); - tmp3 = create_tmp_var (optype, "PROF"); + result = create_tmp_reg (optype, "PROF"); + tmp2 = make_temp_ssa_name (optype, NULL, "PROF"); + tmp3 = make_temp_ssa_name (optype, NULL, "PROF"); stmt2 = gimple_build_assign_with_ops (PLUS_EXPR, tmp2, op2, build_int_cst (optype, -1)); stmt3 = gimple_build_assign_with_ops (BIT_AND_EXPR, tmp3, tmp2, op2); @@ -772,7 +897,7 @@ gimple_mod_pow2 (gimple stmt, int prob, gcov_type count, gcov_type all) e13->count = all - count; remove_edge (e23); - + e24 = make_edge (bb2, bb4, EDGE_FALLTHRU); e24->probability = REG_BR_PROB_BASE; e24->count = count; @@ -803,7 +928,7 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) return false; code = gimple_assign_rhs_code (stmt); - + if (code != TRUNC_MOD_EXPR || !TYPE_UNSIGNED (lhs_type)) return false; @@ -836,13 +961,14 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) return false; if (all > 0) - prob = (count * REG_BR_PROB_BASE + all / 2) / all; + prob = GCOV_COMPUTE_SCALE (count, all); else prob = 0; result = gimple_mod_pow2 (stmt, prob, count, all); gimple_assign_set_rhs_from_tree (si, result); + update_stmt (gsi_stmt (*si)); return true; } @@ -851,8 +977,8 @@ gimple_mod_pow2_value_transform (gimple_stmt_iterator *si) NCOUNTS the number of cases to support. Currently only NCOUNTS==0 or 1 is supported and this is built into this interface. The probabilities of taking the optimal paths are PROB1 and PROB2, which are equivalent to COUNT1/ALL and - COUNT2/ALL respectively within roundoff error). This generates the - result into a temp and returns the temp; it does not replace or alter + COUNT2/ALL respectively within roundoff error). This generates the + result into a temp and returns the temp; it does not replace or alter the original STMT. */ /* FIXME: Generalize the interface to handle NCOUNTS > 1. */ @@ -879,8 +1005,8 @@ gimple_mod_subtract (gimple stmt, int prob1, int prob2, int ncounts, bb = gimple_bb (stmt); gsi = gsi_for_stmt (stmt); - result = create_tmp_var (optype, "PROF"); - tmp1 = create_tmp_var (optype, "PROF"); + result = create_tmp_reg (optype, "PROF"); + tmp1 = make_temp_ssa_name (optype, NULL, "PROF"); stmt1 = gimple_build_assign (result, op1); stmt2 = gimple_build_assign (tmp1, op2); stmt3 = gimple_build_cond (LT_EXPR, result, tmp1, NULL_TREE, NULL_TREE); @@ -911,7 +1037,7 @@ gimple_mod_subtract (gimple stmt, int prob1, int prob2, int ncounts, e12 = split_block (bb, bb1end); bb2 = e12->dest; bb2->count = all - count1; - + if (ncounts) /* Assumed to be 0 or 1. */ { e23 = split_block (bb2, bb2end); @@ -959,7 +1085,7 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) histogram_value histogram; enum tree_code code; gcov_type count, wrong_values, all; - tree lhs_type, result, value; + tree lhs_type, result; gcov_type prob1, prob2; unsigned int i, steps; gcov_type count1, count2; @@ -974,7 +1100,7 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) return false; code = gimple_assign_rhs_code (stmt); - + if (code != TRUNC_MOD_EXPR || !TYPE_UNSIGNED (lhs_type)) return false; @@ -982,7 +1108,6 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) if (!histogram) return false; - value = histogram->hvalue.value; all = 0; wrong_values = 0; for (i = 0; i < histogram->hdata.intvl.steps; i++) @@ -1030,8 +1155,8 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) /* Compute probability of taking the optimal path(s). */ if (all > 0) { - prob1 = (count1 * REG_BR_PROB_BASE + all / 2) / all; - prob2 = (count2 * REG_BR_PROB_BASE + all / 2) / all; + prob1 = GCOV_COMPUTE_SCALE (count1, all); + prob2 = GCOV_COMPUTE_SCALE (count2, all); } else { @@ -1043,39 +1168,76 @@ gimple_mod_subtract_transform (gimple_stmt_iterator *si) result = gimple_mod_subtract (stmt, prob1, prob2, i, count1, count2, all); gimple_assign_set_rhs_from_tree (si, result); + update_stmt (gsi_stmt (*si)); return true; } -static struct cgraph_node** pid_map = NULL; +static vec cgraph_node_map + = vNULL; -/* Initialize map of pids (pid -> cgraph node) */ +/* Initialize map from FUNCDEF_NO to CGRAPH_NODE. */ -static void -init_pid_map (void) +void +init_node_map (void) { struct cgraph_node *n; - if (pid_map != NULL) - return; - - pid_map = XCNEWVEC (struct cgraph_node*, cgraph_max_pid); + if (get_last_funcdef_no ()) + cgraph_node_map.safe_grow_cleared (get_last_funcdef_no ()); - for (n = cgraph_nodes; n; n = n->next) + FOR_EACH_FUNCTION (n) { - if (n->pid != -1) - pid_map [n->pid] = n; + if (DECL_STRUCT_FUNCTION (n->symbol.decl)) + cgraph_node_map[DECL_STRUCT_FUNCTION (n->symbol.decl)->funcdef_no] = n; } } +/* Delete the CGRAPH_NODE_MAP. */ + +void +del_node_map (void) +{ + cgraph_node_map.release (); +} + /* Return cgraph node for function with pid */ static inline struct cgraph_node* -find_func_by_pid (int pid) +find_func_by_funcdef_no (int func_id) { - init_pid_map (); + int max_id = get_last_funcdef_no (); + if (func_id >= max_id || cgraph_node_map[func_id] == NULL) + { + if (flag_profile_correction) + inform (DECL_SOURCE_LOCATION (current_function_decl), + "Inconsistent profile: indirect call target (%d) does not exist", func_id); + else + error ("Inconsistent profile: indirect call target (%d) does not exist", func_id); - return pid_map [pid]; + return NULL; + } + + return cgraph_node_map[func_id]; +} + +/* Perform sanity check on the indirect call target. Due to race conditions, + false function target may be attributed to an indirect call site. If the + call expression type mismatches with the target function's type, expand_call + may ICE. Here we only do very minimal sanity check just to make compiler happy. + Returns true if TARGET is considered ok for call CALL_STMT. */ + +static bool +check_ic_target (gimple call_stmt, struct cgraph_node *target) +{ + location_t locus; + if (gimple_check_call_matching_types (call_stmt, target->symbol.decl, true)) + return true; + + locus = gimple_location (call_stmt); + inform (locus, "Skipping target %s with mismatching types for icall ", + cgraph_node_name (target)); + return false; } /* Do transformation @@ -1087,36 +1249,42 @@ find_func_by_pid (int pid) */ static gimple -gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, +gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, int prob, gcov_type count, gcov_type all) { gimple dcall_stmt, load_stmt, cond_stmt; - tree tmp1, tmpv, tmp; - basic_block cond_bb, dcall_bb, icall_bb, join_bb; + tree tmp0, tmp1, tmp; + basic_block cond_bb, dcall_bb, icall_bb, join_bb = NULL; tree optype = build_pointer_type (void_type_node); - edge e_cd, e_ci, e_di, e_dj, e_ij; + edge e_cd, e_ci, e_di, e_dj = NULL, e_ij; gimple_stmt_iterator gsi; - int lp_nr; + int lp_nr, dflags; cond_bb = gimple_bb (icall_stmt); gsi = gsi_for_stmt (icall_stmt); - tmpv = create_tmp_var (optype, "PROF"); - tmp1 = create_tmp_var (optype, "PROF"); + tmp0 = make_temp_ssa_name (optype, NULL, "PROF"); + tmp1 = make_temp_ssa_name (optype, NULL, "PROF"); tmp = unshare_expr (gimple_call_fn (icall_stmt)); - load_stmt = gimple_build_assign (tmpv, tmp); + load_stmt = gimple_build_assign (tmp0, tmp); gsi_insert_before (&gsi, load_stmt, GSI_SAME_STMT); - tmp = fold_convert (optype, build_addr (direct_call->decl, + tmp = fold_convert (optype, build_addr (direct_call->symbol.decl, current_function_decl)); load_stmt = gimple_build_assign (tmp1, tmp); gsi_insert_before (&gsi, load_stmt, GSI_SAME_STMT); - cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmpv, NULL_TREE, NULL_TREE); + cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE); gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); + gimple_set_vdef (icall_stmt, NULL_TREE); + gimple_set_vuse (icall_stmt, NULL_TREE); + update_stmt (icall_stmt); dcall_stmt = gimple_copy (icall_stmt); - gimple_call_set_fndecl (dcall_stmt, direct_call->decl); + gimple_call_set_fndecl (dcall_stmt, direct_call->symbol.decl); + dflags = flags_from_decl_or_type (direct_call->symbol.decl); + if ((dflags & ECF_NORETURN) != 0) + gimple_call_set_lhs (dcall_stmt, NULL_TREE); gsi_insert_before (&gsi, dcall_stmt, GSI_SAME_STMT); /* Fix CFG. */ @@ -1129,9 +1297,25 @@ gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, icall_bb = e_di->dest; icall_bb->count = all - count; - e_ij = split_block (icall_bb, icall_stmt); - join_bb = e_ij->dest; - join_bb->count = all; + /* Do not disturb existing EH edges from the indirect call. */ + if (!stmt_ends_bb_p (icall_stmt)) + e_ij = split_block (icall_bb, icall_stmt); + else + { + e_ij = find_fallthru_edge (icall_bb->succs); + /* The indirect call might be noreturn. */ + if (e_ij != NULL) + { + e_ij->probability = REG_BR_PROB_BASE; + e_ij->count = all - count; + e_ij = single_pred_edge (split_edge (e_ij)); + } + } + if (e_ij != NULL) + { + join_bb = e_ij->dest; + join_bb->count = all; + } e_cd->flags = (e_cd->flags & ~EDGE_FALLTHRU) | EDGE_TRUE_VALUE; e_cd->probability = prob; @@ -1142,28 +1326,58 @@ gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, e_ci->count = all - count; remove_edge (e_di); - - e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU); - e_dj->probability = REG_BR_PROB_BASE; - e_dj->count = count; - e_ij->probability = REG_BR_PROB_BASE; - e_ij->count = all - count; + if (e_ij != NULL) + { + if ((dflags & ECF_NORETURN) != 0) + e_ij->count = all; + else + { + e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU); + e_dj->probability = REG_BR_PROB_BASE; + e_dj->count = count; - /* Fix eh edges */ + e_ij->count = all - count; + } + e_ij->probability = REG_BR_PROB_BASE; + } + + /* Insert PHI node for the call result if necessary. */ + if (gimple_call_lhs (icall_stmt) + && TREE_CODE (gimple_call_lhs (icall_stmt)) == SSA_NAME + && (dflags & ECF_NORETURN) == 0) + { + tree result = gimple_call_lhs (icall_stmt); + gimple phi = create_phi_node (result, join_bb); + gimple_call_set_lhs (icall_stmt, + duplicate_ssa_name (result, icall_stmt)); + add_phi_arg (phi, gimple_call_lhs (icall_stmt), e_ij, UNKNOWN_LOCATION); + gimple_call_set_lhs (dcall_stmt, + duplicate_ssa_name (result, dcall_stmt)); + add_phi_arg (phi, gimple_call_lhs (dcall_stmt), e_dj, UNKNOWN_LOCATION); + } + + /* Build an EH edge for the direct call if necessary. */ lp_nr = lookup_stmt_eh_lp (icall_stmt); - if (lp_nr != 0) + if (lp_nr != 0 + && stmt_could_throw_p (dcall_stmt)) { - gimple_purge_dead_eh_edges (join_bb); + edge e_eh, e; + edge_iterator ei; + gimple_stmt_iterator psi; - if (stmt_could_throw_p (dcall_stmt)) + add_stmt_to_eh_lp (dcall_stmt, lp_nr); + FOR_EACH_EDGE (e_eh, ei, icall_bb->succs) + if (e_eh->flags & EDGE_EH) + break; + e = make_edge (dcall_bb, e_eh->dest, EDGE_EH); + for (psi = gsi_start_phis (e_eh->dest); + !gsi_end_p (psi); gsi_next (&psi)) { - add_stmt_to_eh_lp (dcall_stmt, lp_nr); - make_eh_edges (dcall_stmt); + gimple phi = gsi_stmt (psi); + SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), + PHI_ARG_DEF_FROM_EDGE (phi, e_eh)); } - - gcc_assert (stmt_could_throw_p (icall_stmt)); - make_eh_edges (icall_stmt); } return dcall_stmt; @@ -1176,21 +1390,22 @@ gimple_ic (gimple icall_stmt, struct cgraph_node *direct_call, */ static bool -gimple_ic_transform (gimple stmt) +gimple_ic_transform (gimple_stmt_iterator *gsi) { + gimple stmt = gsi_stmt (*gsi); histogram_value histogram; gcov_type val, count, all, bb_all; gcov_type prob; - tree callee; gimple modify; struct cgraph_node *direct_call; - + if (gimple_code (stmt) != GIMPLE_CALL) return false; - callee = gimple_call_fn (stmt); + if (gimple_call_fndecl (stmt) != NULL_TREE) + return false; - if (TREE_CODE (callee) == FUNCTION_DECL) + if (gimple_call_internal_p (stmt)) return false; histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_INDIR_CALL); @@ -1206,7 +1421,7 @@ gimple_ic_transform (gimple stmt) return false; bb_all = gimple_bb (stmt)->count; - /* The order of CHECK_COUNTER calls is important - + /* The order of CHECK_COUNTER calls is important - since check_counter can correct the third parameter and we want to make count <= all <= bb_all. */ if ( check_counter (stmt, "ic", &all, &bb_all, bb_all) @@ -1214,14 +1429,17 @@ gimple_ic_transform (gimple stmt) return false; if (all > 0) - prob = (count * REG_BR_PROB_BASE + all / 2) / all; + prob = GCOV_COMPUTE_SCALE (count, all); else prob = 0; - direct_call = find_func_by_pid ((int)val); + direct_call = find_func_by_funcdef_no ((int)val); if (direct_call == NULL) return false; + if (!check_ic_target (stmt, direct_call)) + return false; + modify = gimple_ic (stmt, direct_call, prob, count, all); if (dump_file) @@ -1229,7 +1447,7 @@ gimple_ic_transform (gimple stmt) fprintf (dump_file, "Indirect call -> direct call "); print_generic_expr (dump_file, gimple_call_fn (stmt), TDF_SLIM); fprintf (dump_file, "=> "); - print_generic_expr (dump_file, direct_call->decl, TDF_SLIM); + print_generic_expr (dump_file, direct_call->symbol.decl, TDF_SLIM); fprintf (dump_file, " transformation on insn "); print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM); fprintf (dump_file, " to "); @@ -1275,7 +1493,7 @@ interesting_stringop_to_profile_p (tree fndecl, gimple call, int *size_arg) } /* Convert stringop (..., vcall_size) - into + into if (vcall_size == icall_size) stringop (..., icall_size); else @@ -1287,7 +1505,7 @@ gimple_stringop_fixed_value (gimple vcall_stmt, tree icall_size, int prob, gcov_type count, gcov_type all) { gimple tmp_stmt, cond_stmt, icall_stmt; - tree tmp1, tmpv, vcall_size, optype; + tree tmp0, tmp1, vcall_size, optype; basic_block cond_bb, icall_bb, vcall_bb, join_bb; edge e_ci, e_cv, e_iv, e_ij, e_vj; gimple_stmt_iterator gsi; @@ -1304,17 +1522,20 @@ gimple_stringop_fixed_value (gimple vcall_stmt, tree icall_size, int prob, vcall_size = gimple_call_arg (vcall_stmt, size_arg); optype = TREE_TYPE (vcall_size); - tmpv = create_tmp_var (optype, "PROF"); - tmp1 = create_tmp_var (optype, "PROF"); - tmp_stmt = gimple_build_assign (tmpv, fold_convert (optype, icall_size)); + tmp0 = make_temp_ssa_name (optype, NULL, "PROF"); + tmp1 = make_temp_ssa_name (optype, NULL, "PROF"); + tmp_stmt = gimple_build_assign (tmp0, fold_convert (optype, icall_size)); gsi_insert_before (&gsi, tmp_stmt, GSI_SAME_STMT); tmp_stmt = gimple_build_assign (tmp1, vcall_size); gsi_insert_before (&gsi, tmp_stmt, GSI_SAME_STMT); - cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmpv, NULL_TREE, NULL_TREE); + cond_stmt = gimple_build_cond (EQ_EXPR, tmp1, tmp0, NULL_TREE, NULL_TREE); gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); + gimple_set_vdef (vcall_stmt, NULL); + gimple_set_vuse (vcall_stmt, NULL); + update_stmt (vcall_stmt); icall_stmt = gimple_copy (vcall_stmt); gimple_call_set_arg (icall_stmt, size_arg, icall_size); gsi_insert_before (&gsi, icall_stmt, GSI_SAME_STMT); @@ -1342,7 +1563,7 @@ gimple_stringop_fixed_value (gimple vcall_stmt, tree icall_size, int prob, e_cv->count = all - count; remove_edge (e_iv); - + e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU); e_ij->probability = REG_BR_PROB_BASE; e_ij->count = count; @@ -1350,6 +1571,20 @@ gimple_stringop_fixed_value (gimple vcall_stmt, tree icall_size, int prob, e_vj->probability = REG_BR_PROB_BASE; e_vj->count = all - count; + /* Insert PHI node for the call result if necessary. */ + if (gimple_call_lhs (vcall_stmt) + && TREE_CODE (gimple_call_lhs (vcall_stmt)) == SSA_NAME) + { + tree result = gimple_call_lhs (vcall_stmt); + gimple phi = create_phi_node (result, join_bb); + gimple_call_set_lhs (vcall_stmt, + duplicate_ssa_name (result, vcall_stmt)); + add_phi_arg (phi, gimple_call_lhs (vcall_stmt), e_vj, UNKNOWN_LOCATION); + gimple_call_set_lhs (icall_stmt, + duplicate_ssa_name (result, icall_stmt)); + add_phi_arg (phi, gimple_call_lhs (icall_stmt), e_ij, UNKNOWN_LOCATION); + } + /* Because these are all string op builtins, they're all nothrow. */ gcc_assert (!stmt_could_throw_p (vcall_stmt)); gcc_assert (!stmt_could_throw_p (icall_stmt)); @@ -1366,7 +1601,6 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) enum built_in_function fcode; histogram_value histogram; gcov_type count, all, val; - tree value; tree dest, src; unsigned int dest_align, src_align; gcov_type prob; @@ -1389,7 +1623,6 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) histogram = gimple_histogram_value_of_type (cfun, stmt, HIST_TYPE_SINGLE_VALUE); if (!histogram) return false; - value = histogram->hvalue.value; val = histogram->hvalue.counters[0]; count = histogram->hvalue.counters[1]; all = histogram->hvalue.counters[2]; @@ -1402,17 +1635,17 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count)) return false; if (all > 0) - prob = (count * REG_BR_PROB_BASE + all / 2) / all; + prob = GCOV_COMPUTE_SCALE (count, all); else prob = 0; dest = gimple_call_arg (stmt, 0); - dest_align = get_pointer_alignment (dest, BIGGEST_ALIGNMENT); + dest_align = get_pointer_alignment (dest); switch (fcode) { case BUILT_IN_MEMCPY: case BUILT_IN_MEMPCPY: src = gimple_call_arg (stmt, 1); - src_align = get_pointer_alignment (src, BIGGEST_ALIGNMENT); + src_align = get_pointer_alignment (src); if (!can_move_by_pieces (val, MIN (dest_align, src_align))) return false; break; @@ -1441,7 +1674,7 @@ gimple_stringops_transform (gimple_stmt_iterator *gsi) print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM); } gimple_stringop_fixed_value (stmt, tree_val, prob, count, all); - + return true; } @@ -1494,14 +1727,6 @@ stringop_block_profile (gimple stmt, unsigned int *expected_align, } } -struct value_prof_hooks { - /* Find list of values for which we want to measure histograms. */ - void (*find_values_to_profile) (histogram_values *); - - /* Identify and exploit properties of values that are hard to analyze - statically. See value-prof.c for more detail. */ - bool (*value_profile_transformations) (void); -}; /* Find values inside STMT for that we want to measure histograms for division/modulo optimization. */ @@ -1526,13 +1751,12 @@ gimple_divmod_values_to_profile (gimple stmt, histogram_values *values) divisor = gimple_assign_rhs2 (stmt); op0 = gimple_assign_rhs1 (stmt); - VEC_reserve (histogram_value, heap, *values, 3); + values->reserve (3); - if (is_gimple_reg (divisor)) + if (TREE_CODE (divisor) == SSA_NAME) /* Check for the case where the divisor is the same value most of the time. */ - VEC_quick_push (histogram_value, *values, - gimple_alloc_histogram_value (cfun, + values->quick_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_SINGLE_VALUE, stmt, divisor)); @@ -1543,16 +1767,16 @@ gimple_divmod_values_to_profile (gimple stmt, histogram_values *values) { tree val; /* Check for a special case where the divisor is power of 2. */ - VEC_quick_push (histogram_value, *values, - gimple_alloc_histogram_value (cfun, HIST_TYPE_POW2, - stmt, divisor)); + values->quick_push (gimple_alloc_histogram_value (cfun, + HIST_TYPE_POW2, + stmt, divisor)); val = build2 (TRUNC_DIV_EXPR, type, op0, divisor); hist = gimple_alloc_histogram_value (cfun, HIST_TYPE_INTERVAL, stmt, val); hist->hdata.intvl.int_start = 0; hist->hdata.intvl.steps = 2; - VEC_quick_push (histogram_value, *values, hist); + values->quick_push (hist); } return; @@ -1561,8 +1785,8 @@ gimple_divmod_values_to_profile (gimple stmt, histogram_values *values) } } -/* Find calls inside STMT for that we want to measure histograms for - indirect/virtual call optimization. */ +/* Find calls inside STMT for that we want to measure histograms for + indirect/virtual call optimization. */ static void gimple_indirect_call_to_profile (gimple stmt, histogram_values *values) @@ -1570,16 +1794,16 @@ gimple_indirect_call_to_profile (gimple stmt, histogram_values *values) tree callee; if (gimple_code (stmt) != GIMPLE_CALL + || gimple_call_internal_p (stmt) || gimple_call_fndecl (stmt) != NULL_TREE) return; callee = gimple_call_fn (stmt); - VEC_reserve (histogram_value, heap, *values, 3); + values->reserve (3); - VEC_quick_push (histogram_value, *values, - gimple_alloc_histogram_value (cfun, HIST_TYPE_INDIR_CALL, - stmt, callee)); + values->quick_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_INDIR_CALL, + stmt, callee)); return; } @@ -1592,7 +1816,6 @@ gimple_stringops_values_to_profile (gimple stmt, histogram_values *values) tree fndecl; tree blck_size; tree dest; - enum built_in_function fcode; int size_arg; if (gimple_code (stmt) != GIMPLE_CALL) @@ -1600,7 +1823,6 @@ gimple_stringops_values_to_profile (gimple stmt, histogram_values *values) fndecl = gimple_call_fndecl (stmt); if (!fndecl) return; - fcode = DECL_FUNCTION_CODE (fndecl); if (!interesting_stringop_to_profile_p (fndecl, stmt, &size_arg)) return; @@ -1610,17 +1832,15 @@ gimple_stringops_values_to_profile (gimple stmt, histogram_values *values) if (TREE_CODE (blck_size) != INTEGER_CST) { - VEC_safe_push (histogram_value, heap, *values, - gimple_alloc_histogram_value (cfun, HIST_TYPE_SINGLE_VALUE, - stmt, blck_size)); - VEC_safe_push (histogram_value, heap, *values, - gimple_alloc_histogram_value (cfun, HIST_TYPE_AVERAGE, - stmt, blck_size)); + values->safe_push (gimple_alloc_histogram_value (cfun, + HIST_TYPE_SINGLE_VALUE, + stmt, blck_size)); + values->safe_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_AVERAGE, + stmt, blck_size)); } if (TREE_CODE (blck_size) != INTEGER_CST) - VEC_safe_push (histogram_value, heap, *values, - gimple_alloc_histogram_value (cfun, HIST_TYPE_IOR, - stmt, dest)); + values->safe_push (gimple_alloc_histogram_value (cfun, HIST_TYPE_IOR, + stmt, dest)); } /* Find values inside STMT for that we want to measure histograms and adds @@ -1629,15 +1849,12 @@ gimple_stringops_values_to_profile (gimple stmt, histogram_values *values) static void gimple_values_to_profile (gimple stmt, histogram_values *values) { - if (flag_value_profile_transformations) - { - gimple_divmod_values_to_profile (stmt, values); - gimple_stringops_values_to_profile (stmt, values); - gimple_indirect_call_to_profile (stmt, values); - } + gimple_divmod_values_to_profile (stmt, values); + gimple_stringops_values_to_profile (stmt, values); + gimple_indirect_call_to_profile (stmt, values); } -static void +void gimple_find_values_to_profile (histogram_values *values) { basic_block bb; @@ -1645,12 +1862,12 @@ gimple_find_values_to_profile (histogram_values *values) unsigned i; histogram_value hist = NULL; - *values = NULL; + values->create (0); FOR_EACH_BB (bb) for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) gimple_values_to_profile (gsi_stmt (gsi), values); - - for (i = 0; VEC_iterate (histogram_value, *values, i, hist); i++) + + FOR_EACH_VEC_ELT (*values, i, hist) { switch (hist->type) { @@ -1693,29 +1910,3 @@ gimple_find_values_to_profile (histogram_values *values) } } } - -static struct value_prof_hooks gimple_value_prof_hooks = { - gimple_find_values_to_profile, - gimple_value_profile_transformations -}; - -void -gimple_register_value_prof_hooks (void) -{ - gcc_assert (current_ir_type () == IR_GIMPLE); - value_prof_hooks = &gimple_value_prof_hooks; -} - -/* IR-independent entry points. */ -void -find_values_to_profile (histogram_values *values) -{ - (value_prof_hooks->find_values_to_profile) (values); -} - -bool -value_profile_transformations (void) -{ - return (value_prof_hooks->value_profile_transformations) (); -} -