* asan.c (asan_emit_stack_protection): Update.
(create_cond_insert_point): Update.
* auto-profile.c (afdo_propagate_circuit): Update.
* basic-block.h (struct edge_def): Turn probability to
profile_probability.
(EDGE_FREQUENCY): Update.
* bb-reorder.c (find_traces_1_round): Update.
(better_edge_p): Update.
(sanitize_hot_paths): Update.
* cfg.c (unchecked_make_edge): Initialize probability to uninitialized.
(make_single_succ_edge): Update.
(check_bb_profile): Update.
(dump_edge_info): Update.
(update_bb_profile_for_threading): Update.
* cfganal.c (connect_infinite_loops_to_exit): Initialize new edge
probabilitycount to 0.
* cfgbuild.c (compute_outgoing_frequencies): Update.
* cfgcleanup.c (try_forward_edges): Update.
(outgoing_edges_match): Update.
(try_crossjump_to_edge): Update.
* cfgexpand.c (expand_gimple_cond): Update make_single_succ_edge.
(expand_gimple_tailcall): Update.
(construct_init_block): Use make_single_succ_edge.
(construct_exit_block): Use make_single_succ_edge.
* cfghooks.c (verify_flow_info): Update.
(redirect_edge_succ_nodup): Update.
(split_edge): Update.
(account_profile_record): Update.
* cfgloopanal.c (single_likely_exit): Update.
* cfgloopmanip.c (scale_loop_profile): Update.
(set_zero_probability): Remove.
(duplicate_loop_to_header_edge): Update.
* cfgloopmanip.h (loop_version): Update prototype.
* cfgrtl.c (try_redirect_by_replacing_jump): Update.
(force_nonfallthru_and_redirect): Update.
(update_br_prob_note): Update.
(rtl_verify_edges): Update.
(purge_dead_edges): Update.
(rtl_lv_add_condition_to_bb): Update.
* cgraph.c: (cgraph_edge::redirect_call_stmt_to_calle): Update.
* cgraphunit.c (init_lowered_empty_function): Update.
(cgraph_node::expand_thunk): Update.
* cilk-common.c: Include profile-count.h
* dojump.c (inv): Remove.
(jumpifnot): Update.
(jumpifnot_1): Update.
(do_jump_1): Update.
(do_jump): Update.
(do_jump_by_parts_greater_rtx): Update.
(do_compare_rtx_and_jump): Update.
* dojump.h (jumpifnot, jumpifnot_1, jumpif_1, jumpif, do_jump,
do_jump_1. do_compare_rtx_and_jump): Update prototype.
* dwarf2cfi.c: Include profile-count.h
* except.c (dw2_build_landing_pads): Use make_single_succ_edge.
(sjlj_emit_dispatch_table): Likewise.
* explow.c: Include profile-count.h
* expmed.c (emit_store_flag_force): Update.
(do_cmp_and_jump): Update.
* expr.c (compare_by_pieces_d::generate): Update.
(compare_by_pieces_d::finish_mode): Update.
(emit_block_move_via_loop): Update.
(store_expr_with_bounds): Update.
(store_constructor): Update.
(expand_expr_real_2): Update.
(expand_expr_real_1): Update.
* expr.h (try_casesi, try_tablejump): Update prototypes.
* gimple-pretty-print.c (dump_probability): Update.
(dump_profile): New.
(dump_gimple_label): Update.
(dump_gimple_bb_header): Update.
* graph.c (draw_cfg_node_succ_edges): Update.
* hsa-gen.c (convert_switch_statements): Update.
* ifcvt.c (cheap_bb_rtx_cost_p): Update.
(find_if_case_1): Update.
(find_if_case_2): Update.
* internal-fn.c (expand_arith_overflow_result_store): Update.
(expand_addsub_overflow): Update.
(expand_neg_overflow): Update.
(expand_mul_overflow): Update.
(expand_vector_ubsan_overflow): Update.
* ipa-cp.c (good_cloning_opportunity_p): Update.
* ipa-split.c (split_function): Use make_single_succ_edge.
* ipa-utils.c (ipa_merge_profiles): Update.
* loop-doloop.c (add_test): Update.
(doloop_modify): Update.
* loop-unroll.c (compare_and_jump_seq): Update.
(unroll_loop_runtime_iterations): Update.
* lra-constraints.c (lra_inheritance): Update.
* lto-streamer-in.c (input_cfg): Update.
* lto-streamer-out.c (output_cfg): Update.
* mcf.c (adjust_cfg_counts): Update.
* modulo-sched.c (sms_schedule): Update.
* omp-expand.c (expand_omp_for_init_counts): Update.
(extract_omp_for_update_vars): Update.
(expand_omp_ordered_sink): Update.
(expand_omp_for_ordered_loops): Update.
(expand_omp_for_generic): Update.
(expand_omp_for_static_nochunk): Update.
(expand_omp_for_static_chunk): Update.
(expand_cilk_for): Update.
(expand_omp_simd): Update.
(expand_omp_taskloop_for_outer): Update.
(expand_omp_taskloop_for_inner): Update.
* omp-simd-clone.c (simd_clone_adjust): Update.
* optabs.c (expand_doubleword_shift): Update.
(expand_abs): Update.
(emit_cmp_and_jump_insn_1): Update.
(expand_compare_and_swap_loop): Update.
* optabs.h (emit_cmp_and_jump_insns): Update prototype.
* predict.c (predictable_edge_p): Update.
(edge_probability_reliable_p): Update.
(set_even_probabilities): Update.
(combine_predictions_for_insn): Update.
(combine_predictions_for_bb): Update.
(propagate_freq): Update.
(estimate_bb_frequencies): Update.
(force_edge_cold): Update.
* profile-count.c (profile_count::dump): Add missing space into dump.
(profile_count::debug): Add newline.
(profile_count::differs_from_p): Explicitly convert to unsigned.
(profile_count::stream_in): Update.
(profile_probability::dump): New member function.
(profile_probability::debug): New member function.
(profile_probability::differs_from_p): New member function.
(profile_probability::differs_lot_from_p): New member function.
(profile_probability::stream_in): New member function.
(profile_probability::stream_out): New member function.
* profile-count.h (profile_count_quality): Rename to ...
(profile_quality): ... this one.
(profile_probability): New.
(profile_count): Update.
* profile.c (compute_branch_probabilities): Update.
* recog.c (peep2_attempt): Update.
* sched-ebb.c (schedule_ebbs): Update.
* sched-rgn.c (find_single_block_region): Update.
(compute_dom_prob_ps): Update.
(schedule_region): Update.
* sel-sched-ir.c (compute_succs_info): Update.
* stmt.c (struct case_node): Update.
(do_jump_if_equal): Update.
(get_outgoing_edge_probs): Update.
(conditional_probability): Update.
(emit_case_dispatch_table): Update.
(expand_case): Update.
(expand_sjlj_dispatch_table): Update.
(emit_case_nodes): Update.
* targhooks.c: Update.
* tracer.c (better_p): Update.
(find_best_successor): Update.
* trans-mem.c (expand_transaction): Update.
* tree-call-cdce.c: Update.
* tree-cfg.c (gimple_split_edge): Upate.
(move_sese_region_to_fn): Upate.
* tree-cfgcleanup.c (cleanup_control_expr_graph): Upate.
* tree-eh.c (lower_resx): Upate.
(cleanup_empty_eh_move_lp): Upate.
* tree-if-conv.c (version_loop_for_if_conversion): Update.
* tree-inline.c (copy_edges_for_bb): Update.
(copy_cfg_body): Update.
* tree-parloops.c (gen_parallel_loop): Update.
* tree-profile.c (gimple_gen_ic_func_profiler): Update.
(gimple_gen_time_profiler): Update.
* tree-ssa-dce.c (remove_dead_stmt): Update.
* tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
* tree-ssa-loop-im.c (execute_sm_if_changed): Update.
* tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update.
(unloop_loops): Update.
(try_peel_loop): Update.
* tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
* tree-ssa-loop-split.c (connect_loops): Update.
(split_loop): Update.
* tree-ssa-loop-unswitch.c (tree_unswitch_loop): Update.
(hoist_guard): Update.
* tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update.
* tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update.
(value_replacement): Update.
* tree-ssa-reassoc.c (branch_fixup): Update.
* tree-ssa-tail-merge.c (replace_block_by): Update.
* tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update.
(create_edge_and_update_destination_phis): Update.
(compute_path_counts): Update.
(recompute_probabilities): Update.
(update_joiner_offpath_counts): Update.
(freqs_to_counts_path): Update.
(duplicate_thread_path): Update.
* tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update.
(struct switch_conv_info): Update.
(gen_inbound_check): Update.
* tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
(vect_do_peeling): Update.
(vect_loop_versioning): Update.
* tree-vect-loop.c (scale_profile_for_vect_loop): Update.
(optimize_mask_stores): Update.
* ubsan.c (ubsan_expand_null_ifn): Update.
* value-prof.c (gimple_divmod_fixed_value): Update.
(gimple_divmod_fixed_value_transform): Update.
(gimple_mod_pow2): Update.
(gimple_mod_pow2_value_transform): Update.
(gimple_mod_subtract): Update.
(gimple_mod_subtract_transform): Update.
(gimple_ic): Update.
(gimple_stringop_fixed_value): Update.
(gimple_stringops_transform): Update.
* value-prof.h: Update.
From-SVN: r249800
+2017-06-29 Jan Hubicka <hubicka@ucw.cz>
+
+ * asan.c (asan_emit_stack_protection): Update.
+ (create_cond_insert_point): Update.
+ * auto-profile.c (afdo_propagate_circuit): Update.
+ * basic-block.h (struct edge_def): Turn probability to
+ profile_probability.
+ (EDGE_FREQUENCY): Update.
+ * bb-reorder.c (find_traces_1_round): Update.
+ (better_edge_p): Update.
+ (sanitize_hot_paths): Update.
+ * cfg.c (unchecked_make_edge): Initialize probability to uninitialized.
+ (make_single_succ_edge): Update.
+ (check_bb_profile): Update.
+ (dump_edge_info): Update.
+ (update_bb_profile_for_threading): Update.
+ * cfganal.c (connect_infinite_loops_to_exit): Initialize new edge
+ probabilitycount to 0.
+ * cfgbuild.c (compute_outgoing_frequencies): Update.
+ * cfgcleanup.c (try_forward_edges): Update.
+ (outgoing_edges_match): Update.
+ (try_crossjump_to_edge): Update.
+ * cfgexpand.c (expand_gimple_cond): Update make_single_succ_edge.
+ (expand_gimple_tailcall): Update.
+ (construct_init_block): Use make_single_succ_edge.
+ (construct_exit_block): Use make_single_succ_edge.
+ * cfghooks.c (verify_flow_info): Update.
+ (redirect_edge_succ_nodup): Update.
+ (split_edge): Update.
+ (account_profile_record): Update.
+ * cfgloopanal.c (single_likely_exit): Update.
+ * cfgloopmanip.c (scale_loop_profile): Update.
+ (set_zero_probability): Remove.
+ (duplicate_loop_to_header_edge): Update.
+ * cfgloopmanip.h (loop_version): Update prototype.
+ * cfgrtl.c (try_redirect_by_replacing_jump): Update.
+ (force_nonfallthru_and_redirect): Update.
+ (update_br_prob_note): Update.
+ (rtl_verify_edges): Update.
+ (purge_dead_edges): Update.
+ (rtl_lv_add_condition_to_bb): Update.
+ * cgraph.c: (cgraph_edge::redirect_call_stmt_to_calle): Update.
+ * cgraphunit.c (init_lowered_empty_function): Update.
+ (cgraph_node::expand_thunk): Update.
+ * cilk-common.c: Include profile-count.h
+ * dojump.c (inv): Remove.
+ (jumpifnot): Update.
+ (jumpifnot_1): Update.
+ (do_jump_1): Update.
+ (do_jump): Update.
+ (do_jump_by_parts_greater_rtx): Update.
+ (do_compare_rtx_and_jump): Update.
+ * dojump.h (jumpifnot, jumpifnot_1, jumpif_1, jumpif, do_jump,
+ do_jump_1. do_compare_rtx_and_jump): Update prototype.
+ * dwarf2cfi.c: Include profile-count.h
+ * except.c (dw2_build_landing_pads): Use make_single_succ_edge.
+ (sjlj_emit_dispatch_table): Likewise.
+ * explow.c: Include profile-count.h
+ * expmed.c (emit_store_flag_force): Update.
+ (do_cmp_and_jump): Update.
+ * expr.c (compare_by_pieces_d::generate): Update.
+ (compare_by_pieces_d::finish_mode): Update.
+ (emit_block_move_via_loop): Update.
+ (store_expr_with_bounds): Update.
+ (store_constructor): Update.
+ (expand_expr_real_2): Update.
+ (expand_expr_real_1): Update.
+ * expr.h (try_casesi, try_tablejump): Update prototypes.
+ * gimple-pretty-print.c (dump_probability): Update.
+ (dump_profile): New.
+ (dump_gimple_label): Update.
+ (dump_gimple_bb_header): Update.
+ * graph.c (draw_cfg_node_succ_edges): Update.
+ * hsa-gen.c (convert_switch_statements): Update.
+ * ifcvt.c (cheap_bb_rtx_cost_p): Update.
+ (find_if_case_1): Update.
+ (find_if_case_2): Update.
+ * internal-fn.c (expand_arith_overflow_result_store): Update.
+ (expand_addsub_overflow): Update.
+ (expand_neg_overflow): Update.
+ (expand_mul_overflow): Update.
+ (expand_vector_ubsan_overflow): Update.
+ * ipa-cp.c (good_cloning_opportunity_p): Update.
+ * ipa-split.c (split_function): Use make_single_succ_edge.
+ * ipa-utils.c (ipa_merge_profiles): Update.
+ * loop-doloop.c (add_test): Update.
+ (doloop_modify): Update.
+ * loop-unroll.c (compare_and_jump_seq): Update.
+ (unroll_loop_runtime_iterations): Update.
+ * lra-constraints.c (lra_inheritance): Update.
+ * lto-streamer-in.c (input_cfg): Update.
+ * lto-streamer-out.c (output_cfg): Update.
+ * mcf.c (adjust_cfg_counts): Update.
+ * modulo-sched.c (sms_schedule): Update.
+ * omp-expand.c (expand_omp_for_init_counts): Update.
+ (extract_omp_for_update_vars): Update.
+ (expand_omp_ordered_sink): Update.
+ (expand_omp_for_ordered_loops): Update.
+ (expand_omp_for_generic): Update.
+ (expand_omp_for_static_nochunk): Update.
+ (expand_omp_for_static_chunk): Update.
+ (expand_cilk_for): Update.
+ (expand_omp_simd): Update.
+ (expand_omp_taskloop_for_outer): Update.
+ (expand_omp_taskloop_for_inner): Update.
+ * omp-simd-clone.c (simd_clone_adjust): Update.
+ * optabs.c (expand_doubleword_shift): Update.
+ (expand_abs): Update.
+ (emit_cmp_and_jump_insn_1): Update.
+ (expand_compare_and_swap_loop): Update.
+ * optabs.h (emit_cmp_and_jump_insns): Update prototype.
+ * predict.c (predictable_edge_p): Update.
+ (edge_probability_reliable_p): Update.
+ (set_even_probabilities): Update.
+ (combine_predictions_for_insn): Update.
+ (combine_predictions_for_bb): Update.
+ (propagate_freq): Update.
+ (estimate_bb_frequencies): Update.
+ (force_edge_cold): Update.
+ * profile-count.c (profile_count::dump): Add missing space into dump.
+ (profile_count::debug): Add newline.
+ (profile_count::differs_from_p): Explicitly convert to unsigned.
+ (profile_count::stream_in): Update.
+ (profile_probability::dump): New member function.
+ (profile_probability::debug): New member function.
+ (profile_probability::differs_from_p): New member function.
+ (profile_probability::differs_lot_from_p): New member function.
+ (profile_probability::stream_in): New member function.
+ (profile_probability::stream_out): New member function.
+ * profile-count.h (profile_count_quality): Rename to ...
+ (profile_quality): ... this one.
+ (profile_probability): New.
+ (profile_count): Update.
+ * profile.c (compute_branch_probabilities): Update.
+ * recog.c (peep2_attempt): Update.
+ * sched-ebb.c (schedule_ebbs): Update.
+ * sched-rgn.c (find_single_block_region): Update.
+ (compute_dom_prob_ps): Update.
+ (schedule_region): Update.
+ * sel-sched-ir.c (compute_succs_info): Update.
+ * stmt.c (struct case_node): Update.
+ (do_jump_if_equal): Update.
+ (get_outgoing_edge_probs): Update.
+ (conditional_probability): Update.
+ (emit_case_dispatch_table): Update.
+ (expand_case): Update.
+ (expand_sjlj_dispatch_table): Update.
+ (emit_case_nodes): Update.
+ * targhooks.c: Update.
+ * tracer.c (better_p): Update.
+ (find_best_successor): Update.
+ * trans-mem.c (expand_transaction): Update.
+ * tree-call-cdce.c: Update.
+ * tree-cfg.c (gimple_split_edge): Upate.
+ (move_sese_region_to_fn): Upate.
+ * tree-cfgcleanup.c (cleanup_control_expr_graph): Upate.
+ * tree-eh.c (lower_resx): Upate.
+ (cleanup_empty_eh_move_lp): Upate.
+ * tree-if-conv.c (version_loop_for_if_conversion): Update.
+ * tree-inline.c (copy_edges_for_bb): Update.
+ (copy_cfg_body): Update.
+ * tree-parloops.c (gen_parallel_loop): Update.
+ * tree-profile.c (gimple_gen_ic_func_profiler): Update.
+ (gimple_gen_time_profiler): Update.
+ * tree-ssa-dce.c (remove_dead_stmt): Update.
+ * tree-ssa-ifcombine.c (update_profile_after_ifcombine): Update.
+ * tree-ssa-loop-im.c (execute_sm_if_changed): Update.
+ * tree-ssa-loop-ivcanon.c (remove_exits_and_undefined_stmts): Update.
+ (unloop_loops): Update.
+ (try_peel_loop): Update.
+ * tree-ssa-loop-manip.c (tree_transform_and_unroll_loop): Update.
+ * tree-ssa-loop-split.c (connect_loops): Update.
+ (split_loop): Update.
+ * tree-ssa-loop-unswitch.c (tree_unswitch_loop): Update.
+ (hoist_guard): Update.
+ * tree-ssa-phionlycprop.c (propagate_rhs_into_lhs): Update.
+ * tree-ssa-phiopt.c (replace_phi_edge_with_variable): Update.
+ (value_replacement): Update.
+ * tree-ssa-reassoc.c (branch_fixup): Update.
+ * tree-ssa-tail-merge.c (replace_block_by): Update.
+ * tree-ssa-threadupdate.c (remove_ctrl_stmt_and_useless_edges): Update.
+ (create_edge_and_update_destination_phis): Update.
+ (compute_path_counts): Update.
+ (recompute_probabilities): Update.
+ (update_joiner_offpath_counts): Update.
+ (freqs_to_counts_path): Update.
+ (duplicate_thread_path): Update.
+ * tree-switch-conversion.c (hoist_edge_and_branch_if_true): Update.
+ (struct switch_conv_info): Update.
+ (gen_inbound_check): Update.
+ * tree-vect-loop-manip.c (slpeel_add_loop_guard): Update.
+ (vect_do_peeling): Update.
+ (vect_loop_versioning): Update.
+ * tree-vect-loop.c (scale_profile_for_vect_loop): Update.
+ (optimize_mask_stores): Update.
+ * ubsan.c (ubsan_expand_null_ifn): Update.
+ * value-prof.c (gimple_divmod_fixed_value): Update.
+ (gimple_divmod_fixed_value_transform): Update.
+ (gimple_mod_pow2): Update.
+ (gimple_mod_pow2_value_transform): Update.
+ (gimple_mod_subtract): Update.
+ (gimple_mod_subtract_transform): Update.
+ (gimple_ic): Update.
+ (gimple_stringop_fixed_value): Update.
+ (gimple_stringops_transform): Update.
+ * value-prof.h: Update.
+
2017-06-29 Carl Love <cel@us.ibm.com>
* config/rs6000/rs6000-c.c: Add support for built-in functions
emit_move_insn (orig_base, base);
ret = expand_normal (asan_detect_stack_use_after_return);
lab = gen_label_rtx ();
- int very_likely = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
- VOIDmode, 0, lab, very_likely);
+ VOIDmode, 0, lab,
+ profile_probability::very_likely ());
snprintf (buf, sizeof buf, "__asan_stack_malloc_%d",
use_after_return_class);
ret = init_one_libfunc (buf);
/* __asan_stack_malloc_[n] returns a pointer to fake stack if succeeded
and NULL otherwise. Check RET value is NULL here and jump over the
BASE reassignment in this case. Otherwise, reassign BASE to RET. */
- int very_unlikely = REG_BR_PROB_BASE / 2000 - 1;
emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
- VOIDmode, 0, lab, very_unlikely);
+ VOIDmode, 0, lab,
+ profile_probability:: very_unlikely ());
ret = convert_memory_address (Pmode, ret);
emit_move_insn (base, ret);
emit_label (lab);
{
rtx_code_label *lab2 = gen_label_rtx ();
char c = (char) ASAN_STACK_MAGIC_USE_AFTER_RET;
- int very_likely = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
emit_cmp_and_jump_insns (orig_base, base, EQ, NULL_RTX,
- VOIDmode, 0, lab2, very_likely);
+ VOIDmode, 0, lab2,
+ profile_probability::very_likely ());
shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
set_mem_alias_set (shadow_mem, asan_shadow_set);
mem = gen_rtx_MEM (ptr_mode, base);
= then_more_likely_p
? PROB_VERY_UNLIKELY
: PROB_ALWAYS - PROB_VERY_UNLIKELY;
- e->probability = PROB_ALWAYS - fallthrough_probability;
+ e->probability = profile_probability::from_reg_br_prob_base
+ (PROB_ALWAYS - fallthrough_probability);
if (create_then_fallthru_edge)
make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
e = find_edge (cond_bb, fallthru_bb);
e->flags = EDGE_FALSE_VALUE;
e->count = cond_bb->count;
- e->probability = fallthrough_probability;
+ e->probability
+ = profile_probability::from_reg_br_prob_base (fallthrough_probability);
/* Update dominance info for the newly created then_bb; note that
fallthru_bb's dominance info has already been updated by
continue;
total++;
only_one = ep;
- if (e->probability == 0 && !is_edge_annotated (ep, *annotated_edge))
+ if (!e->probability.initialized_p ()
+ && !is_edge_annotated (ep, *annotated_edge))
{
- ep->probability = 0;
+ ep->probability = profile_probability::never ();
ep->count = profile_count::zero ();
set_edge_annotated (ep, annotated_edge);
}
unsigned int dest_idx;
int flags; /* see cfg-flags.def */
- int probability; /* biased by REG_BR_PROB_BASE */
+ profile_probability probability;
profile_count count; /* Expected number of executions calculated
in profile.c */
};
? EDGE_SUCC ((bb), 1) : EDGE_SUCC ((bb), 0))
/* Return expected execution frequency of the edge E. */
-#define EDGE_FREQUENCY(e) RDIV ((e)->src->frequency * (e)->probability, \
- REG_BR_PROB_BASE)
+#define EDGE_FREQUENCY(e) e->probability.apply (e->src->frequency)
/* Compute a scale factor (or probability) suitable for scaling of
gcov_type values via apply_probability() and apply_scale(). */
int, bb_heap_t **, int);
static basic_block copy_bb (basic_block, edge, basic_block, int);
static long bb_to_key (basic_block);
-static bool better_edge_p (const_basic_block, const_edge, int, int, int, int,
- const_edge);
+static bool better_edge_p (const_basic_block, const_edge, profile_probability,
+ int, profile_probability, int, const_edge);
static bool connect_better_edge_p (const_edge, bool, int, const_edge,
struct trace *);
static void connect_traces (int, struct trace *);
do
{
- int prob, freq;
+ profile_probability prob;
+ int freq;
bool ends_in_call;
/* The probability and frequency of the best edge. */
- int best_prob = INT_MIN / 2;
+ profile_probability best_prob = profile_probability::uninitialized ();
int best_freq = INT_MIN / 2;
best_edge = NULL;
successor (i.e. it is unsuitable successor). When optimizing
for size, ignore the probability and frequency. */
if (!(e->flags & EDGE_CAN_FALLTHRU) || (e->flags & EDGE_COMPLEX)
- || ((prob < branch_th || EDGE_FREQUENCY (e) < exec_th
+ || !prob.initialized_p ()
+ || ((prob.to_reg_br_prob_base () < branch_th
+ || EDGE_FREQUENCY (e) < exec_th
|| e->count < count_th) && (!for_size)))
continue;
if (!(e->flags & EDGE_CAN_FALLTHRU)
|| (e->flags & EDGE_COMPLEX)
- || prob < branch_th || freq < exec_th
+ || !prob.initialized_p ()
+ || prob.to_reg_br_prob_base () < branch_th
+ || freq < exec_th
|| e->count < count_th)
{
/* When partitioning hot/cold basic blocks, make sure
BEST_PROB; similarly for frequency. */
static bool
-better_edge_p (const_basic_block bb, const_edge e, int prob, int freq,
- int best_prob, int best_freq, const_edge cur_best_edge)
+better_edge_p (const_basic_block bb, const_edge e, profile_probability prob,
+ int freq, profile_probability best_prob, int best_freq,
+ const_edge cur_best_edge)
{
bool is_better_edge;
/* The BEST_* values do not have to be best, but can be a bit smaller than
maximum values. */
- int diff_prob = best_prob / 10;
+ profile_probability diff_prob = best_prob.apply_scale (1, 10);
int diff_freq = best_freq / 10;
/* The smaller one is better to keep the original order. */
vec<edge, va_gc> *edges = walk_up ? bb->preds : bb->succs;
edge e;
edge_iterator ei;
- int highest_probability = 0;
+ profile_probability highest_probability
+ = profile_probability::uninitialized ();
int highest_freq = 0;
profile_count highest_count = profile_count::uninitialized ();
bool found = false;
/* The following loop will look for the hottest edge via
the edge count, if it is non-zero, then fallback to the edge
frequency and finally the edge probability. */
- if (e->count > highest_count)
+ if (!highest_count.initialized_p () || e->count > highest_count)
highest_count = e->count;
int edge_freq = EDGE_FREQUENCY (e);
if (edge_freq > highest_freq)
highest_freq = edge_freq;
- if (e->probability > highest_probability)
+ if (!highest_probability.initialized_p ()
+ || e->probability > highest_probability)
highest_probability = e->probability;
}
n_edges_for_fn (cfun)++;
e->count = profile_count::uninitialized ();
+ e->probability = profile_probability::uninitialized ();
e->src = src;
e->dest = dst;
e->flags = flags;
{
edge e = make_edge (src, dest, flags);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = src->count;
return e;
}
check_bb_profile (basic_block bb, FILE * file, int indent)
{
edge e;
- int sum = 0;
edge_iterator ei;
struct function *fun = DECL_STRUCT_FUNCTION (current_function_decl);
char *s_indent = (char *) alloca ((size_t) indent + 1);
if (bb != EXIT_BLOCK_PTR_FOR_FN (fun))
{
bool found = false;
+ profile_probability sum = profile_probability::never ();
+ int isum = 0;
+
FOR_EACH_EDGE (e, ei, bb->succs)
{
- if (!(e->flags & EDGE_EH))
+ if (!(e->flags & (EDGE_EH | EDGE_FAKE)))
found = true;
sum += e->probability;
+ if (e->probability.initialized_p ())
+ isum += e->probability.to_reg_br_prob_base ();
}
/* Only report mismatches for non-EH control flow. If there are only EH
edges it means that the BB ends by noreturn call. Here the control
flow may just terminate. */
if (found)
{
- if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100)
- fprintf (file,
- ";; %sInvalid sum of outgoing probabilities %.1f%%\n",
- s_indent, sum * 100.0 / REG_BR_PROB_BASE);
+ if (sum.differs_from_p (profile_probability::always ()))
+ {
+ fprintf (file,
+ ";; %sInvalid sum of outgoing probabilities ",
+ s_indent);
+ sum.dump (file);
+ fprintf (file, "\n");
+ }
+ /* Probabilities caps to 100% and thus the previous test will never
+ fire if the sum of probabilities is too large. */
+ else if (isum > REG_BR_PROB_BASE + 100)
+ {
+ fprintf (file,
+ ";; %sInvalid sum of outgoing probabilities %.1f%%\n",
+ s_indent, isum * 100.0 / REG_BR_PROB_BASE);
+ }
profile_count lsum = profile_count::zero ();
FOR_EACH_EDGE (e, ei, bb->succs)
lsum += e->count;
}
}
}
- if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun))
+ if (bb != ENTRY_BLOCK_PTR_FOR_FN (fun))
{
- sum = 0;
+ int sum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
sum += EDGE_FREQUENCY (e);
if (abs (sum - bb->frequency) > 100)
else
fprintf (file, " %d", side->index);
- if (e->probability && do_details)
- fprintf (file, " [%.1f%%] ", e->probability * 100.0 / REG_BR_PROB_BASE);
+ if (e->probability.initialized_p () && do_details)
+ {
+ fprintf (file, " [");
+ e->probability.dump (file);
+ fprintf (file, "] ");
+ }
if (e->count.initialized_p () && do_details)
{
profile_count count, edge taken_edge)
{
edge c;
- int prob;
+ profile_probability prob;
edge_iterator ei;
if (bb->count < count)
/* Compute the probability of TAKEN_EDGE being reached via threaded edge.
Watch for overflows. */
if (bb->frequency)
- prob = GCOV_COMPUTE_SCALE (edge_frequency, bb->frequency);
+ /* FIXME: We should get edge frequency as count. */
+ prob = profile_probability::probability_in_gcov_type
+ (edge_frequency, bb->frequency);
else
- prob = 0;
+ prob = profile_probability::never ();
if (prob > taken_edge->probability)
{
if (dump_file)
- fprintf (dump_file, "Jump threading proved probability of edge "
- "%i->%i too small (it is %i, should be %i).\n",
- taken_edge->src->index, taken_edge->dest->index,
- taken_edge->probability, prob);
- prob = taken_edge->probability * 6 / 8;
+ {
+ fprintf (dump_file, "Jump threading proved probability of edge "
+ "%i->%i too small (it is ",
+ taken_edge->src->index, taken_edge->dest->index);
+ taken_edge->probability.dump (dump_file);
+ fprintf (dump_file, " should be ");
+ prob.dump (dump_file);
+ fprintf (dump_file, ")\n");
+ }
+ prob = taken_edge->probability.apply_scale (6, 8);
}
/* Now rescale the probabilities. */
taken_edge->probability -= prob;
- prob = REG_BR_PROB_BASE - prob;
- if (prob <= 0)
+ prob = prob.invert ();
+ if (prob == profile_probability::never ())
{
if (dump_file)
fprintf (dump_file, "Edge frequencies of bb %i has been reset, "
"frequency of block should end up being 0, it is %i\n",
bb->index, bb->frequency);
- EDGE_SUCC (bb, 0)->probability = REG_BR_PROB_BASE;
+ EDGE_SUCC (bb, 0)->probability = profile_probability::guessed_always ();
ei = ei_start (bb->succs);
ei_next (&ei);
for (; (c = ei_safe_edge (ei)); ei_next (&ei))
- c->probability = 0;
+ c->probability = profile_probability::guessed_never ();
}
- else if (prob != REG_BR_PROB_BASE)
+ else if (!(prob == profile_probability::always ()))
{
- int scale = RDIV (65536 * REG_BR_PROB_BASE, prob);
-
FOR_EACH_EDGE (c, ei, bb->succs)
- {
- /* Protect from overflow due to additional scaling. */
- if (c->probability > prob)
- c->probability = REG_BR_PROB_BASE;
- else
- {
- c->probability = RDIV (c->probability * scale, 65536);
- if (c->probability > REG_BR_PROB_BASE)
- c->probability = REG_BR_PROB_BASE;
- }
- }
+ c->probability /= prob;
}
gcc_assert (bb == taken_edge->src);
- if (taken_edge->count < count)
- {
- if (dump_file)
- fprintf (dump_file, "edge %i->%i count became negative after threading",
- taken_edge->src->index, taken_edge->dest->index);
- }
+ if (dump_file && taken_edge->count < count)
+ fprintf (dump_file, "edge %i->%i count became negative after threading",
+ taken_edge->src->index, taken_edge->dest->index);
taken_edge->count -= count;
}
break;
basic_block deadend_block = dfs_find_deadend (unvisited_block);
- make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FAKE);
+ edge e = make_edge (deadend_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
+ EDGE_FAKE);
+ e->count = profile_count::zero ();
+ e->probability = profile_probability::never ();
dfs.add_bb (deadend_block);
}
}
{
probability = XINT (note, 0);
e = BRANCH_EDGE (b);
- e->probability = probability;
+ e->probability
+ = profile_probability::from_reg_br_prob_base (probability);
e->count = b->count.apply_probability (probability);
f = FALLTHRU_EDGE (b);
- f->probability = REG_BR_PROB_BASE - probability;
+ f->probability
+ = profile_probability::from_reg_br_prob_base (REG_BR_PROB_BASE
+ - probability);
f->count = b->count - e->count;
return;
}
else if (single_succ_p (b))
{
e = single_succ_edge (b);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = b->count;
return;
}
}
else
uninitialized_src = true;
- bb->frequency += EDGE_FREQUENCY (e);
+ if (e->probability.initialized_p ())
+ bb->frequency += EDGE_FREQUENCY (e);
}
/* When some edges are missing with read profile, this is
most likely because RTL expansion introduced loop.
{
/* Save the values now, as the edge may get removed. */
profile_count edge_count = e->count;
- int edge_probability = e->probability;
+ profile_probability edge_probability = e->probability;
int edge_frequency;
int n = 0;
/* We successfully forwarded the edge. Now update profile
data: for each edge we traversed in the chain, remove
the original edge's execution count. */
- edge_frequency = apply_probability (b->frequency, edge_probability);
+ edge_frequency = edge_probability.apply (b->frequency);
do
{
&& optimize_bb_for_speed_p (bb1)
&& optimize_bb_for_speed_p (bb2))
{
- int prob2;
+ profile_probability prob2;
if (b1->dest == b2->dest)
prob2 = b2->probability;
else
/* Do not use f2 probability as f2 may be forwarded. */
- prob2 = REG_BR_PROB_BASE - b2->probability;
+ prob2 = b2->probability.invert ();
/* Fail if the difference in probabilities is greater than 50%.
This rules out two well-predicted branches with opposite
outcomes. */
- if (abs (b1->probability - prob2) > REG_BR_PROB_BASE / 2)
+ if (b1->probability.differs_lot_from_p (prob2))
{
if (dump_file)
- fprintf (dump_file,
- "Outcomes of branch in bb %i and %i differ too much (%i %i)\n",
- bb1->index, bb2->index, b1->probability, prob2);
-
+ {
+ fprintf (dump_file,
+ "Outcomes of branch in bb %i and %i differ too"
+ " much (", bb1->index, bb2->index);
+ b1->probability.dump (dump_file);
+ prob2.dump (dump_file);
+ fprintf (dump_file, ")\n");
+ }
return false;
}
}
}
if (!redirect_edges_to->frequency && !src1->frequency)
- s->probability = (s->probability + s2->probability) / 2;
- else
- s->probability
- = ((s->probability * redirect_edges_to->frequency +
- s2->probability * src1->frequency)
- / (redirect_edges_to->frequency + src1->frequency));
+ s->probability = s->probability.combine_with_freq
+ (redirect_edges_to->frequency,
+ s2->probability, src1->frequency);
}
/* Adjust count and frequency for the block. An earlier jump
expand_gimple_cond (basic_block bb, gcond *stmt)
{
basic_block new_bb, dest;
- edge new_edge;
edge true_edge;
edge false_edge;
rtx_insn *last2, *last;
if (loop->latch == bb
&& loop->header == dest)
loop->latch = new_bb;
- new_edge = make_edge (new_bb, dest, 0);
- new_edge->probability = REG_BR_PROB_BASE;
- new_edge->count = new_bb->count;
+ make_single_succ_edge (new_bb, dest, 0);
if (BARRIER_P (BB_END (new_bb)))
BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
update_bb_for_insn (new_bb);
rtx_insn *last2, *last;
edge e;
edge_iterator ei;
- int probability;
+ profile_probability probability;
last2 = last = expand_gimple_stmt (stmt);
all edges here, or redirecting the existing fallthru edge to
the exit block. */
- probability = 0;
+ probability = profile_probability::never ();
profile_count count = profile_count::zero ();
for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
{
first_block = e->dest;
redirect_edge_succ (e, init_block);
- e = make_edge (init_block, first_block, flags);
+ e = make_single_succ_edge (init_block, first_block, flags);
}
else
- e = make_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FALLTHRU);
- e->probability = REG_BR_PROB_BASE;
- e->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
+ e = make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
+ EDGE_FALLTHRU);
update_bb_for_insn (init_block);
return init_block;
ix++;
}
- e = make_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_FALLTHRU);
- e->probability = REG_BR_PROB_BASE;
- e->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
+ e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
+ EDGE_FALLTHRU);
FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
if (e2 != e)
{
e->src->index, e->dest->index);
err = 1;
}
- if (e->probability < 0 || e->probability > REG_BR_PROB_BASE)
+ if (!e->probability.verify ())
{
- error ("verify_flow_info: Wrong probability of edge %i->%i %i",
- e->src->index, e->dest->index, e->probability);
+ error ("verify_flow_info: Wrong probability of edge %i->%i",
+ e->src->index, e->dest->index);
err = 1;
}
if (!e->count.verify ())
{
s->flags |= e->flags;
s->probability += e->probability;
- if (s->probability > REG_BR_PROB_BASE)
- s->probability = REG_BR_PROB_BASE;
s->count += e->count;
/* FIXME: This should be called via a hook and only for IR_GIMPLE. */
redirect_edge_var_map_dup (s, e);
ret = cfg_hooks->split_edge (e);
ret->count = count;
ret->frequency = freq;
- single_succ_edge (ret)->probability = REG_BR_PROB_BASE;
+ single_succ_edge (ret)->probability = profile_probability::always ();
single_succ_edge (ret)->count = count;
if (irr)
basic_block bb;
edge_iterator ei;
edge e;
- int sum;
FOR_ALL_BB_FN (bb, cfun)
{
if (bb != EXIT_BLOCK_PTR_FOR_FN (cfun)
&& profile_status_for_fn (cfun) != PROFILE_ABSENT)
{
- sum = 0;
+ profile_probability sum = profile_probability::never ();
FOR_EACH_EDGE (e, ei, bb->succs)
sum += e->probability;
- if (EDGE_COUNT (bb->succs) && abs (sum - REG_BR_PROB_BASE) > 100)
+ if (EDGE_COUNT (bb->succs)
+ && sum.differs_from_p (profile_probability::always ()))
record->num_mismatched_freq_out[after_pass]++;
profile_count lsum = profile_count::zero ();
FOR_EACH_EDGE (e, ei, bb->succs)
if (bb != ENTRY_BLOCK_PTR_FOR_FN (cfun)
&& profile_status_for_fn (cfun) != PROFILE_ABSENT)
{
- sum = 0;
+ int sum = 0;
FOR_EACH_EDGE (e, ei, bb->preds)
sum += EDGE_FREQUENCY (e);
if (abs (sum - bb->frequency) > 100
reasons.
FIXME: Turn to likely_never_executed */
if ((profile_status_for_fn (cfun) != PROFILE_ABSENT
- && ex->probability < 5)
+ && ex->probability < profile_probability::from_reg_br_prob_base (5))
|| ex->count == profile_count::zero ())
continue;
if (!found)
/* Probability of exit must be 1/iterations. */
freq_delta = EDGE_FREQUENCY (e);
- e->probability = REG_BR_PROB_BASE / iteration_bound;
- other_e->probability = inverse_probability (e->probability);
+ e->probability = profile_probability::from_reg_br_prob_base
+ (REG_BR_PROB_BASE / iteration_bound);
+ other_e->probability = e->probability.invert ();
freq_delta -= EDGE_FREQUENCY (e);
/* Adjust counts accordingly. */
return ret;
}
-/* Sets probability and count of edge E to zero. The probability and count
- is redistributed evenly to the remaining edges coming from E->src. */
-
-static void
-set_zero_probability (edge e)
-{
- basic_block bb = e->src;
- edge_iterator ei;
- edge ae, last = NULL;
- unsigned n = EDGE_COUNT (bb->succs);
- profile_count cnt = e->count, cnt1;
- unsigned prob = e->probability, prob1;
-
- gcc_assert (n > 1);
- cnt1 = cnt.apply_scale (1, (n - 1));
- prob1 = prob / (n - 1);
-
- FOR_EACH_EDGE (ae, ei, bb->succs)
- {
- if (ae == e)
- continue;
-
- ae->probability += prob1;
- ae->count += cnt1;
- last = ae;
- }
-
- /* Move the rest to one of the edges. */
- last->probability += prob % (n - 1);
- /* TODO: Remove once we have fractional counts. */
- if (cnt.initialized_p ())
- last->count += profile_count::from_gcov_type (cnt.to_gcov_type () % (n - 1));
-
- e->probability = 0;
- e->count = profile_count::zero ();
-}
-
/* Duplicates body of LOOP to given edge E NDUPL times. Takes care of updating
loop structure and dominators. E's destination must be LOOP header for
this to work, i.e. it must be entry or latch edge of this loop; these are
prob_pass_wont_exit =
RDIV (REG_BR_PROB_BASE * (freq_le + freq_out_orig), freq_in);
- if (orig
- && REG_BR_PROB_BASE - orig->probability != 0)
+ if (orig && orig->probability.initialized_p ()
+ && !(orig->probability == profile_probability::always ()))
{
/* The blocks that are dominated by a removed exit edge ORIG have
frequencies scaled by this. */
- scale_after_exit
- = GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE,
- REG_BR_PROB_BASE - orig->probability);
+ if (orig->probability.initialized_p ())
+ scale_after_exit
+ = GCOV_COMPUTE_SCALE (REG_BR_PROB_BASE,
+ REG_BR_PROB_BASE
+ - orig->probability.to_reg_br_prob_base ());
+ else
+ scale_after_exit = REG_BR_PROB_BASE;
bbs_to_scale = BITMAP_ALLOC (NULL);
for (i = 0; i < n; i++)
{
{
if (to_remove)
to_remove->safe_push (new_spec_edges[SE_ORIG]);
- set_zero_probability (new_spec_edges[SE_ORIG]);
+ force_edge_cold (new_spec_edges[SE_ORIG], true);
/* Scale the frequencies of the blocks dominated by the exit. */
if (bbs_to_scale)
{
if (to_remove)
to_remove->safe_push (orig);
- set_zero_probability (orig);
+ force_edge_cold (orig, true);
/* Scale the frequencies of the blocks dominated by the exit. */
if (bbs_to_scale)
static basic_block
lv_adjust_loop_entry_edge (basic_block first_head, basic_block second_head,
- edge e, void *cond_expr, unsigned then_prob,
- unsigned else_prob)
+ edge e, void *cond_expr,
+ profile_probability then_prob,
+ profile_probability else_prob)
{
basic_block new_head = NULL;
edge e1;
struct loop *
loop_version (struct loop *loop,
void *cond_expr, basic_block *condition_bb,
- unsigned then_prob, unsigned else_prob,
+ profile_probability then_prob, profile_probability else_prob,
unsigned then_scale, unsigned else_scale,
bool place_after)
{
extern void create_preheaders (int);
extern void force_single_succ_latches (void);
struct loop * loop_version (struct loop *, void *,
- basic_block *, unsigned, unsigned,
+ basic_block *,
+ profile_probability, profile_probability,
unsigned, unsigned, bool);
#endif /* GCC_CFGLOOPMANIP_H */
else
e->flags = 0;
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = src->count;
if (e->dest != target)
{
int prob = XINT (note, 0);
- b->probability = prob;
+ b->probability = profile_probability::from_reg_br_prob_base (prob);
b->count = e->count.apply_probability (prob);
e->probability -= e->probability;
e->count -= b->count;
- if (e->probability < 0)
- e->probability = 0;
}
}
{
rtx_insn *new_head;
profile_count count = e->count;
- int probability = e->probability;
+ profile_probability probability = e->probability;
/* Create the new structures. */
/* If the old block ended with a tablejump, skip its table
/* Redirect old edge. */
redirect_edge_pred (e, jump_block);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
/* If e->src was previously region crossing, it no longer is
and the reg crossing note should be removed. */
add also edge from asm goto bb to target. */
if (asm_goto_edge)
{
- new_edge->probability /= 2;
+ new_edge->probability = new_edge->probability.apply_scale (1, 2);
new_edge->count = new_edge->count.apply_scale (1, 2);
jump_block->count = jump_block->count.apply_scale (1, 2);
jump_block->frequency /= 2;
update_br_prob_note (basic_block bb)
{
rtx note;
- if (!JUMP_P (BB_END (bb)))
+ if (!JUMP_P (BB_END (bb)) || !BRANCH_EDGE (bb)->probability.initialized_p ())
return;
note = find_reg_note (BB_END (bb), REG_BR_PROB, NULL_RTX);
- if (!note || XINT (note, 0) == BRANCH_EDGE (bb)->probability)
+ if (!note
+ || XINT (note, 0) == BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ())
return;
- XINT (note, 0) = BRANCH_EDGE (bb)->probability;
+ XINT (note, 0) = BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ();
}
/* Get the last insn associated with block BB (that includes barriers and
&& EDGE_COUNT (bb->succs) >= 2
&& any_condjump_p (BB_END (bb)))
{
- if (XINT (note, 0) != BRANCH_EDGE (bb)->probability
- && profile_status_for_fn (cfun) != PROFILE_ABSENT)
+ if (!BRANCH_EDGE (bb)->probability.initialized_p ())
+ {
+ /* FIXME: sometimes we create BBs with only branch edge
+ probability defined. */
+ if (0)
+ {
+ error ("verify_flow_info: "
+ "REG_BR_PROB is set but cfg probability is not");
+ err = 1;
+ }
+ }
+ else if (XINT (note, 0)
+ != BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ()
+ && profile_status_for_fn (cfun) != PROFILE_ABSENT)
{
error ("verify_flow_info: REG_BR_PROB does not match cfg %i %i",
- XINT (note, 0), BRANCH_EDGE (bb)->probability);
+ XINT (note, 0),
+ BRANCH_EDGE (bb)->probability.to_reg_br_prob_base ());
err = 1;
}
}
/* Redistribute probabilities. */
if (single_succ_p (bb))
{
- single_succ_edge (bb)->probability = REG_BR_PROB_BASE;
+ single_succ_edge (bb)->probability = profile_probability::always ();
single_succ_edge (bb)->count = bb->count;
}
else
b = BRANCH_EDGE (bb);
f = FALLTHRU_EDGE (bb);
- b->probability = XINT (note, 0);
- f->probability = REG_BR_PROB_BASE - b->probability;
+ b->probability = profile_probability::from_reg_br_prob_base
+ (XINT (note, 0));
+ f->probability = profile_probability::always () - b->probability;
b->count = bb->count.apply_probability (b->probability);
f->count = bb->count.apply_probability (f->probability);
}
gcc_assert (single_succ_p (bb));
- single_succ_edge (bb)->probability = REG_BR_PROB_BASE;
+ single_succ_edge (bb)->probability = profile_probability::always ();
single_succ_edge (bb)->count = bb->count;
if (dump_file)
start_sequence ();
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
- do_compare_rtx_and_jump (op0, op1, comp, 0, mode, NULL_RTX, NULL, label, -1);
+ do_compare_rtx_and_jump (op0, op1, comp, 0, mode, NULL_RTX, NULL, label,
+ profile_probability::uninitialized ());
jump = get_last_insn ();
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
/* This file contains basic routines manipulating call graph
- The call-graph is a data structure designed for intra-procedural optimization.
- It represents a multi-graph where nodes are functions and edges are call sites. */
+ The call-graph is a data structure designed for inter-procedural
+ optimization. It represents a multi-graph where nodes are functions
+ (symbols within symbol table) and edges are call sites. */
#include "config.h"
#include "system.h"
push_cfun (DECL_STRUCT_FUNCTION (e->caller->decl));
new_stmt = gimple_ic (e->call_stmt,
dyn_cast<cgraph_node *> (ref->referred),
+ /* FIXME: cleanup. */
+ profile_probability::from_reg_br_prob_base (
e->count > profile_count::zero ()
|| e2->count > profile_count::zero ()
- ? e->count.probability_in (e->count + e2->count)
+ ? e->count.probability_in
+ (e->count + e2->count).to_reg_br_prob_base ()
: e->frequency || e2->frequency
? RDIV (e->frequency * REG_BR_PROB_BASE,
e->frequency + e2->frequency)
- : REG_BR_PROB_BASE / 2,
+ : REG_BR_PROB_BASE / 2),
e->count, e->count + e2->count);
e->speculative = false;
e->caller->set_call_stmt_including_clones (e->call_stmt, new_stmt,
/* Create BB for body of the function and connect it properly. */
ENTRY_BLOCK_PTR_FOR_FN (cfun)->count = count;
- ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = REG_BR_PROB_BASE;
+ ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX;
EXIT_BLOCK_PTR_FOR_FN (cfun)->count = count;
- EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency = REG_BR_PROB_BASE;
+ EXIT_BLOCK_PTR_FOR_FN (cfun)->frequency = BB_FREQ_MAX;
bb = create_basic_block (NULL, ENTRY_BLOCK_PTR_FOR_FN (cfun));
bb->count = count;
bb->frequency = BB_FREQ_MAX;
e = make_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun), bb, EDGE_FALLTHRU);
e->count = count;
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
e->count = count;
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
add_bb_to_loop (bb, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
return bb;
NULL_TREE, NULL_TREE);
gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
e = make_edge (bb, then_bb, EDGE_TRUE_VALUE);
- e->probability = REG_BR_PROB_BASE - REG_BR_PROB_BASE / 16;
+ e->probability = profile_probability::guessed_always ()
+ .apply_scale (1, 16);
e->count = count - count.apply_scale (1, 16);
e = make_edge (bb, else_bb, EDGE_FALSE_VALUE);
- e->probability = REG_BR_PROB_BASE / 16;
+ e->probability = profile_probability::guessed_always ()
+ .apply_scale (1, 16);
e->count = count.apply_scale (1, 16);
- e = make_edge (return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->probability = REG_BR_PROB_BASE;
- e->count = count;
- e = make_edge (then_bb, return_bb, EDGE_FALLTHRU);
- e->probability = REG_BR_PROB_BASE;
- e->count = count - count.apply_scale (1, 16);
+ make_single_succ_edge (return_bb,
+ EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
+ make_single_succ_edge (then_bb, return_bb, EDGE_FALLTHRU);
e = make_edge (else_bb, return_bb, EDGE_FALLTHRU);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = count.apply_scale (1, 16);
bsi = gsi_last_bb (then_bb);
}
#include "stor-layout.h"
#include "langhooks.h"
#include "explow.h"
+#include "profile-count.h"
#include "expr.h"
#include "tree-iterator.h"
#include "gimplify.h"
paging and cache locality performance.
This optimization is automatically turned off in the presence of
-exception handling, for linkonce sections, for functions with a user-defined
+exception handling or unwind tables (on targets using setjump/longjump or target specific scheme), for linkonce sections, for functions with a user-defined
section attribute and on any architecture that does not support named
sections. When @option{-fsplit-stack} is used this option is not
enabled by default (to avoid linker errors), but may be enabled
static bool prefer_and_bit_test (machine_mode, int);
static void do_jump_by_parts_greater (tree, tree, int,
- rtx_code_label *, rtx_code_label *, int);
+ rtx_code_label *, rtx_code_label *,
+ profile_probability);
static void do_jump_by_parts_equality (tree, tree, rtx_code_label *,
- rtx_code_label *, int);
+ rtx_code_label *, profile_probability);
static void do_compare_and_jump (tree, tree, enum rtx_code, enum rtx_code,
- rtx_code_label *, rtx_code_label *, int);
-
-/* Invert probability if there is any. -1 stands for unknown. */
-
-static inline int
-inv (int prob)
-{
- return prob == -1 ? -1 : REG_BR_PROB_BASE - prob;
-}
+ rtx_code_label *, rtx_code_label *,
+ profile_probability);
/* At the start of a function, record that we have no previously-pushed
arguments waiting to be popped. */
/* Generate code to evaluate EXP and jump to LABEL if the value is zero. */
void
-jumpifnot (tree exp, rtx_code_label *label, int prob)
+jumpifnot (tree exp, rtx_code_label *label, profile_probability prob)
{
- do_jump (exp, label, NULL, inv (prob));
+ do_jump (exp, label, NULL, prob.invert ());
}
void
jumpifnot_1 (enum tree_code code, tree op0, tree op1, rtx_code_label *label,
- int prob)
+ profile_probability prob)
{
- do_jump_1 (code, op0, op1, label, NULL, inv (prob));
+ do_jump_1 (code, op0, op1, label, NULL, prob.invert ());
}
/* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */
void
-jumpif (tree exp, rtx_code_label *label, int prob)
+jumpif (tree exp, rtx_code_label *label, profile_probability prob)
{
do_jump (exp, NULL, label, prob);
}
void
jumpif_1 (enum tree_code code, tree op0, tree op1,
- rtx_code_label *label, int prob)
+ rtx_code_label *label, profile_probability prob)
{
do_jump_1 (code, op0, op1, NULL, label, prob);
}
/* Subroutine of do_jump, dealing with exploded comparisons of the type
OP0 CODE OP1 . IF_FALSE_LABEL and IF_TRUE_LABEL like in do_jump.
- PROB is probability of jump to if_true_label, or -1 if unknown. */
+ PROB is probability of jump to if_true_label. */
void
do_jump_1 (enum tree_code code, tree op0, tree op1,
rtx_code_label *if_false_label, rtx_code_label *if_true_label,
- int prob)
+ profile_probability prob)
{
machine_mode mode;
rtx_code_label *drop_through_label = 0;
!= MODE_COMPLEX_INT);
if (integer_zerop (op1))
- do_jump (op0, if_true_label, if_false_label, inv (prob));
+ do_jump (op0, if_true_label, if_false_label,
+ prob.invert ());
else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
&& !can_compare_p (EQ, TYPE_MODE (inner_type), ccp_jump))
do_jump_by_parts_equality (op0, op1, if_false_label, if_true_label,
else if (GET_MODE_CLASS (TYPE_MODE (inner_type)) == MODE_INT
&& !can_compare_p (NE, TYPE_MODE (inner_type), ccp_jump))
do_jump_by_parts_equality (op0, op1, if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
else
do_compare_and_jump (op0, op1, NE, NE, if_false_label, if_true_label,
prob);
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (LE, mode, ccp_jump))
do_jump_by_parts_greater (op0, op1, 0, if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
else
do_compare_and_jump (op0, op1, LE, LEU, if_false_label, if_true_label,
prob);
if (GET_MODE_CLASS (mode) == MODE_INT
&& ! can_compare_p (GE, mode, ccp_jump))
do_jump_by_parts_greater (op0, op1, 1, if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
else
do_compare_and_jump (op0, op1, GE, GEU, if_false_label, if_true_label,
prob);
half of the total probability of being false, so its jump has a false
probability of half the total, relative to the probability we
reached it (i.e. the first condition was true). */
- int op0_prob = -1;
- int op1_prob = -1;
- if (prob != -1)
+ profile_probability op0_prob = profile_probability::uninitialized ();
+ profile_probability op1_prob = profile_probability::uninitialized ();
+ if (prob.initialized_p ())
{
- int false_prob = inv (prob);
- int op0_false_prob = false_prob / 2;
- int op1_false_prob = GCOV_COMPUTE_SCALE ((false_prob / 2),
- inv (op0_false_prob));
+ profile_probability false_prob = prob.invert ();
+ profile_probability op0_false_prob = false_prob.apply_scale (1, 2);
+ profile_probability op1_false_prob = false_prob.apply_scale (1, 2)
+ / op0_false_prob.invert ();
/* Get the probability that each jump below is true. */
- op0_prob = inv (op0_false_prob);
- op1_prob = inv (op1_false_prob);
+ op0_prob = op0_false_prob.invert ();
+ op1_prob = op1_false_prob.invert ();
}
if (if_false_label == NULL)
{
The second condition has the other half of the total probability,
so its jump has a probability of half the total, relative to
the probability we reached it (i.e. the first condition was false). */
- int op0_prob = -1;
- int op1_prob = -1;
- if (prob != -1)
+ profile_probability op0_prob = profile_probability::uninitialized ();
+ profile_probability op1_prob = profile_probability::uninitialized ();
+ if (prob.initialized_p ())
{
- op0_prob = prob / 2;
- op1_prob = GCOV_COMPUTE_SCALE ((prob / 2), inv (op0_prob));
+ op0_prob = prob.apply_scale (1, 2);
+ op1_prob = prob.apply_scale (1, 2) / op0_prob.invert ();
}
if (if_true_label == NULL)
{
actually perform a jump. An example where there is no jump
is when EXP is `(foo (), 0)' and IF_FALSE_LABEL is null.
- PROB is probability of jump to if_true_label, or -1 if unknown. */
+ PROB is probability of jump to if_true_label. */
void
do_jump (tree exp, rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label, profile_probability prob)
{
enum tree_code code = TREE_CODE (exp);
rtx temp;
case TRUTH_NOT_EXPR:
do_jump (TREE_OPERAND (exp, 0), if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
break;
case COND_EXPR:
}
do_pending_stack_adjust ();
- do_jump (TREE_OPERAND (exp, 0), label1, NULL, -1);
+ do_jump (TREE_OPERAND (exp, 0), label1, NULL,
+ profile_probability::uninitialized ());
do_jump (TREE_OPERAND (exp, 1), if_false_label, if_true_label, prob);
emit_label (label1);
do_jump (TREE_OPERAND (exp, 2), if_false_label, if_true_label, prob);
{
tree exp0 = TREE_OPERAND (exp, 0);
rtx_code_label *set_label, *clr_label;
- int setclr_prob = prob;
+ profile_probability setclr_prob = prob;
/* Strip narrowing integral type conversions. */
while (CONVERT_EXPR_P (exp0)
exp0 = TREE_OPERAND (exp0, 0);
clr_label = if_true_label;
set_label = if_false_label;
- setclr_prob = inv (prob);
+ setclr_prob = prob.invert ();
}
else
{
do_jump_by_parts_greater_rtx (machine_mode mode, int unsignedp, rtx op0,
rtx op1, rtx_code_label *if_false_label,
rtx_code_label *if_true_label,
- int prob)
+ profile_probability prob)
{
int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
rtx_code_label *drop_through_label = 0;
if_false_label = drop_through_label;
drop_through_if_true = false;
drop_through_if_false = true;
- prob = inv (prob);
+ prob = prob.invert ();
}
/* Compare a word at a time, high order first. */
/* Consider lower words only if these are equal. */
do_compare_rtx_and_jump (op0_word, op1_word, NE, unsignedp, word_mode,
- NULL_RTX, NULL, if_false_label, inv (prob));
+ NULL_RTX, NULL, if_false_label,
+ prob.invert ());
}
if (!drop_through_if_false)
static void
do_jump_by_parts_greater (tree treeop0, tree treeop1, int swap,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label,
+ profile_probability prob)
{
rtx op0 = expand_normal (swap ? treeop1 : treeop0);
rtx op1 = expand_normal (swap ? treeop0 : treeop1);
static void
do_jump_by_parts_zero_rtx (machine_mode mode, rtx op0,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label,
+ profile_probability prob)
{
int nwords = GET_MODE_SIZE (mode) / UNITS_PER_WORD;
rtx part;
static void
do_jump_by_parts_equality_rtx (machine_mode mode, rtx op0, rtx op1,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label,
+ profile_probability prob)
{
int nwords = (GET_MODE_SIZE (mode) / UNITS_PER_WORD);
rtx_code_label *drop_through_label = NULL;
static void
do_jump_by_parts_equality (tree treeop0, tree treeop1,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label,
+ profile_probability prob)
{
rtx op0 = expand_normal (treeop0);
rtx op1 = expand_normal (treeop1);
do_compare_rtx_and_jump (rtx op0, rtx op1, enum rtx_code code, int unsignedp,
machine_mode mode, rtx size,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label,
+ profile_probability prob)
{
rtx tem;
rtx_code_label *dummy_label = NULL;
{
std::swap (if_true_label, if_false_label);
code = rcode;
- prob = inv (prob);
+ prob = prob.invert ();
}
}
case LEU:
do_jump_by_parts_greater_rtx (mode, 1, op0, op1,
if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
break;
case GTU:
case GEU:
do_jump_by_parts_greater_rtx (mode, 1, op1, op0,
if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
break;
case LT:
case LE:
do_jump_by_parts_greater_rtx (mode, 0, op0, op1,
if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
break;
case GT:
case GE:
do_jump_by_parts_greater_rtx (mode, 0, op1, op0,
if_true_label, if_false_label,
- inv (prob));
+ prob.invert ());
break;
case EQ:
case NE:
do_jump_by_parts_equality_rtx (mode, op0, op1, if_true_label,
- if_false_label, inv (prob));
+ if_false_label,
+ prob.invert ());
break;
default:
else
{
- int first_prob = prob;
+ profile_probability first_prob = prob;
if (first_code == UNORDERED)
- first_prob = REG_BR_PROB_BASE / 100;
+ first_prob = profile_probability::guessed_always ().apply_scale
+ (1, 100);
else if (first_code == ORDERED)
- first_prob = REG_BR_PROB_BASE - REG_BR_PROB_BASE / 100;
+ first_prob = profile_probability::guessed_always ().apply_scale
+ (99, 100);
if (and_them)
{
rtx_code_label *dest_label;
do_compare_and_jump (tree treeop0, tree treeop1, enum rtx_code signed_code,
enum rtx_code unsigned_code,
rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob)
+ rtx_code_label *if_true_label, profile_probability prob)
{
rtx op0, op1;
tree type;
extern void restore_pending_stack_adjust (saved_pending_stack_adjust *);
/* Generate code to evaluate EXP and jump to LABEL if the value is zero. */
-extern void jumpifnot (tree exp, rtx_code_label *label, int prob);
-extern void jumpifnot_1 (enum tree_code, tree, tree, rtx_code_label *, int);
+extern void jumpifnot (tree exp, rtx_code_label *label,
+ profile_probability prob);
+extern void jumpifnot_1 (enum tree_code, tree, tree, rtx_code_label *,
+ profile_probability);
/* Generate code to evaluate EXP and jump to LABEL if the value is nonzero. */
-extern void jumpif (tree exp, rtx_code_label *label, int prob);
-extern void jumpif_1 (enum tree_code, tree, tree, rtx_code_label *, int);
+extern void jumpif (tree exp, rtx_code_label *label, profile_probability prob);
+extern void jumpif_1 (enum tree_code, tree, tree, rtx_code_label *,
+ profile_probability);
/* Generate code to evaluate EXP and jump to IF_FALSE_LABEL if
the result is zero, or IF_TRUE_LABEL if the result is one. */
extern void do_jump (tree exp, rtx_code_label *if_false_label,
- rtx_code_label *if_true_label, int prob);
+ rtx_code_label *if_true_label, profile_probability prob);
extern void do_jump_1 (enum tree_code, tree, tree, rtx_code_label *,
- rtx_code_label *, int);
+ rtx_code_label *, profile_probability);
extern void do_compare_rtx_and_jump (rtx, rtx, enum rtx_code, int,
machine_mode, rtx, rtx_code_label *,
- rtx_code_label *, int);
+ rtx_code_label *, profile_probability);
extern bool split_comparison (enum rtx_code, machine_mode,
enum rtx_code *, enum rtx_code *);
#include "common/common-target.h"
#include "except.h" /* expand_builtin_dwarf_sp_column */
+#include "profile-count.h" /* For expr.h */
#include "expr.h" /* init_return_column_size */
#include "output.h" /* asm_out_file */
#include "debug.h" /* dwarf2out_do_frame, dwarf2out_do_cfi_asm */
{
basic_block bb;
rtx_insn *seq;
- edge e;
if (lp == NULL || lp->post_landing_pad == NULL)
continue;
end_sequence ();
bb = emit_to_new_bb_before (seq, label_rtx (lp->post_landing_pad));
- e = make_edge (bb, bb->next_bb, e_flags);
- e->count = bb->count;
- e->probability = REG_BR_PROB_BASE;
+ make_single_succ_edge (bb, bb->next_bb, e_flags);
if (current_loops)
{
struct loop *loop = bb->next_bb->loop_father;
rtx_insn *seq;
basic_block bb;
eh_region r;
- edge e;
int i, disp_index;
vec<tree> dispatch_labels = vNULL;
rtx_insn *before = label_rtx (lp->post_landing_pad);
bb = emit_to_new_bb_before (seq2, before);
- e = make_edge (bb, bb->next_bb, EDGE_FALLTHRU);
- e->count = bb->count;
- e->probability = REG_BR_PROB_BASE;
+ make_single_succ_edge (bb, bb->next_bb, EDGE_FALLTHRU);
if (current_loops)
{
struct loop *loop = bb->next_bb->loop_father;
bb = emit_to_new_bb_before (seq, first_reachable_label);
if (num_dispatch == 1)
{
- e = make_edge (bb, bb->next_bb, EDGE_FALLTHRU);
- e->count = bb->count;
- e->probability = REG_BR_PROB_BASE;
+ make_single_succ_edge (bb, bb->next_bb, EDGE_FALLTHRU);
if (current_loops)
{
struct loop *loop = bb->next_bb->loop_father;
#include "memmodel.h"
#include "tm_p.h"
#include "expmed.h"
+#include "profile-count.h"
#include "optabs.h"
#include "emit-rtl.h"
#include "recog.h"
{
label = gen_label_rtx ();
do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
- NULL_RTX, NULL, label, -1);
+ NULL_RTX, NULL, label,
+ profile_probability::uninitialized ());
emit_move_insn (target, trueval);
emit_label (label);
return target;
emit_move_insn (target, trueval);
label = gen_label_rtx ();
do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
- label, -1);
+ label, profile_probability::uninitialized ());
emit_move_insn (target, falseval);
emit_label (label);
{
int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
- NULL, label, -1);
+ NULL, label, profile_probability::uninitialized ());
}
#ifdef PUSH_ROUNDING
static void emit_single_push_insn (machine_mode, rtx, tree);
#endif
-static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx, int);
+static void do_tablejump (rtx, machine_mode, rtx, rtx, rtx,
+ profile_probability);
static rtx const_vector_from_tree (tree);
static rtx const_scalar_mask_from_tree (tree);
static tree tree_expr_size (const_tree);
m_accumulator = NULL_RTX;
}
do_compare_rtx_and_jump (op0, op1, NE, true, mode, NULL_RTX, NULL,
- m_fail_label, -1);
+ m_fail_label, profile_probability::uninitialized ());
}
/* Return true if MODE can be used for a set of moves and comparisons,
{
if (m_accumulator != NULL_RTX)
do_compare_rtx_and_jump (m_accumulator, const0_rtx, NE, true, mode,
- NULL_RTX, NULL, m_fail_label, -1);
+ NULL_RTX, NULL, m_fail_label,
+ profile_probability::uninitialized ());
}
/* Generate several move instructions to compare LEN bytes from blocks
emit_label (cmp_label);
emit_cmp_and_jump_insns (iter, size, LT, NULL_RTX, iter_mode,
- true, top_label, REG_BR_PROB_BASE * 90 / 100);
+ true, top_label,
+ profile_probability::guessed_always ()
+ .apply_scale (9, 10));
}
\f
/* Expand a call to memcpy or memmove or memcmp, and return the result.
do_pending_stack_adjust ();
NO_DEFER_POP;
- jumpifnot (TREE_OPERAND (exp, 0), lab1, -1);
+ jumpifnot (TREE_OPERAND (exp, 0), lab1,
+ profile_probability::uninitialized ());
store_expr_with_bounds (TREE_OPERAND (exp, 1), target, call_param_p,
nontemporal, reverse, btarget);
emit_jump_insn (targetm.gen_jump (lab2));
/* Generate a conditional jump to exit the loop. */
exit_cond = build2 (LT_EXPR, integer_type_node,
index, hi_index);
- jumpif (exit_cond, loop_end, -1);
+ jumpif (exit_cond, loop_end,
+ profile_probability::uninitialized ());
/* Update the loop counter, and jump to the head of
the loop. */
lab = gen_label_rtx ();
do_compare_rtx_and_jump (target, cmpop1, comparison_code,
unsignedp, mode, NULL_RTX, NULL, lab,
- -1);
+ profile_probability::uninitialized ());
}
emit_move_insn (target, op1);
emit_label (lab);
emit_move_insn (target, const0_rtx);
rtx_code_label *lab1 = gen_label_rtx ();
- jumpifnot_1 (code, treeop0, treeop1, lab1, -1);
+ jumpifnot_1 (code, treeop0, treeop1, lab1,
+ profile_probability::uninitialized ());
if (TYPE_PRECISION (type) == 1 && !TYPE_UNSIGNED (type))
emit_move_insn (target, constm1_rtx);
NO_DEFER_POP;
rtx_code_label *lab0 = gen_label_rtx ();
rtx_code_label *lab1 = gen_label_rtx ();
- jumpifnot (treeop0, lab0, -1);
+ jumpifnot (treeop0, lab0,
+ profile_probability::uninitialized ());
store_expr (treeop1, temp,
modifier == EXPAND_STACK_PARM,
false, false);
int value = TREE_CODE (rhs) == BIT_IOR_EXPR;
do_jump (TREE_OPERAND (rhs, 1),
value ? label : 0,
- value ? 0 : label, -1);
+ value ? 0 : label,
+ profile_probability::uninitialized ());
expand_assignment (lhs, build_int_cst (TREE_TYPE (rhs), value),
false);
do_pending_stack_adjust ();
int
try_casesi (tree index_type, tree index_expr, tree minval, tree range,
rtx table_label, rtx default_label, rtx fallback_label,
- int default_probability)
+ profile_probability default_probability)
{
struct expand_operand ops[5];
machine_mode index_mode = SImode;
static void
do_tablejump (rtx index, machine_mode mode, rtx range, rtx table_label,
- rtx default_label, int default_probability)
+ rtx default_label, profile_probability default_probability)
{
rtx temp, vector;
int
try_tablejump (tree index_type, tree index_expr, tree minval, tree range,
- rtx table_label, rtx default_label, int default_probability)
+ rtx table_label, rtx default_label,
+ profile_probability default_probability)
{
rtx index;
extern tree string_constant (tree, tree *);
/* Two different ways of generating switch statements. */
-extern int try_casesi (tree, tree, tree, tree, rtx, rtx, rtx, int);
-extern int try_tablejump (tree, tree, tree, tree, rtx, rtx, int);
+extern int try_casesi (tree, tree, tree, tree, rtx, rtx, rtx, profile_probability);
+extern int try_tablejump (tree, tree, tree, tree, rtx, rtx, profile_probability);
extern int safe_from_p (const_rtx, tree, int);
by xstrdup_for_dump. */
static const char *
-dump_probability (int frequency, profile_count &count)
+dump_profile (int frequency, profile_count &count)
{
float minimum = 0.01f;
return ret;
}
+/* Return formatted string of a VALUE probability
+ (biased by REG_BR_PROB_BASE). Returned string is allocated
+ by xstrdup_for_dump. */
+
+static const char *
+dump_probability (profile_probability probability, profile_count &count)
+{
+ float minimum = 0.01f;
+ float fvalue = -1;
+
+ if (probability.initialized_p ())
+ {
+ fvalue = probability.to_reg_br_prob_base () * 100.0f / REG_BR_PROB_BASE;
+ if (fvalue < minimum && probability.to_reg_br_prob_base ())
+ fvalue = minimum;
+ }
+
+ char *buf;
+ if (count.initialized_p ())
+ asprintf (&buf, "[%.2f%%] [count: %" PRId64 "]", fvalue,
+ count.to_gcov_type ());
+ else if (probability.initialized_p ())
+ asprintf (&buf, "[%.2f%%] [count: INV]", fvalue);
+ else
+ asprintf (&buf, "[INV] [count: INV]");
+
+ const char *ret = xstrdup_for_dump (buf);
+ free (buf);
+
+ return ret;
+}
+
/* Dump E probability to BUFFER. */
static void
dump_generic_node (buffer, label, spc, flags, false);
basic_block bb = gimple_bb (gs);
if (bb && !(flags & TDF_GIMPLE))
- pp_scalar (buffer, " %s", dump_probability (bb->frequency, bb->count));
+ pp_scalar (buffer, " %s", dump_profile (bb->frequency, bb->count));
pp_colon (buffer);
}
if (flags & TDF_GIMPLE)
fprintf (outf, "%*sbb_%d:\n", indent, "", bb->index);
else
fprintf (outf, "%*s<bb %d> %s:\n",
- indent, "", bb->index, dump_probability (bb->frequency,
- bb->count));
+ indent, "", bb->index, dump_profile (bb->frequency,
+ bb->count));
}
}
}
pp_printf (pp,
"\tfn_%d_basic_block_%d:s -> fn_%d_basic_block_%d:n "
- "[style=%s,color=%s,weight=%d,constraint=%s, label=\"[%i%%]\"];\n",
+ "[style=%s,color=%s,weight=%d,constraint=%s];\n",
funcdef_no, e->src->index,
funcdef_no, e->dest->index,
style, color, weight,
- (e->flags & (EDGE_FAKE | EDGE_DFS_BACK)) ? "false" : "true",
- e->probability * 100 / REG_BR_PROB_BASE);
+ (e->flags & (EDGE_FAKE | EDGE_DFS_BACK)) ? "false" : "true");
+ if (e->probability.initialized_p ())
+ pp_printf (pp, ", label=\"[%i%%]\"",
+ e->probability.to_reg_br_prob_base ()
+ * 100 / REG_BR_PROB_BASE);
+ pp_printf (pp, "];\n");
}
pp_flush (pp);
}
auto_vec <edge> new_edges;
auto_vec <phi_definition *> phi_todo_list;
auto_vec <profile_count> edge_counts;
- auto_vec <int> edge_probabilities;
+ auto_vec <profile_probability> edge_probabilities;
/* Investigate all labels that and PHI nodes in these edges which
should be fixed after we add new collection of edges. */
basic_block label_bb
= label_to_block_fn (func, CASE_LABEL (label));
edge new_edge = make_edge (cur_bb, label_bb, EDGE_TRUE_VALUE);
- int prob_sum = sum_slice <int> (edge_probabilities, i, labels, 0) +
- edge_probabilities[0];
+ profile_probability prob_sum = sum_slice <profile_probability>
+ (edge_probabilities, i, labels, profile_probability::never ())
+ + edge_probabilities[0];
- if (prob_sum)
- new_edge->probability
- = RDIV (REG_BR_PROB_BASE * edge_probabilities[i], prob_sum);
+ if (prob_sum.initialized_p ())
+ new_edge->probability = edge_probabilities[i] / prob_sum;
new_edge->count = edge_counts[i];
new_edges.safe_push (new_edge);
}
edge next_edge = make_edge (cur_bb, next_bb, EDGE_FALSE_VALUE);
- next_edge->probability
- = inverse_probability (new_edge->probability);
+ next_edge->probability = new_edge->probability.invert ();
next_edge->count = edge_counts[0]
+ sum_slice <profile_count> (edge_counts, i, labels,
profile_count::zero ());
of the switch. */
{
edge e = make_edge (cur_bb, default_label_bb, EDGE_FALSE_VALUE);
- e->probability = inverse_probability (new_edge->probability);
+ e->probability = new_edge->probability.invert ();
e->count = edge_counts[0];
new_edges.safe_insert (0, e);
}
/* Forward references. */
static int count_bb_insns (const_basic_block);
-static bool cheap_bb_rtx_cost_p (const_basic_block, int, int);
+static bool cheap_bb_rtx_cost_p (const_basic_block, profile_probability, int);
static rtx_insn *first_active_insn (basic_block);
static rtx_insn *last_active_insn (basic_block, int);
static rtx_insn *find_active_insn_before (basic_block, rtx_insn *);
plus a small fudge factor. */
static bool
-cheap_bb_rtx_cost_p (const_basic_block bb, int scale, int max_cost)
+cheap_bb_rtx_cost_p (const_basic_block bb,
+ profile_probability prob, int max_cost)
{
int count = 0;
rtx_insn *insn = BB_HEAD (bb);
bool speed = optimize_bb_for_speed_p (bb);
+ int scale = prob.initialized_p () ? prob.to_reg_br_prob_base ()
+ : REG_BR_PROB_BASE;
/* Set scale to REG_BR_PROB_BASE to void the identical scaling
applied to insn_rtx_cost when optimizing for size. Only do
basic_block then_bb = then_edge->dest;
basic_block else_bb = else_edge->dest;
basic_block new_bb;
- int then_bb_index, then_prob;
+ int then_bb_index;
+ profile_probability then_prob;
rtx else_target = NULL_RTX;
/* If we are partitioning hot/cold basic blocks, we don't want to
"\nIF-CASE-1 found, start %d, then %d\n",
test_bb->index, then_bb->index);
- if (then_edge->probability)
- then_prob = REG_BR_PROB_BASE - then_edge->probability;
- else
- then_prob = REG_BR_PROB_BASE / 2;
+ then_prob = then_edge->probability.invert ();
/* We're speculating from the THEN path, we want to make sure the cost
of speculation is within reason. */
basic_block then_bb = then_edge->dest;
basic_block else_bb = else_edge->dest;
edge else_succ;
- int then_prob, else_prob;
+ profile_probability then_prob, else_prob;
/* We do not want to speculate (empty) loop latches. */
if (current_loops
if (then_bb->index < NUM_FIXED_BLOCKS)
return FALSE;
- if (else_edge->probability)
- {
- else_prob = else_edge->probability;
- then_prob = REG_BR_PROB_BASE - else_prob;
- }
- else
- {
- else_prob = REG_BR_PROB_BASE / 2;
- then_prob = REG_BR_PROB_BASE / 2;
- }
+ else_prob = else_edge->probability;
+ then_prob = else_prob.invert ();
/* ELSE is predicted or SUCC(ELSE) postdominates THEN. */
if (else_prob > then_prob)
gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
EQ, true, mode, NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (done_label);
}
}
do_compare_rtx_and_jump (res, lres,
EQ, true, tgtmode, NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (done_label);
}
tem = op1;
do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
true, mode, NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
goto do_error_label;
}
code == PLUS_EXPR ? res : op0, sgn,
NULL_RTX, false, OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
- done_label, PROB_VERY_LIKELY);
+ done_label, profile_probability::very_likely ());
goto do_error_label;
}
else if (pos_neg == 3)
/* If ARG0 is not known to be always positive, check at runtime. */
do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
- NULL, do_error, PROB_VERY_UNLIKELY);
+ NULL, do_error, profile_probability::very_unlikely ());
do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
- done_label, PROB_VERY_LIKELY);
+ done_label, profile_probability::very_likely ());
goto do_error_label;
}
rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
- done_label, PROB_VERY_LIKELY);
+ done_label, profile_probability::very_likely ());
goto do_error_label;
}
res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
- NULL, do_error, PROB_VERY_UNLIKELY);
+ NULL, do_error, profile_probability::very_unlikely ());
rtx tem = op1;
/* The operation is commutative, so we can pick operand to compare
against. For prec <= BITS_PER_WORD, I think preferring REG operand
: CONST_SCALAR_INT_P (op0))
tem = op0;
do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
- done_label, PROB_VERY_LIKELY);
+ done_label, profile_probability::very_likely ());
goto do_error_label;
}
? and_optab : ior_optab,
op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
}
else
{
do_compare_rtx_and_jump (op1, const0_rtx,
code == MINUS_EXPR ? GE : LT, false, mode,
NULL_RTX, NULL, do_ior_label,
- PROB_EVEN);
+ profile_probability::even ());
tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
emit_jump (do_error);
emit_label (do_ior_label);
tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
}
goto do_error_label;
}
OPTAB_LIB_WIDEN);
rtx_code_label *op0_geu_op1 = gen_label_rtx ();
do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
- op0_geu_op1, PROB_EVEN);
+ op0_geu_op1, profile_probability::even ());
do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
emit_jump (do_error);
emit_label (op0_geu_op1);
do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
goto do_error_label;
}
&& JUMP_P (last)
&& any_condjump_p (last)
&& !find_reg_note (last, REG_BR_PROB, 0))
- add_int_reg_note (last, REG_BR_PROB, PROB_VERY_UNLIKELY);
+ add_int_reg_note (last, REG_BR_PROB, PROB_UNLIKELY);
emit_jump (done_label);
goto do_error_label;
}
/* No overflow if the result has bit sign cleared. */
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
}
/* Compare the result of the operation with the first operand.
do_compare_rtx_and_jump (res, op0,
(pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
false, mode, NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
}
do_error_label:
/* Compare the operand with the most negative value. */
rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
- done_label, PROB_VERY_LIKELY);
+ done_label, profile_probability::very_likely ());
}
emit_label (do_error);
ops.location = loc;
res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
goto do_error_label;
case 3:
rtx_code_label *do_main_label;
do_main_label = gen_label_rtx ();
do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, do_main_label, PROB_VERY_LIKELY);
+ NULL, do_main_label, profile_probability::very_likely ());
do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
- NULL, do_main_label, PROB_VERY_LIKELY);
+ NULL, do_main_label, profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (do_main_label);
goto do_main;
ops.location = loc;
res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
- NULL, do_error, PROB_VERY_UNLIKELY);
+ NULL, do_error, profile_probability::very_unlikely ());
int prec;
prec = GET_MODE_PRECISION (mode);
rtx sgn;
sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
- NULL, done_label, PROB_VERY_LIKELY);
+ NULL, done_label, profile_probability::very_likely ());
goto do_error_label;
case 3:
/* Rest of handling of this case after res is computed. */
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode,
NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
goto do_error_label;
}
/* The general case, do all the needed comparisons at runtime. */
tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, after_negate_label, PROB_VERY_LIKELY);
+ NULL, after_negate_label, profile_probability::very_likely ());
/* Both arguments negative here, negate them and continue with
normal unsigned overflow checking multiplication. */
emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
tem2 = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
OPTAB_LIB_WIDEN);
do_compare_rtx_and_jump (tem2, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, do_main_label, PROB_VERY_LIKELY);
+ NULL, do_main_label, profile_probability::very_likely ());
/* One argument is negative here, the other positive. This
overflows always, unless one of the arguments is 0. But
if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
is, thus we can keep do_main code oring in overflow as is. */
do_compare_rtx_and_jump (tem, const0_rtx, EQ, true, mode, NULL_RTX,
- NULL, do_main_label, PROB_VERY_LIKELY);
+ NULL, do_main_label, profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (do_main_label);
goto do_main;
HIPART is non-zero. */
do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
else
{
rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
HIPART is different from RES < 0 ? -1 : 0. */
do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
NULL_RTX, NULL, done_label,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
}
}
else if (hmode != BLKmode && 2 * GET_MODE_PRECISION (hmode) == prec)
if (!op0_small_p)
do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
NULL_RTX, NULL, large_op0,
- PROB_UNLIKELY);
+ profile_probability::unlikely ());
if (!op1_small_p)
do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
NULL_RTX, NULL, small_op0_large_op1,
- PROB_UNLIKELY);
+ profile_probability::unlikely ());
/* If both op0 and op1 are sign (!uns) or zero (uns) extended from
hmode to mode, the multiplication will never overflow. We can
if (!op1_small_p)
do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
NULL_RTX, NULL, both_ops_large,
- PROB_UNLIKELY);
+ profile_probability::unlikely ());
/* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
but op0 is not, prepare larger, hipart and lopart pseudos and
else if (larger_sign != -1)
do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
NULL_RTX, NULL, after_hipart_neg,
- PROB_EVEN);
+ profile_probability::even ());
tem = convert_modes (mode, hmode, lopart, 1);
tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
else if (smaller_sign != -1)
do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
NULL_RTX, NULL, after_lopart_neg,
- PROB_EVEN);
+ profile_probability::even ());
tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
1, OPTAB_DIRECT);
do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
NULL_RTX, NULL, do_overflow,
- PROB_VERY_UNLIKELY);
+ profile_probability::very_unlikely ());
/* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1; */
rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
NULL_RTX, 1, OPTAB_DIRECT);
do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
NULL_RTX, NULL, do_error,
- PROB_VERY_UNLIKELY);
+ profile_probability::very_unlikely ());
}
if (!op1_medium_p)
NULL_RTX, 1, OPTAB_DIRECT);
do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
NULL_RTX, NULL, do_error,
- PROB_VERY_UNLIKELY);
+ profile_probability::very_unlikely ());
}
/* At this point hipart{0,1} are both in [-1, 0]. If they are
else if (op0_sign == 1 || op1_sign == 1)
do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
NULL_RTX, NULL, hipart_different,
- PROB_EVEN);
+ profile_probability::even ());
do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode,
NULL_RTX, NULL, do_error,
- PROB_VERY_UNLIKELY);
+ profile_probability::very_unlikely ());
emit_jump (done_label);
emit_label (hipart_different);
do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
NULL_RTX, NULL, do_error,
- PROB_VERY_UNLIKELY);
+ profile_probability::very_unlikely ());
emit_jump (done_label);
}
{
rtx_code_label *all_done_label = gen_label_rtx ();
do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, all_done_label, PROB_VERY_LIKELY);
+ NULL, all_done_label, profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
emit_label (all_done_label);
}
rtx_code_label *all_done_label = gen_label_rtx ();
rtx_code_label *set_noovf = gen_label_rtx ();
do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
- NULL, all_done_label, PROB_VERY_LIKELY);
+ NULL, all_done_label, profile_probability::very_likely ());
expand_arith_set_overflow (lhs, target);
do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
- NULL, set_noovf, PROB_VERY_LIKELY);
+ NULL, set_noovf, profile_probability::very_likely ());
do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
- NULL, all_done_label, PROB_VERY_UNLIKELY);
+ NULL, all_done_label, profile_probability::very_unlikely ());
do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
- all_done_label, PROB_VERY_UNLIKELY);
+ all_done_label, profile_probability::very_unlikely ());
emit_label (set_noovf);
write_complex_part (target, const0_rtx, true);
emit_label (all_done_label);
emit_move_insn (cntvar, ret);
do_compare_rtx_and_jump (cntvar, GEN_INT (cnt), NE, false,
TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
- PROB_VERY_LIKELY);
+ profile_probability::very_likely ());
}
if (lhs && resv == NULL_TREE)
{
struct ipa_node_params *info = IPA_NODE_REF (node);
if (max_count > profile_count::zero ())
{
- int factor = RDIV (count_sum.probability_in (max_count)
+ int factor = RDIV (count_sum.probability_in
+ (max_count).to_reg_br_prob_base ()
* 1000, REG_BR_PROB_BASE);
int64_t evaluation = (((int64_t) time_benefit * factor)
/ size_cost);
break;
}
}
- e = make_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->probability = REG_BR_PROB_BASE;
- e->count = new_return_bb->count;
+ e = make_single_succ_edge (new_return_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
add_bb_to_loop (new_return_bb, current_loops->tree_root);
bitmap_set_bit (split_point->split_bbs, new_return_bb->index);
retbnd = find_retbnd (return_bb);
}
else
{
- e = make_edge (call_bb, return_bb,
- return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
- ? 0 : EDGE_FALLTHRU);
- e->count = call_bb->count;
- e->probability = REG_BR_PROB_BASE;
+ e = make_single_succ_edge (call_bb, return_bb,
+ return_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
+ ? 0 : EDGE_FALLTHRU);
/* If there is return basic block, see what value we need to store
return value into and put call just before it. */
}
}
int prob = direct->count.probability_in (direct->count
- + indirect->count);
+ + indirect->count).
+ to_reg_br_prob_base ();
direct->frequency = RDIV (freq * prob, REG_BR_PROB_BASE);
indirect->frequency = RDIV (freq * (REG_BR_PROB_BASE - prob),
REG_BR_PROB_BASE);
e2->speculative_call_info (direct, indirect, ref);
e->count = count;
e->frequency = freq;
- int prob = direct->count.probability_in (e->count);
+ int prob = direct->count.probability_in (e->count)
+ .to_reg_br_prob_base ();
e->make_speculative (direct->callee, direct->count,
RDIV (freq * prob, REG_BR_PROB_BASE));
}
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
label = block_label (dest);
- do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL, label, -1);
+ do_compare_rtx_and_jump (op0, op1, code, 0, mode, NULL_RTX, NULL, label,
+ profile_probability::uninitialized ());
jump = get_last_insn ();
if (!jump || !JUMP_P (jump))
add_reg_note (jump_insn, REG_NONNEG, NULL_RTX);
/* Update the REG_BR_PROB note. */
- if (true_prob_val)
+ if (true_prob_val && desc->in_edge->probability.initialized_p ())
{
/* Seems safer to use the branch probability. */
- add_int_reg_note (jump_insn, REG_BR_PROB, desc->in_edge->probability);
+ add_int_reg_note (jump_insn, REG_BR_PROB,
+ desc->in_edge->probability.to_reg_br_prob_base ());
}
}
static rtx_insn *
compare_and_jump_seq (rtx op0, rtx op1, enum rtx_code comp,
- rtx_code_label *label, int prob, rtx_insn *cinsn)
+ rtx_code_label *label, profile_probability prob,
+ rtx_insn *cinsn)
{
rtx_insn *seq;
rtx_jump_insn *jump;
op0 = force_operand (op0, NULL_RTX);
op1 = force_operand (op1, NULL_RTX);
do_compare_rtx_and_jump (op0, op1, comp, 0,
- mode, NULL_RTX, NULL, label, -1);
+ mode, NULL_RTX, NULL, label,
+ profile_probability::uninitialized ());
jump = as_a <rtx_jump_insn *> (get_last_insn ());
jump->set_jump_target (label);
LABEL_NUSES (label)++;
}
- add_int_reg_note (jump, REG_BR_PROB, prob);
+ if (prob.initialized_p ())
+ add_int_reg_note (jump, REG_BR_PROB, prob.to_reg_br_prob_base ());
seq = get_insns ();
end_sequence ();
{
rtx old_niter, niter, tmp;
rtx_insn *init_code, *branch_code;
- unsigned i, j, p;
+ unsigned i, j;
+ profile_probability p;
basic_block preheader, *body, swtch, ezc_swtch = NULL;
int may_exit_copy, iter_freq, new_freq;
profile_count iter_count, new_count;
/* Create item for switch. */
j = n_peel - i - (extra_zero_check ? 0 : 1);
- p = REG_BR_PROB_BASE / (i + 2);
+ p = profile_probability::always ().apply_scale (1, i + 2);
preheader = split_edge (loop_preheader_edge (loop));
/* Add in frequency/count of edge from switch block. */
swtch = split_edge_and_insert (single_pred_edge (swtch), branch_code);
set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
- single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p;
+ single_succ_edge (swtch)->probability = p.invert ();
single_succ_edge (swtch)->count = new_count;
new_freq += iter_freq;
new_count += iter_count;
if (extra_zero_check)
{
/* Add branch for zero iterations. */
- p = REG_BR_PROB_BASE / (max_unroll + 1);
+ p = profile_probability::always ().apply_scale (1, max_unroll + 1);
swtch = ezc_swtch;
preheader = split_edge (loop_preheader_edge (loop));
/* Recompute frequency/count adjustments since initial peel copy may
swtch = split_edge_and_insert (single_succ_edge (swtch), branch_code);
set_immediate_dominator (CDI_DOMINATORS, preheader, swtch);
- single_succ_edge (swtch)->probability = REG_BR_PROB_BASE - p;
+ single_succ_edge (swtch)->probability = p.invert ();
single_succ_edge (swtch)->count -= iter_count;
e = make_edge (swtch, preheader,
single_succ_edge (swtch)->flags & EDGE_IRREDUCIBLE_LOOP);
e = find_fallthru_edge (bb->succs);
if (! e)
break;
- if (e->probability < EBB_PROBABILITY_CUTOFF)
+ if (e->probability.initialized_p ()
+ && e->probability.to_reg_br_prob_base () < EBB_PROBABILITY_CUTOFF)
break;
bb = bb->next_bb;
}
unsigned int dest_index;
unsigned int edge_flags;
basic_block dest;
- int probability;
+ profile_probability probability;
profile_count count;
edge e;
dest_index = streamer_read_uhwi (ib);
- probability = (int) streamer_read_hwi (ib);
+ probability = profile_probability::stream_in (ib);
count = profile_count::stream_in (ib).apply_scale
(count_materialization_scale, REG_BR_PROB_BASE);
edge_flags = streamer_read_uhwi (ib);
FOR_EACH_EDGE (e, ei, bb->succs)
{
streamer_write_uhwi (ob, e->dest->index);
- streamer_write_hwi (ob, e->probability);
+ e->probability.stream_out (ob);
e->count.stream_out (ob);
streamer_write_uhwi (ob, e->flags);
}
}
if (bb_gcov_count (bb))
- e->probability = RDIV (REG_BR_PROB_BASE * edge_gcov_count (e),
- bb_gcov_count (bb));
+ e->probability = profile_probability::probability_in_gcov_type
+ (edge_gcov_count (e), bb_gcov_count (bb));
if (dump_file)
- fprintf (dump_file, " = %" PRId64 "\t(%.1f%%)\n",
- edge_gcov_count (e),
- e->probability * 100.0 / REG_BR_PROB_BASE);
+ {
+ fprintf (dump_file, " = %" PRId64 "\t",
+ edge_gcov_count (e));
+ e->probability.dump (dump_file);
+ fprintf (dump_file, "\n");
+ }
}
}
if (bb_gcov_count (bb))
{
FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = RDIV (REG_BR_PROB_BASE * edge_gcov_count (e),
- bb_gcov_count (bb));
- }
- else
- {
- int total = 0;
- FOR_EACH_EDGE (e, ei, bb->succs)
- if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE)))
- total++;
- if (total)
- {
- FOR_EACH_EDGE (e, ei, bb->succs)
- {
- if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE)))
- e->probability = REG_BR_PROB_BASE / total;
- else
- e->probability = 0;
- }
- }
- else
- {
- total += EDGE_COUNT (bb->succs);
- FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = REG_BR_PROB_BASE / total;
- }
+ e->probability = profile_probability::probability_in_gcov_type
+ (edge_gcov_count (e), bb_gcov_count (bb));
}
}
rtx comp_rtx = gen_rtx_GT (VOIDmode, count_reg,
gen_int_mode (stage_count,
GET_MODE (count_reg)));
- unsigned prob = (PROB_SMS_ENOUGH_ITERATIONS
- * REG_BR_PROB_BASE) / 100;
+ profile_probability prob = profile_probability::guessed_always ()
+ .apply_scale (PROB_SMS_ENOUGH_ITERATIONS, 100);
loop_version (loop, comp_rtx, &condition_bb,
- prob, REG_BR_PROB_BASE - prob,
- prob, REG_BR_PROB_BASE - prob,
+ prob, prob.invert (),
+ prob.to_reg_br_prob_base (),
+ prob.invert ().to_reg_br_prob_base (),
true);
}
entry_bb);
}
ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE);
- ne->probability = REG_BR_PROB_BASE / 2000 - 1;
+ ne->probability = profile_probability::very_unlikely ();
e->flags = EDGE_TRUE_VALUE;
- e->probability = REG_BR_PROB_BASE - ne->probability;
+ e->probability = ne->probability.invert ();
if (l2_dom_bb == NULL)
l2_dom_bb = entry_bb;
entry_bb = e->dest;
if (i < fd->collapse - 1)
{
e = make_edge (last_bb, bb, EDGE_FALSE_VALUE);
- e->probability = REG_BR_PROB_BASE / 8;
+ e->probability = profile_probability::guessed_always ().apply_scale (1, 8);
t = fd->loops[i + 1].n1;
t = force_gimple_operand_gsi (&gsi, t,
stmt = gimple_build_cond_empty (t);
gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
e = make_edge (bb, body_bb, EDGE_TRUE_VALUE);
- e->probability = REG_BR_PROB_BASE * 7 / 8;
+ e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
}
else
make_edge (bb, body_bb, EDGE_FALLTHRU);
GSI_CONTINUE_LINKING);
gsi_insert_after (gsi, gimple_build_cond_empty (cond), GSI_NEW_STMT);
edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
- e3->probability = REG_BR_PROB_BASE / 8;
- e1->probability = REG_BR_PROB_BASE - e3->probability;
+ e3->probability = profile_probability::guessed_always ().apply_scale (1, 8);
+ e1->probability = e3->probability.invert ();
e1->flags = EDGE_TRUE_VALUE;
set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
remove_edge (e1);
make_edge (body_bb, new_header, EDGE_FALLTHRU);
e3->flags = EDGE_FALSE_VALUE;
- e3->probability = REG_BR_PROB_BASE / 8;
+ e3->probability = profile_probability::guessed_always ().apply_scale (1, 8);
e1 = make_edge (new_header, new_body, EDGE_TRUE_VALUE);
- e1->probability = REG_BR_PROB_BASE - e3->probability;
+ e1->probability = e3->probability.invert ();
set_immediate_dominator (CDI_DOMINATORS, new_header, body_bb);
set_immediate_dominator (CDI_DOMINATORS, new_body, new_header);
e->flags = EDGE_TRUE_VALUE;
if (e)
{
- e->probability = REG_BR_PROB_BASE * 7 / 8;
- find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8;
+ e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
+ find_edge (cont_bb, l2_bb)->probability = e->probability.invert ();
}
else
{
ep = split_block (entry_bb, cond_stmt);
ep->flags = EDGE_TRUE_VALUE;
entry_bb = ep->dest;
- ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
+ ep->probability = profile_probability::very_likely ();
ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE);
- ep->probability = REG_BR_PROB_BASE / 2000 - 1;
+ ep->probability = profile_probability::very_unlikely ();
if (gimple_in_ssa_p (cfun))
{
int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx;
/* Connect all the blocks. */
ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE);
- ep->probability = REG_BR_PROB_BASE / 4 * 3;
+ ep->probability = profile_probability::guessed_always ().apply_scale (3, 4);
ep = find_edge (entry_bb, second_bb);
ep->flags = EDGE_TRUE_VALUE;
- ep->probability = REG_BR_PROB_BASE / 4;
+ ep->probability = profile_probability::guessed_always ().apply_scale (1, 4);
find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE;
find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE;
se = split_block (entry_bb, cond_stmt);
se->flags = EDGE_TRUE_VALUE;
entry_bb = se->dest;
- se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1);
+ se->probability = profile_probability::very_likely ();
se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE);
- se->probability = REG_BR_PROB_BASE / 2000 - 1;
+ se->probability = profile_probability::very_unlikely ();
if (gimple_in_ssa_p (cfun))
{
int dest_idx = find_edge (iter_part_bb, fin_bb)->dest_idx;
}
ne->flags = EDGE_FALSE_VALUE;
- e->probability = REG_BR_PROB_BASE * 7 / 8;
- ne->probability = REG_BR_PROB_BASE / 8;
+ e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
+ ne->probability = e->probability.invert ();
set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
}
ne->flags = EDGE_FALSE_VALUE;
- e->probability = REG_BR_PROB_BASE * 7 / 8;
- ne->probability = REG_BR_PROB_BASE / 8;
+ e->probability = profile_probability::guessed_always ().apply_scale (7, 8);
+ ne->probability = e->probability.invert ();
set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb);
set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb);
gsi_insert_after (&gsi, cond_stmt, GSI_NEW_STMT);
make_edge (entry_bb, l2_bb, EDGE_FALSE_VALUE);
FALLTHRU_EDGE (entry_bb)->flags = EDGE_TRUE_VALUE;
- FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE * 7 / 8;
- BRANCH_EDGE (entry_bb)->probability = REG_BR_PROB_BASE / 8;
+ FALLTHRU_EDGE (entry_bb)->probability
+ = profile_probability::guessed_always ().apply_scale (7, 8);
+ BRANCH_EDGE (entry_bb)->probability
+ = FALLTHRU_EDGE (entry_bb)->probability.invert ();
l2_dom_bb = entry_bb;
}
set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb);
gsi = gsi_last_bb (exit_bb);
gsi_remove (&gsi, true);
- FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE;
+ FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
remove_edge (BRANCH_EDGE (entry_bb));
- FALLTHRU_EDGE (cont_bb)->probability = REG_BR_PROB_BASE;
+ FALLTHRU_EDGE (cont_bb)->probability = profile_probability::always ();
remove_edge (BRANCH_EDGE (cont_bb));
set_immediate_dominator (CDI_DOMINATORS, exit_bb, cont_bb);
set_immediate_dominator (CDI_DOMINATORS, region->entry,
gsi = gsi_last_bb (exit_bb);
gsi_remove (&gsi, true);
- FALLTHRU_EDGE (entry_bb)->probability = REG_BR_PROB_BASE;
+ FALLTHRU_EDGE (entry_bb)->probability = profile_probability::always ();
if (!broken_loop)
remove_edge (BRANCH_EDGE (entry_bb));
else
e = single_succ_edge (store_bb);
e->flags &= ~EDGE_FALLTHRU;
e->flags |= EDGE_FALSE_VALUE;
+ /* Expect no looping. */
+ e->probability = profile_probability::guessed_always ();
e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE);
+ e->probability = profile_probability::guessed_never ();
/* Copy the new value to loadedi (we already did that before the condition
if we are not in SSA). */
if (incr_bb)
{
- edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
- e->probability = REG_BR_PROB_BASE;
+ make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0);
gsi = gsi_last_bb (incr_bb);
iter2 = make_ssa_name (iter);
g = gimple_build_assign (iter2, PLUS_EXPR, iter1,
redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb);
- make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
+ edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE);
+
+ /* FIXME: Do we need to distribute probabilities for the conditional? */
+ new_e->probability = profile_probability::guessed_never ();
/* The successor of incr_bb is already pointing to latch_bb; just
change the flags.
make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */
NO_DEFER_POP;
do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
- 0, 0, subword_label, -1);
+ 0, 0, subword_label,
+ profile_probability::uninitialized ());
OK_DEFER_POP;
if (!expand_superword_shift (binoptab, outof_input, superword_op1,
NO_DEFER_POP;
do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
- NULL_RTX, NULL, op1, -1);
+ NULL_RTX, NULL, op1,
+ profile_probability::uninitialized ());
op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
target, target, 0);
we can do the branch. */
static void
-emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label, int prob)
+emit_cmp_and_jump_insn_1 (rtx test, machine_mode mode, rtx label,
+ profile_probability prob)
{
machine_mode optab_mode;
enum mode_class mclass;
gcc_assert (insn_operand_matches (icode, 0, test));
insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
XEXP (test, 1), label));
- if (prob != -1
+ if (prob.initialized_p ()
&& profile_status_for_fn (cfun) != PROFILE_ABSENT
&& insn
&& JUMP_P (insn)
&& any_condjump_p (insn)
&& !find_reg_note (insn, REG_BR_PROB, 0))
- add_int_reg_note (insn, REG_BR_PROB, prob);
+ add_int_reg_note (insn, REG_BR_PROB, prob.to_reg_br_prob_base ());
}
/* Generate code to compare X with Y so that the condition codes are
void
emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
machine_mode mode, int unsignedp, rtx label,
- int prob)
+ profile_probability prob)
{
rtx op0 = x, op1 = y;
rtx test;
/* Mark this jump predicted not taken. */
emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
- GET_MODE (success), 1, label, 0);
+ GET_MODE (success), 1, label,
+ profile_probability::guessed_never ());
return true;
}
/* Emit a pair of rtl insns to compare two rtx's and to jump
to a label if the comparison is true. */
extern void emit_cmp_and_jump_insns (rtx, rtx, enum rtx_code, rtx,
- machine_mode, int, rtx, int prob=-1);
+ machine_mode, int, rtx,
+ profile_probability prob
+ = profile_probability::uninitialized ());
/* Generate code to indirectly jump to a location given in the rtx LOC. */
extern void emit_indirect_jump (rtx);
#include "ipa-utils.h"
#include "gimple-pretty-print.h"
#include "selftest.h"
+#include "cfgrtl.h"
/* Enum with reasons why a predictor is ignored. */
bool
predictable_edge_p (edge e)
{
- if (profile_status_for_fn (cfun) == PROFILE_ABSENT)
+ if (!e->probability.initialized_p ())
return false;
- if ((e->probability
+ if ((e->probability.to_reg_br_prob_base ()
<= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100)
- || (REG_BR_PROB_BASE - e->probability
+ || (REG_BR_PROB_BASE - e->probability.to_reg_br_prob_base ()
<= PARAM_VALUE (PARAM_PREDICTABLE_BRANCH_OUTCOME) * REG_BR_PROB_BASE / 100))
return true;
return false;
bool
edge_probability_reliable_p (const_edge e)
{
- return probability_reliable_p (e->probability);
+ return e->probability.reliable_p ();
}
/* Same predicate as edge_probability_reliable_p, working on notes. */
if (!unlikely_executed_edge_p (e))
{
if (unlikely_edges != NULL && unlikely_edges->contains (e))
- e->probability = PROB_VERY_UNLIKELY;
+ e->probability = profile_probability::very_unlikely ();
else
- e->probability = (REG_BR_PROB_BASE + c / 2) / c;
+ e->probability = profile_probability::guessed_always ()
+ .apply_scale (1, c);
}
else
- e->probability = 0;
+ e->probability = profile_probability::never ();
}
/* Combine all REG_BR_PRED notes into single probability and attach REG_BR_PROB
conditional jump. */
if (!single_succ_p (bb))
{
- BRANCH_EDGE (bb)->probability = combined_probability;
+ BRANCH_EDGE (bb)->probability
+ = profile_probability::from_reg_br_prob_base (combined_probability);
FALLTHRU_EDGE (bb)->probability
- = REG_BR_PROB_BASE - combined_probability;
+ = BRANCH_EDGE (bb)->probability.invert ();
}
}
else if (!single_succ_p (bb))
{
int prob = XINT (prob_note, 0);
- BRANCH_EDGE (bb)->probability = prob;
- FALLTHRU_EDGE (bb)->probability = REG_BR_PROB_BASE - prob;
+ BRANCH_EDGE (bb)->probability
+ = profile_probability::from_reg_br_prob_base (prob);
+ FALLTHRU_EDGE (bb)->probability
+ = BRANCH_EDGE (bb)->probability.invert ();
}
else
- single_succ_edge (bb)->probability = REG_BR_PROB_BASE;
+ single_succ_edge (bb)->probability = profile_probability::always ();
}
/* Edge prediction hash traits. */
if (!first)
first = e;
}
+ else if (!e->probability.initialized_p ())
+ e->probability = profile_probability::never ();
/* When there is no successor or only one choice, prediction is easy.
nedges, bb->index);
FOR_EACH_EDGE (e, ei, bb->succs)
if (!unlikely_executed_edge_p (e))
- dump_prediction (dump_file, PRED_COMBINED, e->probability,
- bb, REASON_NONE, e);
+ dump_prediction (dump_file, PRED_COMBINED,
+ e->probability.to_reg_br_prob_base (), bb, REASON_NONE, e);
}
}
return;
if (!bb->count.initialized_p () && !dry_run)
{
- first->probability = combined_probability;
- second->probability = REG_BR_PROB_BASE - combined_probability;
+ first->probability
+ = profile_probability::from_reg_br_prob_base (combined_probability);
+ second->probability = first->probability.invert ();
}
}
* BLOCK_INFO (e->src)->frequency /
REG_BR_PROB_BASE); */
- sreal tmp = e->probability;
+ sreal tmp = e->probability.to_reg_br_prob_base ();
tmp *= BLOCK_INFO (e->src)->frequency;
tmp *= real_inv_br_prob_base;
frequency += tmp;
= ((e->probability * BLOCK_INFO (bb)->frequency)
/ REG_BR_PROB_BASE); */
- sreal tmp = e->probability;
+ sreal tmp = e->probability.to_reg_br_prob_base ();
tmp *= BLOCK_INFO (bb)->frequency;
EDGE_INFO (e)->back_edge_prob = tmp * real_inv_br_prob_base;
}
mark_dfs_back_edges ();
single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun))->probability =
- REG_BR_PROB_BASE;
+ profile_probability::always ();
/* Set up block info for each basic block. */
alloc_aux_for_blocks (sizeof (block_info));
FOR_EACH_EDGE (e, ei, bb->succs)
{
- EDGE_INFO (e)->back_edge_prob = e->probability;
+ EDGE_INFO (e)->back_edge_prob
+ = e->probability.to_reg_br_prob_base ();
EDGE_INFO (e)->back_edge_prob *= real_inv_br_prob_base;
}
}
force_edge_cold (edge e, bool impossible)
{
profile_count count_sum = profile_count::zero ();
- int prob_sum = 0;
+ profile_probability prob_sum = profile_probability::never ();
edge_iterator ei;
edge e2;
profile_count old_count = e->count;
- int old_probability = e->probability;
- int prob_scale = REG_BR_PROB_BASE;
+ profile_probability old_probability = e->probability;
bool uninitialized_exit = false;
+ profile_probability goal = (impossible ? profile_probability::never ()
+ : profile_probability::very_unlikely ());
+
/* If edge is already improbably or cold, just return. */
- if (e->probability <= (impossible ? PROB_VERY_UNLIKELY : 0)
+ if (e->probability <= goal
&& (!impossible || e->count == profile_count::zero ()))
return;
FOR_EACH_EDGE (e2, ei, e->src->succs)
count_sum += e2->count;
else
uninitialized_exit = true;
- prob_sum += e2->probability;
+ if (e2->probability.initialized_p ())
+ prob_sum += e2->probability;
}
/* If there are other edges out of e->src, redistribute probabilitity
there. */
- if (prob_sum)
+ if (prob_sum > profile_probability::never ())
{
- e->probability
- = MIN (e->probability, impossible ? 0 : PROB_VERY_UNLIKELY);
+ if (!(e->probability < goal))
+ e->probability = goal;
if (impossible)
e->count = profile_count::zero ();
- else if (old_probability)
- e->count = e->count.apply_scale (e->probability, old_probability);
+ else if (old_probability > profile_probability::never ())
+ e->count = e->count.apply_probability (e->probability
+ / old_probability);
else
e->count = e->count.apply_scale (1, REG_BR_PROB_BASE);
- prob_scale = RDIV ((REG_BR_PROB_BASE - e->probability) * REG_BR_PROB_BASE,
- prob_sum);
+ profile_probability prob_comp = prob_sum / e->probability.invert ();
+
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, "Making edge %i->%i %s by redistributing "
"probability to other edges.\n",
{
if (count_sum > 0)
e2->count.apply_scale (count_sum2, count_sum);
- e2->probability = RDIV (e2->probability * prob_scale,
- REG_BR_PROB_BASE);
+ e2->probability /= prob_comp;
}
+ if (current_ir_type () != IR_GIMPLE)
+ update_br_prob_note (e->src);
}
/* If all edges out of e->src are unlikely, the basic block itself
is unlikely. */
else
{
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
+ if (current_ir_type () != IR_GIMPLE)
+ update_br_prob_note (e->src);
if (e->src->count == profile_count::zero ())
return;
if (count_sum == profile_count::zero () && !uninitialized_exit
&& impossible)
{
bool found = false;
- for (gimple_stmt_iterator gsi = gsi_start_bb (e->src);
- !gsi_end_p (gsi); gsi_next (&gsi))
- {
- if (stmt_can_terminate_bb_p (gsi_stmt (gsi)))
- {
- found = true;
- break;
- }
- }
+ if (e->src == ENTRY_BLOCK_PTR_FOR_FN (cfun))
+ ;
+ else if (current_ir_type () == IR_GIMPLE)
+ for (gimple_stmt_iterator gsi = gsi_start_bb (e->src);
+ !gsi_end_p (gsi); gsi_next (&gsi))
+ {
+ if (stmt_can_terminate_bb_p (gsi_stmt (gsi)))
+ {
+ found = true;
+ break;
+ }
+ }
+ /* FIXME: Implement RTL path. */
+ else
+ found = true;
if (!found)
{
if (dump_file && (dump_flags & TDF_DETAILS))
This in general is difficult task to do, but handle special case when
BB has only one predecestor. This is common case when we are updating
after loop transforms. */
- if (!prob_sum && count_sum == profile_count::zero ()
+ if (!(prob_sum > profile_probability::never ())
+ && count_sum == profile_count::zero ()
&& single_pred_p (e->src) && e->src->frequency > (impossible ? 0 : 1))
{
int old_frequency = e->src->frequency;
#include "data-streamer.h"
#include "cgraph.h"
+/* Dump THIS to F. */
+
void
profile_count::dump (FILE *f) const
{
else
{
fprintf (f, "%" PRId64, m_val);
- if (m_quality == count_adjusted)
- fprintf (f, "(adjusted)");
- else if (m_quality == count_afdo)
- fprintf (f, "(auto FDO)");
- else if (m_quality == count_guessed)
- fprintf (f, "(guessed)");
+ if (m_quality == profile_adjusted)
+ fprintf (f, " (adjusted)");
+ else if (m_quality == profile_afdo)
+ fprintf (f, " (auto FDO)");
+ else if (m_quality == profile_guessed)
+ fprintf (f, " (guessed)");
}
}
+/* Dump THIS to stderr. */
+
void
profile_count::debug () const
{
dump (stderr);
+ fprintf (stderr, "\n");
}
+/* Return true if THIS differs from OTHER; tolerate small diferences. */
+
bool
profile_count::differs_from_p (profile_count other) const
{
if (!initialized_p () || !other.initialized_p ())
return false;
- if (m_val - other.m_val < 100 || other.m_val - m_val < 100)
+ if ((uint64_t)m_val - (uint64_t)other.m_val < 100
+ || (uint64_t)other.m_val - (uint64_t)m_val < 100)
return false;
if (!other.m_val)
return true;
- int64_t ratio = m_val * 100 / other.m_val;
+ int64_t ratio = (int64_t)m_val * 100 / other.m_val;
return ratio < 99 || ratio > 101;
}
+/* Stream THIS from IB. */
+
profile_count
profile_count::stream_in (struct lto_input_block *ib)
{
profile_count ret;
ret.m_val = streamer_read_gcov_count (ib);
- ret.m_quality = (profile_count_quality) streamer_read_uhwi (ib);
+ ret.m_quality = (profile_quality) streamer_read_uhwi (ib);
return ret;
}
+/* Stream THIS to OB. */
+
void
profile_count::stream_out (struct output_block *ob)
{
streamer_write_uhwi (ob, m_quality);
}
+/* Stream THIS to OB. */
+
void
profile_count::stream_out (struct lto_output_stream *ob)
{
streamer_write_gcov_count_stream (ob, m_val);
streamer_write_uhwi_stream (ob, m_quality);
}
+
+/* Dump THIS to F. */
+
+void
+profile_probability::dump (FILE *f) const
+{
+ if (!initialized_p ())
+ fprintf (f, "uninitialized");
+ else
+ {
+ /* Make difference between 0.00 as a roundoff error and actual 0.
+ Similarly for 1. */
+ if (m_val == 0)
+ fprintf (f, "never");
+ else if (m_val == max_probability)
+ fprintf (f, "always");
+ else
+ fprintf (f, "%3.1f%%", (double)m_val * 100 / max_probability);
+ if (m_quality == profile_adjusted)
+ fprintf (f, " (adjusted)");
+ else if (m_quality == profile_afdo)
+ fprintf (f, " (auto FDO)");
+ else if (m_quality == profile_guessed)
+ fprintf (f, " (guessed)");
+ }
+}
+
+/* Dump THIS to stderr. */
+
+void
+profile_probability::debug () const
+{
+ dump (stderr);
+ fprintf (stderr, "\n");
+}
+
+/* Return true if THIS differs from OTHER; tolerate small diferences. */
+
+bool
+profile_probability::differs_from_p (profile_probability other) const
+{
+ if (!initialized_p () || !other.initialized_p ())
+ return false;
+ if ((uint64_t)m_val - (uint64_t)other.m_val < 10
+ || (uint64_t)other.m_val - (uint64_t)m_val < 10)
+ return false;
+ if (!other.m_val)
+ return true;
+ int64_t ratio = m_val * 100 / other.m_val;
+ return ratio < 99 || ratio > 101;
+}
+
+/* Return true if THIS differs significantly from OTHER. */
+
+bool
+profile_probability::differs_lot_from_p (profile_probability other) const
+{
+ if (!initialized_p () || !other.initialized_p ())
+ return false;
+ uint32_t d = m_val > other.m_val ? m_val - other.m_val : other.m_val - m_val;
+ return d > max_probability / 2;
+}
+
+/* Stream THIS from IB. */
+
+profile_probability
+profile_probability::stream_in (struct lto_input_block *ib)
+{
+ profile_probability ret;
+ ret.m_val = streamer_read_uhwi (ib);
+ ret.m_quality = (profile_quality) streamer_read_uhwi (ib);
+ return ret;
+}
+
+/* Stream THIS to OB. */
+
+void
+profile_probability::stream_out (struct output_block *ob)
+{
+ streamer_write_uhwi (ob, m_val);
+ streamer_write_uhwi (ob, m_quality);
+}
+
+/* Stream THIS to OB. */
+
+void
+profile_probability::stream_out (struct lto_output_stream *ob)
+{
+ streamer_write_uhwi_stream (ob, m_val);
+ streamer_write_uhwi_stream (ob, m_quality);
+}
/* Quality of the proflie count. Because gengtype does not support enums
inside of clases, this is in global namespace. */
-enum profile_count_quality {
+enum profile_quality {
/* Profile is based on static branch prediction heuristics. It may or may
not reflect the reality. */
- count_guessed = 0,
+ profile_guessed = 0,
/* Profile was determined by autofdo. */
- count_afdo = 1,
+ profile_afdo = 1,
/* Profile was originally based on feedback but it was adjusted
by code duplicating optimization. It may not precisely reflect the
particular code path. */
- count_adjusted = 2,
+ profile_adjusted = 2,
/* Profile was read from profile feedback or determined by accurate static
method. */
- count_read = 3
+ profile_precise = 3
};
/* The base value for branch probability notes and edge probabilities. */
#define RDIV(X,Y) (((X) + (Y) / 2) / (Y))
+/* Data type to hold probabilities. It implement fixed point arithmetics
+ with capping so probability is always in range [0,1] and scaling requiring
+ values greater than 1 needs to be represented otherwise.
+
+ In addition to actual value the quality of profile is tracked and propagated
+ through all operations. Special value UNINITIALIZED is used for probabilities
+ that has not been detemrined yet (for example bacause of
+ -fno-guess-branch-probability)
+
+ Typically probabilities are derived from profile feedback (via
+ probability_in_gcov_type), autoFDO or guessed statically and then propagated
+ thorough the compilation.
+
+ Named probabilities are available:
+ - never (0 probability)
+ - guessed_never
+ - very_unlikely (1/2000 probability)
+ - unlikely (1/5 probablity)
+ - even (1/2 probability)
+ - likely (4/5 probability)
+ - very_likely (1999/2000 probability)
+ - guessed_always
+ - always
+
+ Named probabilities except for never/always are assumed to be statically
+ guessed and thus not necessarily acurate. The difference between never
+ and guessedn never is that the first one should be used only in case that
+ well behaving program will very likely not execute the "never" path.
+ For example if the path is going to abort () call or it exception handling.
+
+ Alawyas and guessted_always probabilities are symmetric.
+
+ For legacy code we support conversion to/from REG_BR_PROB_BASE based fixpoint
+ integer arithmetics. Once the code is converted to branch probabiitlies,
+ these conversions will probably go away because they are lossy.
+*/
+
+class GTY((user)) profile_probability
+{
+ /* For now use values in range 0...REG_BR_PROB_BASE. Later we can use full
+ precision of 30 bits available. */
+
+ static const int n_bits = 30;
+ static const uint32_t max_probability = REG_BR_PROB_BASE;
+ static const uint32_t uninitialized_probability = ((uint32_t) 1 << n_bits) - 1;
+
+ uint32_t m_val : 30;
+ enum profile_quality m_quality : 2;
+
+ friend class profile_count;
+public:
+
+ /* Named probabilities. */
+ static profile_probability never ()
+ {
+ profile_probability ret;
+ ret.m_val = 0;
+ ret.m_quality = profile_precise;
+ return ret;
+ }
+ static profile_probability guessed_never ()
+ {
+ profile_probability ret;
+ ret.m_val = 0;
+ ret.m_quality = profile_guessed;
+ return ret;
+ }
+ static profile_probability very_unlikely ()
+ {
+ /* Be consistent with PROB_VERY_UNLIKELY in predict.h. */
+ profile_probability r
+ = profile_probability::always ().apply_scale (1, 2000);
+ r.m_val--;
+ return r;
+ }
+ static profile_probability unlikely ()
+ {
+ /* Be consistent with PROB_VERY_LIKELY in predict.h. */
+ profile_probability r
+ = profile_probability::always ().apply_scale (1, 5);
+ r.m_val--;
+ return r;
+ }
+ static profile_probability even ()
+ {
+ return profile_probability::always ().apply_scale (1, 2);
+ }
+ static profile_probability very_likely ()
+ {
+ return profile_probability::always () - very_unlikely ();
+ }
+ static profile_probability likely ()
+ {
+ return profile_probability::always () - unlikely ();
+ }
+ static profile_probability guessed_always ()
+ {
+ profile_probability ret;
+ ret.m_val = max_probability;
+ ret.m_quality = profile_guessed;
+ return ret;
+ }
+ static profile_probability always ()
+ {
+ profile_probability ret;
+ ret.m_val = max_probability;
+ ret.m_quality = profile_precise;
+ return ret;
+ }
+ /* Probabilities which has not been initialized. Either because
+ initialization did not happen yet or because profile is unknown. */
+ static profile_probability uninitialized ()
+ {
+ profile_probability c;
+ c.m_val = uninitialized_probability;
+ c.m_quality = profile_guessed;
+ return c;
+ }
+
+
+ /* Return true if value has been initialized. */
+ bool initialized_p () const
+ {
+ return m_val != uninitialized_probability;
+ }
+ /* Return true if value can be trusted. */
+ bool reliable_p () const
+ {
+ return initialized_p ();
+ }
+
+ /* Conversion from and to REG_BR_PROB_BASE integer fixpoint arithmetics.
+ this is mostly to support legacy code and hsould go away. */
+ static profile_probability from_reg_br_prob_base (int v)
+ {
+ profile_probability ret;
+ gcc_checking_assert (v >= 0 && v <= REG_BR_PROB_BASE);
+ ret.m_val = RDIV (v * max_probability, REG_BR_PROB_BASE);
+ ret.m_quality = profile_guessed;
+ return ret;
+ }
+ int to_reg_br_prob_base () const
+ {
+ gcc_checking_assert (initialized_p ());
+ return RDIV (m_val * REG_BR_PROB_BASE, max_probability);
+ }
+
+ /* Return VAL1/VAL2. */
+ static profile_probability probability_in_gcov_type
+ (gcov_type val1, gcov_type val2)
+ {
+ profile_probability ret;
+ gcc_checking_assert (val1 >= 0 && val2 > 0);
+ if (val1 > val2)
+ ret.m_val = max_probability;
+ else
+ ret.m_val = RDIV (val1 * max_probability, val2);
+ ret.m_quality = profile_precise;
+ return ret;
+ }
+
+ /* Basic operations. */
+ bool operator== (const profile_probability &other) const
+ {
+ return m_val == other.m_val && m_quality == other.m_quality;
+ }
+ profile_probability operator+ (const profile_probability &other) const
+ {
+ if (other == profile_probability::never ())
+ return *this;
+ if (*this == profile_probability::never ())
+ return other;
+ if (!initialized_p () || !other.initialized_p ())
+ return profile_probability::uninitialized ();
+
+ profile_probability ret;
+ ret.m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
+ ret.m_quality = MIN (m_quality, other.m_quality);
+ return ret;
+ }
+ profile_probability &operator+= (const profile_probability &other)
+ {
+ if (other == profile_probability::never ())
+ return *this;
+ if (*this == profile_probability::never ())
+ {
+ *this = other;
+ return *this;
+ }
+ if (!initialized_p () || !other.initialized_p ())
+ return *this = profile_probability::uninitialized ();
+ else
+ {
+ m_val = MIN ((uint32_t)(m_val + other.m_val), max_probability);
+ m_quality = MIN (m_quality, other.m_quality);
+ }
+ return *this;
+ }
+ profile_probability operator- (const profile_probability &other) const
+ {
+ if (*this == profile_probability::never ()
+ || other == profile_probability::never ())
+ return *this;
+ if (!initialized_p () || !other.initialized_p ())
+ return profile_probability::uninitialized ();
+ profile_probability ret;
+ ret.m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
+ ret.m_quality = MIN (m_quality, other.m_quality);
+ return ret;
+ }
+ profile_probability &operator-= (const profile_probability &other)
+ {
+ if (*this == profile_probability::never ()
+ || other == profile_probability::never ())
+ return *this;
+ if (!initialized_p () || !other.initialized_p ())
+ return *this = profile_probability::uninitialized ();
+ else
+ {
+ m_val = m_val >= other.m_val ? m_val - other.m_val : 0;
+ m_quality = MIN (m_quality, other.m_quality);
+ }
+ return *this;
+ }
+ profile_probability operator* (const profile_probability &other) const
+ {
+ if (*this == profile_probability::never ()
+ || other == profile_probability::never ())
+ return profile_probability::never ();
+ if (!initialized_p () || !other.initialized_p ())
+ return profile_probability::uninitialized ();
+ profile_probability ret;
+ ret.m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
+ ret.m_quality = MIN (m_quality, other.m_quality);
+ return ret;
+ }
+ profile_probability &operator*= (const profile_probability &other)
+ {
+ if (*this == profile_probability::never ()
+ || other == profile_probability::never ())
+ return *this = profile_probability::never ();
+ if (!initialized_p () || !other.initialized_p ())
+ return *this = profile_probability::uninitialized ();
+ else
+ {
+ m_val = RDIV ((uint64_t)m_val * other.m_val, max_probability);
+ m_quality = MIN (m_quality, other.m_quality);
+ }
+ return *this;
+ }
+ profile_probability operator/ (const profile_probability &other) const
+ {
+ if (*this == profile_probability::never ())
+ return profile_probability::never ();
+ if (!initialized_p () || !other.initialized_p ())
+ return profile_probability::uninitialized ();
+ profile_probability ret;
+ if (m_val >= other.m_val)
+ ret.m_val = max_probability;
+ else if (!m_val)
+ ret.m_val = 0;
+ else
+ {
+ gcc_checking_assert (other.m_val);
+ ret.m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
+ other.m_val),
+ max_probability);
+ }
+ ret.m_quality = MIN (m_quality, other.m_quality);
+ return ret;
+ }
+ profile_probability &operator/= (const profile_probability &other)
+ {
+ if (*this == profile_probability::never ())
+ return *this = profile_probability::never ();
+ if (!initialized_p () || !other.initialized_p ())
+ return *this = profile_probability::uninitialized ();
+ else
+ {
+ if (m_val > other.m_val)
+ m_val = max_probability;
+ else if (!m_val)
+ ;
+ else
+ {
+ gcc_checking_assert (other.m_val);
+ m_val = MIN (RDIV ((uint64_t)m_val * max_probability,
+ other.m_val),
+ max_probability);
+ }
+ m_quality = MIN (m_quality, other.m_quality);
+ }
+ return *this;
+ }
+
+ gcov_type apply (gcov_type val) const
+ {
+ if (*this == profile_probability::uninitialized ())
+ return val / 2;
+ return RDIV (val * m_val, max_probability);
+ }
+
+ /* Return 1-*THIS. */
+ profile_probability invert () const
+ {
+ return profile_probability::always() - *this;
+ }
+
+ profile_probability combine_with_freq (int freq1, profile_probability other,
+ int freq2) const
+ {
+ profile_probability ret;
+
+ if (*this == profile_probability::uninitialized ()
+ || other == profile_probability::uninitialized ())
+ return profile_probability::uninitialized ();
+
+ gcc_checking_assert (freq1 >= 0 && freq2 >= 0);
+ if (!freq1 && !freq2)
+ {
+ ret.m_val = (m_val + other.m_val) / 2;
+ }
+ else
+ ret.m_val = RDIV (m_val * (uint64_t) freq1
+ + other.m_val * (uint64_t) freq2, freq1 + freq2);
+ ret.m_quality = MIN (m_quality, other.m_quality);
+ return ret;
+ }
+
+ /* Return *THIS * NUM / DEN. */
+ profile_probability apply_scale (int64_t num, int64_t den) const
+ {
+ if (*this == profile_probability::never ())
+ return *this;
+ if (!initialized_p ())
+ return profile_probability::uninitialized ();
+ profile_probability ret;
+ ret.m_val = MIN (RDIV (m_val * num, den),
+ max_probability);
+ ret.m_quality = MIN (m_quality, profile_adjusted);
+ return ret;
+ }
+
+ /* Return true when the probability of edge is reliable.
+
+ The profile guessing code is good at predicting branch outcome (ie.
+ taken/not taken), that is predicted right slightly over 75% of time.
+ It is however notoriously poor on predicting the probability itself.
+ In general the profile appear a lot flatter (with probabilities closer
+ to 50%) than the reality so it is bad idea to use it to drive optimization
+ such as those disabling dynamic branch prediction for well predictable
+ branches.
+
+ There are two exceptions - edges leading to noreturn edges and edges
+ predicted by number of iterations heuristics are predicted well. This macro
+ should be able to distinguish those, but at the moment it simply check for
+ noreturn heuristic that is only one giving probability over 99% or bellow
+ 1%. In future we might want to propagate reliability information across the
+ CFG if we find this information useful on multiple places. */
+
+ bool probably_reliable_p () const
+ {
+ if (m_quality >= profile_adjusted)
+ return true;
+ if (!initialized_p ())
+ return false;
+ return m_val < max_probability / 100
+ || m_val > max_probability - max_probability / 100;
+ }
+
+ /* Return false if profile_probability is bogus. */
+ bool verify () const
+ {
+ if (m_val == uninitialized_probability)
+ return m_quality == profile_guessed;
+ else
+ return m_val <= REG_BR_PROB_BASE;
+ }
+
+ /* Comparsions are three-state and conservative. False is returned if
+ the inequality can not be decided. */
+ bool operator< (const profile_probability &other) const
+ {
+ return initialized_p () && other.initialized_p () && m_val < other.m_val;
+ }
+ bool operator> (const profile_probability &other) const
+ {
+ return initialized_p () && other.initialized_p () && m_val > other.m_val;
+ }
+
+ bool operator<= (const profile_probability &other) const
+ {
+ return initialized_p () && other.initialized_p () && m_val <= other.m_val;
+ }
+ bool operator>= (const profile_probability &other) const
+ {
+ return initialized_p () && other.initialized_p () && m_val >= other.m_val;
+ }
+
+ /* Output THIS to F. */
+ void dump (FILE *f) const;
+
+ /* Print THIS to stderr. */
+ void debug () const;
+
+ /* Return true if THIS is known to differ significantly from OTHER. */
+ bool differs_from_p (profile_probability other) const;
+ /* Return if difference is greater than 50%. */
+ bool differs_lot_from_p (profile_probability other) const;
+
+ /* LTO streaming support. */
+ static profile_probability stream_in (struct lto_input_block *);
+ void stream_out (struct output_block *);
+ void stream_out (struct lto_output_stream *);
+};
+
/* Main data type to hold profile counters in GCC. In most cases profile
counts originate from profile feedback. They are 64bit integers
representing number of executions during the train run.
static const uint64_t uninitialized_count = ((uint64_t) 1 << n_bits) - 1;
uint64_t m_val : n_bits;
- enum profile_count_quality m_quality : 2;
+ enum profile_quality m_quality : 2;
/* Assume numbers smaller than this to multiply. This is set to make
testsuite pass, in future we may implement precise multiplication in higer
{
profile_count c;
c.m_val = uninitialized_count;
- c.m_quality = count_guessed;
+ c.m_quality = profile_guessed;
return c;
}
profile_count ret;
gcc_checking_assert (v >= 0 && (uint64_t) v <= max_count);
ret.m_val = v;
- ret.m_quality = count_read;
+ ret.m_quality = profile_precise;
return ret;
}
/* Return false if profile_count is bogus. */
bool verify () const
{
- return m_val != uninitialized_count || m_quality == count_guessed;
+ return m_val != uninitialized_count || m_quality == profile_guessed;
}
/* Comparsions are three-state and conservative. False is returned if
}
bool operator>= (const profile_count &other) const
{
- return initialized_p () && m_val >= other.m_val;
+ return initialized_p () && other.initialized_p () && m_val >= other.m_val;
}
bool operator<= (const gcov_type other) const
{
return profile_count::uninitialized ();
profile_count ret;
ret.m_val = RDIV (m_val * prob, REG_BR_PROB_BASE);
- ret.m_quality = MIN (m_quality, count_adjusted);
+ ret.m_quality = MIN (m_quality, profile_adjusted);
+ return ret;
+ }
+
+ /* Scale counter according to PROB. */
+ profile_count apply_probability (profile_probability prob) const
+ {
+ if (*this == profile_count::zero ())
+ return *this;
+ if (prob == profile_probability::never ())
+ return profile_count::zero ();
+ if (!initialized_p ())
+ return profile_count::uninitialized ();
+ profile_count ret;
+ ret.m_val = RDIV (m_val * prob.m_val,
+ profile_probability::max_probability);
+ ret.m_quality = MIN (m_quality, prob.m_quality);
return ret;
}
/* Return *THIS * NUM / DEN. */
gcc_checking_assert ((num <= REG_BR_PROB_BASE
|| den <= REG_BR_PROB_BASE) || 1);
ret.m_val = RDIV (m_val * num, den);
- ret.m_quality = MIN (m_quality, count_adjusted);
+ ret.m_quality = MIN (m_quality, profile_adjusted);
return ret;
}
profile_count apply_scale (profile_count num, profile_count den) const
else
ret.m_val = RDIV (m_val * RDIV (num.m_val * max_safe_multiplier,
den.m_val), max_safe_multiplier);
- ret.m_quality = MIN (m_quality, count_adjusted);
+ ret.m_quality = MIN (m_quality, profile_adjusted);
return ret;
}
/* Return probability of event with counter THIS within event with counter
OVERALL. */
- int probability_in (profile_count overall)
+ profile_probability probability_in (const profile_count overall) const
{
if (!m_val)
- return 0;
- if (!initialized_p () || !overall.initialized_p ())
- return REG_BR_PROB_BASE / 2;
- if (overall < *this)
- return REG_BR_PROB_BASE;
- if (!overall.m_val)
- return REG_BR_PROB_BASE / 2;
- return RDIV (m_val * REG_BR_PROB_BASE, overall.m_val);
+ return profile_probability::never ();
+ if (!initialized_p () || !overall.initialized_p ()
+ || !overall.m_val)
+ return profile_probability::uninitialized ();
+ profile_probability ret;
+ if (overall < m_val)
+ ret.m_val = profile_probability::max_probability;
+ else
+ ret.m_val = RDIV (m_val * profile_probability::max_probability,
+ overall.m_val);
+ ret.m_quality = MIN (m_quality, overall.m_quality);
+ return ret;
}
/* Output THIS to F. */
if (bb_gcov_count (bb))
{
FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = GCOV_COMPUTE_SCALE (edge_gcov_count (e),
- bb_gcov_count (bb));
+ e->probability = profile_probability::probability_in_gcov_type
+ (edge_gcov_count (e), bb_gcov_count (bb));
if (bb->index >= NUM_FIXED_BLOCKS
&& block_ends_with_condjump_p (bb)
&& EDGE_COUNT (bb->succs) >= 2)
if (!(e->flags & (EDGE_FAKE | EDGE_FALLTHRU)))
break;
- prob = e->probability;
+ prob = e->probability.to_reg_br_prob_base ();
index = prob * 20 / REG_BR_PROB_BASE;
if (index == 20)
{
FOR_EACH_EDGE (e, ei, bb->succs)
if (!(e->flags & (EDGE_COMPLEX | EDGE_FAKE)))
- e->probability = REG_BR_PROB_BASE / total;
+ e->probability
+ = profile_probability::guessed_always ().apply_scale (1, total);
else
- e->probability = 0;
+ e->probability = profile_probability::never ();
}
else
{
total += EDGE_COUNT (bb->succs);
FOR_EACH_EDGE (e, ei, bb->succs)
- e->probability = REG_BR_PROB_BASE / total;
+ e->probability
+ = profile_probability::guessed_always ().apply_scale (1, total);
}
if (bb->index >= NUM_FIXED_BLOCKS
&& block_ends_with_condjump_p (bb)
flags);
nehe->probability = eh_edge->probability;
- nfte->probability
- = REG_BR_PROB_BASE - nehe->probability;
+ nfte->probability = nehe->probability.invert ();
peep2_do_cleanup_cfg |= purge_dead_edges (nfte->dest);
bb = nfte->src;
e = find_fallthru_edge (bb->succs);
if (! e)
break;
- if (e->probability <= probability_cutoff)
+ if (e->probability.initialized_p ()
+ && e->probability.to_reg_br_prob_base () <= probability_cutoff)
break;
if (e->dest->flags & BB_DISABLE_SCHEDULE)
break;
e = find_fallthru_edge (bb->succs);
if (! e)
break;
- if (e->probability <= probability_cutoff)
+ if (e->probability.initialized_p ()
+ && e->probability.to_reg_br_prob_base () <= probability_cutoff)
break;
}
FOR_EACH_EDGE (out_edge, out_ei, in_edge->src->succs)
bitmap_set_bit (pot_split[bb], EDGE_TO_BIT (out_edge));
- prob[bb] += combine_probabilities (prob[pred_bb], in_edge->probability);
+ prob[bb] += combine_probabilities
+ (prob[pred_bb],
+ in_edge->probability.initialized_p ()
+ ? in_edge->probability.to_reg_br_prob_base ()
+ : 0);
// The rounding divide in combine_probabilities can result in an extra
// probability increment propagating along 50-50 edges. Eventually when
// the edges re-merge, the accumulated probability can go slightly above
sched_rgn_n_insns += sched_n_insns;
realloc_bb_state_array (saved_last_basic_block);
f = find_fallthru_edge (last_bb->succs);
- if (f && f->probability * 100 / REG_BR_PROB_BASE >=
- PARAM_VALUE (PARAM_SCHED_STATE_EDGE_PROB_CUTOFF))
+ if (f
+ && (!f->probability.initialized_p ()
+ || f->probability.to_reg_br_prob_base () * 100 / REG_BR_PROB_BASE >=
+ PARAM_VALUE (PARAM_SCHED_STATE_EDGE_PROB_CUTOFF)))
{
memcpy (bb_state[f->dest->index], curr_state,
dfa_state_size);
sinfo->probs_ok.safe_push (
/* FIXME: Improve calculation when skipping
inner loop to exits. */
- si.bb_end ? si.e1->probability : REG_BR_PROB_BASE);
+ si.bb_end && si.e1->probability.initialized_p ()
+ ? si.e1->probability.to_reg_br_prob_base ()
+ : REG_BR_PROB_BASE);
sinfo->succs_ok_n++;
}
else
/* Compute all_prob. */
if (!si.bb_end)
sinfo->all_prob = REG_BR_PROB_BASE;
- else
- sinfo->all_prob += si.e1->probability;
+ else if (si.e1->probability.initialized_p ())
+ sinfo->all_prob += si.e1->probability.to_reg_br_prob_base ();
sinfo->all_succs_n++;
}
tree low; /* Lowest index value for this label */
tree high; /* Highest index value for this label */
tree code_label; /* Label to jump to when node matches */
- int prob; /* Probability of taking this case. */
+ profile_probability prob; /* Probability of taking this case. */
/* Probability of reaching subtree rooted at this node */
- int subtree_prob;
+ profile_probability subtree_prob;
};
typedef struct case_node *case_node_ptr;
static int node_has_low_bound (case_node_ptr, tree);
static int node_has_high_bound (case_node_ptr, tree);
static int node_is_bounded (case_node_ptr, tree);
-static void emit_case_nodes (rtx, case_node_ptr, rtx_code_label *, int, tree);
+static void emit_case_nodes (rtx, case_node_ptr, rtx_code_label *,
+ profile_probability, tree);
\f
/* Return the rtx-label that corresponds to a LABEL_DECL,
creating it if necessary. */
is the probability of jumping to LABEL. */
static void
do_jump_if_equal (machine_mode mode, rtx op0, rtx op1, rtx_code_label *label,
- int unsignedp, int prob)
+ int unsignedp, profile_probability prob)
{
- gcc_assert (prob <= REG_BR_PROB_BASE);
do_compare_rtx_and_jump (op0, op1, EQ, unsignedp, mode,
NULL_RTX, NULL, label, prob);
}
static struct case_node *
add_case_node (struct case_node *head, tree low, tree high,
- tree label, int prob,
+ tree label, profile_probability prob,
object_allocator<case_node> &case_node_pool)
{
struct case_node *r;
static void
emit_case_decision_tree (tree index_expr, tree index_type,
case_node_ptr case_list, rtx_code_label *default_label,
- int default_prob)
+ profile_probability default_prob)
{
rtx index = expand_normal (index_expr);
/* Return the sum of probabilities of outgoing edges of basic block BB. */
-static int
+static profile_probability
get_outgoing_edge_probs (basic_block bb)
{
edge e;
edge_iterator ei;
- int prob_sum = 0;
+ profile_probability prob_sum = profile_probability::never ();
if (!bb)
- return 0;
+ return profile_probability::never ();
FOR_EACH_EDGE (e, ei, bb->succs)
prob_sum += e->probability;
return prob_sum;
BASE_PROB is the probability of reaching the branch instruction relative
to the same basic block BB. */
-static inline int
-conditional_probability (int target_prob, int base_prob)
+static inline profile_probability
+conditional_probability (profile_probability target_prob,
+ profile_probability base_prob)
{
- if (base_prob > 0)
- {
- gcc_assert (target_prob >= 0);
- gcc_assert (target_prob <= base_prob);
- return GCOV_COMPUTE_SCALE (target_prob, base_prob);
- }
- return -1;
+ return target_prob / base_prob;
}
/* Generate a dispatch tabler, switching on INDEX_EXPR and jumping to
rtx_code_label *table_label = gen_label_rtx ();
bool has_gaps = false;
edge default_edge = stmt_bb ? EDGE_SUCC (stmt_bb, 0) : NULL;
- int default_prob = default_edge ? default_edge->probability : 0;
- int base = get_outgoing_edge_probs (stmt_bb);
+ profile_probability default_prob = default_edge ? default_edge->probability
+ : profile_probability::never ();
+ profile_probability base = get_outgoing_edge_probs (stmt_bb);
bool try_with_tablejump = false;
- int new_default_prob = conditional_probability (default_prob,
- base);
+ profile_probability new_default_prob = conditional_probability (default_prob,
+ base);
if (! try_casesi (index_type, index_expr, minval, range,
table_label, default_label, fallback_label,
through the indirect jump or the direct conditional jump
before that. Split the probability of reaching the
default label among these two jumps. */
- new_default_prob = conditional_probability (default_prob/2,
+ new_default_prob = conditional_probability (default_prob.apply_scale
+ (1, 2),
base);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
base -= default_prob;
}
else
{
base -= default_prob;
- default_prob = 0;
+ default_prob = profile_probability::never ();
}
if (default_edge)
/* We have altered the probability of the default edge. So the probabilities
of all other edges need to be adjusted so that it sums up to
REG_BR_PROB_BASE. */
- if (base)
+ if (base > profile_probability::never ())
{
edge e;
edge_iterator ei;
FOR_EACH_EDGE (e, ei, stmt_bb->succs)
- e->probability = GCOV_COMPUTE_SCALE (e->probability, base);
+ e->probability /= base;
}
if (try_with_tablejump)
default_label = jump_target_rtx
(CASE_LABEL (gimple_switch_default_label (stmt)));
edge default_edge = EDGE_SUCC (bb, 0);
- int default_prob = default_edge->probability;
+ profile_probability default_prob = default_edge->probability;
/* Get upper and lower bounds of case values. */
elt = gimple_switch_label (stmt, 1);
edge case_edge = find_edge (bb, case_bb);
case_list = add_case_node (
case_list, low, high, lab,
- case_edge->probability / (intptr_t)(case_edge->aux),
+ case_edge->probability.apply_scale (1, (intptr_t)(case_edge->aux)),
case_node_pool);
}
reset_out_edges_aux (bb);
{
tree elt = dispatch_table[i];
rtx_code_label *lab = jump_target_rtx (CASE_LABEL (elt));
- do_jump_if_equal (index_mode, index, zero, lab, 0, -1);
+ do_jump_if_equal (index_mode, index, zero, lab, 0,
+ profile_probability::uninitialized ());
force_expand_binop (index_mode, sub_optab,
index, CONST1_RTX (index_mode),
index, 0, OPTAB_DIRECT);
tree elt = dispatch_table[i];
tree low = CASE_LOW (elt);
tree lab = CASE_LABEL (elt);
- case_list = add_case_node (case_list, low, low, lab, 0, case_node_pool);
+ case_list = add_case_node (case_list, low, low, lab,
+ profile_probability::guessed_always ()
+ .apply_scale (1, ncases),
+ case_node_pool);
}
emit_case_dispatch_table (index_expr, index_type,
static void
emit_case_nodes (rtx index, case_node_ptr node, rtx_code_label *default_label,
- int default_prob, tree index_type)
+ profile_probability default_prob, tree index_type)
{
/* If INDEX has an unsigned type, we must make unsigned branches. */
int unsignedp = TYPE_UNSIGNED (index_type);
- int probability;
- int prob = node->prob, subtree_prob = node->subtree_prob;
+ profile_probability probability;
+ profile_probability prob = node->prob, subtree_prob = node->subtree_prob;
machine_mode mode = GET_MODE (index);
machine_mode imode = TYPE_MODE (index_type);
subtree or the left subtree. Divide the probability
equally. */
probability = conditional_probability (
- node->right->subtree_prob + default_prob/2,
+ node->right->subtree_prob + default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
/* See if the value is on the right. */
emit_cmp_and_jump_insns (index,
GT, NULL_RTX, mode, unsignedp,
label_rtx (test_label),
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
/* Value must be on the left.
Handle the left-hand subtree. */
if (!node_has_low_bound (node, index_type))
{
probability = conditional_probability (
- default_prob/2,
+ default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
emit_cmp_and_jump_insns (index,
convert_modes
LT, NULL_RTX, mode, unsignedp,
default_label,
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
}
emit_case_nodes (index, node->right, default_label, default_prob, index_type);
if (!node_has_high_bound (node, index_type))
{
probability = conditional_probability (
- default_prob/2,
+ default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
emit_cmp_and_jump_insns (index,
convert_modes
GT, NULL_RTX, mode, unsignedp,
default_label,
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
}
emit_case_nodes (index, node->left, default_label,
test_label = build_decl (curr_insn_location (),
LABEL_DECL, NULL_TREE, void_type_node);
probability = conditional_probability (
- node->right->subtree_prob + default_prob/2,
+ node->right->subtree_prob + default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
emit_cmp_and_jump_insns (index,
convert_modes
GT, NULL_RTX, mode, unsignedp,
label_rtx (test_label),
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
}
/* Value belongs to this node or to the left-hand subtree. */
if (!node_has_low_bound (node, index_type))
{
probability = conditional_probability (
- default_prob/2,
+ default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
emit_cmp_and_jump_insns (index,
convert_modes
LT, NULL_RTX, mode, unsignedp,
default_label,
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
}
/* Value belongs to this node or to the right-hand subtree. */
if (!node_has_high_bound (node, index_type))
{
probability = conditional_probability (
- default_prob/2,
+ default_prob.apply_scale (1, 2),
subtree_prob + default_prob);
emit_cmp_and_jump_insns (index,
convert_modes
GT, NULL_RTX, mode, unsignedp,
default_label,
probability);
- default_prob /= 2;
+ default_prob = default_prob.apply_scale (1, 2);
}
/* Value belongs to this node or to the left-hand subtree. */
#include "stringpool.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
+#include "profile-count.h"
#include "optabs.h"
#include "regs.h"
#include "recog.h"
if (e1->count.initialized_p () && e2->count.initialized_p ()
&& !(e1->count == e2->count))
return e1->count > e2->count;
- if (e1->src->frequency * e1->probability !=
- e2->src->frequency * e2->probability)
- return (e1->src->frequency * e1->probability
- > e2->src->frequency * e2->probability);
+ if (EDGE_FREQUENCY (e1) != EDGE_FREQUENCY (e2))
+ return EDGE_FREQUENCY (e1) > EDGE_FREQUENCY (e2);
/* This is needed to avoid changes in the decision after
CFG is modified. */
if (e1->src != e2->src)
best = e;
if (!best || ignore_bb_p (best->dest))
return NULL;
- if (best->probability <= probability_cutoff)
+ if (best->probability.initialized_p ()
+ && best->probability.to_reg_br_prob_base () <= probability_cutoff)
return NULL;
return best;
}
join_bb->frequency = test_bb->frequency = transaction_bb->frequency;
join_bb->count = test_bb->count = transaction_bb->count;
- ei->probability = PROB_ALWAYS;
- et->probability = PROB_LIKELY;
- ef->probability = PROB_UNLIKELY;
+ ei->probability = profile_probability::always ();
+ et->probability = profile_probability::likely ();
+ ef->probability = profile_probability::unlikely ();
et->count = test_bb->count.apply_probability (et->probability);
ef->count = test_bb->count.apply_probability (ef->probability);
edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU);
test_bb->frequency = transaction_bb->frequency;
test_bb->count = transaction_bb->count;
- ei->probability = PROB_ALWAYS;
+ ei->probability = profile_probability::always ();
// Not abort edge. If both are live, chose one at random as we'll
// we'll be fixing that up below.
redirect_edge_pred (fallthru_edge, test_bb);
fallthru_edge->flags = EDGE_FALSE_VALUE;
- fallthru_edge->probability = PROB_VERY_LIKELY;
+ fallthru_edge->probability = profile_probability::very_likely ();
fallthru_edge->count = test_bb->count.apply_probability
(fallthru_edge->probability);
// Abort/over edge.
redirect_edge_pred (abort_edge, test_bb);
abort_edge->flags = EDGE_TRUE_VALUE;
- abort_edge->probability = PROB_VERY_UNLIKELY;
+ abort_edge->probability = profile_probability::unlikely ();
abort_edge->count = test_bb->count.apply_probability
(abort_edge->probability);
// use the uninst path when falling back to serial mode.
redirect_edge_pred (inst_edge, test_bb);
inst_edge->flags = EDGE_FALSE_VALUE;
- inst_edge->probability = REG_BR_PROB_BASE / 2;
+ inst_edge->probability = profile_probability::even ();
inst_edge->count
= test_bb->count.apply_probability (inst_edge->probability);
redirect_edge_pred (uninst_edge, test_bb);
uninst_edge->flags = EDGE_TRUE_VALUE;
- uninst_edge->probability = REG_BR_PROB_BASE / 2;
+ uninst_edge->probability = profile_probability::even ();
uninst_edge->count
= test_bb->count.apply_probability (uninst_edge->probability);
}
return;
}
-
-/* Probability of the branch (to the call) is taken. */
-#define ERR_PROB 0.01
-
/* Shrink-wrap BI_CALL so that it is only called when one of the NCONDS
conditions in CONDS is false. */
basic_block src_bb = call_edge->src;
gcc_assert (src_bb == nocall_edge->src);
- call_edge->probability = REG_BR_PROB_BASE * ERR_PROB;
+ call_edge->probability = profile_probability::very_unlikely ();
call_edge->count
= src_bb->count.apply_probability (call_edge->probability);
- nocall_edge->probability = inverse_probability (call_edge->probability);
+ nocall_edge->probability = profile_probability::always ()
+ - call_edge->probability;
nocall_edge->count = src_bb->count - call_edge->count;
- unsigned int call_frequency = apply_probability (src_bb->frequency,
- call_edge->probability);
+ unsigned int call_frequency
+ = call_edge->probability.apply (src_bb->frequency);
bi_call_bb->count += call_edge->count;
bi_call_bb->frequency += call_frequency;
new_bb = create_empty_bb (after_bb);
new_bb->frequency = EDGE_FREQUENCY (edge_in);
new_bb->count = edge_in->count;
- new_edge = make_edge (new_bb, dest, EDGE_FALLTHRU);
- new_edge->probability = REG_BR_PROB_BASE;
- new_edge->count = edge_in->count;
+ new_edge = make_single_succ_edge (new_bb, dest, EDGE_FALLTHRU);
e = redirect_edge_and_branch (edge_in, new_bb);
gcc_assert (e == edge_in);
basic_block after, bb, *entry_pred, *exit_succ, abb;
struct function *saved_cfun = cfun;
int *entry_flag, *exit_flag;
- unsigned *entry_prob, *exit_prob;
+ profile_probability *entry_prob, *exit_prob;
unsigned i, num_entry_edges, num_exit_edges, num_nodes;
edge e;
edge_iterator ei;
num_entry_edges = EDGE_COUNT (entry_bb->preds);
entry_pred = XNEWVEC (basic_block, num_entry_edges);
entry_flag = XNEWVEC (int, num_entry_edges);
- entry_prob = XNEWVEC (unsigned, num_entry_edges);
+ entry_prob = XNEWVEC (profile_probability, num_entry_edges);
i = 0;
for (ei = ei_start (entry_bb->preds); (e = ei_safe_edge (ei)) != NULL;)
{
num_exit_edges = EDGE_COUNT (exit_bb->succs);
exit_succ = XNEWVEC (basic_block, num_exit_edges);
exit_flag = XNEWVEC (int, num_exit_edges);
- exit_prob = XNEWVEC (unsigned, num_exit_edges);
+ exit_prob = XNEWVEC (profile_probability, num_exit_edges);
i = 0;
for (ei = ei_start (exit_bb->succs); (e = ei_safe_edge (ei)) != NULL;)
{
}
if (!warned)
fold_undefer_and_ignore_overflow_warnings ();
- if (taken_edge->probability > REG_BR_PROB_BASE)
- taken_edge->probability = REG_BR_PROB_BASE;
}
else
taken_edge = single_succ_edge (bb);
}
gcc_assert (EDGE_COUNT (bb->succs) == 0);
- e = make_edge (bb, new_bb, EDGE_FALLTHRU);
- e->count = bb->count;
- e->probability = REG_BR_PROB_BASE;
+ e = make_single_succ_edge (bb, new_bb, EDGE_FALLTHRU);
}
else
{
e = single_succ_edge (bb);
gcc_assert (e->flags & EDGE_EH);
e->flags = (e->flags & ~EDGE_EH) | EDGE_FALLTHRU;
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = bb->count;
/* If there are no more EH users of the landing pad, delete it. */
/* Clean up E_OUT for the fallthru. */
e_out->flags = (e_out->flags & ~EDGE_EH) | EDGE_FALLTHRU;
- e_out->probability = REG_BR_PROB_BASE;
+ e_out->probability = profile_probability::always ();
e_out->count = e_out->src->count;
}
/* At this point we invalidate porfile confistency until IFN_LOOP_VECTORIZED
is re-merged in the vectorizer. */
new_loop = loop_version (loop, cond, &cond_bb,
- REG_BR_PROB_BASE, REG_BR_PROB_BASE,
+ profile_probability::always (),
+ profile_probability::always (),
REG_BR_PROB_BASE, REG_BR_PROB_BASE, true);
free_original_copy_tables ();
}
}
+ bool update_probs = false;
+
if (gimple_code (copy_stmt) == GIMPLE_EH_DISPATCH)
- make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt));
+ {
+ make_eh_dispatch_edges (as_a <geh_dispatch *> (copy_stmt));
+ update_probs = true;
+ }
else if (can_throw)
- make_eh_edges (copy_stmt);
+ {
+ make_eh_edges (copy_stmt);
+ update_probs = true;
+ }
+
+ /* EH edges may not match old edges. Copy as much as possible. */
+ if (update_probs)
+ {
+ edge e;
+ edge_iterator ei;
+ basic_block copy_stmt_bb = gimple_bb (copy_stmt);
+
+ FOR_EACH_EDGE (old_edge, ei, bb->succs)
+ if ((old_edge->flags & EDGE_EH)
+ && (e = find_edge (copy_stmt_bb,
+ (basic_block) old_edge->dest->aux))
+ && (e->flags & EDGE_EH))
+ {
+ e->probability = old_edge->probability;
+ e->count = old_edge->count;
+ }
+
+ FOR_EACH_EDGE (e, ei, copy_stmt_bb->succs)
+ if ((e->flags & EDGE_EH) && !e->probability.initialized_p ())
+ {
+ e->probability = profile_probability::never ();
+ e->count = profile_count::zero ();
+ }
+ }
+
/* If the call we inline cannot make abnormal goto do not add
additional abnormal edges but only retain those already present
&& gimple_call_arg (copy_stmt, 0) == boolean_true_node)
nonlocal_goto = false;
else
- make_edge (copy_stmt_bb, abnormal_goto_dest, EDGE_ABNORMAL);
+ make_single_succ_edge (copy_stmt_bb, abnormal_goto_dest,
+ EDGE_ABNORMAL);
}
if ((can_throw || nonlocal_goto)
if (new_entry)
{
edge e = make_edge (entry_block_map, (basic_block)new_entry->aux, EDGE_FALLTHRU);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = incoming_count;
}
gcc_assert (exit == single_dom_exit (loop));
guard = make_edge (for_bb, ex_bb, 0);
+ /* FIXME: What is the probability? */
+ guard->probability = profile_probability::guessed_never ();
/* Split the latch edge, so LOOPS_HAVE_SIMPLE_LATCHES is still valid. */
loop->latch = split_edge (single_succ_edge (loop->latch));
single_pred_edge (loop->latch)->flags = 0;
- end = make_edge (single_pred (loop->latch), ex_bb, EDGE_FALLTHRU);
+ end = make_single_succ_edge (single_pred (loop->latch), ex_bb, EDGE_FALLTHRU);
rescan_loop_exit (end, true, false);
for (gphi_iterator gpi = gsi_start_phis (ex_bb);
/* We assume that the loop usually iterates a lot. */
prob = 4 * REG_BR_PROB_BASE / 5;
loop_version (loop, many_iterations_cond, NULL,
- prob, REG_BR_PROB_BASE - prob,
+ profile_probability::from_reg_br_prob_base (prob),
+ profile_probability::from_reg_br_prob_base
+ (REG_BR_PROB_BASE - prob),
prob, REG_BR_PROB_BASE - prob, true);
update_ssa (TODO_update_ssa);
free_original_copy_tables ();
gsi_insert_after (&gsi2, cond, GSI_NEW_STMT);
edge e3 = make_edge (bb, bb3, EDGE_FALSE_VALUE);
+ /* FIXME: What is the probability? */
+ e3->probability = profile_probability::guessed_never ();
e->flags = EDGE_TRUE_VALUE;
tree vdef = gimple_vdef (stmt);
edge true_edge = single_succ_edge (cond_bb);
true_edge->flags = EDGE_TRUE_VALUE;
- int probability;
+ profile_probability probability;
if (DECL_VIRTUAL_P (current_function_decl))
- probability = PROB_VERY_LIKELY;
+ probability = profile_probability::very_likely ();
else
- probability = PROB_UNLIKELY;
+ probability = profile_probability::unlikely ();
true_edge->probability = probability;
edge e = make_edge (cond_bb, single_succ_edge (update_bb)->dest,
EDGE_FALSE_VALUE);
- e->probability = REG_BR_PROB_BASE - true_edge->probability;
+ e->probability = true_edge->probability.invert ();
/* Insert code:
edge true_edge = single_succ_edge (cond_bb);
true_edge->flags = EDGE_TRUE_VALUE;
- true_edge->probability = PROB_UNLIKELY;
+ true_edge->probability = profile_probability::unlikely ();
edge e
= make_edge (cond_bb, single_succ_edge (update_bb)->dest, EDGE_FALSE_VALUE);
- e->probability = REG_BR_PROB_BASE - true_edge->probability;
+ e->probability = true_edge->probability.invert ();
gimple_stmt_iterator gsi = gsi_start_bb (cond_bb);
tree original_ref = tree_coverage_counter_ref (tag, base);
e = e2;
}
gcc_assert (e);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = bb->count;
/* The edge is no longer associated with a conditional, so it does
inner_taken->count += outer2->count;
outer2->count = profile_count::zero ();
- inner_taken->probability = outer2->probability
- + RDIV (outer_to_inner->probability
- * inner_taken->probability,
- REG_BR_PROB_BASE);
- if (inner_taken->probability > REG_BR_PROB_BASE)
- inner_taken->probability = REG_BR_PROB_BASE;
- inner_not_taken->probability = REG_BR_PROB_BASE
+ inner_taken->probability = outer2->probability + outer_to_inner->probability
+ * inner_taken->probability;
+ inner_not_taken->probability = profile_probability::always ()
- inner_taken->probability;
- outer_to_inner->probability = REG_BR_PROB_BASE;
+ outer_to_inner->probability = profile_probability::always ();
inner_cond_bb->frequency = outer_cond_bb->frequency;
- outer2->probability = 0;
+ outer2->probability = profile_probability::never ();
}
/* If-convert on a and pattern with a common else block. The inner
int freq_sum = 0;
profile_count count_sum = profile_count::zero ();
int nbbs = 0, ncount = 0;
- int flag_probability = -1;
+ profile_probability flag_probability = profile_probability::uninitialized ();
/* Flag is set in FLAG_BBS. Determine probability that flag will be true
at loop exit.
if ((*it)->count.initialized_p ())
count_sum += (*it)->count, ncount ++;
if (dominated_by_p (CDI_DOMINATORS, ex->src, *it))
- flag_probability = REG_BR_PROB_BASE;
+ flag_probability = profile_probability::always ();
nbbs++;
}
- if (flag_probability != -1)
+ profile_probability cap = profile_probability::always ().apply_scale (2, 3);
+
+ if (flag_probability.initialized_p ())
;
else if (ncount == nbbs && count_sum > 0 && preheader->count >= count_sum)
{
flag_probability = count_sum.probability_in (preheader->count);
- if (flag_probability > REG_BR_PROB_BASE * 2 / 3)
- flag_probability = REG_BR_PROB_BASE * 2 / 3;
+ if (flag_probability > cap)
+ flag_probability = cap;
}
else if (freq_sum > 0 && EDGE_FREQUENCY (preheader) >= freq_sum)
{
- flag_probability = GCOV_COMPUTE_SCALE (freq_sum,
- EDGE_FREQUENCY (preheader));
- if (flag_probability > REG_BR_PROB_BASE * 2 / 3)
- flag_probability = REG_BR_PROB_BASE * 2 / 3;
+ flag_probability = profile_probability::from_reg_br_prob_base
+ (GCOV_COMPUTE_SCALE (freq_sum, EDGE_FREQUENCY (preheader)));
+ if (flag_probability > cap)
+ flag_probability = cap;
}
else
- flag_probability = REG_BR_PROB_BASE * 2 / 3;
+ flag_probability = cap;
/* ?? Insert store after previous store if applicable. See note
below. */
old_dest = ex->dest;
new_bb = split_edge (ex);
then_bb = create_empty_bb (new_bb);
- then_bb->frequency = apply_probability (new_bb->frequency, flag_probability);
+ then_bb->frequency = flag_probability.apply (new_bb->frequency);
then_bb->count = new_bb->count.apply_probability (flag_probability);
if (irr)
then_bb->flags = BB_IRREDUCIBLE_LOOP;
e1->flags |= EDGE_FALSE_VALUE | (irr ? EDGE_IRREDUCIBLE_LOOP : 0);
e1->flags &= ~EDGE_FALLTHRU;
- e1->probability = REG_BR_PROB_BASE - flag_probability;
+ e1->probability = flag_probability.invert ();
e1->count = new_bb->count - then_bb->count;
- then_old_edge = make_edge (then_bb, old_dest,
+ then_old_edge = make_single_succ_edge (then_bb, old_dest,
EDGE_FALLTHRU | (irr ? EDGE_IRREDUCIBLE_LOOP : 0));
- then_old_edge->probability = REG_BR_PROB_BASE;
- then_old_edge->count = then_bb->count;
set_immediate_dominator (CDI_DOMINATORS, then_bb, new_bb);
}
if (!loop_exit_edge_p (loop, exit_edge))
exit_edge = EDGE_SUCC (bb, 1);
- exit_edge->probability = REG_BR_PROB_BASE;
+ exit_edge->probability = profile_probability::always ();
exit_edge->count = exit_edge->src->count;
gcc_checking_assert (loop_exit_edge_p (loop, exit_edge));
gcond *cond_stmt = as_a <gcond *> (elt->stmt);
it in. */
stmt = gimple_build_call (builtin_decl_implicit (BUILT_IN_UNREACHABLE), 0);
latch_edge = make_edge (latch, create_basic_block (NULL, NULL, latch), flags);
- latch_edge->probability = 0;
+ latch_edge->probability = profile_probability::never ();
latch_edge->count = profile_count::zero ();
latch_edge->flags |= flags;
latch_edge->goto_locus = locus;
}
int scale = 1;
if (loop->header->count > 0)
- scale = entry_count.probability_in (loop->header->count);
+ scale = entry_count.probability_in (loop->header->count).to_reg_br_prob_base ();
else if (loop->header->frequency)
scale = RDIV (entry_freq * REG_BR_PROB_BASE, loop->header->frequency);
scale_loop_profile (loop, scale, 0);
scale_rest = REG_BR_PROB_BASE;
new_loop = loop_version (loop, enter_main_cond, NULL,
- prob_entry, REG_BR_PROB_BASE - prob_entry,
+ profile_probability::from_reg_br_prob_base
+ (prob_entry),
+ profile_probability::from_reg_br_prob_base
+ (REG_BR_PROB_BASE - prob_entry),
scale_unrolled, scale_rest, true);
gcc_assert (new_loop != NULL);
update_ssa (TODO_update_ssa);
/* Since the exit edge will be removed, the frequency of all the blocks
in the loop that are dominated by it must be scaled by
1 / (1 - exit->probability). */
- scale_dominated_blocks_in_loop (loop, exit->src,
- REG_BR_PROB_BASE,
- REG_BR_PROB_BASE - exit->probability);
+ if (exit->probability.initialized_p ())
+ scale_dominated_blocks_in_loop (loop, exit->src,
+ REG_BR_PROB_BASE,
+ REG_BR_PROB_BASE
+ - exit->probability.to_reg_br_prob_base ());
bsi = gsi_last_bb (exit_bb);
exit_if = gimple_build_cond (EQ_EXPR, integer_zero_node,
new_exit->count = exit->count;
new_exit->probability = exit->probability;
new_nonexit = single_pred_edge (loop->latch);
- new_nonexit->probability = REG_BR_PROB_BASE - exit->probability;
+ new_nonexit->probability = exit->probability.invert ();
new_nonexit->flags = EDGE_TRUE_VALUE;
new_nonexit->count -= exit->count;
- scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability,
- REG_BR_PROB_BASE);
+ if (new_nonexit->probability.initialized_p ())
+ scale_bbs_frequencies_int (&loop->latch, 1,
+ new_nonexit->probability.to_reg_br_prob_base (),
+ REG_BR_PROB_BASE);
old_entry = loop_preheader_edge (loop);
new_entry = loop_preheader_edge (new_loop);
if (freq_e == profile_count::zero ())
freq_e = profile_count::from_gcov_type (1);
/* This should not overflow. */
- scale = freq_e.probability_in (freq_h);
+ scale = freq_e.probability_in (freq_h).to_reg_br_prob_base ();
scale_loop_frequencies (loop, scale, REG_BR_PROB_BASE);
}
exit_bb = single_pred (loop->latch);
new_exit = find_edge (exit_bb, rest);
new_exit->count = loop_preheader_edge (loop)->count;
- new_exit->probability = REG_BR_PROB_BASE / (new_est_niter + 1);
+ new_exit->probability = profile_probability::always ()
+ .apply_scale (1, new_est_niter + 1);
rest->count += new_exit->count;
rest->frequency += EDGE_FREQUENCY (new_exit);
new_nonexit = single_pred_edge (loop->latch);
- prob = new_nonexit->probability;
- new_nonexit->probability = REG_BR_PROB_BASE - new_exit->probability;
+ if (new_nonexit->probability.initialized_p ())
+ prob = new_nonexit->probability.to_reg_br_prob_base ();
+ else
+ prob = 0;
+ new_nonexit->probability = new_exit->probability.invert ();
new_nonexit->count = exit_bb->count - new_exit->count;
if (prob > 0)
- scale_bbs_frequencies_int (&loop->latch, 1, new_nonexit->probability,
+ scale_bbs_frequencies_int (&loop->latch, 1,
+ new_nonexit->probability.to_reg_br_prob_base (),
prob);
/* Finally create the new counter for number of iterations and add the new
}
new_e->count = skip_bb->count;
- new_e->probability = PROB_LIKELY;
+ new_e->probability = profile_probability::likely ();
new_e->count = skip_e->count.apply_probability (PROB_LIKELY);
skip_e->count -= new_e->count;
- skip_e->probability = inverse_probability (PROB_LIKELY);
+ skip_e->probability = profile_probability::unlikely ();
return new_e;
}
them, and fix up SSA form for that. */
initialize_original_copy_tables ();
basic_block cond_bb;
+
+ /* FIXME: probabilities seems wrong here. */
struct loop *loop2 = loop_version (loop1, cond, &cond_bb,
- REG_BR_PROB_BASE, REG_BR_PROB_BASE,
+ profile_probability::always (),
+ profile_probability::always (),
REG_BR_PROB_BASE, REG_BR_PROB_BASE,
true);
gcc_assert (loop2);
tree_unswitch_loop (struct loop *loop,
basic_block unswitch_on, tree cond)
{
- unsigned prob_true;
+ profile_probability prob_true;
edge edge_true, edge_false;
/* Some sanity checking. */
extract_true_false_edges_from_block (unswitch_on, &edge_true, &edge_false);
prob_true = edge_true->probability;
+ int p = prob_true.initialized_p () ? prob_true.to_reg_br_prob_base ()
+ : REG_BR_PROB_BASE / 2;
return loop_version (loop, unshare_expr (cond),
- NULL, prob_true, REG_BR_PROB_BASE - prob_true, prob_true,
- REG_BR_PROB_BASE - prob_true, false);
+ NULL, prob_true,
+ prob_true.invert (),
+ p, REG_BR_PROB_BASE - p,
+ false);
}
/* Unswitch outer loops by hoisting invariant guard on
/* Create new loop pre-header. */
e = split_block (pre_header, last_stmt (pre_header));
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, " Moving guard %i->%i (prob %i) to bb %i, "
- "new preheader is %i\n",
- guard->src->index, guard->dest->index, guard->probability,
- e->src->index, e->dest->index);
+ {
+ fprintf (dump_file, " Moving guard %i->%i (prob ",
+ guard->src->index, guard->dest->index);
+ guard->probability.dump (dump_file);
+ fprintf (dump_file, ") to bb %i, new preheader is %i\n",
+ e->src->index, e->dest->index);
+ }
gcc_assert (loop_preheader_edge (loop)->src == e->dest);
}
new_edge->count = skip_count;
if (dump_file && (dump_flags & TDF_DETAILS))
- fprintf (dump_file, " Estimated probability of skipping loop is %i\n",
- new_edge->probability);
+ {
+ fprintf (dump_file, " Estimated probability of skipping loop is ");
+ new_edge->probability.dump (dump_file);
+ fprintf (dump_file, "\n");
+ }
/* Update profile after the transform:
First decrease count of path from newly hoisted loop guard
to loop header... */
e->count -= skip_count;
- e->probability = REG_BR_PROB_BASE - new_edge->probability;
+ e->probability = new_edge->probability.invert ();
e->dest->count = e->count;
e->dest->frequency = EDGE_FREQUENCY (e);
/* ... now update profile to represent that original guard will be optimized
away ... */
- guard->probability = 0;
+ guard->probability = profile_probability::never ();
guard->count = profile_count::zero ();
- not_guard->probability = REG_BR_PROB_BASE;
+ not_guard->probability = profile_probability::always ();
/* This count is wrong (frequency of not_guard does not change),
but will be scaled later. */
not_guard->count = guard->src->count;
{
if (dump_file && (dump_flags & TDF_DETAILS))
fprintf (dump_file, " %i", bb->index);
- scale_bbs_frequencies_int (&bb, 1, e->probability, REG_BR_PROB_BASE);
+ if (e->probability.initialized_p ())
+ scale_bbs_frequencies_int (&bb, 1,
+ e->probability.to_reg_br_prob_base (),
+ REG_BR_PROB_BASE);
}
}
te->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
te->flags &= ~EDGE_ABNORMAL;
te->flags |= EDGE_FALLTHRU;
- if (te->probability > REG_BR_PROB_BASE)
- te->probability = REG_BR_PROB_BASE;
}
}
}
{
EDGE_SUCC (cond_block, 0)->flags |= EDGE_FALLTHRU;
EDGE_SUCC (cond_block, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
- EDGE_SUCC (cond_block, 0)->probability = REG_BR_PROB_BASE;
+ EDGE_SUCC (cond_block, 0)->probability = profile_probability::always ();
EDGE_SUCC (cond_block, 0)->count += EDGE_SUCC (cond_block, 1)->count;
block_to_remove = EDGE_SUCC (cond_block, 1)->dest;
EDGE_SUCC (cond_block, 1)->flags |= EDGE_FALLTHRU;
EDGE_SUCC (cond_block, 1)->flags
&= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
- EDGE_SUCC (cond_block, 1)->probability = REG_BR_PROB_BASE;
+ EDGE_SUCC (cond_block, 1)->probability = profile_probability::always ();
EDGE_SUCC (cond_block, 1)->count += EDGE_SUCC (cond_block, 0)->count;
block_to_remove = EDGE_SUCC (cond_block, 0)->dest;
if (optimize_bb_for_speed_p (cond_bb)
/* The special case is useless if it has a low probability. */
&& profile_status_for_fn (cfun) != PROFILE_ABSENT
- && EDGE_PRED (middle_bb, 0)->probability < PROB_EVEN
+ && EDGE_PRED (middle_bb, 0)->probability < profile_probability::even ()
/* If assign is cheap, there is no point avoiding it. */
&& estimate_num_insns (assign, &eni_time_weights)
>= 3 * estimate_num_insns (cond, &eni_time_weights))
gsi_insert_after (&gsi, g, GSI_NEW_STMT);
edge etrue = make_edge (cond_bb, merge_bb, EDGE_TRUE_VALUE);
- etrue->probability = REG_BR_PROB_BASE / 2;
+ etrue->probability = profile_probability::even ();
etrue->count = cond_bb->count.apply_scale (1, 2);
edge efalse = find_edge (cond_bb, then_bb);
efalse->flags = EDGE_FALSE_VALUE;
else if (bb2->frequency && !bb1->frequency)
;
else if (out_freq_sum)
- e2->probability = GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1)
- + EDGE_FREQUENCY (e2),
- out_freq_sum);
+ e2->probability = profile_probability::from_reg_br_prob_base
+ (GCOV_COMPUTE_SCALE (EDGE_FREQUENCY (e1)
+ + EDGE_FREQUENCY (e2),
+ out_freq_sum));
out_sum += e2->count;
}
bb2->frequency += bb1->frequency;
}
else
{
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = bb->count;
ei_next (&ei);
}
create_edge_and_update_destination_phis (struct redirection_data *rd,
basic_block bb, int idx)
{
- edge e = make_edge (bb, rd->path->last ()->e->dest, EDGE_FALLTHRU);
+ edge e = make_single_succ_edge (bb, rd->path->last ()->e->dest, EDGE_FALLTHRU);
rescan_loop_exit (e, true, false);
- e->probability = REG_BR_PROB_BASE;
- e->count = bb->count;
/* We used to copy the thread path here. That was added in 2007
and dutifully updated through the representation changes in 2013.
/* Handle incoming profile insanities. */
if (total_count < path_in_count)
path_in_count = total_count;
- int onpath_scale = path_in_count.probability_in (total_count);
+ int onpath_scale
+ = path_in_count.probability_in (total_count).to_reg_br_prob_base ();
/* Walk the entire path to do some more computation in order to estimate
how much of the path_in_count will flow out of the duplicated threading
get a flow verification error.
Not much we can do to make counts/freqs sane without
redoing the profile estimation. */
- esucc->probability = REG_BR_PROB_BASE;
+ esucc->probability = profile_probability::guessed_always ();
}
}
among the duplicated off-path edges based on their original
ratio to the full off-path count (total_orig_off_path_count).
*/
- int scale = enonpath->count.probability_in (total_orig_off_path_count);
+ int scale = enonpath->count.probability_in (total_orig_off_path_count)
+ .to_reg_br_prob_base ();
/* Give the duplicated offpath edge a portion of the duplicated
total. */
enonpathdup->count = total_dup_off_path_count.apply_probability (scale);
/* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
errors applying the probability when the frequencies are very
small. */
- ein->count = profile_count::from_gcov_type
- (apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
- ein->probability));
+ if (ein->probability.initialized_p ())
+ ein->count = profile_count::from_gcov_type
+ (apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
+ ein->probability.to_reg_br_prob_base ()));
+ else
+ /* FIXME: this is hack; we should track uninitialized values. */
+ ein->count = profile_count::zero ();
}
for (unsigned int i = 1; i < path->length (); i++)
if (e)
{
rescan_loop_exit (e, true, false);
- e->probability = REG_BR_PROB_BASE;
+ e->probability = profile_probability::always ();
e->count = region_copy[n_region - 1]->count;
}
e_false->flags &= ~EDGE_FALLTHRU;
e_false->flags |= EDGE_FALSE_VALUE;
- e_false->probability = REG_BR_PROB_BASE - e_true->probability;
+ e_false->probability = e_true->probability.invert ();
e_false->count = split_bb->count - e_true->count;
new_bb->count = e_false->count;
basic_block final_bb;
/* The probability of the default edge in the replaced switch. */
- int default_prob;
+ profile_probability default_prob;
/* The count of the default edge in the replaced switch. */
profile_count default_count;
/* flags and profiles of the edge for in-range values */
if (!info->default_case_nonstandard)
e01 = make_edge (bb0, bb1, EDGE_TRUE_VALUE);
- e01->probability = REG_BR_PROB_BASE - info->default_prob;
+ e01->probability = info->default_prob.invert ();
e01->count = info->other_count;
/* flags and profiles of the edge taking care of out-of-range values */
bbf = info->final_bb;
e1f = make_edge (bb1, bbf, EDGE_FALLTHRU);
- e1f->probability = REG_BR_PROB_BASE;
+ e1f->probability = profile_probability::always ();
e1f->count = info->other_count;
if (info->default_case_nonstandard)
else
{
e2f = make_edge (bb2, bbf, EDGE_FALLTHRU);
- e2f->probability = REG_BR_PROB_BASE;
+ e2f->probability = profile_probability::always ();
e2f->count = info->default_count;
}
static edge
slpeel_add_loop_guard (basic_block guard_bb, tree cond,
basic_block guard_to, basic_block dom_bb,
- int probability, bool irreducible_p)
+ profile_probability probability, bool irreducible_p)
{
gimple_stmt_iterator gsi;
edge new_e, enter_e;
new_e->flags |= EDGE_IRREDUCIBLE_LOOP;
enter_e->count -= new_e->count;
- enter_e->probability = inverse_probability (probability);
+ enter_e->probability = probability.invert ();
set_immediate_dominator (CDI_DOMINATORS, guard_to, dom_bb);
/* Split enter_e to preserve LOOPS_HAVE_PREHEADERS. */
edge e, guard_e;
tree type = TREE_TYPE (niters), guard_cond;
basic_block guard_bb, guard_to;
- int prob_prolog, prob_vector, prob_epilog;
+ profile_probability prob_prolog, prob_vector, prob_epilog;
int bound_prolog = 0, bound_scalar = 0, bound = 0;
int vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
int prolog_peeling = LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo);
if (!prolog_peeling && !epilog_peeling)
return NULL;
- prob_vector = 9 * REG_BR_PROB_BASE / 10;
+ prob_vector = profile_probability::guessed_always ().apply_scale (9, 10);
if ((vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo)) == 2)
vf = 3;
- prob_prolog = prob_epilog = (vf - 1) * REG_BR_PROB_BASE / vf;
+ prob_prolog = prob_epilog = profile_probability::guessed_always ()
+ .apply_scale (vf - 1, vf);
vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
struct loop *prolog, *epilog = NULL, *loop = LOOP_VINFO_LOOP (loop_vinfo);
separately. Note in this case, the probability of epilog loop
needs to be scaled back later. */
basic_block bb_before_loop = loop_preheader_edge (loop)->src;
- scale_bbs_frequencies_int (&bb_before_loop, 1, prob_vector,
+ if (prob_vector.initialized_p ())
+ scale_bbs_frequencies_int (&bb_before_loop, 1,
+ prob_vector.to_reg_br_prob_base (),
REG_BR_PROB_BASE);
- scale_loop_profile (loop, prob_vector, bound);
+ scale_loop_profile (loop, prob_vector.to_reg_br_prob_base (), bound);
}
tree niters_prolog = build_int_cst (type, 0);
guard_to = split_edge (loop_preheader_edge (loop));
guard_e = slpeel_add_loop_guard (guard_bb, guard_cond,
guard_to, guard_bb,
- inverse_probability (prob_prolog),
+ prob_prolog.invert (),
irred_flag);
e = EDGE_PRED (guard_to, 0);
e = (e != guard_e ? e : EDGE_PRED (guard_to, 1));
slpeel_update_phi_nodes_for_guard1 (prolog, loop, guard_e, e);
- scale_bbs_frequencies_int (&bb_after_prolog, 1, prob_prolog,
+ scale_bbs_frequencies_int (&bb_after_prolog, 1,
+ prob_prolog.to_reg_br_prob_base (),
REG_BR_PROB_BASE);
- scale_loop_profile (prolog, prob_prolog, bound_prolog);
+ scale_loop_profile (prolog, prob_prolog.to_reg_br_prob_base (),
+ bound_prolog);
}
/* Update init address of DRs. */
vect_update_inits_of_drs (loop_vinfo, niters_prolog);
guard_to = split_edge (loop_preheader_edge (epilog));
guard_e = slpeel_add_loop_guard (guard_bb, guard_cond,
guard_to, guard_bb,
- inverse_probability (prob_vector),
+ prob_vector.invert (),
irred_flag);
e = EDGE_PRED (guard_to, 0);
e = (e != guard_e ? e : EDGE_PRED (guard_to, 1));
guard_to->count = guard_bb->count;
single_succ_edge (guard_to)->count = guard_to->count;
/* Scale probability of epilog loop back. */
- int scale_up = REG_BR_PROB_BASE * REG_BR_PROB_BASE / prob_vector;
+ int scale_up = REG_BR_PROB_BASE * REG_BR_PROB_BASE
+ / prob_vector.to_reg_br_prob_base ();
scale_loop_frequencies (epilog, scale_up, REG_BR_PROB_BASE);
}
guard_to = split_edge (single_exit (epilog));
guard_e = slpeel_add_loop_guard (guard_bb, guard_cond, guard_to,
skip_vector ? anchor : guard_bb,
- inverse_probability (prob_epilog),
+ prob_epilog.invert (),
irred_flag);
slpeel_update_phi_nodes_for_guard2 (loop, epilog, guard_e,
single_exit (epilog));
the guard_bb, which is the case when skip_vector is true. */
if (guard_bb != bb_before_epilog)
{
- prob_epilog = (combine_probabilities (prob_vector, prob_epilog)
- + inverse_probability (prob_vector));
+ prob_epilog = prob_vector * prob_epilog + prob_vector.invert ();
- scale_bbs_frequencies_int (&bb_before_epilog, 1, prob_epilog,
+ scale_bbs_frequencies_int (&bb_before_epilog, 1,
+ prob_epilog.to_reg_br_prob_base (),
REG_BR_PROB_BASE);
}
- scale_loop_profile (epilog, prob_epilog, bound);
+ scale_loop_profile (epilog, prob_epilog.to_reg_br_prob_base (), bound);
}
else
slpeel_update_phi_nodes_for_lcssa (epilog);
/* We don't want to scale SCALAR_LOOP's frequencies, we need to
scale LOOP's frequencies instead. */
nloop = loop_version (scalar_loop, cond_expr, &condition_bb,
- prob, REG_BR_PROB_BASE - prob,
+ profile_probability::guessed_always ().apply_scale
+ (prob, REG_BR_PROB_BASE),
+ profile_probability::guessed_always ().apply_scale
+ (REG_BR_PROB_BASE - prob, REG_BR_PROB_BASE),
REG_BR_PROB_BASE, REG_BR_PROB_BASE - prob, true);
scale_loop_frequencies (loop, prob, REG_BR_PROB_BASE);
/* CONDITION_BB was created above SCALAR_LOOP's preheader,
}
else
nloop = loop_version (loop, cond_expr, &condition_bb,
- prob, REG_BR_PROB_BASE - prob,
+ profile_probability::guessed_always ().apply_scale
+ (prob, REG_BR_PROB_BASE),
+ profile_probability::guessed_always ().apply_scale
+ (REG_BR_PROB_BASE - prob, REG_BR_PROB_BASE),
prob, REG_BR_PROB_BASE - prob, true);
if (version_niter)
if (!(freq_e > profile_count::from_gcov_type (1)))
freq_e = profile_count::from_gcov_type (1);
/* This should not overflow. */
- scale = freq_e.apply_scale (new_est_niter + 1, 1).probability_in (freq_h);
+ scale = freq_e.apply_scale (new_est_niter + 1, 1).probability_in (freq_h)
+ .to_reg_br_prob_base ();
scale_loop_frequencies (loop, scale, REG_BR_PROB_BASE);
}
basic_block exit_bb = single_pred (loop->latch);
edge exit_e = single_exit (loop);
exit_e->count = loop_preheader_edge (loop)->count;
- exit_e->probability = REG_BR_PROB_BASE / (new_est_niter + 1);
+ exit_e->probability = profile_probability::always ()
+ .apply_scale (1, new_est_niter + 1);
edge exit_l = single_pred_edge (loop->latch);
- int prob = exit_l->probability;
- exit_l->probability = REG_BR_PROB_BASE - exit_e->probability;
+ int prob = exit_l->probability.initialized_p ()
+ ? exit_l->probability.to_reg_br_prob_base () : 0;
+ exit_l->probability = exit_e->probability.invert ();
exit_l->count = exit_bb->count - exit_e->count;
if (prob > 0)
- scale_bbs_frequencies_int (&loop->latch, 1, exit_l->probability, prob);
+ scale_bbs_frequencies_int (&loop->latch, 1,
+ exit_l->probability.to_reg_br_prob_base (), prob);
}
/* Function vect_transform_loop.
e->flags = EDGE_TRUE_VALUE;
efalse = make_edge (bb, store_bb, EDGE_FALSE_VALUE);
/* Put STORE_BB to likely part. */
- efalse->probability = PROB_UNLIKELY;
+ efalse->probability = profile_probability::unlikely ();
store_bb->frequency = PROB_ALWAYS - EDGE_FREQUENCY (efalse);
make_edge (store_bb, join_bb, EDGE_FALLTHRU);
if (dom_info_available_p (CDI_DOMINATORS))
/* Make an edge coming from the 'cond block' into the 'then block';
this edge is unlikely taken, so set up the probability accordingly. */
e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
- e->probability = PROB_VERY_UNLIKELY;
+ e->probability = profile_probability::very_unlikely ();
/* Connect 'then block' with the 'else block'. This is needed
as the ubsan routines we call in the 'then block' are not noreturn.
e = find_edge (cond_bb, fallthru_bb);
e->flags = EDGE_FALSE_VALUE;
e->count = cond_bb->count;
- e->probability = REG_BR_PROB_BASE - PROB_VERY_UNLIKELY;
+ e->probability = profile_probability::very_likely ();
/* Update dominance info for the newly created then_bb; note that
fallthru_bb's dominance info has already been updated by
this edge is unlikely taken, so set up the probability
accordingly. */
e = make_edge (cond1_bb, then_bb, EDGE_TRUE_VALUE);
- e->probability = PROB_VERY_UNLIKELY;
+ e->probability = profile_probability::very_unlikely ();
/* Set up the fallthrough basic block. */
e = find_edge (cond1_bb, cond2_bb);
e->flags = EDGE_FALSE_VALUE;
e->count = cond1_bb->count;
- e->probability = REG_BR_PROB_BASE - PROB_VERY_UNLIKELY;
+ e->probability = profile_probability::very_likely ();
/* Update dominance info. */
if (dom_info_available_p (CDI_DOMINATORS))
and gimple_value_profile_transformations table-driven, perhaps...
*/
-static tree gimple_divmod_fixed_value (gassign *, tree, int, gcov_type,
- gcov_type);
-static tree gimple_mod_pow2 (gassign *, int, gcov_type, gcov_type);
-static tree gimple_mod_subtract (gassign *, int, int, int, gcov_type,
- gcov_type, gcov_type);
static bool gimple_divmod_fixed_value_transform (gimple_stmt_iterator *);
static bool gimple_mod_pow2_value_transform (gimple_stmt_iterator *);
static bool gimple_mod_subtract_transform (gimple_stmt_iterator *);
alter the original STMT. */
static tree
-gimple_divmod_fixed_value (gassign *stmt, tree value, int prob,
+gimple_divmod_fixed_value (gassign *stmt, tree value, profile_probability prob,
gcov_type count, gcov_type all)
{
gassign *stmt1, *stmt2;
e12->count = profile_count::from_gcov_type (count);
e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
- e13->probability = REG_BR_PROB_BASE - prob;
+ e13->probability = prob.invert ();
e13->count = profile_count::from_gcov_type (all - count);
remove_edge (e23);
e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
- e24->probability = REG_BR_PROB_BASE;
+ e24->probability = profile_probability::always ();
e24->count = profile_count::from_gcov_type (count);
- e34->probability = REG_BR_PROB_BASE;
+ e34->probability = profile_probability::always ();
e34->count = profile_count::from_gcov_type (all - count);
return tmp2;
enum tree_code code;
gcov_type val, count, all;
tree result, value, tree_val;
- gcov_type prob;
+ profile_probability prob;
gassign *stmt;
stmt = dyn_cast <gassign *> (gsi_stmt (*si));
/* Compute probability of taking the optimal path. */
if (all > 0)
- prob = GCOV_COMPUTE_SCALE (count, all);
+ prob = profile_probability::probability_in_gcov_type (count, all);
else
- prob = 0;
+ prob = profile_probability::never ();
if (sizeof (gcov_type) == sizeof (HOST_WIDE_INT))
tree_val = build_int_cst (get_gcov_type (), val);
the temp; it does not replace or alter the original STMT. */
static tree
-gimple_mod_pow2 (gassign *stmt, int prob, gcov_type count, gcov_type all)
+gimple_mod_pow2 (gassign *stmt, profile_probability prob, gcov_type count, gcov_type all)
{
gassign *stmt1, *stmt2, *stmt3;
gcond *stmt4;
e12->count = profile_count::from_gcov_type (count);
e13 = make_edge (bb, bb3, EDGE_TRUE_VALUE);
- e13->probability = REG_BR_PROB_BASE - prob;
+ e13->probability = prob.invert ();
e13->count = profile_count::from_gcov_type (all - count);
remove_edge (e23);
e24 = make_edge (bb2, bb4, EDGE_FALLTHRU);
- e24->probability = REG_BR_PROB_BASE;
+ e24->probability = profile_probability::always ();
e24->count = profile_count::from_gcov_type (count);
- e34->probability = REG_BR_PROB_BASE;
+ e34->probability = profile_probability::always ();
e34->count = profile_count::from_gcov_type (all - count);
return result;
enum tree_code code;
gcov_type count, wrong_values, all;
tree lhs_type, result, value;
- gcov_type prob;
+ profile_probability prob;
gassign *stmt;
stmt = dyn_cast <gassign *> (gsi_stmt (*si));
return false;
if (all > 0)
- prob = GCOV_COMPUTE_SCALE (count, all);
+ prob = profile_probability::probability_in_gcov_type (count, all);
else
- prob = 0;
+ prob = profile_probability::never ();
result = gimple_mod_pow2 (stmt, prob, count, all);
/* FIXME: Generalize the interface to handle NCOUNTS > 1. */
static tree
-gimple_mod_subtract (gassign *stmt, int prob1, int prob2, int ncounts,
+gimple_mod_subtract (gassign *stmt, profile_probability prob1,
+ profile_probability prob2, int ncounts,
gcov_type count1, gcov_type count2, gcov_type all)
{
gassign *stmt1;
e12->flags &= ~EDGE_FALLTHRU;
e12->flags |= EDGE_FALSE_VALUE;
- e12->probability = REG_BR_PROB_BASE - prob1;
+ e12->probability = prob1.invert ();
e12->count = profile_count::from_gcov_type (all - count1);
e14 = make_edge (bb, bb4, EDGE_TRUE_VALUE);
e23->flags &= ~EDGE_FALLTHRU;
e23->flags |= EDGE_FALSE_VALUE;
e23->count = profile_count::from_gcov_type (all - count1 - count2);
- e23->probability = REG_BR_PROB_BASE - prob2;
+ e23->probability = prob2.invert ();
e24 = make_edge (bb2, bb4, EDGE_TRUE_VALUE);
e24->probability = prob2;
e24->count = profile_count::from_gcov_type (count2);
}
- e34->probability = REG_BR_PROB_BASE;
+ e34->probability = profile_probability::always ();
e34->count = profile_count::from_gcov_type (all - count1 - count2);
return result;
enum tree_code code;
gcov_type count, wrong_values, all;
tree lhs_type, result;
- gcov_type prob1, prob2;
+ profile_probability prob1, prob2;
unsigned int i, steps;
gcov_type count1, count2;
gassign *stmt;
/* Compute probability of taking the optimal path(s). */
if (all > 0)
{
- prob1 = GCOV_COMPUTE_SCALE (count1, all);
- prob2 = GCOV_COMPUTE_SCALE (count2, all);
+ prob1 = profile_probability::probability_in_gcov_type (count1, all);
+ prob2 = profile_probability::probability_in_gcov_type (count2, all);
}
else
{
- prob1 = prob2 = 0;
+ prob1 = prob2 = profile_probability::never ();
}
/* In practice, "steps" is always 2. This interface reflects this,
gcall *
gimple_ic (gcall *icall_stmt, struct cgraph_node *direct_call,
- int prob, profile_count count, profile_count all)
+ profile_probability prob, profile_count count, profile_count all)
{
gcall *dcall_stmt;
gassign *load_stmt;
/* The indirect call might be noreturn. */
if (e_ij != NULL)
{
- e_ij->probability = REG_BR_PROB_BASE;
+ e_ij->probability = profile_probability::always ();
e_ij->count = all - count;
e_ij = single_pred_edge (split_edge (e_ij));
}
e_cd->count = count;
e_ci = make_edge (cond_bb, icall_bb, EDGE_FALSE_VALUE);
- e_ci->probability = REG_BR_PROB_BASE - prob;
+ e_ci->probability = prob.invert ();
e_ci->count = all - count;
remove_edge (e_di);
else
{
e_dj = make_edge (dcall_bb, join_bb, EDGE_FALLTHRU);
- e_dj->probability = REG_BR_PROB_BASE;
+ e_dj->probability = profile_probability::always ();
e_dj->count = count;
e_ij->count = all - count;
}
- e_ij->probability = REG_BR_PROB_BASE;
+ e_ij->probability = profile_probability::always ();
}
/* Insert PHI node for the call result if necessary. */
if (e_eh->flags & (EDGE_EH | EDGE_ABNORMAL))
{
e = make_edge (dcall_bb, e_eh->dest, e_eh->flags);
+ e->probability = e_eh->probability;
+ e->count = e_eh->count;
for (gphi_iterator psi = gsi_start_phis (e_eh->dest);
!gsi_end_p (psi); gsi_next (&psi))
{
assuming we'll propagate a true constant into ICALL_SIZE later. */
static void
-gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, int prob,
+gimple_stringop_fixed_value (gcall *vcall_stmt, tree icall_size, profile_probability prob,
gcov_type count, gcov_type all)
{
gassign *tmp_stmt;
e_ci->count = profile_count::from_gcov_type (count);
e_cv = make_edge (cond_bb, vcall_bb, EDGE_FALSE_VALUE);
- e_cv->probability = REG_BR_PROB_BASE - prob;
+ e_cv->probability = prob.invert ();
e_cv->count = profile_count::from_gcov_type (all - count);
remove_edge (e_iv);
e_ij = make_edge (icall_bb, join_bb, EDGE_FALLTHRU);
- e_ij->probability = REG_BR_PROB_BASE;
+ e_ij->probability = profile_probability::always ();
e_ij->count = profile_count::from_gcov_type (count);
- e_vj->probability = REG_BR_PROB_BASE;
+ e_vj->probability = profile_probability::always ();
e_vj->count = profile_count::from_gcov_type (all - count);
/* Insert PHI node for the call result if necessary. */
gcov_type count, all, val;
tree dest, src;
unsigned int dest_align, src_align;
- gcov_type prob;
+ profile_probability prob;
tree tree_val;
int size_arg;
if (check_counter (stmt, "value", &count, &all, gimple_bb (stmt)->count))
return false;
if (all > 0)
- prob = GCOV_COMPUTE_SCALE (count, all);
+ prob = profile_probability::probability_in_gcov_type (count, all);
else
- prob = 0;
+ prob = profile_probability::never ();
dest = gimple_call_arg (stmt, 0);
dest_align = get_pointer_alignment (dest);
void verify_histograms (void);
void free_histograms (function *);
void stringop_block_profile (gimple *, unsigned int *, HOST_WIDE_INT *);
-gcall *gimple_ic (gcall *, struct cgraph_node *, int, profile_count,
- profile_count);
+gcall *gimple_ic (gcall *, struct cgraph_node *, profile_probability,
+ profile_count, profile_count);
bool check_ic_target (gcall *, struct cgraph_node *);
projects / gcc.git / commitdiff