#include "coretypes.h"
#include "tree.h"
#include "flags.h"
-#include "basic-block.h"
+#include "predict.h"
+#include "vec.h"
+#include "hashtab.h"
+#include "hash-set.h"
+#include "machmode.h"
+#include "tm.h"
+#include "hard-reg-set.h"
+#include "input.h"
#include "function.h"
+#include "dominance.h"
+#include "cfg.h"
+#include "cfganal.h"
+#include "basic-block.h"
#include "hash-table.h"
#include "tree-ssa-alias.h"
#include "internal-fn.h"
nonpath_count += ein->count;
}
}
+
+ /* This is needed due to insane incoming frequencies. */
+ if (path_in_freq > BB_FREQ_MAX)
+ path_in_freq = BB_FREQ_MAX;
+
BITMAP_FREE (in_edge_srcs);
/* Now compute the fraction of the total count coming into the first
edge_iterator ei;
FOR_EACH_EDGE (esucc, ei, bb->succs)
{
- if (bb->count)
+ if (!bb->count)
+ continue;
+
+ /* Prevent overflow computation due to insane profiles. */
+ if (esucc->count < bb->count)
esucc->probability = GCOV_COMPUTE_SCALE (esucc->count,
bb->count);
- if (esucc->probability > REG_BR_PROB_BASE)
- {
- /* Can happen with missing/guessed probabilities, since we
- may determine that more is flowing along duplicated
- path than joiner succ probabilities allowed.
- Counts and freqs will be insane after jump threading,
- at least make sure probability is sane or we will
- get a flow verification error.
- Not much we can do to make counts/freqs sane without
- redoing the profile estimation. */
- esucc->probability = REG_BR_PROB_BASE;
- }
+ else
+ /* Can happen with missing/guessed probabilities, since we
+ may determine that more is flowing along duplicated
+ path than joiner succ probabilities allowed.
+ Counts and freqs will be insane after jump threading,
+ at least make sure probability is sane or we will
+ get a flow verification error.
+ Not much we can do to make counts/freqs sane without
+ redoing the profile estimation. */
+ esucc->probability = REG_BR_PROB_BASE;
}
}
}
+/* Check if the paths through RD all have estimated frequencies but zero
+ profile counts. This is more accurate than checking the entry block
+ for a zero profile count, since profile insanities sometimes creep in. */
+
+static bool
+estimated_freqs_path (struct redirection_data *rd)
+{
+ edge e = rd->incoming_edges->e;
+ vec<jump_thread_edge *> *path = THREAD_PATH (e);
+ edge ein;
+ edge_iterator ei;
+ bool non_zero_freq = false;
+ FOR_EACH_EDGE (ein, ei, e->dest->preds)
+ {
+ if (ein->count)
+ return false;
+ non_zero_freq |= ein->src->frequency != 0;
+ }
+
+ for (unsigned int i = 1; i < path->length (); i++)
+ {
+ edge epath = (*path)[i]->e;
+ if (epath->src->count)
+ return false;
+ non_zero_freq |= epath->src->frequency != 0;
+ edge esucc;
+ FOR_EACH_EDGE (esucc, ei, epath->src->succs)
+ {
+ if (esucc->count)
+ return false;
+ non_zero_freq |= esucc->src->frequency != 0;
+ }
+ }
+ return non_zero_freq;
+}
+
+
/* Invoked for routines that have guessed frequencies and no profile
counts to record the block and edge frequencies for paths through RD
in the profile count fields of those blocks and edges. This is because
edge ein;
edge_iterator ei;
FOR_EACH_EDGE (ein, ei, e->dest->preds)
- ein->count = EDGE_FREQUENCY (ein);
+ {
+ /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
+ errors applying the probability when the frequencies are very
+ small. */
+ ein->count = apply_probability (ein->src->frequency * REG_BR_PROB_BASE,
+ ein->probability);
+ }
for (unsigned int i = 1; i < path->length (); i++)
{
edge epath = (*path)[i]->e;
edge esucc;
+ /* Scale up the frequency by REG_BR_PROB_BASE, to avoid rounding
+ errors applying the edge probability when the frequencies are very
+ small. */
+ epath->src->count = epath->src->frequency * REG_BR_PROB_BASE;
FOR_EACH_EDGE (esucc, ei, epath->src->succs)
- esucc->count = EDGE_FREQUENCY (esucc);
- epath->src->count = epath->src->frequency;
+ esucc->count = apply_probability (esucc->src->count,
+ esucc->probability);
}
}
data we first take a snapshot of the existing block and edge frequencies
by copying them into the empty profile count fields. These counts are
then used to do the incremental updates, and cleared at the end of this
- routine. */
+ routine. If the function is marked as having a profile, we still check
+ to see if the paths through RD are using estimated frequencies because
+ the routine had zero profile counts. */
bool do_freqs_to_counts = (profile_status_for_fn (cfun) != PROFILE_READ
- || !ENTRY_BLOCK_PTR_FOR_FN (cfun)->count);
+ || estimated_freqs_path (rd));
if (do_freqs_to_counts)
freqs_to_counts_path (rd);
/* Now iterate again, converting cases where we want to thread
through a joiner block, but only if no other edge on the path
- already has a jump thread attached to it. */
+ already has a jump thread attached to it. We do this in two passes,
+ to avoid situations where the order in the paths vec can hide overlapping
+ threads (the path is recorded on the incoming edge, so we would miss
+ cases where the second path starts at a downstream edge on the same
+ path). First record all joiner paths, deleting any in the unexpected
+ case where there is already a path for that incoming edge. */
for (i = 0; i < paths.length (); i++)
{
vec<jump_thread_edge *> *path = paths[i];
if ((*path)[1]->type == EDGE_COPY_SRC_JOINER_BLOCK)
+ {
+ /* Attach the path to the starting edge if none is yet recorded. */
+ if ((*path)[0]->e->aux == NULL)
+ (*path)[0]->e->aux = path;
+ else if (dump_file && (dump_flags & TDF_DETAILS))
+ dump_jump_thread_path (dump_file, *path, false);
+ }
+ }
+ /* Second, look for paths that have any other jump thread attached to
+ them, and either finish converting them or cancel them. */
+ for (i = 0; i < paths.length (); i++)
+ {
+ vec<jump_thread_edge *> *path = paths[i];
+ edge e = (*path)[0]->e;
+
+ if ((*path)[1]->type == EDGE_COPY_SRC_JOINER_BLOCK && e->aux == path)
{
unsigned int j;
-
- for (j = 0; j < path->length (); j++)
+ for (j = 1; j < path->length (); j++)
if ((*path)[j]->e->aux != NULL)
break;
/* If we iterated through the entire path without exiting the loop,
- then we are good to go, attach the path to the starting edge. */
+ then we are good to go, record it. */
if (j == path->length ())
+ bitmap_set_bit (tmp, e->dest->index);
+ else
{
- edge e = (*path)[0]->e;
- e->aux = path;
- bitmap_set_bit (tmp, e->dest->index);
- }
- else if (dump_file && (dump_flags & TDF_DETAILS))
- {
- dump_jump_thread_path (dump_file, *path, false);
+ e->aux = NULL;
+ if (dump_file && (dump_flags & TDF_DETAILS))
+ dump_jump_thread_path (dump_file, *path, false);
}
}
}
-
/* If optimizing for size, only thread through block if we don't have
to duplicate it or it's an otherwise empty redirection block. */
if (optimize_function_for_size_p (cfun))