/* Instruction scheduling pass.
- Copyright (C) 1992-2014 Free Software Foundation, Inc.
+ Copyright (C) 1992-2015 Free Software Foundation, Inc.
Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
and currently maintained by, Jim Wilson (wilson@cygnus.com)
#include "insn-attr.h"
#include "except.h"
#include "recog.h"
+#include "dominance.h"
+#include "cfg.h"
+#include "cfgrtl.h"
+#include "cfgbuild.h"
+#include "predict.h"
+#include "basic-block.h"
#include "sched-int.h"
#include "target.h"
#include "common/common-target.h"
#include "cfgloop.h"
#include "ira.h"
#include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
-#include "hash-table.h"
#include "dumpfile.h"
#ifdef INSN_SCHEDULING
/* The minimal value of the INSN_TICK of an instruction. */
#define MIN_TICK (-max_insn_queue_index)
+/* Original order of insns in the ready list.
+ Used to keep order of normal insns while separating DEBUG_INSNs. */
+#define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
+
+/* The deciding reason for INSN's place in the ready list. */
+#define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
+
/* List of important notes we must keep around. This is a pointer to the
last element in the list. */
-rtx note_list;
+rtx_insn *note_list;
static struct spec_info_def spec_info_var;
/* Description of the speculative part of the scheduling.
static int nr_begin_data, nr_be_in_data, nr_begin_control, nr_be_in_control;
/* Array used in {unlink, restore}_bb_notes. */
-static rtx *bb_header = 0;
+static rtx_insn **bb_header = 0;
/* Basic block after which recovery blocks will be created. */
static basic_block before_recovery;
the base maximal time of functional unit reservations and getting a
result. This is the longest time an insn may be queued. */
-static rtx *insn_queue;
+static rtx_insn_list **insn_queue;
static int q_ptr = 0;
static int q_size = 0;
#define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
/* This records the actual schedule. It is built up during the main phase
of schedule_block, and afterwards used to reorder the insns in the RTL. */
-static vec<rtx> scheduled_insns;
+static vec<rtx_insn *> scheduled_insns;
static int may_trap_exp (const_rtx, int);
/* Return the number of cycles until INSN is expected to be ready.
Return zero if it already is. */
static int
-insn_delay (rtx insn)
+insn_delay (rtx_insn *insn)
{
return MAX (INSN_TICK (insn) - clock_var, 0);
}
struct delay_pair
{
struct delay_pair *next_same_i1;
- rtx i1, i2;
+ rtx_insn *i1, *i2;
int cycles;
/* When doing modulo scheduling, we a delay_pair can also be used to
show that I1 and I2 are the same insn in a different stage. If that
/* Helpers for delay hashing. */
-struct delay_i1_hasher : typed_noop_remove <delay_pair>
+struct delay_i1_hasher : nofree_ptr_hash <delay_pair>
{
- typedef delay_pair value_type;
- typedef void compare_type;
- static inline hashval_t hash (const value_type *);
- static inline bool equal (const value_type *, const compare_type *);
+ typedef void *compare_type;
+ static inline hashval_t hash (const delay_pair *);
+ static inline bool equal (const delay_pair *, const void *);
};
/* Returns a hash value for X, based on hashing just I1. */
inline hashval_t
-delay_i1_hasher::hash (const value_type *x)
+delay_i1_hasher::hash (const delay_pair *x)
{
return htab_hash_pointer (x->i1);
}
/* Return true if I1 of pair X is the same as that of pair Y. */
inline bool
-delay_i1_hasher::equal (const value_type *x, const compare_type *y)
+delay_i1_hasher::equal (const delay_pair *x, const void *y)
{
return x->i1 == y;
}
-struct delay_i2_hasher : typed_free_remove <delay_pair>
+struct delay_i2_hasher : free_ptr_hash <delay_pair>
{
- typedef delay_pair value_type;
- typedef void compare_type;
- static inline hashval_t hash (const value_type *);
- static inline bool equal (const value_type *, const compare_type *);
+ typedef void *compare_type;
+ static inline hashval_t hash (const delay_pair *);
+ static inline bool equal (const delay_pair *, const void *);
};
/* Returns a hash value for X, based on hashing just I2. */
inline hashval_t
-delay_i2_hasher::hash (const value_type *x)
+delay_i2_hasher::hash (const delay_pair *x)
{
return htab_hash_pointer (x->i2);
}
/* Return true if I2 of pair X is the same as that of pair Y. */
inline bool
-delay_i2_hasher::equal (const value_type *x, const compare_type *y)
+delay_i2_hasher::equal (const delay_pair *x, const void *y)
{
return x->i2 == y;
}
scheduling. */
void
-record_delay_slot_pair (rtx i1, rtx i2, int cycles, int stages)
+record_delay_slot_pair (rtx_insn *i1, rtx_insn *i2, int cycles, int stages)
{
struct delay_pair *p = XNEW (struct delay_pair);
struct delay_pair **slot;
/* Examine the delay pair hashtable to see if INSN is a shadow for another,
and return the other insn if so. Return NULL otherwise. */
-rtx
-real_insn_for_shadow (rtx insn)
+rtx_insn *
+real_insn_for_shadow (rtx_insn *insn)
{
struct delay_pair *pair;
if (!delay_htab)
- return NULL_RTX;
+ return NULL;
pair = delay_htab_i2->find_with_hash (insn, htab_hash_pointer (insn));
if (!pair || pair->stages > 0)
- return NULL_RTX;
+ return NULL;
return pair->i1;
}
and add dependencies to the real insns to limit the amount of backtracking
needed. */
void
-add_delay_dependencies (rtx insn)
+add_delay_dependencies (rtx_insn *insn)
{
struct delay_pair *pair;
sd_iterator_def sd_it;
FOR_EACH_DEP (pair->i2, SD_LIST_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
struct delay_pair *other_pair
= delay_htab_i2->find_with_hash (pro, htab_hash_pointer (pro));
if (!other_pair || other_pair->stages)
\f
/* Forward declarations. */
-static int priority (rtx);
+static int priority (rtx_insn *);
+static int autopref_rank_for_schedule (const rtx_insn *, const rtx_insn *);
static int rank_for_schedule (const void *, const void *);
-static void swap_sort (rtx *, int);
-static void queue_insn (rtx, int, const char *);
-static int schedule_insn (rtx);
-static void adjust_priority (rtx);
+static void swap_sort (rtx_insn **, int);
+static void queue_insn (rtx_insn *, int, const char *);
+static int schedule_insn (rtx_insn *);
+static void adjust_priority (rtx_insn *);
static void advance_one_cycle (void);
static void extend_h_i_d (void);
unlink_other_notes ()). After scheduling the block, these notes are
inserted at the beginning of the block (in schedule_block()). */
-static void ready_add (struct ready_list *, rtx, bool);
-static rtx ready_remove_first (struct ready_list *);
-static rtx ready_remove_first_dispatch (struct ready_list *ready);
+static void ready_add (struct ready_list *, rtx_insn *, bool);
+static rtx_insn *ready_remove_first (struct ready_list *);
+static rtx_insn *ready_remove_first_dispatch (struct ready_list *ready);
static void queue_to_ready (struct ready_list *);
static int early_queue_to_ready (state_t, struct ready_list *);
/* The following functions are used to implement multi-pass scheduling
on the first cycle. */
-static rtx ready_remove (struct ready_list *, int);
-static void ready_remove_insn (rtx);
+static rtx_insn *ready_remove (struct ready_list *, int);
+static void ready_remove_insn (rtx_insn *);
-static void fix_inter_tick (rtx, rtx);
-static int fix_tick_ready (rtx);
-static void change_queue_index (rtx, int);
+static void fix_inter_tick (rtx_insn *, rtx_insn *);
+static int fix_tick_ready (rtx_insn *);
+static void change_queue_index (rtx_insn *, int);
/* The following functions are used to implement scheduling of data/control
speculative instructions. */
static void extend_h_i_d (void);
-static void init_h_i_d (rtx);
-static int haifa_speculate_insn (rtx, ds_t, rtx *);
-static void generate_recovery_code (rtx);
-static void process_insn_forw_deps_be_in_spec (rtx, rtx, ds_t);
-static void begin_speculative_block (rtx);
-static void add_to_speculative_block (rtx);
+static void init_h_i_d (rtx_insn *);
+static int haifa_speculate_insn (rtx_insn *, ds_t, rtx *);
+static void generate_recovery_code (rtx_insn *);
+static void process_insn_forw_deps_be_in_spec (rtx_insn *, rtx_insn *, ds_t);
+static void begin_speculative_block (rtx_insn *);
+static void add_to_speculative_block (rtx_insn *);
static void init_before_recovery (basic_block *);
-static void create_check_block_twin (rtx, bool);
+static void create_check_block_twin (rtx_insn *, bool);
static void fix_recovery_deps (basic_block);
-static bool haifa_change_pattern (rtx, rtx);
-static void dump_new_block_header (int, basic_block, rtx, rtx);
+static bool haifa_change_pattern (rtx_insn *, rtx);
+static void dump_new_block_header (int, basic_block, rtx_insn *, rtx_insn *);
static void restore_bb_notes (basic_block);
-static void fix_jump_move (rtx);
-static void move_block_after_check (rtx);
+static void fix_jump_move (rtx_insn *);
+static void move_block_after_check (rtx_insn *);
static void move_succs (vec<edge, va_gc> **, basic_block);
-static void sched_remove_insn (rtx);
-static void clear_priorities (rtx, rtx_vec_t *);
+static void sched_remove_insn (rtx_insn *);
+static void clear_priorities (rtx_insn *, rtx_vec_t *);
static void calc_priorities (rtx_vec_t);
-static void add_jump_dependencies (rtx, rtx);
+static void add_jump_dependencies (rtx_insn *, rtx_insn *);
#endif /* INSN_SCHEDULING */
\f
/* Registers mentioned in the current region. */
static bitmap region_ref_regs;
+/* Effective number of available registers of a given class (see comment
+ in sched_pressure_start_bb). */
+static int sched_class_regs_num[N_REG_CLASSES];
+/* Number of call_used_regs. This is a helper for calculating of
+ sched_class_regs_num. */
+static int call_used_regs_num[N_REG_CLASSES];
+
/* Initiate register pressure relative info for scheduling the current
region. Currently it is only clearing register mentioned in the
current region. */
static void
setup_ref_regs (rtx x)
{
- int i, j, regno;
+ int i, j;
const RTX_CODE code = GET_CODE (x);
const char *fmt;
if (REG_P (x))
{
- regno = REGNO (x);
- if (HARD_REGISTER_NUM_P (regno))
- bitmap_set_range (region_ref_regs, regno,
- hard_regno_nregs[regno][GET_MODE (x)]);
- else
- bitmap_set_bit (region_ref_regs, REGNO (x));
+ bitmap_set_range (region_ref_regs, REGNO (x), REG_NREGS (x));
return;
}
fmt = GET_RTX_FORMAT (code);
initiate_bb_reg_pressure_info (basic_block bb)
{
unsigned int i ATTRIBUTE_UNUSED;
- rtx insn;
+ rtx_insn *insn;
if (current_nr_blocks > 1)
FOR_BB_INSNS (bb, insn)
if (NONDEBUG_INSN_P (insn))
setup_ref_regs (PATTERN (insn));
initiate_reg_pressure_info (df_get_live_in (bb));
-#ifdef EH_RETURN_DATA_REGNO
if (bb_has_eh_pred (bb))
for (i = 0; ; ++i)
{
mark_regno_birth_or_death (curr_reg_live, curr_reg_pressure,
regno, true);
}
-#endif
}
/* Save current register pressure related info. */
gcc_assert (curr_reg_pressure[cl] >= 0);
fprintf (sched_dump, " %s:%d(%d)", reg_class_names[cl],
curr_reg_pressure[cl],
- curr_reg_pressure[cl] - ira_class_hard_regs_num[cl]);
+ curr_reg_pressure[cl] - sched_class_regs_num[cl]);
}
fprintf (sched_dump, "\n");
}
/* Determine if INSN has a condition that is clobbered if a register
in SET_REGS is modified. */
static bool
-cond_clobbered_p (rtx insn, HARD_REG_SET set_regs)
+cond_clobbered_p (rtx_insn *insn, HARD_REG_SET set_regs)
{
rtx pat = PATTERN (insn);
gcc_assert (GET_CODE (pat) == COND_EXEC);
/* This function should be called after modifying the pattern of INSN,
to update scheduler data structures as needed. */
static void
-update_insn_after_change (rtx insn)
+update_insn_after_change (rtx_insn *insn)
{
sd_iterator_def sd_it;
dep_t dep;
INSN_COST (insn) = -1;
/* Invalidate INSN_TICK, so it'll be recalculated. */
INSN_TICK (insn) = INVALID_TICK;
+
+ /* Invalidate autoprefetch data entry. */
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
}
false. */
static ds_t
-recompute_todo_spec (rtx next, bool for_backtrack)
+recompute_todo_spec (rtx_insn *next, bool for_backtrack)
{
ds_t new_ds;
sd_iterator_def sd_it;
if (!sd_lists_empty_p (next, SD_LIST_HARD_BACK))
return HARD_DEP;
+ /* If NEXT is intended to sit adjacent to this instruction, we don't
+ want to try to break any dependencies. Treat it as a HARD_DEP. */
+ if (SCHED_GROUP_P (next))
+ return HARD_DEP;
+
/* Now we've got NEXT with speculative deps only.
1. Look at the deps to see what we have to do.
2. Check if we can do 'todo'. */
FOR_EACH_DEP (next, SD_LIST_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
ds_t ds = DEP_STATUS (dep) & SPECULATIVE;
if (DEBUG_INSN_P (pro) && !DEBUG_INSN_P (next))
else if (n_control == 1 && n_replace == 0 && n_spec == 0)
{
- rtx pro, other, new_pat;
+ rtx_insn *pro, *other;
+ rtx new_pat;
rtx cond = NULL_RTX;
bool success;
- rtx prev = NULL_RTX;
+ rtx_insn *prev = NULL;
int i;
unsigned regno;
}
\f
/* Pointer to the last instruction scheduled. */
-static rtx last_scheduled_insn;
+static rtx_insn *last_scheduled_insn;
/* Pointer to the last nondebug instruction scheduled within the
block, or the prev_head of the scheduling block. Used by
rank_for_schedule, so that insns independent of the last scheduled
insn will be preferred over dependent instructions. */
-static rtx last_nondebug_scheduled_insn;
+static rtx_insn *last_nondebug_scheduled_insn;
/* Pointer that iterates through the list of unscheduled insns if we
have a dbg_cnt enabled. It always points at an insn prior to the
first unscheduled one. */
-static rtx nonscheduled_insns_begin;
+static rtx_insn *nonscheduled_insns_begin;
/* Compute cost of executing INSN.
This is the number of cycles between instruction issue and
instruction results. */
int
-insn_cost (rtx insn)
+insn_cost (rtx_insn *insn)
{
int cost;
+ if (sched_fusion)
+ return 0;
+
if (sel_sched_p ())
{
if (recog_memoized (insn) < 0)
int
dep_cost_1 (dep_t link, dw_t dw)
{
- rtx insn = DEP_PRO (link);
- rtx used = DEP_CON (link);
+ rtx_insn *insn = DEP_PRO (link);
+ rtx_insn *used = DEP_CON (link);
int cost;
if (DEP_COST (link) != UNKNOWN_DEP_COST)
{
/* This variable is used for backward compatibility with the
targets. */
- rtx dep_cost_rtx_link = alloc_INSN_LIST (NULL_RTX, NULL_RTX);
+ rtx_insn_list *dep_cost_rtx_link =
+ alloc_INSN_LIST (NULL_RTX, NULL);
/* Make it self-cycled, so that if some tries to walk over this
incomplete list he/she will be caught in an endless loop. */
/* Use this sel-sched.c friendly function in reorder2 instead of increasing
INSN_PRIORITY explicitly. */
void
-increase_insn_priority (rtx insn, int amount)
+increase_insn_priority (rtx_insn *insn, int amount)
{
if (!sel_sched_p ())
{
/* Compute the number of nondebug deps in list LIST for INSN. */
static int
-dep_list_size (rtx insn, sd_list_types_def list)
+dep_list_size (rtx_insn *insn, sd_list_types_def list)
{
sd_iterator_def sd_it;
dep_t dep;
return nodbgcount;
}
+bool sched_fusion;
+
/* Compute the priority number for INSN. */
static int
-priority (rtx insn)
+priority (rtx_insn *insn)
{
if (! INSN_P (insn))
return 0;
{
int this_priority = -1;
- if (dep_list_size (insn, SD_LIST_FORW) == 0)
+ if (sched_fusion)
+ {
+ int this_fusion_priority;
+
+ targetm.sched.fusion_priority (insn, FUSION_MAX_PRIORITY,
+ &this_fusion_priority, &this_priority);
+ INSN_FUSION_PRIORITY (insn) = this_fusion_priority;
+ }
+ else if (dep_list_size (insn, SD_LIST_FORW) == 0)
/* ??? We should set INSN_PRIORITY to insn_cost when and insn has
some forward deps but all of them are ignored by
contributes_to_priority hook. At the moment we set priority of
this_priority = insn_cost (insn);
else
{
- rtx prev_first, twin;
+ rtx_insn *prev_first, *twin;
basic_block rec;
/* For recovery check instructions we calculate priority slightly
FOR_EACH_DEP (twin, SD_LIST_FORW, sd_it, dep)
{
- rtx next;
+ rtx_insn *next;
int next_priority;
next = DEP_CON (dep);
/* Macros and functions for keeping the priority queue sorted, and
dealing with queuing and dequeuing of instructions. */
-#define SCHED_SORT(READY, N_READY) \
-do { if ((N_READY) == 2) \
- swap_sort (READY, N_READY); \
- else if ((N_READY) > 2) \
- qsort (READY, N_READY, sizeof (rtx), rank_for_schedule); } \
-while (0)
-
/* For each pressure class CL, set DEATH[CL] to the number of registers
in that class that die in INSN. */
static void
-calculate_reg_deaths (rtx insn, int *death)
+calculate_reg_deaths (rtx_insn *insn, int *death)
{
int i;
struct reg_use_data *use;
/* Setup info about the current register pressure impact of scheduling
INSN at the current scheduling point. */
static void
-setup_insn_reg_pressure_info (rtx insn)
+setup_insn_reg_pressure_info (rtx_insn *insn)
{
int i, change, before, after, hard_regno;
int excess_cost_change;
- enum machine_mode mode;
+ machine_mode mode;
enum reg_class cl;
struct reg_pressure_data *pressure_info;
int *max_reg_pressure;
cl = ira_pressure_classes[i];
gcc_assert (curr_reg_pressure[cl] >= 0);
change = (int) pressure_info[i].set_increase - death[cl];
- before = MAX (0, max_reg_pressure[i] - ira_class_hard_regs_num[cl]);
+ before = MAX (0, max_reg_pressure[i] - sched_class_regs_num[cl]);
after = MAX (0, max_reg_pressure[i] + change
- - ira_class_hard_regs_num[cl]);
+ - sched_class_regs_num[cl]);
hard_regno = ira_class_hard_regs[cl][0];
gcc_assert (hard_regno >= 0);
mode = reg_raw_mode[hard_regno];
than the main schedule. */
struct model_insn_info {
/* The instruction itself. */
- rtx insn;
+ rtx_insn *insn;
/* If this instruction is in model_worklist, these fields link to the
previous (higher-priority) and next (lower-priority) instructions
/* Index POINT gives the instruction at point POINT of the model schedule.
This array doesn't change during main scheduling. */
-static vec<rtx> model_schedule;
+static vec<rtx_insn *> model_schedule;
/* The list of instructions in the model worklist, sorted in order of
decreasing priority. */
doesn't belong to that schedule. */
static int
-model_index (rtx insn)
+model_index (rtx_insn *insn)
{
if (INSN_MODEL_INDEX (insn) == 0)
return model_num_insns;
/* Check whether the maximum pressure in the overall schedule
has increased. (This means that the MODEL_MAX_PRESSURE of
- every point <= POINT will need to increae too; see below.) */
+ every point <= POINT will need to increase too; see below.) */
if (group->limits[pci].pressure < ref_pressure)
group->limits[pci].pressure = ref_pressure;
/* INSN has just been scheduled. Update the model schedule accordingly. */
static void
-model_recompute (rtx insn)
+model_recompute (rtx_insn *insn)
{
struct {
int last_use;
/* After DEP, which was cancelled, has been resolved for insn NEXT,
check whether the insn's pattern needs restoring. */
static bool
-must_restore_pattern_p (rtx next, dep_t dep)
+must_restore_pattern_p (rtx_insn *next, dep_t dep)
{
if (QUEUE_INDEX (next) == QUEUE_SCHEDULED)
return false;
/* Return the cost of increasing the pressure in class CL from FROM to TO.
Here we use the very simplistic cost model that every register above
- ira_class_hard_regs_num[CL] has a spill cost of 1. We could use other
+ sched_class_regs_num[CL] has a spill cost of 1. We could use other
measures instead, such as one based on MEMORY_MOVE_COST. However:
(1) In order for an instruction to be scheduled, the higher cost
static int
model_spill_cost (int cl, int from, int to)
{
- from = MAX (from, ira_class_hard_regs_num[cl]);
+ from = MAX (from, sched_class_regs_num[cl]);
return MAX (to, from) - from;
}
if PRINT_P. */
static int
-model_excess_cost (rtx insn, bool print_p)
+model_excess_cost (rtx_insn *insn, bool print_p)
{
int point, pci, cl, cost, this_cost, delta;
struct reg_pressure_data *insn_reg_pressure;
/* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
static void
-model_set_excess_costs (rtx *insns, int count)
+model_set_excess_costs (rtx_insn **insns, int count)
{
int i, cost, priority_base, priority;
bool print_p;
/* Record the baseECC value for each instruction in the model schedule,
- except that negative costs are converted to zero ones now rather thatn
+ except that negative costs are converted to zero ones now rather than
later. Do not assign a cost to debug instructions, since they must
not change code-generation decisions. Experiments suggest we also
get better results by not assigning a cost to instructions from
}
}
\f
+
+/* Enum of rank_for_schedule heuristic decisions. */
+enum rfs_decision {
+ RFS_LIVE_RANGE_SHRINK1, RFS_LIVE_RANGE_SHRINK2,
+ RFS_SCHED_GROUP, RFS_PRESSURE_DELAY, RFS_PRESSURE_TICK,
+ RFS_FEEDS_BACKTRACK_INSN, RFS_PRIORITY, RFS_SPECULATION,
+ RFS_SCHED_RANK, RFS_LAST_INSN, RFS_PRESSURE_INDEX,
+ RFS_DEP_COUNT, RFS_TIE, RFS_FUSION, RFS_N };
+
+/* Corresponding strings for print outs. */
+static const char *rfs_str[RFS_N] = {
+ "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
+ "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
+ "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
+ "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
+ "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
+
+/* Statistical breakdown of rank_for_schedule decisions. */
+typedef struct { unsigned stats[RFS_N]; } rank_for_schedule_stats_t;
+static rank_for_schedule_stats_t rank_for_schedule_stats;
+
+/* Return the result of comparing insns TMP and TMP2 and update
+ Rank_For_Schedule statistics. */
+static int
+rfs_result (enum rfs_decision decision, int result, rtx tmp, rtx tmp2)
+{
+ ++rank_for_schedule_stats.stats[decision];
+ if (result < 0)
+ INSN_LAST_RFS_WIN (tmp) = decision;
+ else if (result > 0)
+ INSN_LAST_RFS_WIN (tmp2) = decision;
+ else
+ gcc_unreachable ();
+ return result;
+}
+
+/* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
+ keeping normal insns in original order. */
+
+static int
+rank_for_schedule_debug (const void *x, const void *y)
+{
+ rtx_insn *tmp = *(rtx_insn * const *) y;
+ rtx_insn *tmp2 = *(rtx_insn * const *) x;
+
+ /* Schedule debug insns as early as possible. */
+ if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
+ return -1;
+ else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
+ return 1;
+ else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
+ return INSN_LUID (tmp) - INSN_LUID (tmp2);
+ else
+ return INSN_RFS_DEBUG_ORIG_ORDER (tmp2) - INSN_RFS_DEBUG_ORIG_ORDER (tmp);
+}
+
/* Returns a positive value if x is preferred; returns a negative value if
y is preferred. Should never return 0, since that will make the sort
unstable. */
static int
rank_for_schedule (const void *x, const void *y)
{
- rtx tmp = *(const rtx *) y;
- rtx tmp2 = *(const rtx *) x;
+ rtx_insn *tmp = *(rtx_insn * const *) y;
+ rtx_insn *tmp2 = *(rtx_insn * const *) x;
int tmp_class, tmp2_class;
int val, priority_val, info_val, diff;
- if (MAY_HAVE_DEBUG_INSNS)
- {
- /* Schedule debug insns as early as possible. */
- if (DEBUG_INSN_P (tmp) && !DEBUG_INSN_P (tmp2))
- return -1;
- else if (!DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
- return 1;
- else if (DEBUG_INSN_P (tmp) && DEBUG_INSN_P (tmp2))
- return INSN_LUID (tmp) - INSN_LUID (tmp2);
- }
-
if (live_range_shrinkage_p)
{
/* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
|| INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2) < 0)
&& (diff = (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp)
- INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2))) != 0)
- return diff;
+ return rfs_result (RFS_LIVE_RANGE_SHRINK1, diff, tmp, tmp2);
/* Sort by INSN_LUID (original insn order), so that we make the
sort stable. This minimizes instruction movement, thus
minimizing sched's effect on debugging and cross-jumping. */
- return INSN_LUID (tmp) - INSN_LUID (tmp2);
+ return rfs_result (RFS_LIVE_RANGE_SHRINK2,
+ INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
}
/* The insn in a schedule group should be issued the first. */
if (flag_sched_group_heuristic &&
SCHED_GROUP_P (tmp) != SCHED_GROUP_P (tmp2))
- return SCHED_GROUP_P (tmp2) ? 1 : -1;
+ return rfs_result (RFS_SCHED_GROUP, SCHED_GROUP_P (tmp2) ? 1 : -1,
+ tmp, tmp2);
/* Make sure that priority of TMP and TMP2 are initialized. */
gcc_assert (INSN_PRIORITY_KNOWN (tmp) && INSN_PRIORITY_KNOWN (tmp2));
+ if (sched_fusion)
+ {
+ /* The instruction that has the same fusion priority as the last
+ instruction is the instruction we picked next. If that is not
+ the case, we sort ready list firstly by fusion priority, then
+ by priority, and at last by INSN_LUID. */
+ int a = INSN_FUSION_PRIORITY (tmp);
+ int b = INSN_FUSION_PRIORITY (tmp2);
+ int last = -1;
+
+ if (last_nondebug_scheduled_insn
+ && !NOTE_P (last_nondebug_scheduled_insn)
+ && BLOCK_FOR_INSN (tmp)
+ == BLOCK_FOR_INSN (last_nondebug_scheduled_insn))
+ last = INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn);
+
+ if (a != last && b != last)
+ {
+ if (a == b)
+ {
+ a = INSN_PRIORITY (tmp);
+ b = INSN_PRIORITY (tmp2);
+ }
+ if (a != b)
+ return rfs_result (RFS_FUSION, b - a, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION,
+ INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
+ }
+ else if (a == b)
+ {
+ gcc_assert (last_nondebug_scheduled_insn
+ && !NOTE_P (last_nondebug_scheduled_insn));
+ last = INSN_PRIORITY (last_nondebug_scheduled_insn);
+
+ a = abs (INSN_PRIORITY (tmp) - last);
+ b = abs (INSN_PRIORITY (tmp2) - last);
+ if (a != b)
+ return rfs_result (RFS_FUSION, a - b, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION,
+ INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
+ }
+ else if (a == last)
+ return rfs_result (RFS_FUSION, -1, tmp, tmp2);
+ else
+ return rfs_result (RFS_FUSION, 1, tmp, tmp2);
+ }
+
if (sched_pressure != SCHED_PRESSURE_NONE)
{
/* Prefer insn whose scheduling results in the smallest register
+ insn_delay (tmp)
- INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2)
- insn_delay (tmp2))))
- return diff;
+ return rfs_result (RFS_PRESSURE_DELAY, diff, tmp, tmp2);
}
if (sched_pressure != SCHED_PRESSURE_NONE
- && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var))
+ && (INSN_TICK (tmp2) > clock_var || INSN_TICK (tmp) > clock_var)
+ && INSN_TICK (tmp2) != INSN_TICK (tmp))
{
- if (INSN_TICK (tmp) <= clock_var)
- return -1;
- else if (INSN_TICK (tmp2) <= clock_var)
- return 1;
- else
- return INSN_TICK (tmp) - INSN_TICK (tmp2);
+ diff = INSN_TICK (tmp) - INSN_TICK (tmp2);
+ return rfs_result (RFS_PRESSURE_TICK, diff, tmp, tmp2);
}
/* If we are doing backtracking in this schedule, prefer insns that
{
priority_val = FEEDS_BACKTRACK_INSN (tmp2) - FEEDS_BACKTRACK_INSN (tmp);
if (priority_val)
- return priority_val;
+ return rfs_result (RFS_FEEDS_BACKTRACK_INSN, priority_val, tmp, tmp2);
}
/* Prefer insn with higher priority. */
priority_val = INSN_PRIORITY (tmp2) - INSN_PRIORITY (tmp);
if (flag_sched_critical_path_heuristic && priority_val)
- return priority_val;
+ return rfs_result (RFS_PRIORITY, priority_val, tmp, tmp2);
+
+ if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) >= 0)
+ {
+ int autopref = autopref_rank_for_schedule (tmp, tmp2);
+ if (autopref != 0)
+ return autopref;
+ }
/* Prefer speculative insn with greater dependencies weakness. */
if (flag_sched_spec_insn_heuristic && spec_info)
dw = dw2 - dw1;
if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8))
- return dw;
+ return rfs_result (RFS_SPECULATION, dw, tmp, tmp2);
}
info_val = (*current_sched_info->rank) (tmp, tmp2);
if (flag_sched_rank_heuristic && info_val)
- return info_val;
+ return rfs_result (RFS_SCHED_RANK, info_val, tmp, tmp2);
/* Compare insns based on their relation to the last scheduled
non-debug insn. */
{
dep_t dep1;
dep_t dep2;
- rtx last = last_nondebug_scheduled_insn;
+ rtx_insn *last = last_nondebug_scheduled_insn;
/* Classify the instructions into three classes:
1) Data dependent on last schedule insn.
tmp2_class = 2;
if ((val = tmp2_class - tmp_class))
- return val;
+ return rfs_result (RFS_LAST_INSN, val, tmp, tmp2);
}
/* Prefer instructions that occur earlier in the model schedule. */
- if (sched_pressure == SCHED_PRESSURE_MODEL)
+ if (sched_pressure == SCHED_PRESSURE_MODEL
+ && INSN_BB (tmp) == target_bb && INSN_BB (tmp2) == target_bb)
{
- int diff;
-
diff = model_index (tmp) - model_index (tmp2);
- if (diff != 0)
- return diff;
+ gcc_assert (diff != 0);
+ return rfs_result (RFS_PRESSURE_INDEX, diff, tmp, tmp2);
}
/* Prefer the insn which has more later insns that depend on it.
- dep_list_size (tmp, SD_LIST_FORW));
if (flag_sched_dep_count_heuristic && val != 0)
- return val;
+ return rfs_result (RFS_DEP_COUNT, val, tmp, tmp2);
/* If insns are equally good, sort by INSN_LUID (original insn order),
so that we make the sort stable. This minimizes instruction movement,
thus minimizing sched's effect on debugging and cross-jumping. */
- return INSN_LUID (tmp) - INSN_LUID (tmp2);
+ return rfs_result (RFS_TIE, INSN_LUID (tmp) - INSN_LUID (tmp2), tmp, tmp2);
}
/* Resort the array A in which only element at index N may be out of order. */
HAIFA_INLINE static void
-swap_sort (rtx *a, int n)
+swap_sort (rtx_insn **a, int n)
{
- rtx insn = a[n - 1];
+ rtx_insn *insn = a[n - 1];
int i = n - 2;
while (i >= 0 && rank_for_schedule (a + i, &insn) >= 0)
output. */
HAIFA_INLINE static void
-queue_insn (rtx insn, int n_cycles, const char *reason)
+queue_insn (rtx_insn *insn, int n_cycles, const char *reason)
{
int next_q = NEXT_Q_AFTER (q_ptr, n_cycles);
- rtx link = alloc_INSN_LIST (insn, insn_queue[next_q]);
+ rtx_insn_list *link = alloc_INSN_LIST (insn, insn_queue[next_q]);
int new_tick;
gcc_assert (n_cycles <= max_insn_queue_index);
/* Remove INSN from queue. */
static void
-queue_remove (rtx insn)
+queue_remove (rtx_insn *insn)
{
gcc_assert (QUEUE_INDEX (insn) >= 0);
remove_free_INSN_LIST_elem (insn, &insn_queue[QUEUE_INDEX (insn)]);
/* Return a pointer to the bottom of the ready list, i.e. the insn
with the lowest priority. */
-rtx *
+rtx_insn **
ready_lastpos (struct ready_list *ready)
{
gcc_assert (ready->n_ready >= 1);
lowest/highest priority depending on FIRST_P. */
HAIFA_INLINE static void
-ready_add (struct ready_list *ready, rtx insn, bool first_p)
+ready_add (struct ready_list *ready, rtx_insn *insn, bool first_p)
{
if (!first_p)
{
/* Remove the element with the highest priority from the ready list and
return it. */
-HAIFA_INLINE static rtx
+HAIFA_INLINE static rtx_insn *
ready_remove_first (struct ready_list *ready)
{
- rtx t;
+ rtx_insn *t;
gcc_assert (ready->n_ready);
t = ready->vec[ready->first--];
insn with the highest priority is 0, and the lowest priority has
N_READY - 1. */
-rtx
+rtx_insn *
ready_element (struct ready_list *ready, int index)
{
gcc_assert (ready->n_ready && index < ready->n_ready);
for insn with the highest priority is 0, and the lowest priority
has N_READY - 1. */
-HAIFA_INLINE static rtx
+HAIFA_INLINE static rtx_insn *
ready_remove (struct ready_list *ready, int index)
{
- rtx t;
+ rtx_insn *t;
int i;
if (index == 0)
/* Remove INSN from the ready list. */
static void
-ready_remove_insn (rtx insn)
+ready_remove_insn (rtx_insn *insn)
{
int i;
gcc_unreachable ();
}
-/* Sort the ready list READY by ascending priority, using the SCHED_SORT
- macro. */
+/* Calculate difference of two statistics set WAS and NOW.
+ Result returned in WAS. */
+static void
+rank_for_schedule_stats_diff (rank_for_schedule_stats_t *was,
+ const rank_for_schedule_stats_t *now)
+{
+ for (int i = 0; i < RFS_N; ++i)
+ was->stats[i] = now->stats[i] - was->stats[i];
+}
-void
-ready_sort (struct ready_list *ready)
+/* Print rank_for_schedule statistics. */
+static void
+print_rank_for_schedule_stats (const char *prefix,
+ const rank_for_schedule_stats_t *stats,
+ struct ready_list *ready)
+{
+ for (int i = 0; i < RFS_N; ++i)
+ if (stats->stats[i])
+ {
+ fprintf (sched_dump, "%s%20s: %u", prefix, rfs_str[i], stats->stats[i]);
+
+ if (ready != NULL)
+ /* Print out insns that won due to RFS_<I>. */
+ {
+ rtx_insn **p = ready_lastpos (ready);
+
+ fprintf (sched_dump, ":");
+ /* Start with 1 since least-priority insn didn't have any wins. */
+ for (int j = 1; j < ready->n_ready; ++j)
+ if (INSN_LAST_RFS_WIN (p[j]) == i)
+ fprintf (sched_dump, " %s",
+ (*current_sched_info->print_insn) (p[j], 0));
+ }
+ fprintf (sched_dump, "\n");
+ }
+}
+
+/* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
+ of array. */
+static void
+ready_sort_debug (struct ready_list *ready)
{
int i;
- rtx *first = ready_lastpos (ready);
+ rtx_insn **first = ready_lastpos (ready);
+
+ for (i = 0; i < ready->n_ready; ++i)
+ if (!DEBUG_INSN_P (first[i]))
+ INSN_RFS_DEBUG_ORIG_ORDER (first[i]) = i;
+
+ qsort (first, ready->n_ready, sizeof (rtx), rank_for_schedule_debug);
+}
+
+/* Sort non-debug insns in the ready list READY by ascending priority.
+ Assumes that all debug insns are separated from the real insns. */
+static void
+ready_sort_real (struct ready_list *ready)
+{
+ int i;
+ rtx_insn **first = ready_lastpos (ready);
+ int n_ready_real = ready->n_ready - ready->n_debug;
if (sched_pressure == SCHED_PRESSURE_WEIGHTED)
+ for (i = 0; i < n_ready_real; ++i)
+ setup_insn_reg_pressure_info (first[i]);
+ else if (sched_pressure == SCHED_PRESSURE_MODEL
+ && model_curr_point < model_num_insns)
+ model_set_excess_costs (first, n_ready_real);
+
+ rank_for_schedule_stats_t stats1;
+ if (sched_verbose >= 4)
+ stats1 = rank_for_schedule_stats;
+
+ if (n_ready_real == 2)
+ swap_sort (first, n_ready_real);
+ else if (n_ready_real > 2)
+ qsort (first, n_ready_real, sizeof (rtx), rank_for_schedule);
+
+ if (sched_verbose >= 4)
{
- for (i = 0; i < ready->n_ready; i++)
- if (!DEBUG_INSN_P (first[i]))
- setup_insn_reg_pressure_info (first[i]);
+ rank_for_schedule_stats_diff (&stats1, &rank_for_schedule_stats);
+ print_rank_for_schedule_stats (";;\t\t", &stats1, ready);
}
- if (sched_pressure == SCHED_PRESSURE_MODEL
- && model_curr_point < model_num_insns)
- model_set_excess_costs (first, ready->n_ready);
- SCHED_SORT (first, ready->n_ready);
+}
+
+/* Sort the ready list READY by ascending priority. */
+static void
+ready_sort (struct ready_list *ready)
+{
+ if (ready->n_debug > 0)
+ ready_sort_debug (ready);
+ else
+ ready_sort_real (ready);
}
/* PREV is an insn that is ready to execute. Adjust its priority if that
provide a hook for the target to tweak itself. */
HAIFA_INLINE static void
-adjust_priority (rtx prev)
+adjust_priority (rtx_insn *prev)
{
/* ??? There used to be code here to try and estimate how an insn
affected register lifetimes, but it did it by looking at REG_DEAD
{
advance_state (curr_state);
if (sched_verbose >= 4)
- fprintf (sched_dump, ";;\tAdvanced a state.\n");
+ fprintf (sched_dump, ";;\tAdvance the current state.\n");
}
/* Update register pressure after scheduling INSN. */
static void
-update_register_pressure (rtx insn)
+update_register_pressure (rtx_insn *insn)
{
struct reg_use_data *use;
struct reg_set_data *set;
meaning in sched-int.h::_haifa_insn_data) for all current BB insns
after insn AFTER. */
static void
-setup_insn_max_reg_pressure (rtx after, bool update_p)
+setup_insn_max_reg_pressure (rtx_insn *after, bool update_p)
{
int i, p;
bool eq_p;
- rtx insn;
+ rtx_insn *insn;
static int max_reg_pressure[N_REG_CLASSES];
save_reg_pressure ();
also max register pressure for unscheduled insns of the current
BB. */
static void
-update_reg_and_insn_max_reg_pressure (rtx insn)
+update_reg_and_insn_max_reg_pressure (rtx_insn *insn)
{
int i;
int before[N_REG_CLASSES];
insns starting after insn AFTER. Set up also max register pressure
for all insns of the basic block. */
void
-sched_setup_bb_reg_pressure_info (basic_block bb, rtx after)
+sched_setup_bb_reg_pressure_info (basic_block bb, rtx_insn *after)
{
gcc_assert (sched_pressure == SCHED_PRESSURE_WEIGHTED);
initiate_bb_reg_pressure_info (bb);
only be scheduled once their control dependency is resolved. */
static void
-check_clobbered_conditions (rtx insn)
+check_clobbered_conditions (rtx_insn *insn)
{
HARD_REG_SET t;
int i;
restart:
for (i = 0; i < ready.n_ready; i++)
{
- rtx x = ready_element (&ready, i);
+ rtx_insn *x = ready_element (&ready, i);
if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
{
ready_remove_insn (x);
}
for (i = 0; i <= max_insn_queue_index; i++)
{
- rtx link;
+ rtx_insn_list *link;
int q = NEXT_Q_AFTER (q_ptr, i);
restart_queue:
- for (link = insn_queue[q]; link; link = XEXP (link, 1))
+ for (link = insn_queue[q]; link; link = link->next ())
{
- rtx x = XEXP (link, 0);
+ rtx_insn *x = link->insn ();
if (TODO_SPEC (x) == DEP_CONTROL && cond_clobbered_p (x, t))
{
queue_remove (x);
/* Add INSN to the end of the model schedule. */
static void
-model_add_to_schedule (rtx insn)
+model_add_to_schedule (rtx_insn *insn)
{
unsigned int point;
static void
model_analyze_insns (void)
{
- rtx start, end, iter;
+ rtx_insn *start, *end, *iter;
sd_iterator_def sd_it;
dep_t dep;
struct model_insn_info *insn, *con;
model_reset_queue_indices (void)
{
unsigned int i;
- rtx insn;
+ rtx_insn *insn;
FOR_EACH_VEC_ELT (model_schedule, i, insn)
QUEUE_INDEX (insn) = MODEL_INSN_INFO (insn)->old_queue;
scheduling region. */
static void
-model_start_schedule (void)
+model_start_schedule (basic_block bb)
{
- basic_block bb;
-
model_next_priority = 1;
model_schedule.create (sched_max_luid);
model_insns = XCNEWVEC (struct model_insn_info, sched_max_luid);
- bb = BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head));
+ gcc_assert (bb == BLOCK_FOR_INSN (NEXT_INSN (current_sched_info->prev_head)));
initiate_reg_pressure_info (df_get_live_in (bb));
model_analyze_insns ();
model_finalize_pressure_group (&model_before_pressure);
model_schedule.release ();
}
+
+/* Prepare reg pressure scheduling for basic block BB. */
+static void
+sched_pressure_start_bb (basic_block bb)
+{
+ /* Set the number of available registers for each class taking into account
+ relative probability of current basic block versus function prologue and
+ epilogue.
+ * If the basic block executes much more often than the prologue/epilogue
+ (e.g., inside a hot loop), then cost of spill in the prologue is close to
+ nil, so the effective number of available registers is
+ (ira_class_hard_regs_num[cl] - 0).
+ * If the basic block executes as often as the prologue/epilogue,
+ then spill in the block is as costly as in the prologue, so the effective
+ number of available registers is
+ (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
+ Note that all-else-equal, we prefer to spill in the prologue, since that
+ allows "extra" registers for other basic blocks of the function.
+ * If the basic block is on the cold path of the function and executes
+ rarely, then we should always prefer to spill in the block, rather than
+ in the prologue/epilogue. The effective number of available register is
+ (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
+ {
+ int i;
+ int entry_freq = ENTRY_BLOCK_PTR_FOR_FN (cfun)->frequency;
+ int bb_freq = bb->frequency;
+
+ if (bb_freq == 0)
+ {
+ if (entry_freq == 0)
+ entry_freq = bb_freq = 1;
+ }
+ if (bb_freq < entry_freq)
+ bb_freq = entry_freq;
+
+ for (i = 0; i < ira_pressure_classes_num; ++i)
+ {
+ enum reg_class cl = ira_pressure_classes[i];
+ sched_class_regs_num[cl] = ira_class_hard_regs_num[cl];
+ sched_class_regs_num[cl]
+ -= (call_used_regs_num[cl] * entry_freq) / bb_freq;
+ }
+ }
+
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
+ model_start_schedule (bb);
+}
\f
/* A structure that holds local state for the loop in schedule_block. */
struct sched_block_state
zero for insns in a schedule group). */
static int
-schedule_insn (rtx insn)
+schedule_insn (rtx_insn *insn)
{
sd_iterator_def sd_it;
dep_t dep;
for (sd_it = sd_iterator_start (insn, SD_LIST_BACK);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx dbg = DEP_PRO (dep);
+ rtx_insn *dbg = DEP_PRO (dep);
struct reg_use_data *use, *next;
if (DEP_STATUS (dep) & DEP_CANCELLED)
gcc_assert (INSN_TICK (insn) >= MIN_TICK);
if (INSN_TICK (insn) > clock_var)
/* INSN has been prematurely moved from the queue to the ready list.
- This is possible only if following flag is set. */
- gcc_assert (flag_sched_stalled_insns);
+ This is possible only if following flags are set. */
+ gcc_assert (flag_sched_stalled_insns || sched_fusion);
/* ??? Probably, if INSN is scheduled prematurely, we should leave
INSN_TICK untouched. This is a machine-dependent issue, actually. */
sd_iterator_cond (&sd_it, &dep); sd_iterator_next (&sd_it))
{
struct dep_replacement *desc = DEP_REPLACE (dep);
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
if (QUEUE_INDEX (pro) != QUEUE_SCHEDULED
&& desc != NULL && desc->insn == pro)
apply_replacement (dep, false);
for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx next = DEP_CON (dep);
+ rtx_insn *next = DEP_CON (dep);
bool cancelled = (DEP_STATUS (dep) & DEP_CANCELLED) != 0;
/* Resolve the dependence between INSN and NEXT.
/* Add note list that ends on FROM_END to the end of TO_ENDP. */
void
-concat_note_lists (rtx from_end, rtx *to_endp)
+concat_note_lists (rtx_insn *from_end, rtx_insn **to_endp)
{
- rtx from_start;
+ rtx_insn *from_start;
/* It's easy when have nothing to concat. */
if (from_end == NULL)
while (PREV_INSN (from_start) != NULL)
from_start = PREV_INSN (from_start);
- PREV_INSN (from_start) = *to_endp;
- NEXT_INSN (*to_endp) = from_start;
+ SET_PREV_INSN (from_start) = *to_endp;
+ SET_NEXT_INSN (*to_endp) = from_start;
*to_endp = from_end;
}
/* Delete notes between HEAD and TAIL and put them in the chain
of notes ended by NOTE_LIST. */
void
-remove_notes (rtx head, rtx tail)
+remove_notes (rtx_insn *head, rtx_insn *tail)
{
- rtx next_tail, insn, next;
+ rtx_insn *next_tail, *insn, *next;
note_list = 0;
if (head == tail && !INSN_P (head))
remove_insn (insn);
/* Add the note to list that ends at NOTE_LIST. */
- PREV_INSN (insn) = note_list;
- NEXT_INSN (insn) = NULL_RTX;
+ SET_PREV_INSN (insn) = note_list;
+ SET_NEXT_INSN (insn) = NULL_RTX;
if (note_list)
- NEXT_INSN (note_list) = insn;
+ SET_NEXT_INSN (note_list) = insn;
note_list = insn;
break;
}
struct ready_list ready;
state_t curr_state;
- rtx last_scheduled_insn;
- rtx last_nondebug_scheduled_insn;
- rtx nonscheduled_insns_begin;
+ rtx_insn *last_scheduled_insn;
+ rtx_insn *last_nondebug_scheduled_insn;
+ rtx_insn *nonscheduled_insns_begin;
int cycle_issued_insns;
/* Copies of state used in the inner loop of schedule_block. */
/* We don't need to save q_ptr, as its value is arbitrary and we can set it
to 0 when restoring. */
int q_size;
- rtx *insn_queue;
+ rtx_insn_list **insn_queue;
/* Describe pattern replacements that occurred since this backtrack point
was queued. */
/* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
to SET_P. */
static void
-mark_backtrack_feeds (rtx insn, int set_p)
+mark_backtrack_feeds (rtx_insn *insn, int set_p)
{
sd_iterator_def sd_it;
dep_t dep;
save->ready.n_ready = ready.n_ready;
save->ready.n_debug = ready.n_debug;
save->ready.veclen = ready.veclen;
- save->ready.vec = XNEWVEC (rtx, ready.veclen);
+ save->ready.vec = XNEWVEC (rtx_insn *, ready.veclen);
memcpy (save->ready.vec, ready.vec, ready.veclen * sizeof (rtx));
- save->insn_queue = XNEWVEC (rtx, max_insn_queue_index + 1);
+ save->insn_queue = XNEWVEC (rtx_insn_list *, max_insn_queue_index + 1);
save->q_size = q_size;
for (i = 0; i <= max_insn_queue_index; i++)
{
if (ready.n_ready > 0)
{
- rtx *first = ready_lastpos (&ready);
+ rtx_insn **first = ready_lastpos (&ready);
for (i = 0; i < ready.n_ready; i++)
FOR_EACH_DEP (first[i], SD_LIST_BACK, sd_it, dep)
if (!DEBUG_INSN_P (DEP_PRO (dep)))
for (i = 0; i <= max_insn_queue_index; i++)
{
int q = NEXT_Q_AFTER (q_ptr, i);
- rtx link;
- for (link = insn_queue[q]; link; link = XEXP (link, 1))
+ rtx_insn_list *link;
+ for (link = insn_queue[q]; link; link = link->next ())
{
- rtx insn = XEXP (link, 0);
+ rtx_insn *insn = link->insn ();
FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
if (!DEBUG_INSN_P (DEP_PRO (dep)))
{
queued nowhere. */
static void
-unschedule_insns_until (rtx insn)
+unschedule_insns_until (rtx_insn *insn)
{
- auto_vec<rtx> recompute_vec;
+ auto_vec<rtx_insn *> recompute_vec;
/* Make two passes over the insns to be unscheduled. First, we clear out
dependencies and other trivial bookkeeping. */
for (;;)
{
- rtx last;
+ rtx_insn *last;
sd_iterator_def sd_it;
dep_t dep;
for (sd_it = sd_iterator_start (last, SD_LIST_RES_FORW);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx con = DEP_CON (dep);
+ rtx_insn *con = DEP_CON (dep);
sd_unresolve_dep (sd_it);
if (!MUST_RECOMPUTE_SPEC_P (con))
{
up-to-date. */
while (!recompute_vec.is_empty ())
{
- rtx con;
+ rtx_insn *con;
con = recompute_vec.pop ();
MUST_RECOMPUTE_SPEC_P (con) = 0;
static void
restore_last_backtrack_point (struct sched_block_state *psched_block)
{
- rtx link;
int i;
struct haifa_saved_data *save = backtrack_queue;
of the queues. */
if (ready.n_ready > 0)
{
- rtx *first = ready_lastpos (&ready);
+ rtx_insn **first = ready_lastpos (&ready);
for (i = 0; i < ready.n_ready; i++)
{
- rtx insn = first[i];
+ rtx_insn *insn = first[i];
QUEUE_INDEX (insn) = QUEUE_NOWHERE;
INSN_TICK (insn) = INVALID_TICK;
}
{
int q = NEXT_Q_AFTER (q_ptr, i);
- for (link = insn_queue[q]; link; link = XEXP (link, 1))
+ for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
{
- rtx x = XEXP (link, 0);
+ rtx_insn *x = link->insn ();
QUEUE_INDEX (x) = QUEUE_NOWHERE;
INSN_TICK (x) = INVALID_TICK;
}
if (ready.n_ready > 0)
{
- rtx *first = ready_lastpos (&ready);
+ rtx_insn **first = ready_lastpos (&ready);
for (i = 0; i < ready.n_ready; i++)
{
- rtx insn = first[i];
+ rtx_insn *insn = first[i];
QUEUE_INDEX (insn) = QUEUE_READY;
TODO_SPEC (insn) = recompute_todo_spec (insn, true);
INSN_TICK (insn) = save->clock_var;
insn_queue[q] = save->insn_queue[q];
- for (link = insn_queue[q]; link; link = XEXP (link, 1))
+ for (rtx_insn_list *link = insn_queue[q]; link; link = link->next ())
{
- rtx x = XEXP (link, 0);
+ rtx_insn *x = link->insn ();
QUEUE_INDEX (x) = i;
TODO_SPEC (x) = recompute_todo_spec (x, true);
INSN_TICK (x) = save->clock_var + i;
static void
restore_pattern (dep_t dep, bool immediately)
{
- rtx next = DEP_CON (dep);
+ rtx_insn *next = DEP_CON (dep);
int tick = INSN_TICK (next);
/* If we already scheduled the insn, the modified version is
reduced on recursive calls. Return true if we produced a good
estimate, or false if we exceeded the budget. */
static bool
-estimate_insn_tick (bitmap processed, rtx insn, int budget)
+estimate_insn_tick (bitmap processed, rtx_insn *insn, int budget)
{
sd_iterator_def sd_it;
dep_t dep;
FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
int t;
if (DEP_STATUS (dep) & DEP_CANCELLED)
/* If INSN has no unresolved backwards dependencies, add it to the schedule and
recursively resolve all its forward dependencies. */
static void
-resolve_dependencies (rtx insn)
+resolve_dependencies (rtx_insn *insn)
{
sd_iterator_def sd_it;
dep_t dep;
for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx next = DEP_CON (dep);
+ rtx_insn *next = DEP_CON (dep);
if (sched_verbose >= 4)
fprintf (sched_dump, ";;\t\tdep %d against %d\n", INSN_UID (insn),
/* Return the head and tail pointers of ebb starting at BEG and ending
at END. */
void
-get_ebb_head_tail (basic_block beg, basic_block end, rtx *headp, rtx *tailp)
+get_ebb_head_tail (basic_block beg, basic_block end,
+ rtx_insn **headp, rtx_insn **tailp)
{
- rtx beg_head = BB_HEAD (beg);
- rtx beg_tail = BB_END (beg);
- rtx end_head = BB_HEAD (end);
- rtx end_tail = BB_END (end);
+ rtx_insn *beg_head = BB_HEAD (beg);
+ rtx_insn * beg_tail = BB_END (beg);
+ rtx_insn * end_head = BB_HEAD (end);
+ rtx_insn * end_tail = BB_END (end);
/* Don't include any notes or labels at the beginning of the BEG
basic block, or notes at the end of the END basic blocks. */
beg_head = NEXT_INSN (beg_head);
else if (DEBUG_INSN_P (beg_head))
{
- rtx note, next;
+ rtx_insn * note, *next;
for (note = NEXT_INSN (beg_head);
note != beg_tail;
end_tail = PREV_INSN (end_tail);
else if (DEBUG_INSN_P (end_tail))
{
- rtx note, prev;
+ rtx_insn * note, *prev;
for (note = PREV_INSN (end_tail);
note != end_head;
/* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
int
-no_real_insns_p (const_rtx head, const_rtx tail)
+no_real_insns_p (const rtx_insn *head, const rtx_insn *tail)
{
while (head != NEXT_INSN (tail))
{
/* Restore-other-notes: NOTE_LIST is the end of a chain of notes
previously found among the insns. Insert them just before HEAD. */
-rtx
-restore_other_notes (rtx head, basic_block head_bb)
+rtx_insn *
+restore_other_notes (rtx_insn *head, basic_block head_bb)
{
if (note_list != 0)
{
- rtx note_head = note_list;
+ rtx_insn *note_head = note_list;
if (head)
head_bb = BLOCK_FOR_INSN (head);
/* In the above cycle we've missed this note. */
set_block_for_insn (note_head, head_bb);
- PREV_INSN (note_head) = PREV_INSN (head);
- NEXT_INSN (PREV_INSN (head)) = note_head;
- PREV_INSN (head) = note_list;
- NEXT_INSN (note_list) = head;
+ SET_PREV_INSN (note_head) = PREV_INSN (head);
+ SET_NEXT_INSN (PREV_INSN (head)) = note_head;
+ SET_PREV_INSN (head) = note_list;
+ SET_NEXT_INSN (note_list) = head;
if (BLOCK_FOR_INSN (head) != head_bb)
BB_END (head_bb) = note_list;
static void
undo_all_replacements (void)
{
- rtx insn;
+ rtx_insn *insn;
int i;
FOR_EACH_VEC_ELT (scheduled_insns, i, insn)
/* Return first non-scheduled insn in the current scheduling block.
This is mostly used for debug-counter purposes. */
-static rtx
+static rtx_insn *
first_nonscheduled_insn (void)
{
- rtx insn = (nonscheduled_insns_begin != NULL_RTX
- ? nonscheduled_insns_begin
- : current_sched_info->prev_head);
+ rtx_insn *insn = (nonscheduled_insns_begin != NULL_RTX
+ ? nonscheduled_insns_begin
+ : current_sched_info->prev_head);
do
{
static void
queue_to_ready (struct ready_list *ready)
{
- rtx insn;
- rtx link;
- rtx skip_insn;
+ rtx_insn *insn;
+ rtx_insn_list *link;
+ rtx_insn *skip_insn;
q_ptr = NEXT_Q (q_ptr);
nonscheduled insn. */
skip_insn = first_nonscheduled_insn ();
else
- skip_insn = NULL_RTX;
+ skip_insn = NULL;
/* Add all pending insns that can be scheduled without stalls to the
ready list. */
- for (link = insn_queue[q_ptr]; link; link = XEXP (link, 1))
+ for (link = insn_queue[q_ptr]; link; link = link->next ())
{
- insn = XEXP (link, 0);
+ insn = link->insn ();
q_size -= 1;
if (sched_verbose >= 2)
{
if ((link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)]))
{
- for (; link; link = XEXP (link, 1))
+ for (; link; link = link->next ())
{
- insn = XEXP (link, 0);
+ insn = link->insn ();
q_size -= 1;
if (sched_verbose >= 2)
addition) depending on user flags and target hooks. */
static bool
-ok_for_early_queue_removal (rtx insn)
+ok_for_early_queue_removal (rtx_insn *insn)
{
if (targetm.sched.is_costly_dependence)
{
- rtx prev_insn;
int n_cycles;
int i = scheduled_insns.length ();
for (n_cycles = flag_sched_stalled_insns_dep; n_cycles; n_cycles--)
{
int cost;
- prev_insn = scheduled_insns[i];
+ rtx_insn *prev_insn = scheduled_insns[i];
if (!NOTE_P (prev_insn))
{
static int
early_queue_to_ready (state_t state, struct ready_list *ready)
{
- rtx insn;
- rtx link;
- rtx next_link;
- rtx prev_link;
+ rtx_insn *insn;
+ rtx_insn_list *link;
+ rtx_insn_list *next_link;
+ rtx_insn_list *prev_link;
bool move_to_ready;
int cost;
state_t temp_state = alloca (dfa_state_size);
prev_link = 0;
while (link)
{
- next_link = XEXP (link, 1);
- insn = XEXP (link, 0);
+ next_link = link->next ();
+ insn = link->insn ();
if (insn && sched_verbose > 6)
print_rtl_single (sched_dump, insn);
static void
debug_ready_list_1 (struct ready_list *ready, signed char *ready_try)
{
- rtx *p;
+ rtx_insn **p;
int i;
if (ready->n_ready == 0)
fprintf (sched_dump, ":prio=%d", INSN_PRIORITY (p[i]));
if (INSN_TICK (p[i]) > clock_var)
fprintf (sched_dump, ":delay=%d", INSN_TICK (p[i]) - clock_var);
+ if (sched_pressure == SCHED_PRESSURE_MODEL)
+ fprintf (sched_dump, ":idx=%d",
+ model_index (p[i]));
if (sched_pressure != SCHED_PRESSURE_NONE)
fprintf (sched_dump, ")");
}
NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
replaces the epilogue note in the correct basic block. */
void
-reemit_notes (rtx insn)
+reemit_notes (rtx_insn *insn)
{
- rtx note, last = insn;
+ rtx note;
+ rtx_insn *last = insn;
for (note = REG_NOTES (insn); note; note = XEXP (note, 1))
{
/* Move INSN. Reemit notes if needed. Update CFG, if needed. */
static void
-move_insn (rtx insn, rtx last, rtx nt)
+move_insn (rtx_insn *insn, rtx_insn *last, rtx nt)
{
if (PREV_INSN (insn) != last)
{
basic_block bb;
- rtx note;
+ rtx_insn *note;
int jump_p = 0;
bb = BLOCK_FOR_INSN (insn);
else
note = insn;
- NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
- PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
+ SET_NEXT_INSN (PREV_INSN (insn)) = NEXT_INSN (note);
+ SET_PREV_INSN (NEXT_INSN (note)) = PREV_INSN (insn);
- NEXT_INSN (note) = NEXT_INSN (last);
- PREV_INSN (NEXT_INSN (last)) = note;
+ SET_NEXT_INSN (note) = NEXT_INSN (last);
+ SET_PREV_INSN (NEXT_INSN (last)) = note;
- NEXT_INSN (last) = insn;
- PREV_INSN (insn) = last;
+ SET_NEXT_INSN (last) = insn;
+ SET_PREV_INSN (insn) = last;
bb = BLOCK_FOR_INSN (last);
/* Return true if scheduling INSN will finish current clock cycle. */
static bool
-insn_finishes_cycle_p (rtx insn)
+insn_finishes_cycle_p (rtx_insn *insn)
{
if (SCHED_GROUP_P (insn))
/* After issuing INSN, rest of the sched_group will be forced to issue
return false;
}
+/* Functions to model cache auto-prefetcher.
+
+ Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
+ memory prefetches if it sees instructions with consequitive memory accesses
+ in the instruction stream. Details of such hardware units are not published,
+ so we can only guess what exactly is going on there.
+ In the scheduler, we model abstract auto-prefetcher. If there are memory
+ insns in the ready list (or the queue) that have same memory base, but
+ different offsets, then we delay the insns with larger offsets until insns
+ with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
+ is "1", then we look at the ready list; if it is N>1, then we also look
+ through N-1 queue entries.
+ If the param is N>=0, then rank_for_schedule will consider auto-prefetching
+ among its heuristics.
+ Param value of "-1" disables modelling of the auto-prefetcher. */
+
+/* Initialize autoprefetcher model data for INSN. */
+static void
+autopref_multipass_init (const rtx_insn *insn, int write)
+{
+ autopref_multipass_data_t data = &INSN_AUTOPREF_MULTIPASS_DATA (insn)[write];
+
+ gcc_assert (data->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED);
+ data->base = NULL_RTX;
+ data->offset = 0;
+ /* Set insn entry initialized, but not relevant for auto-prefetcher. */
+ data->status = AUTOPREF_MULTIPASS_DATA_IRRELEVANT;
+
+ rtx set = single_set (insn);
+ if (set == NULL_RTX)
+ return;
+
+ rtx mem = write ? SET_DEST (set) : SET_SRC (set);
+ if (!MEM_P (mem))
+ return;
+
+ struct address_info info;
+ decompose_mem_address (&info, mem);
+
+ /* TODO: Currently only (base+const) addressing is supported. */
+ if (info.base == NULL || !REG_P (*info.base)
+ || (info.disp != NULL && !CONST_INT_P (*info.disp)))
+ return;
+
+ /* This insn is relevant for auto-prefetcher. */
+ data->base = *info.base;
+ data->offset = info.disp ? INTVAL (*info.disp) : 0;
+ data->status = AUTOPREF_MULTIPASS_DATA_NORMAL;
+}
+
+/* Helper function for rank_for_schedule sorting. */
+static int
+autopref_rank_for_schedule (const rtx_insn *insn1, const rtx_insn *insn2)
+{
+ for (int write = 0; write < 2; ++write)
+ {
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+ autopref_multipass_data_t data2
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
+
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn1, write);
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ continue;
+
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn2, write);
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ continue;
+
+ if (!rtx_equal_p (data1->base, data2->base))
+ continue;
+
+ return data1->offset - data2->offset;
+ }
+
+ return 0;
+}
+
+/* True if header of debug dump was printed. */
+static bool autopref_multipass_dfa_lookahead_guard_started_dump_p;
+
+/* Helper for autopref_multipass_dfa_lookahead_guard.
+ Return "1" if INSN1 should be delayed in favor of INSN2. */
+static int
+autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn *insn1,
+ const rtx_insn *insn2, int write)
+{
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+ autopref_multipass_data_t data2
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn2)[write];
+
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn2, write);
+ if (data2->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ return 0;
+
+ if (rtx_equal_p (data1->base, data2->base)
+ && data1->offset > data2->offset)
+ {
+ if (sched_verbose >= 2)
+ {
+ if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
+ {
+ fprintf (sched_dump,
+ ";;\t\tnot trying in max_issue due to autoprefetch "
+ "model: ");
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
+ }
+
+ fprintf (sched_dump, " %d(%d)", INSN_UID (insn1), INSN_UID (insn2));
+ }
+
+ return 1;
+ }
+
+ return 0;
+}
+
+/* General note:
+
+ We could have also hooked autoprefetcher model into
+ first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
+ to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
+ (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
+ unblocked). We don't bother about this yet because target of interest
+ (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
+
+/* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
+ Return "1" if INSN1 should not be considered in max_issue due to
+ auto-prefetcher considerations. */
+int
+autopref_multipass_dfa_lookahead_guard (rtx_insn *insn1, int ready_index)
+{
+ int r = 0;
+
+ if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) <= 0)
+ return 0;
+
+ if (sched_verbose >= 2 && ready_index == 0)
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = false;
+
+ for (int write = 0; write < 2; ++write)
+ {
+ autopref_multipass_data_t data1
+ = &INSN_AUTOPREF_MULTIPASS_DATA (insn1)[write];
+
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_UNINITIALIZED)
+ autopref_multipass_init (insn1, write);
+ if (data1->status == AUTOPREF_MULTIPASS_DATA_IRRELEVANT)
+ continue;
+
+ if (ready_index == 0
+ && data1->status == AUTOPREF_MULTIPASS_DATA_DONT_DELAY)
+ /* We allow only a single delay on priviledged instructions.
+ Doing otherwise would cause infinite loop. */
+ {
+ if (sched_verbose >= 2)
+ {
+ if (!autopref_multipass_dfa_lookahead_guard_started_dump_p)
+ {
+ fprintf (sched_dump,
+ ";;\t\tnot trying in max_issue due to autoprefetch "
+ "model: ");
+ autopref_multipass_dfa_lookahead_guard_started_dump_p = true;
+ }
+
+ fprintf (sched_dump, " *%d*", INSN_UID (insn1));
+ }
+ continue;
+ }
+
+ for (int i2 = 0; i2 < ready.n_ready; ++i2)
+ {
+ rtx_insn *insn2 = get_ready_element (i2);
+ if (insn1 == insn2)
+ continue;
+ r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2, write);
+ if (r)
+ {
+ if (ready_index == 0)
+ {
+ r = -1;
+ data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
+ }
+ goto finish;
+ }
+ }
+
+ if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) == 1)
+ continue;
+
+ /* Everything from the current queue slot should have been moved to
+ the ready list. */
+ gcc_assert (insn_queue[NEXT_Q_AFTER (q_ptr, 0)] == NULL_RTX);
+
+ int n_stalls = PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH) - 1;
+ if (n_stalls > max_insn_queue_index)
+ n_stalls = max_insn_queue_index;
+
+ for (int stalls = 1; stalls <= n_stalls; ++stalls)
+ {
+ for (rtx_insn_list *link = insn_queue[NEXT_Q_AFTER (q_ptr, stalls)];
+ link != NULL_RTX;
+ link = link->next ())
+ {
+ rtx_insn *insn2 = link->insn ();
+ r = autopref_multipass_dfa_lookahead_guard_1 (insn1, insn2,
+ write);
+ if (r)
+ {
+ /* Queue INSN1 until INSN2 can issue. */
+ r = -stalls;
+ if (ready_index == 0)
+ data1->status = AUTOPREF_MULTIPASS_DATA_DONT_DELAY;
+ goto finish;
+ }
+ }
+ }
+ }
+
+ finish:
+ if (sched_verbose >= 2
+ && autopref_multipass_dfa_lookahead_guard_started_dump_p
+ && (ready_index == ready.n_ready - 1 || r < 0))
+ /* This does not /always/ trigger. We don't output EOL if the last
+ insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
+ called. We can live with this. */
+ fprintf (sched_dump, "\n");
+
+ return r;
+}
+
/* Define type for target data used in multipass scheduling. */
#ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
# define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
static int max_lookahead_tries;
-/* The following value is value of hook
- `first_cycle_multipass_dfa_lookahead' at the last call of
- `max_issue'. */
-static int cached_first_cycle_multipass_dfa_lookahead = 0;
-
-/* The following value is value of `issue_rate' at the last call of
- `sched_init'. */
-static int cached_issue_rate = 0;
-
/* The following function returns maximal (or close to maximal) number
of insns which can be issued on the same cycle and one of which
insns is insns with the best rank (the first insn in READY). To
int n, i, all, n_ready, best, delay, tries_num;
int more_issue;
struct choice_entry *top;
- rtx insn;
+ rtx_insn *insn;
+
+ if (sched_fusion)
+ return 0;
n_ready = ready->n_ready;
gcc_assert (dfa_lookahead >= 1 && privileged_n >= 0
&& privileged_n <= n_ready);
/* Init MAX_LOOKAHEAD_TRIES. */
- if (cached_first_cycle_multipass_dfa_lookahead != dfa_lookahead)
+ if (max_lookahead_tries == 0)
{
- cached_first_cycle_multipass_dfa_lookahead = dfa_lookahead;
max_lookahead_tries = 100;
for (i = 0; i < issue_rate; i++)
max_lookahead_tries *= dfa_lookahead;
1 if choose_ready () should be restarted without advancing the cycle. */
static int
choose_ready (struct ready_list *ready, bool first_cycle_insn_p,
- rtx *insn_ptr)
+ rtx_insn **insn_ptr)
{
- int lookahead;
-
if (dbg_cnt (sched_insn) == false)
{
if (nonscheduled_insns_begin == NULL_RTX)
nonscheduled_insns_begin = current_sched_info->prev_head;
- rtx insn = first_nonscheduled_insn ();
+ rtx_insn *insn = first_nonscheduled_insn ();
if (QUEUE_INDEX (insn) == QUEUE_READY)
/* INSN is in the ready_list. */
return -1;
}
- lookahead = 0;
-
- if (targetm.sched.first_cycle_multipass_dfa_lookahead)
- lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
- if (lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0))
+ if (dfa_lookahead <= 0 || SCHED_GROUP_P (ready_element (ready, 0))
|| DEBUG_INSN_P (ready_element (ready, 0)))
{
- if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON))
+ if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
*insn_ptr = ready_remove_first_dispatch (ready);
else
*insn_ptr = ready_remove_first (ready);
{
/* Try to choose the best insn. */
int index = 0, i;
- rtx insn;
+ rtx_insn *insn;
insn = ready_element (ready, 0);
if (INSN_CODE (insn) < 0)
block. TARGET_BB is the argument passed to schedule_block. */
static void
-commit_schedule (rtx prev_head, rtx tail, basic_block *target_bb)
+commit_schedule (rtx_insn *prev_head, rtx_insn *tail, basic_block *target_bb)
{
unsigned int i;
- rtx insn;
+ rtx_insn *insn;
last_scheduled_insn = prev_head;
for (i = 0;
if (sched_verbose)
{
- rtx x;
+ rtx_insn *x;
x = next_real_insn (last_scheduled_insn);
gcc_assert (x);
bool sched_group_found = false;
int min_cost_group = 1;
+ if (sched_fusion)
+ return;
+
for (i = 0; i < ready.n_ready; i++)
{
- rtx insn = ready_element (&ready, i);
+ rtx_insn *insn = ready_element (&ready, i);
if (SCHED_GROUP_P (insn))
{
sched_group_found = true;
int n = ready.n_ready;
for (i = 0; i < n; i++)
{
- rtx insn = ready_element (&ready, i);
+ rtx_insn *insn = ready_element (&ready, i);
int cost = 0;
const char *reason = "resource conflict";
if (SCHED_GROUP_P (insn) && cost > min_cost_group)
min_cost_group = cost;
ready_remove (&ready, i);
- queue_insn (insn, cost, reason);
+ /* Normally we'd want to queue INSN for COST cycles. However,
+ if SCHED_GROUP_P is set, then we must ensure that nothing
+ else comes between INSN and its predecessor. If there is
+ some other insn ready to fire on the next cycle, then that
+ invariant would be broken.
+
+ So when SCHED_GROUP_P is set, just queue this insn for a
+ single cycle. */
+ queue_insn (insn, SCHED_GROUP_P (insn) ? 1 : cost, reason);
if (i + 1 < n)
break;
}
{
int t;
struct delay_pair *pair = save->delay_pair;
- rtx i1 = pair->i1;
+ rtx_insn *i1 = pair->i1;
for (; pair; pair = pair->next_same_i1)
{
- rtx i2 = pair->i2;
+ rtx_insn *i2 = pair->i2;
if (QUEUE_INDEX (i2) == QUEUE_SCHEDULED)
continue;
/* Print instructions together with useful scheduling information between
HEAD and TAIL (inclusive). */
static void
-dump_insn_stream (rtx head, rtx tail)
+dump_insn_stream (rtx_insn *head, rtx_insn *tail)
{
fprintf (sched_dump, ";;\t| insn | prio |\n");
- rtx next_tail = NEXT_INSN (tail);
- for (rtx insn = head; insn != next_tail; insn = NEXT_INSN (insn))
+ rtx_insn *next_tail = NEXT_INSN (tail);
+ for (rtx_insn *insn = head; insn != next_tail; insn = NEXT_INSN (insn))
{
int priority = NOTE_P (insn) ? 0 : INSN_PRIORITY (insn);
const char *pattern = (NOTE_P (insn)
int sort_p, advance, start_clock_var;
/* Head/tail info for this block. */
- rtx prev_head = current_sched_info->prev_head;
- rtx next_tail = current_sched_info->next_tail;
- rtx head = NEXT_INSN (prev_head);
- rtx tail = PREV_INSN (next_tail);
+ rtx_insn *prev_head = current_sched_info->prev_head;
+ rtx_insn *next_tail = current_sched_info->next_tail;
+ rtx_insn *head = NEXT_INSN (prev_head);
+ rtx_insn *tail = PREV_INSN (next_tail);
if ((current_sched_info->flags & DONT_BREAK_DEPENDENCIES) == 0
- && sched_pressure != SCHED_PRESSURE_MODEL)
+ && sched_pressure != SCHED_PRESSURE_MODEL && !sched_fusion)
find_modifiable_mems (head, tail);
/* We used to have code to avoid getting parameters moved from hard
dump_new_block_header (0, *target_bb, head, tail);
if (sched_verbose >= 2)
- dump_insn_stream (head, tail);
+ {
+ dump_insn_stream (head, tail);
+ memset (&rank_for_schedule_stats, 0,
+ sizeof (rank_for_schedule_stats));
+ }
}
if (init_state == NULL)
/* We start inserting insns after PREV_HEAD. */
last_scheduled_insn = prev_head;
- last_nondebug_scheduled_insn = NULL_RTX;
- nonscheduled_insns_begin = NULL_RTX;
+ last_nondebug_scheduled_insn = NULL;
+ nonscheduled_insns_begin = NULL;
gcc_assert ((NOTE_P (last_scheduled_insn)
|| DEBUG_INSN_P (last_scheduled_insn))
q_ptr = 0;
q_size = 0;
- insn_queue = XALLOCAVEC (rtx, max_insn_queue_index + 1);
+ insn_queue = XALLOCAVEC (rtx_insn_list *, max_insn_queue_index + 1);
memset (insn_queue, 0, (max_insn_queue_index + 1) * sizeof (rtx));
/* Start just before the beginning of time. */
in try_ready () (which is called through init_ready_list ()). */
(*current_sched_info->init_ready_list) ();
- if (sched_pressure == SCHED_PRESSURE_MODEL)
- model_start_schedule ();
+ if (sched_pressure)
+ sched_pressure_start_bb (*target_bb);
/* The algorithm is O(n^2) in the number of ready insns at any given
time in the worst case. Before reload we are more likely to have
if (!reload_completed
&& ready.n_ready - ready.n_debug > MAX_SCHED_READY_INSNS)
{
- ready_sort (&ready);
+ ready_sort_debug (&ready);
+ ready_sort_real (&ready);
/* Find first free-standing insn past MAX_SCHED_READY_INSNS.
If there are debug insns, we know they're first. */
if (sched_verbose >= 2)
{
fprintf (sched_dump,
- ";;\t\tReady list on entry: %d insns\n", ready.n_ready);
+ ";;\t\tReady list on entry: %d insns: ", ready.n_ready);
+ debug_ready_list (&ready);
fprintf (sched_dump,
";;\t\t before reload => truncated to %d insns\n", i);
}
activated make an exception for the insn right after
nonscheduled_insns_begin. */
{
- rtx skip_insn;
+ rtx_insn *skip_insn;
if (dbg_cnt (sched_insn) == false)
skip_insn = first_nonscheduled_insn ();
else
- skip_insn = NULL_RTX;
+ skip_insn = NULL;
while (i < ready.n_ready)
{
- rtx insn;
+ rtx_insn *insn;
insn = ready_remove (&ready, i);
modulo_insns_scheduled = 0;
ls.modulo_epilogue = false;
+ ls.first_cycle_insn_p = true;
/* Loop until all the insns in BB are scheduled. */
while ((*current_sched_info->schedule_more_p) ())
if (must_backtrack)
goto do_backtrack;
- ls.first_cycle_insn_p = true;
ls.shadows_only_p = false;
cycle_issued_insns = 0;
ls.can_issue_more = issue_rate;
for (;;)
{
- rtx insn;
+ rtx_insn *insn;
int cost;
bool asm_p;
{
while (ready.n_ready && DEBUG_INSN_P (ready_element (&ready, 0)))
{
- rtx insn = ready_remove_first (&ready);
+ rtx_insn *insn = ready_remove_first (&ready);
gcc_assert (DEBUG_INSN_P (insn));
(*current_sched_info->begin_schedule_ready) (insn);
scheduled_insns.safe_push (insn);
{
int res;
- insn = NULL_RTX;
+ insn = NULL;
res = choose_ready (&ready, ls.first_cycle_insn_p, &insn);
if (res < 0)
if (TODO_SPEC (insn) & SPECULATIVE)
generate_recovery_code (insn);
- if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON))
+ if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
targetm.sched.dispatch_do (insn, ADD_TO_DISPATCH_WINDOW);
/* Update counters, etc in the scheduler's front end. */
{
memcpy (temp_state, curr_state, dfa_state_size);
cost = state_transition (curr_state, insn);
- if (sched_pressure != SCHED_PRESSURE_WEIGHTED)
+ if (sched_pressure != SCHED_PRESSURE_WEIGHTED && !sched_fusion)
gcc_assert (cost < 0);
if (memcmp (temp_state, curr_state, dfa_state_size) != 0)
cycle_issued_insns++;
if (!must_backtrack)
for (i = 0; i < ready.n_ready; i++)
{
- rtx insn = ready_element (&ready, i);
+ rtx_insn *insn = ready_element (&ready, i);
if (INSN_EXACT_TICK (insn) == clock_var)
{
must_backtrack = true;
while (must_backtrack)
{
struct haifa_saved_data *failed;
- rtx failed_insn;
+ rtx_insn *failed_insn;
must_backtrack = false;
failed = verify_shadows ();
break;
}
}
+ ls.first_cycle_insn_p = true;
}
if (ls.modulo_epilogue)
success = true;
end_schedule:
- advance_one_cycle ();
+ if (!ls.first_cycle_insn_p || advance)
+ advance_one_cycle ();
perform_replacements_new_cycle ();
if (modulo_ii > 0)
{
restart_debug_insn_loop:
for (i = ready.n_ready - 1; i >= 0; i--)
{
- rtx x;
+ rtx_insn *x;
x = ready_element (&ready, i);
if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x)) != NULL
}
for (i = ready.n_ready - 1; i >= 0; i--)
{
- rtx x;
+ rtx_insn *x;
x = ready_element (&ready, i);
resolve_dependencies (x);
}
for (i = 0; i <= max_insn_queue_index; i++)
{
- rtx link;
+ rtx_insn_list *link;
while ((link = insn_queue[i]) != NULL)
{
- rtx x = XEXP (link, 0);
- insn_queue[i] = XEXP (link, 1);
+ rtx_insn *x = link->insn ();
+ insn_queue[i] = link->next ();
QUEUE_INDEX (x) = QUEUE_NOWHERE;
free_INSN_LIST_node (link);
resolve_dependencies (x);
/* We must maintain QUEUE_INDEX between blocks in region. */
for (i = ready.n_ready - 1; i >= 0; i--)
{
- rtx x;
+ rtx_insn *x;
x = ready_element (&ready, i);
QUEUE_INDEX (x) = QUEUE_NOWHERE;
if (q_size)
for (i = 0; i <= max_insn_queue_index; i++)
{
- rtx link;
- for (link = insn_queue[i]; link; link = XEXP (link, 1))
+ rtx_insn_list *link;
+ for (link = insn_queue[i]; link; link = link->next ())
{
- rtx x;
+ rtx_insn *x;
- x = XEXP (link, 0);
+ x = link->insn ();
QUEUE_INDEX (x) = QUEUE_NOWHERE;
TODO_SPEC (x) = HARD_DEP;
}
INSN_UID (head), INSN_UID (tail));
if (sched_verbose >= 2)
- dump_insn_stream (head, tail);
+ {
+ dump_insn_stream (head, tail);
+ print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats,
+ NULL);
+ }
fprintf (sched_dump, "\n");
}
/* Set_priorities: compute priority of each insn in the block. */
int
-set_priorities (rtx head, rtx tail)
+set_priorities (rtx_insn *head, rtx_insn *tail)
{
- rtx insn;
+ rtx_insn *insn;
int n_insn;
int sched_max_insns_priority =
current_sched_info->sched_max_insns_priority;
- rtx prev_head;
+ rtx_insn *prev_head;
if (head == tail && ! INSN_P (head))
gcc_unreachable ();
saved_reg_live = BITMAP_ALLOC (NULL);
region_ref_regs = BITMAP_ALLOC (NULL);
}
+
+ /* Calculate number of CALL_USED_REGS in register classes that
+ we calculate register pressure for. */
+ for (int c = 0; c < ira_pressure_classes_num; ++c)
+ {
+ enum reg_class cl = ira_pressure_classes[c];
+
+ call_used_regs_num[cl] = 0;
+
+ for (int i = 0; i < ira_class_hard_regs_num[cl]; ++i)
+ if (call_used_regs[ira_class_hard_regs[cl][i]])
+ ++call_used_regs_num[cl];
+ }
}
}
sched_init (void)
{
/* Disable speculative loads in their presence if cc0 defined. */
-#ifdef HAVE_cc0
+ if (HAVE_cc0)
flag_schedule_speculative_load = 0;
-#endif
- if (targetm.sched.dispatch (NULL_RTX, IS_DISPATCH_ON))
- targetm.sched.dispatch_do (NULL_RTX, DISPATCH_INIT);
+ if (targetm.sched.dispatch (NULL, IS_DISPATCH_ON))
+ targetm.sched.dispatch_do (NULL, DISPATCH_INIT);
if (live_range_shrinkage_p)
sched_pressure = SCHED_PRESSURE_WEIGHTED;
else
issue_rate = 1;
- if (cached_issue_rate != issue_rate)
- {
- cached_issue_rate = issue_rate;
- /* To invalidate max_lookahead_tries: */
- cached_first_cycle_multipass_dfa_lookahead = 0;
- }
-
- if (targetm.sched.first_cycle_multipass_dfa_lookahead)
+ if (targetm.sched.first_cycle_multipass_dfa_lookahead
+ /* Don't use max_issue with reg_pressure scheduling. Multipass
+ scheduling and reg_pressure scheduling undo each other's decisions. */
+ && sched_pressure == SCHED_PRESSURE_NONE)
dfa_lookahead = targetm.sched.first_cycle_multipass_dfa_lookahead ();
else
dfa_lookahead = 0;
+ /* Set to "0" so that we recalculate. */
+ max_lookahead_tries = 0;
+
if (targetm.sched.init_dfa_pre_cycle_insn)
targetm.sched.init_dfa_pre_cycle_insn ();
INSN_TICKs of their dependents.
HEAD and TAIL are the begin and the end of the current scheduled block. */
static void
-fix_inter_tick (rtx head, rtx tail)
+fix_inter_tick (rtx_insn *head, rtx_insn *tail)
{
/* Set of instructions with corrected INSN_TICK. */
bitmap_head processed;
FOR_EACH_DEP (head, SD_LIST_RES_FORW, sd_it, dep)
{
- rtx next;
+ rtx_insn *next;
next = DEP_CON (dep);
tick = INSN_TICK (next);
0 - added to the ready list,
0 < N - queued for N cycles. */
int
-try_ready (rtx next)
+try_ready (rtx_insn *next)
{
ds_t old_ts, new_ts;
/* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
static int
-fix_tick_ready (rtx next)
+fix_tick_ready (rtx_insn *next)
{
int tick, delay;
FOR_EACH_DEP (next, SD_LIST_RES_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
int tick1;
gcc_assert (INSN_TICK (pro) >= MIN_TICK);
INSN_TICK (next) = tick;
delay = tick - clock_var;
- if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE)
+ if (delay <= 0 || sched_pressure != SCHED_PRESSURE_NONE || sched_fusion)
delay = QUEUE_READY;
change_queue_index (next, delay);
or add it to the ready list (DELAY == QUEUE_READY),
or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
static void
-change_queue_index (rtx next, int delay)
+change_queue_index (rtx_insn *next, int delay)
{
int i = QUEUE_INDEX (next);
i = sched_ready_n_insns + 1;
ready.veclen = new_sched_ready_n_insns + issue_rate;
- ready.vec = XRESIZEVEC (rtx, ready.vec, ready.veclen);
+ ready.vec = XRESIZEVEC (rtx_insn *, ready.vec, ready.veclen);
gcc_assert (new_sched_ready_n_insns >= sched_ready_n_insns);
/* Generates recovery code for INSN. */
static void
-generate_recovery_code (rtx insn)
+generate_recovery_code (rtx_insn *insn)
{
if (TODO_SPEC (insn) & BEGIN_SPEC)
begin_speculative_block (insn);
Tries to add speculative dependencies of type FS between instructions
in deps_list L and TWIN. */
static void
-process_insn_forw_deps_be_in_spec (rtx insn, rtx twin, ds_t fs)
+process_insn_forw_deps_be_in_spec (rtx_insn *insn, rtx_insn *twin, ds_t fs)
{
sd_iterator_def sd_it;
dep_t dep;
FOR_EACH_DEP (insn, SD_LIST_FORW, sd_it, dep)
{
ds_t ds;
- rtx consumer;
+ rtx_insn *consumer;
consumer = DEP_CON (dep);
/* Generates recovery code for BEGIN speculative INSN. */
static void
-begin_speculative_block (rtx insn)
+begin_speculative_block (rtx_insn *insn)
{
if (TODO_SPEC (insn) & BEGIN_DATA)
nr_begin_data++;
TODO_SPEC (insn) &= ~BEGIN_SPEC;
}
-static void haifa_init_insn (rtx);
+static void haifa_init_insn (rtx_insn *);
/* Generates recovery code for BE_IN speculative INSN. */
static void
-add_to_speculative_block (rtx insn)
+add_to_speculative_block (rtx_insn *insn)
{
ds_t ts;
sd_iterator_def sd_it;
dep_t dep;
- rtx twins = NULL;
+ rtx_insn_list *twins = NULL;
rtx_vec_t priorities_roots;
ts = TODO_SPEC (insn);
for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx check = DEP_PRO (dep);
+ rtx_insn *check = DEP_PRO (dep);
if (IS_SPECULATION_SIMPLE_CHECK_P (check))
{
while (1)
{
- rtx check, twin;
+ rtx_insn *check, *twin;
basic_block rec;
/* Get the first backward dependency of INSN. */
instructions from REC. */
FOR_EACH_DEP (insn, SD_LIST_SPEC_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
gcc_assert (DEP_TYPE (dep) == REG_DEP_TRUE);
for (sd_it = sd_iterator_start (insn, SD_LIST_SPEC_BACK);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
if (BLOCK_FOR_INSN (pro) == rec)
sd_delete_dep (sd_it);
because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
while (twins)
{
- rtx twin;
+ rtx_insn *twin;
+ rtx_insn_list *next_node;
- twin = XEXP (twins, 0);
+ twin = twins->insn ();
{
dep_def _new_dep, *new_dep = &_new_dep;
sd_add_dep (new_dep, false);
}
- twin = XEXP (twins, 1);
+ next_node = twins->next ();
free_INSN_LIST_node (twins);
- twins = twin;
+ twins = next_node;
}
calc_priorities (priorities_roots);
sched_extend_bb (void)
{
/* The following is done to keep current_sched_info->next_tail non null. */
- rtx end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
- rtx insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
+ rtx_insn *end = BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb);
+ rtx_insn *insn = DEBUG_INSN_P (end) ? prev_nondebug_insn (end) : end;
if (NEXT_INSN (end) == 0
|| (!NOTE_P (insn)
&& !LABEL_P (insn)
/* Don't emit a NOTE if it would end up before a BARRIER. */
&& !BARRIER_P (NEXT_INSN (end))))
{
- rtx note = emit_note_after (NOTE_INSN_DELETED, end);
+ rtx_note *note = emit_note_after (NOTE_INSN_DELETED, end);
/* Make note appear outside BB. */
set_block_for_insn (note, NULL);
BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb) = end;
Between these two blocks recovery blocks will be emitted. */
basic_block single, empty;
- rtx x, label;
/* If the fallthrough edge to exit we've found is from the block we've
created before, don't do anything more. */
make_single_succ_edge (empty, EXIT_BLOCK_PTR_FOR_FN (cfun),
EDGE_FALLTHRU);
- label = block_label (empty);
- x = emit_jump_insn_after (gen_jump (label), BB_END (single));
+ rtx_code_label *label = block_label (empty);
+ rtx_jump_insn *x = emit_jump_insn_after (gen_jump (label),
+ BB_END (single));
JUMP_LABEL (x) = label;
LABEL_NUSES (label)++;
haifa_init_insn (x);
basic_block
sched_create_recovery_block (basic_block *before_recovery_ptr)
{
- rtx label;
- rtx barrier;
+ rtx_insn *barrier;
basic_block rec;
haifa_recovery_bb_recently_added_p = true;
barrier = get_last_bb_insn (before_recovery);
gcc_assert (BARRIER_P (barrier));
- label = emit_label_after (gen_label_rtx (), barrier);
+ rtx_insn *label = emit_label_after (gen_label_rtx (), barrier);
rec = create_basic_block (label, label, before_recovery);
sched_create_recovery_edges (basic_block first_bb, basic_block rec,
basic_block second_bb)
{
- rtx label;
- rtx jump;
int edge_flags;
/* This is fixing of incoming edge. */
edge_flags = 0;
make_edge (first_bb, rec, edge_flags);
- label = block_label (second_bb);
- jump = emit_jump_insn_after (gen_jump (label), BB_END (rec));
+ rtx_code_label *label = block_label (second_bb);
+ rtx_jump_insn *jump = emit_jump_insn_after (gen_jump (label), BB_END (rec));
JUMP_LABEL (jump) = label;
LABEL_NUSES (label)++;
/* This function creates recovery code for INSN. If MUTATE_P is nonzero,
INSN is a simple check, that should be converted to branchy one. */
static void
-create_check_block_twin (rtx insn, bool mutate_p)
+create_check_block_twin (rtx_insn *insn, bool mutate_p)
{
basic_block rec;
- rtx label, check, twin;
+ rtx_insn *label, *check, *twin;
+ rtx check_pat;
ds_t fs;
sd_iterator_def sd_it;
dep_t dep;
else
{
rec = EXIT_BLOCK_PTR_FOR_FN (cfun);
- label = NULL_RTX;
+ label = NULL;
}
/* Emit CHECK. */
- check = targetm.sched.gen_spec_check (insn, label, todo_spec);
+ check_pat = targetm.sched.gen_spec_check (insn, label, todo_spec);
if (rec != EXIT_BLOCK_PTR_FOR_FN (cfun))
{
we emit check BEFORE insn, so insn after splitting
insn will be at the beginning of second_bb, which will
provide us with the correct life information. */
- check = emit_jump_insn_before (check, insn);
+ check = emit_jump_insn_before (check_pat, insn);
JUMP_LABEL (check) = label;
LABEL_NUSES (label)++;
}
else
- check = emit_insn_before (check, insn);
+ check = emit_insn_before (check_pat, insn);
/* Extend data structures. */
haifa_init_insn (check);
/* In case of branchy check, fix CFG. */
{
basic_block first_bb, second_bb;
- rtx jump;
+ rtx_insn *jump;
first_bb = BLOCK_FOR_INSN (check);
second_bb = sched_split_block (first_bb, check);
/* First, create dependencies between INSN's producers and CHECK & TWIN. */
FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
ds_t ds;
/* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
static void
fix_recovery_deps (basic_block rec)
{
- rtx note, insn, jump, ready_list = 0;
+ rtx_insn *note, *insn, *jump;
+ rtx_insn_list *ready_list = 0;
bitmap_head in_ready;
- rtx link;
+ rtx_insn_list *link;
bitmap_initialize (&in_ready, 0);
for (sd_it = sd_iterator_start (insn, SD_LIST_FORW);
sd_iterator_cond (&sd_it, &dep);)
{
- rtx consumer = DEP_CON (dep);
+ rtx_insn *consumer = DEP_CON (dep);
if (BLOCK_FOR_INSN (consumer) != rec)
{
bitmap_clear (&in_ready);
/* Try to add instructions to the ready or queue list. */
- for (link = ready_list; link; link = XEXP (link, 1))
- try_ready (XEXP (link, 0));
+ for (link = ready_list; link; link = link->next ())
+ try_ready (link->insn ());
free_INSN_LIST_list (&ready_list);
/* Fixing jump's dependences. */
/* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
instruction data. */
static bool
-haifa_change_pattern (rtx insn, rtx new_pat)
+haifa_change_pattern (rtx_insn *insn, rtx new_pat)
{
int t;
current instruction pattern,
1 - need to change pattern for *NEW_PAT to be speculative. */
int
-sched_speculate_insn (rtx insn, ds_t request, rtx *new_pat)
+sched_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
{
gcc_assert (current_sched_info->flags & DO_SPECULATION
&& (request & SPECULATIVE)
}
static int
-haifa_speculate_insn (rtx insn, ds_t request, rtx *new_pat)
+haifa_speculate_insn (rtx_insn *insn, ds_t request, rtx *new_pat)
{
gcc_assert (sched_deps_info->generate_spec_deps
&& !IS_SPECULATION_CHECK_P (insn));
ends with TAIL, before scheduling it.
I is zero, if scheduler is about to start with the fresh ebb. */
static void
-dump_new_block_header (int i, basic_block bb, rtx head, rtx tail)
+dump_new_block_header (int i, basic_block bb, rtx_insn *head, rtx_insn *tail)
{
if (!i)
fprintf (sched_dump,
if (first == last)
return;
- bb_header = XNEWVEC (rtx, last_basic_block_for_fn (cfun));
+ bb_header = XNEWVEC (rtx_insn *, last_basic_block_for_fn (cfun));
/* Make a sentinel. */
if (last->next_bb != EXIT_BLOCK_PTR_FOR_FN (cfun))
first = first->next_bb;
do
{
- rtx prev, label, note, next;
+ rtx_insn *prev, *label, *note, *next;
label = BB_HEAD (last);
if (LABEL_P (label))
next = NEXT_INSN (note);
gcc_assert (prev && next);
- NEXT_INSN (prev) = next;
- PREV_INSN (next) = prev;
+ SET_NEXT_INSN (prev) = next;
+ SET_PREV_INSN (next) = prev;
bb_header[last->index] = label;
while (first != EXIT_BLOCK_PTR_FOR_FN (cfun)
&& bb_header[first->index])
{
- rtx prev, label, note, next;
+ rtx_insn *prev, *label, *note, *next;
label = bb_header[first->index];
prev = PREV_INSN (label);
bb_header[first->index] = 0;
- NEXT_INSN (prev) = label;
- NEXT_INSN (note) = next;
- PREV_INSN (next) = note;
+ SET_NEXT_INSN (prev) = label;
+ SET_NEXT_INSN (note) = next;
+ SET_PREV_INSN (next) = note;
first = first->next_bb;
}
Fix CFG after both in- and inter-block movement of
control_flow_insn_p JUMP. */
static void
-fix_jump_move (rtx jump)
+fix_jump_move (rtx_insn *jump)
{
basic_block bb, jump_bb, jump_bb_next;
/* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
static void
-move_block_after_check (rtx jump)
+move_block_after_check (rtx_insn *jump)
{
basic_block bb, jump_bb, jump_bb_next;
vec<edge, va_gc> *t;
/* Remove INSN from the instruction stream.
INSN should have any dependencies. */
static void
-sched_remove_insn (rtx insn)
+sched_remove_insn (rtx_insn *insn)
{
sd_finish_insn (insn);
Store in vector pointed to by ROOTS_PTR insns on which priority () should
be invoked to initialize all cleared priorities. */
static void
-clear_priorities (rtx insn, rtx_vec_t *roots_ptr)
+clear_priorities (rtx_insn *insn, rtx_vec_t *roots_ptr)
{
sd_iterator_def sd_it;
dep_t dep;
FOR_EACH_DEP (insn, SD_LIST_BACK, sd_it, dep)
{
- rtx pro = DEP_PRO (dep);
+ rtx_insn *pro = DEP_PRO (dep);
if (INSN_PRIORITY_STATUS (pro) >= 0
&& QUEUE_INDEX (insn) != QUEUE_SCHEDULED)
calc_priorities (rtx_vec_t roots)
{
int i;
- rtx insn;
+ rtx_insn *insn;
FOR_EACH_VEC_ELT (roots, i, insn)
priority (insn);
/* Add dependences between JUMP and other instructions in the recovery
block. INSN is the first insn the recovery block. */
static void
-add_jump_dependencies (rtx insn, rtx jump)
+add_jump_dependencies (rtx_insn *insn, rtx_insn *jump)
{
do
{
/* Initialize LUID for INSN. */
void
-sched_init_insn_luid (rtx insn)
+sched_init_insn_luid (rtx_insn *insn)
{
int i = INSN_P (insn) ? 1 : common_sched_info->luid_for_non_insn (insn);
int luid;
sched_extend_luids ();
FOR_EACH_VEC_ELT (bbs, i, bb)
{
- rtx insn;
+ rtx_insn *insn;
FOR_BB_INSNS (bb, insn)
sched_init_insn_luid (insn);
/* Return logical uid of INSN. Helpful while debugging. */
int
-insn_luid (rtx insn)
+insn_luid (rtx_insn *insn)
{
return INSN_LUID (insn);
}
/* Initialize h_i_d entry of the INSN with default values.
Values, that are not explicitly initialized here, hold zero. */
static void
-init_h_i_d (rtx insn)
+init_h_i_d (rtx_insn *insn)
{
if (INSN_LUID (insn) > 0)
{
INSN_EXACT_TICK (insn) = INVALID_TICK;
INTER_TICK (insn) = INVALID_TICK;
TODO_SPEC (insn) = HARD_DEP;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[0].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
+ INSN_AUTOPREF_MULTIPASS_DATA (insn)[1].status
+ = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED;
}
}
extend_h_i_d ();
FOR_EACH_VEC_ELT (bbs, i, bb)
{
- rtx insn;
+ rtx_insn *insn;
FOR_BB_INSNS (bb, insn)
init_h_i_d (insn);
/* Init data for the new insn INSN. */
static void
-haifa_init_insn (rtx insn)
+haifa_init_insn (rtx_insn *insn)
{
gcc_assert (insn != NULL);
/* Insert PAT as an INSN into the schedule and update the necessary data
structures to account for it. */
-rtx
+rtx_insn *
sched_emit_insn (rtx pat)
{
- rtx insn = emit_insn_before (pat, first_nonscheduled_insn ());
+ rtx_insn *insn = emit_insn_before (pat, first_nonscheduled_insn ());
haifa_init_insn (insn);
if (current_sched_info->add_remove_insn)
/* This function returns a candidate satisfying dispatch constraints from
the ready list. */
-static rtx
+static rtx_insn *
ready_remove_first_dispatch (struct ready_list *ready)
{
int i;
- rtx insn = ready_element (ready, 0);
+ rtx_insn *insn = ready_element (ready, 0);
if (ready->n_ready == 1
|| !INSN_P (insn)
}
}
- if (targetm.sched.dispatch (NULL_RTX, DISPATCH_VIOLATION))
+ if (targetm.sched.dispatch (NULL, DISPATCH_VIOLATION))
return ready_remove_first (ready);
for (i = 1; i < ready->n_ready; i++)
/* Get number of ready's in the ready list. */
-rtx
+rtx_insn *
get_ready_element (int i)
{
return ready_element (&ready, i);