1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
129 #include "diagnostic-core.h"
130 #include "hard-reg-set.h"
134 #include "function.h"
136 #include "insn-config.h"
137 #include "insn-attr.h"
140 #include "dominance.h"
143 #include "cfgbuild.h"
145 #include "basic-block.h"
146 #include "sched-int.h"
148 #include "common/common-target.h"
153 #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
154 #include "dumpfile.h"
156 #ifdef INSN_SCHEDULING
158 /* True if we do register pressure relief through live-range
160 static bool live_range_shrinkage_p
;
162 /* Switch on live range shrinkage. */
164 initialize_live_range_shrinkage (void)
166 live_range_shrinkage_p
= true;
169 /* Switch off live range shrinkage. */
171 finish_live_range_shrinkage (void)
173 live_range_shrinkage_p
= false;
176 /* issue_rate is the number of insns that can be scheduled in the same
177 machine cycle. It can be defined in the config/mach/mach.h file,
178 otherwise we set it to 1. */
182 /* This can be set to true by a backend if the scheduler should not
183 enable a DCE pass. */
186 /* The current initiation interval used when modulo scheduling. */
187 static int modulo_ii
;
189 /* The maximum number of stages we are prepared to handle. */
190 static int modulo_max_stages
;
192 /* The number of insns that exist in each iteration of the loop. We use this
193 to detect when we've scheduled all insns from the first iteration. */
194 static int modulo_n_insns
;
196 /* The current count of insns in the first iteration of the loop that have
197 already been scheduled. */
198 static int modulo_insns_scheduled
;
200 /* The maximum uid of insns from the first iteration of the loop. */
201 static int modulo_iter0_max_uid
;
203 /* The number of times we should attempt to backtrack when modulo scheduling.
204 Decreased each time we have to backtrack. */
205 static int modulo_backtracks_left
;
207 /* The stage in which the last insn from the original loop was
209 static int modulo_last_stage
;
211 /* sched-verbose controls the amount of debugging output the
212 scheduler prints. It is controlled by -fsched-verbose=N:
213 N>0 and no -DSR : the output is directed to stderr.
214 N>=10 will direct the printouts to stderr (regardless of -dSR).
216 N=2: bb's probabilities, detailed ready list info, unit/insn info.
217 N=3: rtl at abort point, control-flow, regions info.
218 N=5: dependences info. */
220 int sched_verbose
= 0;
222 /* Debugging file. All printouts are sent to dump, which is always set,
223 either to stderr, or to the dump listing file (-dRS). */
224 FILE *sched_dump
= 0;
226 /* This is a placeholder for the scheduler parameters common
227 to all schedulers. */
228 struct common_sched_info_def
*common_sched_info
;
230 #define INSN_TICK(INSN) (HID (INSN)->tick)
231 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
232 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
233 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
234 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
235 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
236 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
237 /* Cached cost of the instruction. Use insn_cost to get cost of the
238 insn. -1 here means that the field is not initialized. */
239 #define INSN_COST(INSN) (HID (INSN)->cost)
241 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
242 then it should be recalculated from scratch. */
243 #define INVALID_TICK (-(max_insn_queue_index + 1))
244 /* The minimal value of the INSN_TICK of an instruction. */
245 #define MIN_TICK (-max_insn_queue_index)
247 /* Original order of insns in the ready list.
248 Used to keep order of normal insns while separating DEBUG_INSNs. */
249 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
251 /* The deciding reason for INSN's place in the ready list. */
252 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
254 /* List of important notes we must keep around. This is a pointer to the
255 last element in the list. */
258 static struct spec_info_def spec_info_var
;
259 /* Description of the speculative part of the scheduling.
260 If NULL - no speculation. */
261 spec_info_t spec_info
= NULL
;
263 /* True, if recovery block was added during scheduling of current block.
264 Used to determine, if we need to fix INSN_TICKs. */
265 static bool haifa_recovery_bb_recently_added_p
;
267 /* True, if recovery block was added during this scheduling pass.
268 Used to determine if we should have empty memory pools of dependencies
269 after finishing current region. */
270 bool haifa_recovery_bb_ever_added_p
;
272 /* Counters of different types of speculative instructions. */
273 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
275 /* Array used in {unlink, restore}_bb_notes. */
276 static rtx_insn
**bb_header
= 0;
278 /* Basic block after which recovery blocks will be created. */
279 static basic_block before_recovery
;
281 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
283 basic_block after_recovery
;
285 /* FALSE if we add bb to another region, so we don't need to initialize it. */
286 bool adding_bb_to_current_region_p
= true;
290 /* An instruction is ready to be scheduled when all insns preceding it
291 have already been scheduled. It is important to ensure that all
292 insns which use its result will not be executed until its result
293 has been computed. An insn is maintained in one of four structures:
295 (P) the "Pending" set of insns which cannot be scheduled until
296 their dependencies have been satisfied.
297 (Q) the "Queued" set of insns that can be scheduled when sufficient
299 (R) the "Ready" list of unscheduled, uncommitted insns.
300 (S) the "Scheduled" list of insns.
302 Initially, all insns are either "Pending" or "Ready" depending on
303 whether their dependencies are satisfied.
305 Insns move from the "Ready" list to the "Scheduled" list as they
306 are committed to the schedule. As this occurs, the insns in the
307 "Pending" list have their dependencies satisfied and move to either
308 the "Ready" list or the "Queued" set depending on whether
309 sufficient time has passed to make them ready. As time passes,
310 insns move from the "Queued" set to the "Ready" list.
312 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
313 unscheduled insns, i.e., those that are ready, queued, and pending.
314 The "Queued" set (Q) is implemented by the variable `insn_queue'.
315 The "Ready" list (R) is implemented by the variables `ready' and
317 The "Scheduled" list (S) is the new insn chain built by this pass.
319 The transition (R->S) is implemented in the scheduling loop in
320 `schedule_block' when the best insn to schedule is chosen.
321 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
322 insns move from the ready list to the scheduled list.
323 The transition (Q->R) is implemented in 'queue_to_insn' as time
324 passes or stalls are introduced. */
326 /* Implement a circular buffer to delay instructions until sufficient
327 time has passed. For the new pipeline description interface,
328 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
329 than maximal time of instruction execution computed by genattr.c on
330 the base maximal time of functional unit reservations and getting a
331 result. This is the longest time an insn may be queued. */
333 static rtx_insn_list
**insn_queue
;
334 static int q_ptr
= 0;
335 static int q_size
= 0;
336 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
337 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
339 #define QUEUE_SCHEDULED (-3)
340 #define QUEUE_NOWHERE (-2)
341 #define QUEUE_READY (-1)
342 /* QUEUE_SCHEDULED - INSN is scheduled.
343 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
345 QUEUE_READY - INSN is in ready list.
346 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
348 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
350 /* The following variable value refers for all current and future
351 reservations of the processor units. */
354 /* The following variable value is size of memory representing all
355 current and future reservations of the processor units. */
356 size_t dfa_state_size
;
358 /* The following array is used to find the best insn from ready when
359 the automaton pipeline interface is used. */
360 signed char *ready_try
= NULL
;
362 /* The ready list. */
363 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
365 /* The pointer to the ready list (to be removed). */
366 static struct ready_list
*readyp
= &ready
;
368 /* Scheduling clock. */
369 static int clock_var
;
371 /* Clock at which the previous instruction was issued. */
372 static int last_clock_var
;
374 /* Set to true if, when queuing a shadow insn, we discover that it would be
375 scheduled too late. */
376 static bool must_backtrack
;
378 /* The following variable value is number of essential insns issued on
379 the current cycle. An insn is essential one if it changes the
381 int cycle_issued_insns
;
383 /* This records the actual schedule. It is built up during the main phase
384 of schedule_block, and afterwards used to reorder the insns in the RTL. */
385 static vec
<rtx_insn
*> scheduled_insns
;
387 static int may_trap_exp (const_rtx
, int);
389 /* Nonzero iff the address is comprised from at most 1 register. */
390 #define CONST_BASED_ADDRESS_P(x) \
392 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
393 || (GET_CODE (x) == LO_SUM)) \
394 && (CONSTANT_P (XEXP (x, 0)) \
395 || CONSTANT_P (XEXP (x, 1)))))
397 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
398 as found by analyzing insn's expression. */
401 static int haifa_luid_for_non_insn (rtx x
);
403 /* Haifa version of sched_info hooks common to all headers. */
404 const struct common_sched_info_def haifa_common_sched_info
=
406 NULL
, /* fix_recovery_cfg */
407 NULL
, /* add_block */
408 NULL
, /* estimate_number_of_insns */
409 haifa_luid_for_non_insn
, /* luid_for_non_insn */
410 SCHED_PASS_UNKNOWN
/* sched_pass_id */
413 /* Mapping from instruction UID to its Logical UID. */
414 vec
<int> sched_luids
= vNULL
;
416 /* Next LUID to assign to an instruction. */
417 int sched_max_luid
= 1;
419 /* Haifa Instruction Data. */
420 vec
<haifa_insn_data_def
> h_i_d
= vNULL
;
422 void (* sched_init_only_bb
) (basic_block
, basic_block
);
424 /* Split block function. Different schedulers might use different functions
425 to handle their internal data consistent. */
426 basic_block (* sched_split_block
) (basic_block
, rtx
);
428 /* Create empty basic block after the specified block. */
429 basic_block (* sched_create_empty_bb
) (basic_block
);
431 /* Return the number of cycles until INSN is expected to be ready.
432 Return zero if it already is. */
434 insn_delay (rtx_insn
*insn
)
436 return MAX (INSN_TICK (insn
) - clock_var
, 0);
440 may_trap_exp (const_rtx x
, int is_store
)
449 if (code
== MEM
&& may_trap_p (x
))
456 /* The insn uses memory: a volatile load. */
457 if (MEM_VOLATILE_P (x
))
459 /* An exception-free load. */
462 /* A load with 1 base register, to be further checked. */
463 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
464 return PFREE_CANDIDATE
;
465 /* No info on the load, to be further checked. */
466 return PRISKY_CANDIDATE
;
471 int i
, insn_class
= TRAP_FREE
;
473 /* Neither store nor load, check if it may cause a trap. */
476 /* Recursive step: walk the insn... */
477 fmt
= GET_RTX_FORMAT (code
);
478 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
482 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
483 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
485 else if (fmt
[i
] == 'E')
488 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
490 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
491 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
492 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
496 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
503 /* Classifies rtx X of an insn for the purpose of verifying that X can be
504 executed speculatively (and consequently the insn can be moved
505 speculatively), by examining X, returning:
506 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
507 TRAP_FREE: non-load insn.
508 IFREE: load from a globally safe location.
509 IRISKY: volatile load.
510 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
511 being either PFREE or PRISKY. */
514 haifa_classify_rtx (const_rtx x
)
516 int tmp_class
= TRAP_FREE
;
517 int insn_class
= TRAP_FREE
;
520 if (GET_CODE (x
) == PARALLEL
)
522 int i
, len
= XVECLEN (x
, 0);
524 for (i
= len
- 1; i
>= 0; i
--)
526 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
527 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
528 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
538 /* Test if it is a 'store'. */
539 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
542 /* Test if it is a store. */
543 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
544 if (tmp_class
== TRAP_RISKY
)
546 /* Test if it is a load. */
548 WORST_CLASS (tmp_class
,
549 may_trap_exp (SET_SRC (x
), 0));
552 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
553 if (tmp_class
== TRAP_RISKY
)
555 tmp_class
= WORST_CLASS (tmp_class
,
556 may_trap_exp (COND_EXEC_TEST (x
), 0));
559 tmp_class
= TRAP_RISKY
;
563 insn_class
= tmp_class
;
570 haifa_classify_insn (const_rtx insn
)
572 return haifa_classify_rtx (PATTERN (insn
));
575 /* After the scheduler initialization function has been called, this function
576 can be called to enable modulo scheduling. II is the initiation interval
577 we should use, it affects the delays for delay_pairs that were recorded as
578 separated by a given number of stages.
580 MAX_STAGES provides us with a limit
581 after which we give up scheduling; the caller must have unrolled at least
582 as many copies of the loop body and recorded delay_pairs for them.
584 INSNS is the number of real (non-debug) insns in one iteration of
585 the loop. MAX_UID can be used to test whether an insn belongs to
586 the first iteration of the loop; all of them have a uid lower than
589 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
592 modulo_max_stages
= max_stages
;
593 modulo_n_insns
= insns
;
594 modulo_iter0_max_uid
= max_uid
;
595 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
598 /* A structure to record a pair of insns where the first one is a real
599 insn that has delay slots, and the second is its delayed shadow.
600 I1 is scheduled normally and will emit an assembly instruction,
601 while I2 describes the side effect that takes place at the
602 transition between cycles CYCLES and (CYCLES + 1) after I1. */
605 struct delay_pair
*next_same_i1
;
608 /* When doing modulo scheduling, we a delay_pair can also be used to
609 show that I1 and I2 are the same insn in a different stage. If that
610 is the case, STAGES will be nonzero. */
614 /* Helpers for delay hashing. */
616 struct delay_i1_hasher
: nofree_ptr_hash
<delay_pair
>
618 typedef void *compare_type
;
619 static inline hashval_t
hash (const delay_pair
*);
620 static inline bool equal (const delay_pair
*, const void *);
623 /* Returns a hash value for X, based on hashing just I1. */
626 delay_i1_hasher::hash (const delay_pair
*x
)
628 return htab_hash_pointer (x
->i1
);
631 /* Return true if I1 of pair X is the same as that of pair Y. */
634 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
639 struct delay_i2_hasher
: typed_free_remove
<delay_pair
>
641 typedef delay_pair
*value_type
;
642 typedef void *compare_type
;
643 static inline hashval_t
hash (const delay_pair
*);
644 static inline bool equal (const delay_pair
*, const void *);
647 /* Returns a hash value for X, based on hashing just I2. */
650 delay_i2_hasher::hash (const delay_pair
*x
)
652 return htab_hash_pointer (x
->i2
);
655 /* Return true if I2 of pair X is the same as that of pair Y. */
658 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
663 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
665 static hash_table
<delay_i1_hasher
> *delay_htab
;
666 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
668 /* Called through htab_traverse. Walk the hashtable using I2 as
669 index, and delete all elements involving an UID higher than
670 that pointed to by *DATA. */
672 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
675 struct delay_pair
*p
= *slot
;
676 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
678 delay_htab_i2
->clear_slot (slot
);
683 /* Called through htab_traverse. Walk the hashtable using I2 as
684 index, and delete all elements involving an UID higher than
685 that pointed to by *DATA. */
687 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
690 struct delay_pair
*p
, *first
, **pprev
;
692 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
694 delay_htab
->clear_slot (pslot
);
698 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
700 if (INSN_UID (p
->i2
) < maxuid
)
703 pprev
= &p
->next_same_i1
;
708 delay_htab
->clear_slot (pslot
);
714 /* Discard all delay pairs which involve an insn with an UID higher
717 discard_delay_pairs_above (int max_uid
)
719 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
720 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
723 /* This function can be called by a port just before it starts the final
724 scheduling pass. It records the fact that an instruction with delay
725 slots has been split into two insns, I1 and I2. The first one will be
726 scheduled normally and initiates the operation. The second one is a
727 shadow which must follow a specific number of cycles after I1; its only
728 purpose is to show the side effect that occurs at that cycle in the RTL.
729 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
730 while I2 retains the original insn type.
732 There are two ways in which the number of cycles can be specified,
733 involving the CYCLES and STAGES arguments to this function. If STAGES
734 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
735 which is multiplied by MODULO_II to give the number of cycles. This is
736 only useful if the caller also calls set_modulo_params to enable modulo
740 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
742 struct delay_pair
*p
= XNEW (struct delay_pair
);
743 struct delay_pair
**slot
;
752 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
753 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
755 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
756 p
->next_same_i1
= *slot
;
758 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
762 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
763 and return the other insn if so. Return NULL otherwise. */
765 real_insn_for_shadow (rtx_insn
*insn
)
767 struct delay_pair
*pair
;
772 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
773 if (!pair
|| pair
->stages
> 0)
778 /* For a pair P of insns, return the fixed distance in cycles from the first
779 insn after which the second must be scheduled. */
781 pair_delay (struct delay_pair
*p
)
786 return p
->stages
* modulo_ii
;
789 /* Given an insn INSN, add a dependence on its delayed shadow if it
790 has one. Also try to find situations where shadows depend on each other
791 and add dependencies to the real insns to limit the amount of backtracking
794 add_delay_dependencies (rtx_insn
*insn
)
796 struct delay_pair
*pair
;
797 sd_iterator_def sd_it
;
803 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
806 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
810 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
812 rtx_insn
*pro
= DEP_PRO (dep
);
813 struct delay_pair
*other_pair
814 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
815 if (!other_pair
|| other_pair
->stages
)
817 if (pair_delay (other_pair
) >= pair_delay (pair
))
819 if (sched_verbose
>= 4)
821 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
822 INSN_UID (other_pair
->i1
),
823 INSN_UID (pair
->i1
));
824 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
828 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
829 INSN_UID (other_pair
->i1
),
830 INSN_UID (other_pair
->i2
),
831 pair_delay (other_pair
));
833 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
838 /* Forward declarations. */
840 static int priority (rtx_insn
*);
841 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
842 static int rank_for_schedule (const void *, const void *);
843 static void swap_sort (rtx_insn
**, int);
844 static void queue_insn (rtx_insn
*, int, const char *);
845 static int schedule_insn (rtx_insn
*);
846 static void adjust_priority (rtx_insn
*);
847 static void advance_one_cycle (void);
848 static void extend_h_i_d (void);
851 /* Notes handling mechanism:
852 =========================
853 Generally, NOTES are saved before scheduling and restored after scheduling.
854 The scheduler distinguishes between two types of notes:
856 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
857 Before scheduling a region, a pointer to the note is added to the insn
858 that follows or precedes it. (This happens as part of the data dependence
859 computation). After scheduling an insn, the pointer contained in it is
860 used for regenerating the corresponding note (in reemit_notes).
862 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
863 these notes are put in a list (in rm_other_notes() and
864 unlink_other_notes ()). After scheduling the block, these notes are
865 inserted at the beginning of the block (in schedule_block()). */
867 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
868 static rtx_insn
*ready_remove_first (struct ready_list
*);
869 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
871 static void queue_to_ready (struct ready_list
*);
872 static int early_queue_to_ready (state_t
, struct ready_list
*);
874 /* The following functions are used to implement multi-pass scheduling
875 on the first cycle. */
876 static rtx_insn
*ready_remove (struct ready_list
*, int);
877 static void ready_remove_insn (rtx_insn
*);
879 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
880 static int fix_tick_ready (rtx_insn
*);
881 static void change_queue_index (rtx_insn
*, int);
883 /* The following functions are used to implement scheduling of data/control
884 speculative instructions. */
886 static void extend_h_i_d (void);
887 static void init_h_i_d (rtx_insn
*);
888 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
889 static void generate_recovery_code (rtx_insn
*);
890 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
891 static void begin_speculative_block (rtx_insn
*);
892 static void add_to_speculative_block (rtx_insn
*);
893 static void init_before_recovery (basic_block
*);
894 static void create_check_block_twin (rtx_insn
*, bool);
895 static void fix_recovery_deps (basic_block
);
896 static bool haifa_change_pattern (rtx_insn
*, rtx
);
897 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
898 static void restore_bb_notes (basic_block
);
899 static void fix_jump_move (rtx_insn
*);
900 static void move_block_after_check (rtx_insn
*);
901 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
902 static void sched_remove_insn (rtx_insn
*);
903 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
904 static void calc_priorities (rtx_vec_t
);
905 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
907 #endif /* INSN_SCHEDULING */
909 /* Point to state used for the current scheduling pass. */
910 struct haifa_sched_info
*current_sched_info
;
912 #ifndef INSN_SCHEDULING
914 schedule_insns (void)
919 /* Do register pressure sensitive insn scheduling if the flag is set
921 enum sched_pressure_algorithm sched_pressure
;
923 /* Map regno -> its pressure class. The map defined only when
924 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
925 enum reg_class
*sched_regno_pressure_class
;
927 /* The current register pressure. Only elements corresponding pressure
928 classes are defined. */
929 static int curr_reg_pressure
[N_REG_CLASSES
];
931 /* Saved value of the previous array. */
932 static int saved_reg_pressure
[N_REG_CLASSES
];
934 /* Register living at given scheduling point. */
935 static bitmap curr_reg_live
;
937 /* Saved value of the previous array. */
938 static bitmap saved_reg_live
;
940 /* Registers mentioned in the current region. */
941 static bitmap region_ref_regs
;
943 /* Effective number of available registers of a given class (see comment
944 in sched_pressure_start_bb). */
945 static int sched_class_regs_num
[N_REG_CLASSES
];
946 /* Number of call_used_regs. This is a helper for calculating of
947 sched_class_regs_num. */
948 static int call_used_regs_num
[N_REG_CLASSES
];
950 /* Initiate register pressure relative info for scheduling the current
951 region. Currently it is only clearing register mentioned in the
954 sched_init_region_reg_pressure_info (void)
956 bitmap_clear (region_ref_regs
);
959 /* PRESSURE[CL] describes the pressure on register class CL. Update it
960 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
961 LIVE tracks the set of live registers; if it is null, assume that
962 every birth or death is genuine. */
964 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
966 enum reg_class pressure_class
;
968 pressure_class
= sched_regno_pressure_class
[regno
];
969 if (regno
>= FIRST_PSEUDO_REGISTER
)
971 if (pressure_class
!= NO_REGS
)
975 if (!live
|| bitmap_set_bit (live
, regno
))
976 pressure
[pressure_class
]
977 += (ira_reg_class_max_nregs
978 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
982 if (!live
|| bitmap_clear_bit (live
, regno
))
983 pressure
[pressure_class
]
984 -= (ira_reg_class_max_nregs
985 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
989 else if (pressure_class
!= NO_REGS
990 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
994 if (!live
|| bitmap_set_bit (live
, regno
))
995 pressure
[pressure_class
]++;
999 if (!live
|| bitmap_clear_bit (live
, regno
))
1000 pressure
[pressure_class
]--;
1005 /* Initiate current register pressure related info from living
1006 registers given by LIVE. */
1008 initiate_reg_pressure_info (bitmap live
)
1014 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1015 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1016 bitmap_clear (curr_reg_live
);
1017 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1018 if (sched_pressure
== SCHED_PRESSURE_MODEL
1019 || current_nr_blocks
== 1
1020 || bitmap_bit_p (region_ref_regs
, j
))
1021 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1024 /* Mark registers in X as mentioned in the current region. */
1026 setup_ref_regs (rtx x
)
1029 const RTX_CODE code
= GET_CODE (x
);
1034 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1037 fmt
= GET_RTX_FORMAT (code
);
1038 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1040 setup_ref_regs (XEXP (x
, i
));
1041 else if (fmt
[i
] == 'E')
1043 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1044 setup_ref_regs (XVECEXP (x
, i
, j
));
1048 /* Initiate current register pressure related info at the start of
1051 initiate_bb_reg_pressure_info (basic_block bb
)
1053 unsigned int i ATTRIBUTE_UNUSED
;
1056 if (current_nr_blocks
> 1)
1057 FOR_BB_INSNS (bb
, insn
)
1058 if (NONDEBUG_INSN_P (insn
))
1059 setup_ref_regs (PATTERN (insn
));
1060 initiate_reg_pressure_info (df_get_live_in (bb
));
1061 if (bb_has_eh_pred (bb
))
1064 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1066 if (regno
== INVALID_REGNUM
)
1068 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1069 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1074 /* Save current register pressure related info. */
1076 save_reg_pressure (void)
1080 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1081 saved_reg_pressure
[ira_pressure_classes
[i
]]
1082 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1083 bitmap_copy (saved_reg_live
, curr_reg_live
);
1086 /* Restore saved register pressure related info. */
1088 restore_reg_pressure (void)
1092 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1093 curr_reg_pressure
[ira_pressure_classes
[i
]]
1094 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1095 bitmap_copy (curr_reg_live
, saved_reg_live
);
1098 /* Return TRUE if the register is dying after its USE. */
1100 dying_use_p (struct reg_use_data
*use
)
1102 struct reg_use_data
*next
;
1104 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1105 if (NONDEBUG_INSN_P (next
->insn
)
1106 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1111 /* Print info about the current register pressure and its excess for
1112 each pressure class. */
1114 print_curr_reg_pressure (void)
1119 fprintf (sched_dump
, ";;\t");
1120 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1122 cl
= ira_pressure_classes
[i
];
1123 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1124 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1125 curr_reg_pressure
[cl
],
1126 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1128 fprintf (sched_dump
, "\n");
1131 /* Determine if INSN has a condition that is clobbered if a register
1132 in SET_REGS is modified. */
1134 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1136 rtx pat
= PATTERN (insn
);
1137 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1138 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1140 sd_iterator_def sd_it
;
1142 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1143 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1144 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1145 TODO_SPEC (insn
) = HARD_DEP
;
1146 if (sched_verbose
>= 2)
1147 fprintf (sched_dump
,
1148 ";;\t\tdequeue insn %s because of clobbered condition\n",
1149 (*current_sched_info
->print_insn
) (insn
, 0));
1156 /* This function should be called after modifying the pattern of INSN,
1157 to update scheduler data structures as needed. */
1159 update_insn_after_change (rtx_insn
*insn
)
1161 sd_iterator_def sd_it
;
1164 dfa_clear_single_insn_cache (insn
);
1166 sd_it
= sd_iterator_start (insn
,
1167 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1168 while (sd_iterator_cond (&sd_it
, &dep
))
1170 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1171 sd_iterator_next (&sd_it
);
1174 /* Invalidate INSN_COST, so it'll be recalculated. */
1175 INSN_COST (insn
) = -1;
1176 /* Invalidate INSN_TICK, so it'll be recalculated. */
1177 INSN_TICK (insn
) = INVALID_TICK
;
1179 /* Invalidate autoprefetch data entry. */
1180 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1181 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1182 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1183 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1187 /* Two VECs, one to hold dependencies for which pattern replacements
1188 need to be applied or restored at the start of the next cycle, and
1189 another to hold an integer that is either one, to apply the
1190 corresponding replacement, or zero to restore it. */
1191 static vec
<dep_t
> next_cycle_replace_deps
;
1192 static vec
<int> next_cycle_apply
;
1194 static void apply_replacement (dep_t
, bool);
1195 static void restore_pattern (dep_t
, bool);
1197 /* Look at the remaining dependencies for insn NEXT, and compute and return
1198 the TODO_SPEC value we should use for it. This is called after one of
1199 NEXT's dependencies has been resolved.
1200 We also perform pattern replacements for predication, and for broken
1201 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1205 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1208 sd_iterator_def sd_it
;
1209 dep_t dep
, modify_dep
= NULL
;
1213 bool first_p
= true;
1215 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1216 /* NEXT has all its dependencies resolved. */
1219 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1222 /* If NEXT is intended to sit adjacent to this instruction, we don't
1223 want to try to break any dependencies. Treat it as a HARD_DEP. */
1224 if (SCHED_GROUP_P (next
))
1227 /* Now we've got NEXT with speculative deps only.
1228 1. Look at the deps to see what we have to do.
1229 2. Check if we can do 'todo'. */
1232 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1234 rtx_insn
*pro
= DEP_PRO (dep
);
1235 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1237 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1250 new_ds
= ds_merge (new_ds
, ds
);
1252 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1254 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1259 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1261 else if (DEP_REPLACE (dep
) != NULL
)
1263 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1268 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1272 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1274 if (!dbg_cnt (sched_breakdep
))
1276 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1278 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1281 if (desc
->insn
== next
&& !for_backtrack
)
1283 gcc_assert (n_replace
== 1);
1284 apply_replacement (dep
, true);
1286 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1292 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1294 rtx_insn
*pro
, *other
;
1296 rtx cond
= NULL_RTX
;
1298 rtx_insn
*prev
= NULL
;
1302 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1303 || (ORIG_PAT (next
) != NULL_RTX
1304 && PREDICATED_PAT (next
) == NULL_RTX
))
1307 pro
= DEP_PRO (modify_dep
);
1308 other
= real_insn_for_shadow (pro
);
1309 if (other
!= NULL_RTX
)
1312 cond
= sched_get_reverse_condition_uncached (pro
);
1313 regno
= REGNO (XEXP (cond
, 0));
1315 /* Find the last scheduled insn that modifies the condition register.
1316 We can stop looking once we find the insn we depend on through the
1317 REG_DEP_CONTROL; if the condition register isn't modified after it,
1318 we know that it still has the right value. */
1319 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1320 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1324 find_all_hard_reg_sets (prev
, &t
, true);
1325 if (TEST_HARD_REG_BIT (t
, regno
))
1330 if (ORIG_PAT (next
) == NULL_RTX
)
1332 ORIG_PAT (next
) = PATTERN (next
);
1334 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1335 success
= haifa_change_pattern (next
, new_pat
);
1338 PREDICATED_PAT (next
) = new_pat
;
1340 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1342 bool success
= haifa_change_pattern (next
,
1343 PREDICATED_PAT (next
));
1344 gcc_assert (success
);
1346 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1350 if (PREDICATED_PAT (next
) != NULL_RTX
)
1352 int tick
= INSN_TICK (next
);
1353 bool success
= haifa_change_pattern (next
,
1355 INSN_TICK (next
) = tick
;
1356 gcc_assert (success
);
1359 /* We can't handle the case where there are both speculative and control
1360 dependencies, so we return HARD_DEP in such a case. Also fail if
1361 we have speculative dependencies with not enough points, or more than
1362 one control dependency. */
1363 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1365 /* Too few points? */
1366 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1374 /* Pointer to the last instruction scheduled. */
1375 static rtx_insn
*last_scheduled_insn
;
1377 /* Pointer to the last nondebug instruction scheduled within the
1378 block, or the prev_head of the scheduling block. Used by
1379 rank_for_schedule, so that insns independent of the last scheduled
1380 insn will be preferred over dependent instructions. */
1381 static rtx_insn
*last_nondebug_scheduled_insn
;
1383 /* Pointer that iterates through the list of unscheduled insns if we
1384 have a dbg_cnt enabled. It always points at an insn prior to the
1385 first unscheduled one. */
1386 static rtx_insn
*nonscheduled_insns_begin
;
1388 /* Compute cost of executing INSN.
1389 This is the number of cycles between instruction issue and
1390 instruction results. */
1392 insn_cost (rtx_insn
*insn
)
1401 if (recog_memoized (insn
) < 0)
1404 cost
= insn_default_latency (insn
);
1411 cost
= INSN_COST (insn
);
1415 /* A USE insn, or something else we don't need to
1416 understand. We can't pass these directly to
1417 result_ready_cost or insn_default_latency because it will
1418 trigger a fatal error for unrecognizable insns. */
1419 if (recog_memoized (insn
) < 0)
1421 INSN_COST (insn
) = 0;
1426 cost
= insn_default_latency (insn
);
1430 INSN_COST (insn
) = cost
;
1437 /* Compute cost of dependence LINK.
1438 This is the number of cycles between instruction issue and
1439 instruction results.
1440 ??? We also use this function to call recog_memoized on all insns. */
1442 dep_cost_1 (dep_t link
, dw_t dw
)
1444 rtx_insn
*insn
= DEP_PRO (link
);
1445 rtx_insn
*used
= DEP_CON (link
);
1448 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1449 return DEP_COST (link
);
1453 struct delay_pair
*delay_entry
;
1455 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1458 if (delay_entry
->i1
== insn
)
1460 DEP_COST (link
) = pair_delay (delay_entry
);
1461 return DEP_COST (link
);
1466 /* A USE insn should never require the value used to be computed.
1467 This allows the computation of a function's result and parameter
1468 values to overlap the return and call. We don't care about the
1469 dependence cost when only decreasing register pressure. */
1470 if (recog_memoized (used
) < 0)
1473 recog_memoized (insn
);
1477 enum reg_note dep_type
= DEP_TYPE (link
);
1479 cost
= insn_cost (insn
);
1481 if (INSN_CODE (insn
) >= 0)
1483 if (dep_type
== REG_DEP_ANTI
)
1485 else if (dep_type
== REG_DEP_OUTPUT
)
1487 cost
= (insn_default_latency (insn
)
1488 - insn_default_latency (used
));
1492 else if (bypass_p (insn
))
1493 cost
= insn_latency (insn
, used
);
1497 if (targetm
.sched
.adjust_cost_2
)
1498 cost
= targetm
.sched
.adjust_cost_2 (used
, (int) dep_type
, insn
, cost
,
1500 else if (targetm
.sched
.adjust_cost
!= NULL
)
1502 /* This variable is used for backward compatibility with the
1504 rtx_insn_list
*dep_cost_rtx_link
=
1505 alloc_INSN_LIST (NULL_RTX
, NULL
);
1507 /* Make it self-cycled, so that if some tries to walk over this
1508 incomplete list he/she will be caught in an endless loop. */
1509 XEXP (dep_cost_rtx_link
, 1) = dep_cost_rtx_link
;
1511 /* Targets use only REG_NOTE_KIND of the link. */
1512 PUT_REG_NOTE_KIND (dep_cost_rtx_link
, DEP_TYPE (link
));
1514 cost
= targetm
.sched
.adjust_cost (used
, dep_cost_rtx_link
,
1517 free_INSN_LIST_node (dep_cost_rtx_link
);
1524 DEP_COST (link
) = cost
;
1528 /* Compute cost of dependence LINK.
1529 This is the number of cycles between instruction issue and
1530 instruction results. */
1532 dep_cost (dep_t link
)
1534 return dep_cost_1 (link
, 0);
1537 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1538 INSN_PRIORITY explicitly. */
1540 increase_insn_priority (rtx_insn
*insn
, int amount
)
1542 if (!sel_sched_p ())
1544 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1545 if (INSN_PRIORITY_KNOWN (insn
))
1546 INSN_PRIORITY (insn
) += amount
;
1550 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1551 Use EXPR_PRIORITY instead. */
1552 sel_add_to_insn_priority (insn
, amount
);
1556 /* Return 'true' if DEP should be included in priority calculations. */
1558 contributes_to_priority_p (dep_t dep
)
1560 if (DEBUG_INSN_P (DEP_CON (dep
))
1561 || DEBUG_INSN_P (DEP_PRO (dep
)))
1564 /* Critical path is meaningful in block boundaries only. */
1565 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1569 if (DEP_REPLACE (dep
) != NULL
)
1572 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1573 then speculative instructions will less likely be
1574 scheduled. That is because the priority of
1575 their producers will increase, and, thus, the
1576 producers will more likely be scheduled, thus,
1577 resolving the dependence. */
1578 if (sched_deps_info
->generate_spec_deps
1579 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1580 && (DEP_STATUS (dep
) & SPECULATIVE
))
1586 /* Compute the number of nondebug deps in list LIST for INSN. */
1589 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1591 sd_iterator_def sd_it
;
1593 int dbgcount
= 0, nodbgcount
= 0;
1595 if (!MAY_HAVE_DEBUG_INSNS
)
1596 return sd_lists_size (insn
, list
);
1598 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1600 if (DEBUG_INSN_P (DEP_CON (dep
)))
1602 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1606 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1613 /* Compute the priority number for INSN. */
1615 priority (rtx_insn
*insn
)
1617 if (! INSN_P (insn
))
1620 /* We should not be interested in priority of an already scheduled insn. */
1621 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1623 if (!INSN_PRIORITY_KNOWN (insn
))
1625 int this_priority
= -1;
1629 int this_fusion_priority
;
1631 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1632 &this_fusion_priority
, &this_priority
);
1633 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1635 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1636 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1637 some forward deps but all of them are ignored by
1638 contributes_to_priority hook. At the moment we set priority of
1640 this_priority
= insn_cost (insn
);
1643 rtx_insn
*prev_first
, *twin
;
1646 /* For recovery check instructions we calculate priority slightly
1647 different than that of normal instructions. Instead of walking
1648 through INSN_FORW_DEPS (check) list, we walk through
1649 INSN_FORW_DEPS list of each instruction in the corresponding
1652 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1653 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1654 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1656 prev_first
= PREV_INSN (insn
);
1661 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1662 twin
= PREV_INSN (BB_END (rec
));
1667 sd_iterator_def sd_it
;
1670 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1675 next
= DEP_CON (dep
);
1677 if (BLOCK_FOR_INSN (next
) != rec
)
1681 if (!contributes_to_priority_p (dep
))
1685 cost
= dep_cost (dep
);
1688 struct _dep _dep1
, *dep1
= &_dep1
;
1690 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1692 cost
= dep_cost (dep1
);
1695 next_priority
= cost
+ priority (next
);
1697 if (next_priority
> this_priority
)
1698 this_priority
= next_priority
;
1702 twin
= PREV_INSN (twin
);
1704 while (twin
!= prev_first
);
1707 if (this_priority
< 0)
1709 gcc_assert (this_priority
== -1);
1711 this_priority
= insn_cost (insn
);
1714 INSN_PRIORITY (insn
) = this_priority
;
1715 INSN_PRIORITY_STATUS (insn
) = 1;
1718 return INSN_PRIORITY (insn
);
1721 /* Macros and functions for keeping the priority queue sorted, and
1722 dealing with queuing and dequeuing of instructions. */
1724 /* For each pressure class CL, set DEATH[CL] to the number of registers
1725 in that class that die in INSN. */
1728 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1731 struct reg_use_data
*use
;
1733 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1734 death
[ira_pressure_classes
[i
]] = 0;
1735 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1736 if (dying_use_p (use
))
1737 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1740 /* Setup info about the current register pressure impact of scheduling
1741 INSN at the current scheduling point. */
1743 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1745 int i
, change
, before
, after
, hard_regno
;
1746 int excess_cost_change
;
1749 struct reg_pressure_data
*pressure_info
;
1750 int *max_reg_pressure
;
1751 static int death
[N_REG_CLASSES
];
1753 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1755 excess_cost_change
= 0;
1756 calculate_reg_deaths (insn
, death
);
1757 pressure_info
= INSN_REG_PRESSURE (insn
);
1758 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1759 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1760 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1762 cl
= ira_pressure_classes
[i
];
1763 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1764 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1765 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1766 after
= MAX (0, max_reg_pressure
[i
] + change
1767 - sched_class_regs_num
[cl
]);
1768 hard_regno
= ira_class_hard_regs
[cl
][0];
1769 gcc_assert (hard_regno
>= 0);
1770 mode
= reg_raw_mode
[hard_regno
];
1771 excess_cost_change
+= ((after
- before
)
1772 * (ira_memory_move_cost
[mode
][cl
][0]
1773 + ira_memory_move_cost
[mode
][cl
][1]));
1775 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1778 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1779 It tries to make the scheduler take register pressure into account
1780 without introducing too many unnecessary stalls. It hooks into the
1781 main scheduling algorithm at several points:
1783 - Before scheduling starts, model_start_schedule constructs a
1784 "model schedule" for the current block. This model schedule is
1785 chosen solely to keep register pressure down. It does not take the
1786 target's pipeline or the original instruction order into account,
1787 except as a tie-breaker. It also doesn't work to a particular
1790 This model schedule gives us an idea of what pressure can be
1791 achieved for the block and gives us an example of a schedule that
1792 keeps to that pressure. It also makes the final schedule less
1793 dependent on the original instruction order. This is important
1794 because the original order can either be "wide" (many values live
1795 at once, such as in user-scheduled code) or "narrow" (few values
1796 live at once, such as after loop unrolling, where several
1797 iterations are executed sequentially).
1799 We do not apply this model schedule to the rtx stream. We simply
1800 record it in model_schedule. We also compute the maximum pressure,
1801 MP, that was seen during this schedule.
1803 - Instructions are added to the ready queue even if they require
1804 a stall. The length of the stall is instead computed as:
1806 MAX (INSN_TICK (INSN) - clock_var, 0)
1808 (= insn_delay). This allows rank_for_schedule to choose between
1809 introducing a deliberate stall or increasing pressure.
1811 - Before sorting the ready queue, model_set_excess_costs assigns
1812 a pressure-based cost to each ready instruction in the queue.
1813 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1814 (ECC for short) and is effectively measured in cycles.
1816 - rank_for_schedule ranks instructions based on:
1818 ECC (insn) + insn_delay (insn)
1824 So, for example, an instruction X1 with an ECC of 1 that can issue
1825 now will win over an instruction X0 with an ECC of zero that would
1826 introduce a stall of one cycle. However, an instruction X2 with an
1827 ECC of 2 that can issue now will lose to both X0 and X1.
1829 - When an instruction is scheduled, model_recompute updates the model
1830 schedule with the new pressures (some of which might now exceed the
1831 original maximum pressure MP). model_update_limit_points then searches
1832 for the new point of maximum pressure, if not already known. */
1834 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1835 from surrounding debug information. */
1837 ";;\t\t+------------------------------------------------------\n"
1839 /* Information about the pressure on a particular register class at a
1840 particular point of the model schedule. */
1841 struct model_pressure_data
{
1842 /* The pressure at this point of the model schedule, or -1 if the
1843 point is associated with an instruction that has already been
1847 /* The maximum pressure during or after this point of the model schedule. */
1851 /* Per-instruction information that is used while building the model
1852 schedule. Here, "schedule" refers to the model schedule rather
1853 than the main schedule. */
1854 struct model_insn_info
{
1855 /* The instruction itself. */
1858 /* If this instruction is in model_worklist, these fields link to the
1859 previous (higher-priority) and next (lower-priority) instructions
1861 struct model_insn_info
*prev
;
1862 struct model_insn_info
*next
;
1864 /* While constructing the schedule, QUEUE_INDEX describes whether an
1865 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1866 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1867 old_queue records the value that QUEUE_INDEX had before scheduling
1868 started, so that we can restore it once the schedule is complete. */
1871 /* The relative importance of an unscheduled instruction. Higher
1872 values indicate greater importance. */
1873 unsigned int model_priority
;
1875 /* The length of the longest path of satisfied true dependencies
1876 that leads to this instruction. */
1879 /* The length of the longest path of dependencies of any kind
1880 that leads from this instruction. */
1883 /* The number of predecessor nodes that must still be scheduled. */
1884 int unscheduled_preds
;
1887 /* Information about the pressure limit for a particular register class.
1888 This structure is used when applying a model schedule to the main
1890 struct model_pressure_limit
{
1891 /* The maximum register pressure seen in the original model schedule. */
1894 /* The maximum register pressure seen in the current model schedule
1895 (which excludes instructions that have already been scheduled). */
1898 /* The point of the current model schedule at which PRESSURE is first
1899 reached. It is set to -1 if the value needs to be recomputed. */
1903 /* Describes a particular way of measuring register pressure. */
1904 struct model_pressure_group
{
1905 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1906 struct model_pressure_limit limits
[N_REG_CLASSES
];
1908 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1909 on register class ira_pressure_classes[PCI] at point POINT of the
1910 current model schedule. A POINT of model_num_insns describes the
1911 pressure at the end of the schedule. */
1912 struct model_pressure_data
*model
;
1915 /* Index POINT gives the instruction at point POINT of the model schedule.
1916 This array doesn't change during main scheduling. */
1917 static vec
<rtx_insn
*> model_schedule
;
1919 /* The list of instructions in the model worklist, sorted in order of
1920 decreasing priority. */
1921 static struct model_insn_info
*model_worklist
;
1923 /* Index I describes the instruction with INSN_LUID I. */
1924 static struct model_insn_info
*model_insns
;
1926 /* The number of instructions in the model schedule. */
1927 static int model_num_insns
;
1929 /* The index of the first instruction in model_schedule that hasn't yet been
1930 added to the main schedule, or model_num_insns if all of them have. */
1931 static int model_curr_point
;
1933 /* Describes the pressure before each instruction in the model schedule. */
1934 static struct model_pressure_group model_before_pressure
;
1936 /* The first unused model_priority value (as used in model_insn_info). */
1937 static unsigned int model_next_priority
;
1940 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1941 at point POINT of the model schedule. */
1942 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1943 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1945 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1946 after point POINT of the model schedule. */
1947 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1948 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1950 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1951 of the model schedule. */
1952 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1953 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1955 /* Information about INSN that is used when creating the model schedule. */
1956 #define MODEL_INSN_INFO(INSN) \
1957 (&model_insns[INSN_LUID (INSN)])
1959 /* The instruction at point POINT of the model schedule. */
1960 #define MODEL_INSN(POINT) \
1961 (model_schedule[POINT])
1964 /* Return INSN's index in the model schedule, or model_num_insns if it
1965 doesn't belong to that schedule. */
1968 model_index (rtx_insn
*insn
)
1970 if (INSN_MODEL_INDEX (insn
) == 0)
1971 return model_num_insns
;
1972 return INSN_MODEL_INDEX (insn
) - 1;
1975 /* Make sure that GROUP->limits is up-to-date for the current point
1976 of the model schedule. */
1979 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1981 int pci
, max_pressure
, point
;
1983 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1985 /* We may have passed the final point at which the pressure in
1986 group->limits[pci].pressure was reached. Update the limit if so. */
1987 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1988 group
->limits
[pci
].pressure
= max_pressure
;
1990 /* Find the point at which MAX_PRESSURE is first reached. We need
1991 to search in three cases:
1993 - We've already moved past the previous pressure point.
1994 In this case we search forward from model_curr_point.
1996 - We scheduled the previous point of maximum pressure ahead of
1997 its position in the model schedule, but doing so didn't bring
1998 the pressure point earlier. In this case we search forward
1999 from that previous pressure point.
2001 - Scheduling an instruction early caused the maximum pressure
2002 to decrease. In this case we will have set the pressure
2003 point to -1, and we search forward from model_curr_point. */
2004 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
2005 while (point
< model_num_insns
2006 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
2008 group
->limits
[pci
].point
= point
;
2010 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
2011 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
2015 /* Make sure that all register-pressure limits are up-to-date for the
2016 current position in the model schedule. */
2019 model_update_limit_points (void)
2021 model_update_limit_points_in_group (&model_before_pressure
);
2024 /* Return the model_index of the last unscheduled use in chain USE
2025 outside of USE's instruction. Return -1 if there are no other uses,
2026 or model_num_insns if the register is live at the end of the block. */
2029 model_last_use_except (struct reg_use_data
*use
)
2031 struct reg_use_data
*next
;
2035 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2036 if (NONDEBUG_INSN_P (next
->insn
)
2037 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2039 index
= model_index (next
->insn
);
2040 if (index
== model_num_insns
)
2041 return model_num_insns
;
2048 /* An instruction with model_index POINT has just been scheduled, and it
2049 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2050 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2051 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2054 model_start_update_pressure (struct model_pressure_group
*group
,
2055 int point
, int pci
, int delta
)
2057 int next_max_pressure
;
2059 if (point
== model_num_insns
)
2061 /* The instruction wasn't part of the model schedule; it was moved
2062 from a different block. Update the pressure for the end of
2063 the model schedule. */
2064 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2065 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2069 /* Record that this instruction has been scheduled. Nothing now
2070 changes between POINT and POINT + 1, so get the maximum pressure
2071 from the latter. If the maximum pressure decreases, the new
2072 pressure point may be before POINT. */
2073 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2074 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2075 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2077 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2078 if (group
->limits
[pci
].point
== point
)
2079 group
->limits
[pci
].point
= -1;
2084 /* Record that scheduling a later instruction has changed the pressure
2085 at point POINT of the model schedule by DELTA (which might be 0).
2086 Update GROUP accordingly. Return nonzero if these changes might
2087 trigger changes to previous points as well. */
2090 model_update_pressure (struct model_pressure_group
*group
,
2091 int point
, int pci
, int delta
)
2093 int ref_pressure
, max_pressure
, next_max_pressure
;
2095 /* If POINT hasn't yet been scheduled, update its pressure. */
2096 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2097 if (ref_pressure
>= 0 && delta
!= 0)
2099 ref_pressure
+= delta
;
2100 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2102 /* Check whether the maximum pressure in the overall schedule
2103 has increased. (This means that the MODEL_MAX_PRESSURE of
2104 every point <= POINT will need to increase too; see below.) */
2105 if (group
->limits
[pci
].pressure
< ref_pressure
)
2106 group
->limits
[pci
].pressure
= ref_pressure
;
2108 /* If we are at maximum pressure, and the maximum pressure
2109 point was previously unknown or later than POINT,
2110 bring it forward. */
2111 if (group
->limits
[pci
].pressure
== ref_pressure
2112 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2113 group
->limits
[pci
].point
= point
;
2115 /* If POINT used to be the point of maximum pressure, but isn't
2116 any longer, we need to recalculate it using a forward walk. */
2117 if (group
->limits
[pci
].pressure
> ref_pressure
2118 && group
->limits
[pci
].point
== point
)
2119 group
->limits
[pci
].point
= -1;
2122 /* Update the maximum pressure at POINT. Changes here might also
2123 affect the maximum pressure at POINT - 1. */
2124 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2125 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2126 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2128 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2134 /* INSN has just been scheduled. Update the model schedule accordingly. */
2137 model_recompute (rtx_insn
*insn
)
2142 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2143 struct reg_use_data
*use
;
2144 struct reg_pressure_data
*reg_pressure
;
2145 int delta
[N_REG_CLASSES
];
2146 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2147 unsigned int i
, num_uses
, num_pending_births
;
2150 /* The destinations of INSN were previously live from POINT onwards, but are
2151 now live from model_curr_point onwards. Set up DELTA accordingly. */
2152 point
= model_index (insn
);
2153 reg_pressure
= INSN_REG_PRESSURE (insn
);
2154 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2156 cl
= ira_pressure_classes
[pci
];
2157 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2160 /* Record which registers previously died at POINT, but which now die
2161 before POINT. Adjust DELTA so that it represents the effect of
2162 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2163 registers that will be born in the range [model_curr_point, POINT). */
2165 num_pending_births
= 0;
2166 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2168 new_last
= model_last_use_except (use
);
2169 if (new_last
< point
)
2171 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2172 uses
[num_uses
].last_use
= new_last
;
2173 uses
[num_uses
].regno
= use
->regno
;
2174 /* This register is no longer live after POINT - 1. */
2175 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2178 num_pending_births
++;
2182 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2183 Also set each group pressure limit for POINT. */
2184 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2186 cl
= ira_pressure_classes
[pci
];
2187 model_start_update_pressure (&model_before_pressure
,
2188 point
, pci
, delta
[cl
]);
2191 /* Walk the model schedule backwards, starting immediately before POINT. */
2193 if (point
!= model_curr_point
)
2197 insn
= MODEL_INSN (point
);
2198 queue
= QUEUE_INDEX (insn
);
2200 if (queue
!= QUEUE_SCHEDULED
)
2202 /* DELTA describes the effect of the move on the register pressure
2203 after POINT. Make it describe the effect on the pressure
2206 while (i
< num_uses
)
2208 if (uses
[i
].last_use
== point
)
2210 /* This register is now live again. */
2211 mark_regno_birth_or_death (NULL
, delta
,
2212 uses
[i
].regno
, true);
2214 /* Remove this use from the array. */
2215 uses
[i
] = uses
[num_uses
- 1];
2217 num_pending_births
--;
2223 if (sched_verbose
>= 5)
2227 fprintf (sched_dump
, MODEL_BAR
);
2228 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2230 fprintf (sched_dump
, MODEL_BAR
);
2234 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2235 point
, INSN_UID (insn
),
2236 str_pattern_slim (PATTERN (insn
)));
2237 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2239 cl
= ira_pressure_classes
[pci
];
2240 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2242 fprintf (sched_dump
, " %s:[%d->%d]",
2243 reg_class_names
[ira_pressure_classes
[pci
]],
2244 ref_pressure
, ref_pressure
+ delta
[cl
]);
2246 fprintf (sched_dump
, "\n");
2250 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2251 might have changed as well. */
2252 mix
= num_pending_births
;
2253 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2255 cl
= ira_pressure_classes
[pci
];
2257 mix
|= model_update_pressure (&model_before_pressure
,
2258 point
, pci
, delta
[cl
]);
2261 while (mix
&& point
> model_curr_point
);
2264 fprintf (sched_dump
, MODEL_BAR
);
2267 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2268 check whether the insn's pattern needs restoring. */
2270 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2272 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2275 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2277 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2278 gcc_assert (next
== DEP_CON (dep
));
2282 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2283 if (desc
->insn
!= next
)
2285 gcc_assert (*desc
->loc
== desc
->orig
);
2292 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2293 pressure on CL from P to P'. We use this to calculate a "base ECC",
2294 baseECC (CL, X), for each pressure class CL and each instruction X.
2295 Supposing X changes the pressure on CL from P to P', and that the
2296 maximum pressure on CL in the current model schedule is MP', then:
2298 * if X occurs before or at the next point of maximum pressure in
2299 the model schedule and P' > MP', then:
2301 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2303 The idea is that the pressure after scheduling a fixed set of
2304 instructions -- in this case, the set up to and including the
2305 next maximum pressure point -- is going to be the same regardless
2306 of the order; we simply want to keep the intermediate pressure
2307 under control. Thus X has a cost of zero unless scheduling it
2308 now would exceed MP'.
2310 If all increases in the set are by the same amount, no zero-cost
2311 instruction will ever cause the pressure to exceed MP'. However,
2312 if X is instead moved past an instruction X' with pressure in the
2313 range (MP' - (P' - P), MP'), the pressure at X' will increase
2314 beyond MP'. Since baseECC is very much a heuristic anyway,
2315 it doesn't seem worth the overhead of tracking cases like these.
2317 The cost of exceeding MP' is always based on the original maximum
2318 pressure MP. This is so that going 2 registers over the original
2319 limit has the same cost regardless of whether it comes from two
2320 separate +1 deltas or from a single +2 delta.
2322 * if X occurs after the next point of maximum pressure in the model
2323 schedule and P' > P, then:
2325 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2327 That is, if we move X forward across a point of maximum pressure,
2328 and if X increases the pressure by P' - P, then we conservatively
2329 assume that scheduling X next would increase the maximum pressure
2330 by P' - P. Again, the cost of doing this is based on the original
2331 maximum pressure MP, for the same reason as above.
2333 * if P' < P, P > MP, and X occurs at or after the next point of
2334 maximum pressure, then:
2336 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2338 That is, if we have already exceeded the original maximum pressure MP,
2339 and if X might reduce the maximum pressure again -- or at least push
2340 it further back, and thus allow more scheduling freedom -- it is given
2341 a negative cost to reflect the improvement.
2347 In this case, X is not expected to affect the maximum pressure MP',
2348 so it has zero cost.
2350 We then create a combined value baseECC (X) that is the sum of
2351 baseECC (CL, X) for each pressure class CL.
2353 baseECC (X) could itself be used as the ECC value described above.
2354 However, this is often too conservative, in the sense that it
2355 tends to make high-priority instructions that increase pressure
2356 wait too long in cases where introducing a spill would be better.
2357 For this reason the final ECC is a priority-adjusted form of
2358 baseECC (X). Specifically, we calculate:
2360 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2361 baseP = MAX { P (X) | baseECC (X) <= 0 }
2365 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2367 Thus an instruction's effect on pressure is ignored if it has a high
2368 enough priority relative to the ones that don't increase pressure.
2369 Negative values of baseECC (X) do not increase the priority of X
2370 itself, but they do make it harder for other instructions to
2371 increase the pressure further.
2373 This pressure cost is deliberately timid. The intention has been
2374 to choose a heuristic that rarely interferes with the normal list
2375 scheduler in cases where that scheduler would produce good code.
2376 We simply want to curb some of its worst excesses. */
2378 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2380 Here we use the very simplistic cost model that every register above
2381 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2382 measures instead, such as one based on MEMORY_MOVE_COST. However:
2384 (1) In order for an instruction to be scheduled, the higher cost
2385 would need to be justified in a single saving of that many stalls.
2386 This is overly pessimistic, because the benefit of spilling is
2387 often to avoid a sequence of several short stalls rather than
2390 (2) The cost is still arbitrary. Because we are not allocating
2391 registers during scheduling, we have no way of knowing for
2392 sure how many memory accesses will be required by each spill,
2393 where the spills will be placed within the block, or even
2394 which block(s) will contain the spills.
2396 So a higher cost than 1 is often too conservative in practice,
2397 forcing blocks to contain unnecessary stalls instead of spill code.
2398 The simple cost below seems to be the best compromise. It reduces
2399 the interference with the normal list scheduler, which helps make
2400 it more suitable for a default-on option. */
2403 model_spill_cost (int cl
, int from
, int to
)
2405 from
= MAX (from
, sched_class_regs_num
[cl
]);
2406 return MAX (to
, from
) - from
;
2409 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2410 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2414 model_excess_group_cost (struct model_pressure_group
*group
,
2415 int point
, int pci
, int delta
)
2419 cl
= ira_pressure_classes
[pci
];
2420 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2422 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2423 curr_reg_pressure
[cl
] + delta
);
2424 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2429 if (point
> group
->limits
[pci
].point
)
2430 pressure
= group
->limits
[pci
].pressure
+ delta
;
2432 pressure
= curr_reg_pressure
[cl
] + delta
;
2434 if (pressure
> group
->limits
[pci
].pressure
)
2435 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2442 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2446 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2448 int point
, pci
, cl
, cost
, this_cost
, delta
;
2449 struct reg_pressure_data
*insn_reg_pressure
;
2450 int insn_death
[N_REG_CLASSES
];
2452 calculate_reg_deaths (insn
, insn_death
);
2453 point
= model_index (insn
);
2454 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2458 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2459 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2461 /* Sum up the individual costs for each register class. */
2462 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2464 cl
= ira_pressure_classes
[pci
];
2465 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2466 this_cost
= model_excess_group_cost (&model_before_pressure
,
2470 fprintf (sched_dump
, " %s:[%d base cost %d]",
2471 reg_class_names
[cl
], delta
, this_cost
);
2475 fprintf (sched_dump
, "\n");
2480 /* Dump the next points of maximum pressure for GROUP. */
2483 model_dump_pressure_points (struct model_pressure_group
*group
)
2487 fprintf (sched_dump
, ";;\t\t| pressure points");
2488 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2490 cl
= ira_pressure_classes
[pci
];
2491 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2492 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2493 if (group
->limits
[pci
].point
< model_num_insns
)
2494 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2495 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2497 fprintf (sched_dump
, "end]");
2499 fprintf (sched_dump
, "\n");
2502 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2505 model_set_excess_costs (rtx_insn
**insns
, int count
)
2507 int i
, cost
, priority_base
, priority
;
2510 /* Record the baseECC value for each instruction in the model schedule,
2511 except that negative costs are converted to zero ones now rather than
2512 later. Do not assign a cost to debug instructions, since they must
2513 not change code-generation decisions. Experiments suggest we also
2514 get better results by not assigning a cost to instructions from
2517 Set PRIORITY_BASE to baseP in the block comment above. This is the
2518 maximum priority of the "cheap" instructions, which should always
2519 include the next model instruction. */
2522 for (i
= 0; i
< count
; i
++)
2523 if (INSN_MODEL_INDEX (insns
[i
]))
2525 if (sched_verbose
>= 6 && !print_p
)
2527 fprintf (sched_dump
, MODEL_BAR
);
2528 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2529 model_dump_pressure_points (&model_before_pressure
);
2530 fprintf (sched_dump
, MODEL_BAR
);
2533 cost
= model_excess_cost (insns
[i
], print_p
);
2536 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2537 priority_base
= MAX (priority_base
, priority
);
2540 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2543 fprintf (sched_dump
, MODEL_BAR
);
2545 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2547 for (i
= 0; i
< count
; i
++)
2549 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2550 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2551 if (cost
> 0 && priority
> priority_base
)
2553 cost
+= priority_base
- priority
;
2554 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2560 /* Enum of rank_for_schedule heuristic decisions. */
2562 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2563 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2564 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2565 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2566 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_N
};
2568 /* Corresponding strings for print outs. */
2569 static const char *rfs_str
[RFS_N
] = {
2570 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2571 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2572 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2573 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2574 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2576 /* Statistical breakdown of rank_for_schedule decisions. */
2577 typedef struct { unsigned stats
[RFS_N
]; } rank_for_schedule_stats_t
;
2578 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2580 /* Return the result of comparing insns TMP and TMP2 and update
2581 Rank_For_Schedule statistics. */
2583 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2585 ++rank_for_schedule_stats
.stats
[decision
];
2587 INSN_LAST_RFS_WIN (tmp
) = decision
;
2588 else if (result
> 0)
2589 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2595 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2596 keeping normal insns in original order. */
2599 rank_for_schedule_debug (const void *x
, const void *y
)
2601 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2602 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2604 /* Schedule debug insns as early as possible. */
2605 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2607 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2609 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2610 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2612 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2615 /* Returns a positive value if x is preferred; returns a negative value if
2616 y is preferred. Should never return 0, since that will make the sort
2620 rank_for_schedule (const void *x
, const void *y
)
2622 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2623 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2624 int tmp_class
, tmp2_class
;
2625 int val
, priority_val
, info_val
, diff
;
2627 if (live_range_shrinkage_p
)
2629 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2631 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2632 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2633 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2634 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2635 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2636 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2637 /* Sort by INSN_LUID (original insn order), so that we make the
2638 sort stable. This minimizes instruction movement, thus
2639 minimizing sched's effect on debugging and cross-jumping. */
2640 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2641 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2644 /* The insn in a schedule group should be issued the first. */
2645 if (flag_sched_group_heuristic
&&
2646 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2647 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2650 /* Make sure that priority of TMP and TMP2 are initialized. */
2651 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2655 /* The instruction that has the same fusion priority as the last
2656 instruction is the instruction we picked next. If that is not
2657 the case, we sort ready list firstly by fusion priority, then
2658 by priority, and at last by INSN_LUID. */
2659 int a
= INSN_FUSION_PRIORITY (tmp
);
2660 int b
= INSN_FUSION_PRIORITY (tmp2
);
2663 if (last_nondebug_scheduled_insn
2664 && !NOTE_P (last_nondebug_scheduled_insn
)
2665 && BLOCK_FOR_INSN (tmp
)
2666 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2667 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2669 if (a
!= last
&& b
!= last
)
2673 a
= INSN_PRIORITY (tmp
);
2674 b
= INSN_PRIORITY (tmp2
);
2677 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2679 return rfs_result (RFS_FUSION
,
2680 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2684 gcc_assert (last_nondebug_scheduled_insn
2685 && !NOTE_P (last_nondebug_scheduled_insn
));
2686 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2688 a
= abs (INSN_PRIORITY (tmp
) - last
);
2689 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2691 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2693 return rfs_result (RFS_FUSION
,
2694 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2697 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2699 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2702 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2704 /* Prefer insn whose scheduling results in the smallest register
2706 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2708 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2709 - insn_delay (tmp2
))))
2710 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2713 if (sched_pressure
!= SCHED_PRESSURE_NONE
2714 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2715 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2717 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2718 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2721 /* If we are doing backtracking in this schedule, prefer insns that
2722 have forward dependencies with negative cost against an insn that
2723 was already scheduled. */
2724 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2726 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2728 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2731 /* Prefer insn with higher priority. */
2732 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2734 if (flag_sched_critical_path_heuristic
&& priority_val
)
2735 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2737 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2739 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2744 /* Prefer speculative insn with greater dependencies weakness. */
2745 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2751 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2753 dw1
= ds_weak (ds1
);
2757 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2759 dw2
= ds_weak (ds2
);
2764 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2765 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2768 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2769 if (flag_sched_rank_heuristic
&& info_val
)
2770 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2772 /* Compare insns based on their relation to the last scheduled
2774 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2778 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2780 /* Classify the instructions into three classes:
2781 1) Data dependent on last schedule insn.
2782 2) Anti/Output dependent on last scheduled insn.
2783 3) Independent of last scheduled insn, or has latency of one.
2784 Choose the insn from the highest numbered class if different. */
2785 dep1
= sd_find_dep_between (last
, tmp
, true);
2787 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2789 else if (/* Data dependence. */
2790 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2795 dep2
= sd_find_dep_between (last
, tmp2
, true);
2797 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2799 else if (/* Data dependence. */
2800 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2805 if ((val
= tmp2_class
- tmp_class
))
2806 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2809 /* Prefer instructions that occur earlier in the model schedule. */
2810 if (sched_pressure
== SCHED_PRESSURE_MODEL
2811 && INSN_BB (tmp
) == target_bb
&& INSN_BB (tmp2
) == target_bb
)
2813 diff
= model_index (tmp
) - model_index (tmp2
);
2814 gcc_assert (diff
!= 0);
2815 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2818 /* Prefer the insn which has more later insns that depend on it.
2819 This gives the scheduler more freedom when scheduling later
2820 instructions at the expense of added register pressure. */
2822 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2823 - dep_list_size (tmp
, SD_LIST_FORW
));
2825 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2826 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2828 /* If insns are equally good, sort by INSN_LUID (original insn order),
2829 so that we make the sort stable. This minimizes instruction movement,
2830 thus minimizing sched's effect on debugging and cross-jumping. */
2831 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2834 /* Resort the array A in which only element at index N may be out of order. */
2836 HAIFA_INLINE
static void
2837 swap_sort (rtx_insn
**a
, int n
)
2839 rtx_insn
*insn
= a
[n
- 1];
2842 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2850 /* Add INSN to the insn queue so that it can be executed at least
2851 N_CYCLES after the currently executing insn. Preserve insns
2852 chain for debugging purposes. REASON will be printed in debugging
2855 HAIFA_INLINE
static void
2856 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2858 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2859 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2862 gcc_assert (n_cycles
<= max_insn_queue_index
);
2863 gcc_assert (!DEBUG_INSN_P (insn
));
2865 insn_queue
[next_q
] = link
;
2868 if (sched_verbose
>= 2)
2870 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2871 (*current_sched_info
->print_insn
) (insn
, 0));
2873 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2876 QUEUE_INDEX (insn
) = next_q
;
2878 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2880 new_tick
= clock_var
+ n_cycles
;
2881 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2882 INSN_TICK (insn
) = new_tick
;
2884 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2885 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2887 must_backtrack
= true;
2888 if (sched_verbose
>= 2)
2889 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2894 /* Remove INSN from queue. */
2896 queue_remove (rtx_insn
*insn
)
2898 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2899 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2901 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2904 /* Return a pointer to the bottom of the ready list, i.e. the insn
2905 with the lowest priority. */
2908 ready_lastpos (struct ready_list
*ready
)
2910 gcc_assert (ready
->n_ready
>= 1);
2911 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2914 /* Add an element INSN to the ready list so that it ends up with the
2915 lowest/highest priority depending on FIRST_P. */
2917 HAIFA_INLINE
static void
2918 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2922 if (ready
->first
== ready
->n_ready
)
2924 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2925 ready_lastpos (ready
),
2926 ready
->n_ready
* sizeof (rtx
));
2927 ready
->first
= ready
->veclen
- 1;
2929 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2933 if (ready
->first
== ready
->veclen
- 1)
2936 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2937 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2938 ready_lastpos (ready
),
2939 ready
->n_ready
* sizeof (rtx
));
2940 ready
->first
= ready
->veclen
- 2;
2942 ready
->vec
[++(ready
->first
)] = insn
;
2946 if (DEBUG_INSN_P (insn
))
2949 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2950 QUEUE_INDEX (insn
) = QUEUE_READY
;
2952 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2953 && INSN_EXACT_TICK (insn
) < clock_var
)
2955 must_backtrack
= true;
2959 /* Remove the element with the highest priority from the ready list and
2962 HAIFA_INLINE
static rtx_insn
*
2963 ready_remove_first (struct ready_list
*ready
)
2967 gcc_assert (ready
->n_ready
);
2968 t
= ready
->vec
[ready
->first
--];
2970 if (DEBUG_INSN_P (t
))
2972 /* If the queue becomes empty, reset it. */
2973 if (ready
->n_ready
== 0)
2974 ready
->first
= ready
->veclen
- 1;
2976 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2977 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2982 /* The following code implements multi-pass scheduling for the first
2983 cycle. In other words, we will try to choose ready insn which
2984 permits to start maximum number of insns on the same cycle. */
2986 /* Return a pointer to the element INDEX from the ready. INDEX for
2987 insn with the highest priority is 0, and the lowest priority has
2991 ready_element (struct ready_list
*ready
, int index
)
2993 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2995 return ready
->vec
[ready
->first
- index
];
2998 /* Remove the element INDEX from the ready list and return it. INDEX
2999 for insn with the highest priority is 0, and the lowest priority
3002 HAIFA_INLINE
static rtx_insn
*
3003 ready_remove (struct ready_list
*ready
, int index
)
3009 return ready_remove_first (ready
);
3010 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3011 t
= ready
->vec
[ready
->first
- index
];
3013 if (DEBUG_INSN_P (t
))
3015 for (i
= index
; i
< ready
->n_ready
; i
++)
3016 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3017 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3021 /* Remove INSN from the ready list. */
3023 ready_remove_insn (rtx_insn
*insn
)
3027 for (i
= 0; i
< readyp
->n_ready
; i
++)
3028 if (ready_element (readyp
, i
) == insn
)
3030 ready_remove (readyp
, i
);
3036 /* Calculate difference of two statistics set WAS and NOW.
3037 Result returned in WAS. */
3039 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3040 const rank_for_schedule_stats_t
*now
)
3042 for (int i
= 0; i
< RFS_N
; ++i
)
3043 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3046 /* Print rank_for_schedule statistics. */
3048 print_rank_for_schedule_stats (const char *prefix
,
3049 const rank_for_schedule_stats_t
*stats
,
3050 struct ready_list
*ready
)
3052 for (int i
= 0; i
< RFS_N
; ++i
)
3053 if (stats
->stats
[i
])
3055 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3058 /* Print out insns that won due to RFS_<I>. */
3060 rtx_insn
**p
= ready_lastpos (ready
);
3062 fprintf (sched_dump
, ":");
3063 /* Start with 1 since least-priority insn didn't have any wins. */
3064 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3065 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3066 fprintf (sched_dump
, " %s",
3067 (*current_sched_info
->print_insn
) (p
[j
], 0));
3069 fprintf (sched_dump
, "\n");
3073 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3076 ready_sort_debug (struct ready_list
*ready
)
3079 rtx_insn
**first
= ready_lastpos (ready
);
3081 for (i
= 0; i
< ready
->n_ready
; ++i
)
3082 if (!DEBUG_INSN_P (first
[i
]))
3083 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3085 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3088 /* Sort non-debug insns in the ready list READY by ascending priority.
3089 Assumes that all debug insns are separated from the real insns. */
3091 ready_sort_real (struct ready_list
*ready
)
3094 rtx_insn
**first
= ready_lastpos (ready
);
3095 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3097 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3098 for (i
= 0; i
< n_ready_real
; ++i
)
3099 setup_insn_reg_pressure_info (first
[i
]);
3100 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3101 && model_curr_point
< model_num_insns
)
3102 model_set_excess_costs (first
, n_ready_real
);
3104 rank_for_schedule_stats_t stats1
;
3105 if (sched_verbose
>= 4)
3106 stats1
= rank_for_schedule_stats
;
3108 if (n_ready_real
== 2)
3109 swap_sort (first
, n_ready_real
);
3110 else if (n_ready_real
> 2)
3111 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3113 if (sched_verbose
>= 4)
3115 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3116 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3120 /* Sort the ready list READY by ascending priority. */
3122 ready_sort (struct ready_list
*ready
)
3124 if (ready
->n_debug
> 0)
3125 ready_sort_debug (ready
);
3127 ready_sort_real (ready
);
3130 /* PREV is an insn that is ready to execute. Adjust its priority if that
3131 will help shorten or lengthen register lifetimes as appropriate. Also
3132 provide a hook for the target to tweak itself. */
3134 HAIFA_INLINE
static void
3135 adjust_priority (rtx_insn
*prev
)
3137 /* ??? There used to be code here to try and estimate how an insn
3138 affected register lifetimes, but it did it by looking at REG_DEAD
3139 notes, which we removed in schedule_region. Nor did it try to
3140 take into account register pressure or anything useful like that.
3142 Revisit when we have a machine model to work with and not before. */
3144 if (targetm
.sched
.adjust_priority
)
3145 INSN_PRIORITY (prev
) =
3146 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3149 /* Advance DFA state STATE on one cycle. */
3151 advance_state (state_t state
)
3153 if (targetm
.sched
.dfa_pre_advance_cycle
)
3154 targetm
.sched
.dfa_pre_advance_cycle ();
3156 if (targetm
.sched
.dfa_pre_cycle_insn
)
3157 state_transition (state
,
3158 targetm
.sched
.dfa_pre_cycle_insn ());
3160 state_transition (state
, NULL
);
3162 if (targetm
.sched
.dfa_post_cycle_insn
)
3163 state_transition (state
,
3164 targetm
.sched
.dfa_post_cycle_insn ());
3166 if (targetm
.sched
.dfa_post_advance_cycle
)
3167 targetm
.sched
.dfa_post_advance_cycle ();
3170 /* Advance time on one cycle. */
3171 HAIFA_INLINE
static void
3172 advance_one_cycle (void)
3174 advance_state (curr_state
);
3175 if (sched_verbose
>= 4)
3176 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3179 /* Update register pressure after scheduling INSN. */
3181 update_register_pressure (rtx_insn
*insn
)
3183 struct reg_use_data
*use
;
3184 struct reg_set_data
*set
;
3186 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3188 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3189 if (dying_use_p (use
))
3190 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3192 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3193 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3197 /* Set up or update (if UPDATE_P) max register pressure (see its
3198 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3199 after insn AFTER. */
3201 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3206 static int max_reg_pressure
[N_REG_CLASSES
];
3208 save_reg_pressure ();
3209 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3210 max_reg_pressure
[ira_pressure_classes
[i
]]
3211 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3212 for (insn
= NEXT_INSN (after
);
3213 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3214 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3215 insn
= NEXT_INSN (insn
))
3216 if (NONDEBUG_INSN_P (insn
))
3219 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3221 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3222 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3225 INSN_MAX_REG_PRESSURE (insn
)[i
]
3226 = max_reg_pressure
[ira_pressure_classes
[i
]];
3229 if (update_p
&& eq_p
)
3231 update_register_pressure (insn
);
3232 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3233 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3234 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3235 max_reg_pressure
[ira_pressure_classes
[i
]]
3236 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3238 restore_reg_pressure ();
3241 /* Update the current register pressure after scheduling INSN. Update
3242 also max register pressure for unscheduled insns of the current
3245 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3248 int before
[N_REG_CLASSES
];
3250 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3251 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3252 update_register_pressure (insn
);
3253 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3254 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3256 if (i
< ira_pressure_classes_num
)
3257 setup_insn_max_reg_pressure (insn
, true);
3260 /* Set up register pressure at the beginning of basic block BB whose
3261 insns starting after insn AFTER. Set up also max register pressure
3262 for all insns of the basic block. */
3264 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3266 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3267 initiate_bb_reg_pressure_info (bb
);
3268 setup_insn_max_reg_pressure (after
, false);
3271 /* If doing predication while scheduling, verify whether INSN, which
3272 has just been scheduled, clobbers the conditions of any
3273 instructions that must be predicated in order to break their
3274 dependencies. If so, remove them from the queues so that they will
3275 only be scheduled once their control dependency is resolved. */
3278 check_clobbered_conditions (rtx_insn
*insn
)
3283 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3286 find_all_hard_reg_sets (insn
, &t
, true);
3289 for (i
= 0; i
< ready
.n_ready
; i
++)
3291 rtx_insn
*x
= ready_element (&ready
, i
);
3292 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3294 ready_remove_insn (x
);
3298 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3300 rtx_insn_list
*link
;
3301 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3304 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3306 rtx_insn
*x
= link
->insn ();
3307 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3316 /* Return (in order):
3318 - positive if INSN adversely affects the pressure on one
3321 - negative if INSN reduces the pressure on one register class
3323 - 0 if INSN doesn't affect the pressure on any register class. */
3326 model_classify_pressure (struct model_insn_info
*insn
)
3328 struct reg_pressure_data
*reg_pressure
;
3329 int death
[N_REG_CLASSES
];
3332 calculate_reg_deaths (insn
->insn
, death
);
3333 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3335 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3337 cl
= ira_pressure_classes
[pci
];
3338 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3340 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3345 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3348 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3350 unsigned int height1
, height2
;
3351 unsigned int priority1
, priority2
;
3353 /* Prefer instructions with a higher model priority. */
3354 if (insn1
->model_priority
!= insn2
->model_priority
)
3355 return insn1
->model_priority
> insn2
->model_priority
;
3357 /* Combine the length of the longest path of satisfied true dependencies
3358 that leads to each instruction (depth) with the length of the longest
3359 path of any dependencies that leads from the instruction (alap).
3360 Prefer instructions with the greatest combined length. If the combined
3361 lengths are equal, prefer instructions with the greatest depth.
3363 The idea is that, if we have a set S of "equal" instructions that each
3364 have ALAP value X, and we pick one such instruction I, any true-dependent
3365 successors of I that have ALAP value X - 1 should be preferred over S.
3366 This encourages the schedule to be "narrow" rather than "wide".
3367 However, if I is a low-priority instruction that we decided to
3368 schedule because of its model_classify_pressure, and if there
3369 is a set of higher-priority instructions T, the aforementioned
3370 successors of I should not have the edge over T. */
3371 height1
= insn1
->depth
+ insn1
->alap
;
3372 height2
= insn2
->depth
+ insn2
->alap
;
3373 if (height1
!= height2
)
3374 return height1
> height2
;
3375 if (insn1
->depth
!= insn2
->depth
)
3376 return insn1
->depth
> insn2
->depth
;
3378 /* We have no real preference between INSN1 an INSN2 as far as attempts
3379 to reduce pressure go. Prefer instructions with higher priorities. */
3380 priority1
= INSN_PRIORITY (insn1
->insn
);
3381 priority2
= INSN_PRIORITY (insn2
->insn
);
3382 if (priority1
!= priority2
)
3383 return priority1
> priority2
;
3385 /* Use the original rtl sequence as a tie-breaker. */
3386 return insn1
< insn2
;
3389 /* Add INSN to the model worklist immediately after PREV. Add it to the
3390 beginning of the list if PREV is null. */
3393 model_add_to_worklist_at (struct model_insn_info
*insn
,
3394 struct model_insn_info
*prev
)
3396 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3397 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3402 insn
->next
= prev
->next
;
3407 insn
->next
= model_worklist
;
3408 model_worklist
= insn
;
3411 insn
->next
->prev
= insn
;
3414 /* Remove INSN from the model worklist. */
3417 model_remove_from_worklist (struct model_insn_info
*insn
)
3419 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3420 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3423 insn
->prev
->next
= insn
->next
;
3425 model_worklist
= insn
->next
;
3427 insn
->next
->prev
= insn
->prev
;
3430 /* Add INSN to the model worklist. Start looking for a suitable position
3431 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3432 insns either side. A null PREV indicates the beginning of the list and
3433 a null NEXT indicates the end. */
3436 model_add_to_worklist (struct model_insn_info
*insn
,
3437 struct model_insn_info
*prev
,
3438 struct model_insn_info
*next
)
3442 count
= MAX_SCHED_READY_INSNS
;
3443 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3449 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3451 while (count
> 0 && next
&& model_order_p (next
, insn
))
3457 model_add_to_worklist_at (insn
, prev
);
3460 /* INSN may now have a higher priority (in the model_order_p sense)
3461 than before. Move it up the worklist if necessary. */
3464 model_promote_insn (struct model_insn_info
*insn
)
3466 struct model_insn_info
*prev
;
3470 count
= MAX_SCHED_READY_INSNS
;
3471 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3476 if (prev
!= insn
->prev
)
3478 model_remove_from_worklist (insn
);
3479 model_add_to_worklist_at (insn
, prev
);
3483 /* Add INSN to the end of the model schedule. */
3486 model_add_to_schedule (rtx_insn
*insn
)
3490 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3491 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3493 point
= model_schedule
.length ();
3494 model_schedule
.quick_push (insn
);
3495 INSN_MODEL_INDEX (insn
) = point
+ 1;
3498 /* Analyze the instructions that are to be scheduled, setting up
3499 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3500 instructions to model_worklist. */
3503 model_analyze_insns (void)
3505 rtx_insn
*start
, *end
, *iter
;
3506 sd_iterator_def sd_it
;
3508 struct model_insn_info
*insn
, *con
;
3510 model_num_insns
= 0;
3511 start
= PREV_INSN (current_sched_info
->next_tail
);
3512 end
= current_sched_info
->prev_head
;
3513 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3514 if (NONDEBUG_INSN_P (iter
))
3516 insn
= MODEL_INSN_INFO (iter
);
3518 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3520 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3521 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3522 insn
->alap
= con
->alap
+ 1;
3525 insn
->old_queue
= QUEUE_INDEX (iter
);
3526 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3528 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3529 if (insn
->unscheduled_preds
== 0)
3530 model_add_to_worklist (insn
, NULL
, model_worklist
);
3536 /* The global state describes the register pressure at the start of the
3537 model schedule. Initialize GROUP accordingly. */
3540 model_init_pressure_group (struct model_pressure_group
*group
)
3544 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3546 cl
= ira_pressure_classes
[pci
];
3547 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3548 group
->limits
[pci
].point
= 0;
3550 /* Use index model_num_insns to record the state after the last
3551 instruction in the model schedule. */
3552 group
->model
= XNEWVEC (struct model_pressure_data
,
3553 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3556 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3557 Update the maximum pressure for the whole schedule. */
3560 model_record_pressure (struct model_pressure_group
*group
,
3561 int point
, int pci
, int pressure
)
3563 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3564 if (group
->limits
[pci
].pressure
< pressure
)
3566 group
->limits
[pci
].pressure
= pressure
;
3567 group
->limits
[pci
].point
= point
;
3571 /* INSN has just been added to the end of the model schedule. Record its
3572 register-pressure information. */
3575 model_record_pressures (struct model_insn_info
*insn
)
3577 struct reg_pressure_data
*reg_pressure
;
3578 int point
, pci
, cl
, delta
;
3579 int death
[N_REG_CLASSES
];
3581 point
= model_index (insn
->insn
);
3582 if (sched_verbose
>= 2)
3586 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3587 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3589 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3590 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3591 insn
->depth
+ insn
->alap
, insn
->depth
,
3592 INSN_PRIORITY (insn
->insn
),
3593 str_pattern_slim (PATTERN (insn
->insn
)));
3595 calculate_reg_deaths (insn
->insn
, death
);
3596 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3597 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3599 cl
= ira_pressure_classes
[pci
];
3600 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3601 if (sched_verbose
>= 2)
3602 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3603 curr_reg_pressure
[cl
], delta
);
3604 model_record_pressure (&model_before_pressure
, point
, pci
,
3605 curr_reg_pressure
[cl
]);
3607 if (sched_verbose
>= 2)
3608 fprintf (sched_dump
, "\n");
3611 /* All instructions have been added to the model schedule. Record the
3612 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3615 model_record_final_pressures (struct model_pressure_group
*group
)
3617 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3619 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3621 /* Record the final pressure for this class. */
3622 cl
= ira_pressure_classes
[pci
];
3623 point
= model_num_insns
;
3624 ref_pressure
= curr_reg_pressure
[cl
];
3625 model_record_pressure (group
, point
, pci
, ref_pressure
);
3627 /* Record the original maximum pressure. */
3628 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3630 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3631 max_pressure
= ref_pressure
;
3632 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3636 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3637 max_pressure
= MAX (max_pressure
, ref_pressure
);
3638 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3643 /* Update all successors of INSN, given that INSN has just been scheduled. */
3646 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3648 sd_iterator_def sd_it
;
3649 struct model_insn_info
*con
;
3652 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3654 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3655 /* Ignore debug instructions, and instructions from other blocks. */
3658 con
->unscheduled_preds
--;
3660 /* Update the depth field of each true-dependent successor.
3661 Increasing the depth gives them a higher priority than
3663 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3665 con
->depth
= insn
->depth
+ 1;
3666 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3667 model_promote_insn (con
);
3670 /* If this is a true dependency, or if there are no remaining
3671 dependencies for CON (meaning that CON only had non-true
3672 dependencies), make sure that CON is on the worklist.
3673 We don't bother otherwise because it would tend to fill the
3674 worklist with a lot of low-priority instructions that are not
3675 yet ready to issue. */
3676 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3677 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3678 model_add_to_worklist (con
, insn
, insn
->next
);
3683 /* Give INSN a higher priority than any current instruction, then give
3684 unscheduled predecessors of INSN a higher priority still. If any of
3685 those predecessors are not on the model worklist, do the same for its
3686 predecessors, and so on. */
3689 model_promote_predecessors (struct model_insn_info
*insn
)
3691 struct model_insn_info
*pro
, *first
;
3692 sd_iterator_def sd_it
;
3695 if (sched_verbose
>= 7)
3696 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3697 INSN_UID (insn
->insn
), model_next_priority
);
3698 insn
->model_priority
= model_next_priority
++;
3699 model_remove_from_worklist (insn
);
3700 model_add_to_worklist_at (insn
, NULL
);
3705 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3707 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3708 /* The first test is to ignore debug instructions, and instructions
3709 from other blocks. */
3711 && pro
->model_priority
!= model_next_priority
3712 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3714 pro
->model_priority
= model_next_priority
;
3715 if (sched_verbose
>= 7)
3716 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3717 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3719 /* PRO is already in the worklist, but it now has
3720 a higher priority than before. Move it at the
3721 appropriate place. */
3722 model_remove_from_worklist (pro
);
3723 model_add_to_worklist (pro
, NULL
, model_worklist
);
3727 /* PRO isn't in the worklist. Recursively process
3728 its predecessors until we find one that is. */
3739 if (sched_verbose
>= 7)
3740 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3741 model_next_priority
++;
3744 /* Pick one instruction from model_worklist and process it. */
3747 model_choose_insn (void)
3749 struct model_insn_info
*insn
, *fallback
;
3752 if (sched_verbose
>= 7)
3754 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3755 insn
= model_worklist
;
3756 count
= MAX_SCHED_READY_INSNS
;
3757 while (count
> 0 && insn
)
3759 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3760 INSN_UID (insn
->insn
), insn
->model_priority
,
3761 insn
->depth
+ insn
->alap
, insn
->depth
,
3762 INSN_PRIORITY (insn
->insn
));
3768 /* Look for a ready instruction whose model_classify_priority is zero
3769 or negative, picking the highest-priority one. Adding such an
3770 instruction to the schedule now should do no harm, and may actually
3773 Failing that, see whether there is an instruction with the highest
3774 extant model_priority that is not yet ready, but which would reduce
3775 pressure if it became ready. This is designed to catch cases like:
3777 (set (mem (reg R1)) (reg R2))
3779 where the instruction is the last remaining use of R1 and where the
3780 value of R2 is not yet available (or vice versa). The death of R1
3781 means that this instruction already reduces pressure. It is of
3782 course possible that the computation of R2 involves other registers
3783 that are hard to kill, but such cases are rare enough for this
3784 heuristic to be a win in general.
3786 Failing that, just pick the highest-priority instruction in the
3788 count
= MAX_SCHED_READY_INSNS
;
3789 insn
= model_worklist
;
3793 if (count
== 0 || !insn
)
3795 insn
= fallback
? fallback
: model_worklist
;
3798 if (insn
->unscheduled_preds
)
3800 if (model_worklist
->model_priority
== insn
->model_priority
3802 && model_classify_pressure (insn
) < 0)
3807 if (model_classify_pressure (insn
) <= 0)
3814 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3816 if (insn
->unscheduled_preds
)
3817 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3818 INSN_UID (insn
->insn
));
3820 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3821 INSN_UID (insn
->insn
));
3823 if (insn
->unscheduled_preds
)
3824 /* INSN isn't yet ready to issue. Give all its predecessors the
3825 highest priority. */
3826 model_promote_predecessors (insn
);
3829 /* INSN is ready. Add it to the end of model_schedule and
3830 process its successors. */
3831 model_add_successors_to_worklist (insn
);
3832 model_remove_from_worklist (insn
);
3833 model_add_to_schedule (insn
->insn
);
3834 model_record_pressures (insn
);
3835 update_register_pressure (insn
->insn
);
3839 /* Restore all QUEUE_INDEXs to the values that they had before
3840 model_start_schedule was called. */
3843 model_reset_queue_indices (void)
3848 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3849 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3852 /* We have calculated the model schedule and spill costs. Print a summary
3856 model_dump_pressure_summary (void)
3860 fprintf (sched_dump
, ";; Pressure summary:");
3861 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3863 cl
= ira_pressure_classes
[pci
];
3864 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3865 model_before_pressure
.limits
[pci
].pressure
);
3867 fprintf (sched_dump
, "\n\n");
3870 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3871 scheduling region. */
3874 model_start_schedule (basic_block bb
)
3876 model_next_priority
= 1;
3877 model_schedule
.create (sched_max_luid
);
3878 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3880 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3881 initiate_reg_pressure_info (df_get_live_in (bb
));
3883 model_analyze_insns ();
3884 model_init_pressure_group (&model_before_pressure
);
3885 while (model_worklist
)
3886 model_choose_insn ();
3887 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3888 if (sched_verbose
>= 2)
3889 fprintf (sched_dump
, "\n");
3891 model_record_final_pressures (&model_before_pressure
);
3892 model_reset_queue_indices ();
3894 XDELETEVEC (model_insns
);
3896 model_curr_point
= 0;
3897 initiate_reg_pressure_info (df_get_live_in (bb
));
3898 if (sched_verbose
>= 1)
3899 model_dump_pressure_summary ();
3902 /* Free the information associated with GROUP. */
3905 model_finalize_pressure_group (struct model_pressure_group
*group
)
3907 XDELETEVEC (group
->model
);
3910 /* Free the information created by model_start_schedule. */
3913 model_end_schedule (void)
3915 model_finalize_pressure_group (&model_before_pressure
);
3916 model_schedule
.release ();
3919 /* Prepare reg pressure scheduling for basic block BB. */
3921 sched_pressure_start_bb (basic_block bb
)
3923 /* Set the number of available registers for each class taking into account
3924 relative probability of current basic block versus function prologue and
3926 * If the basic block executes much more often than the prologue/epilogue
3927 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3928 nil, so the effective number of available registers is
3929 (ira_class_hard_regs_num[cl] - 0).
3930 * If the basic block executes as often as the prologue/epilogue,
3931 then spill in the block is as costly as in the prologue, so the effective
3932 number of available registers is
3933 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3934 Note that all-else-equal, we prefer to spill in the prologue, since that
3935 allows "extra" registers for other basic blocks of the function.
3936 * If the basic block is on the cold path of the function and executes
3937 rarely, then we should always prefer to spill in the block, rather than
3938 in the prologue/epilogue. The effective number of available register is
3939 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3942 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
3943 int bb_freq
= bb
->frequency
;
3947 if (entry_freq
== 0)
3948 entry_freq
= bb_freq
= 1;
3950 if (bb_freq
< entry_freq
)
3951 bb_freq
= entry_freq
;
3953 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3955 enum reg_class cl
= ira_pressure_classes
[i
];
3956 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
];
3957 sched_class_regs_num
[cl
]
3958 -= (call_used_regs_num
[cl
] * entry_freq
) / bb_freq
;
3962 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3963 model_start_schedule (bb
);
3966 /* A structure that holds local state for the loop in schedule_block. */
3967 struct sched_block_state
3969 /* True if no real insns have been scheduled in the current cycle. */
3970 bool first_cycle_insn_p
;
3971 /* True if a shadow insn has been scheduled in the current cycle, which
3972 means that no more normal insns can be issued. */
3973 bool shadows_only_p
;
3974 /* True if we're winding down a modulo schedule, which means that we only
3975 issue insns with INSN_EXACT_TICK set. */
3976 bool modulo_epilogue
;
3977 /* Initialized with the machine's issue rate every cycle, and updated
3978 by calls to the variable_issue hook. */
3982 /* INSN is the "currently executing insn". Launch each insn which was
3983 waiting on INSN. READY is the ready list which contains the insns
3984 that are ready to fire. CLOCK is the current cycle. The function
3985 returns necessary cycle advance after issuing the insn (it is not
3986 zero for insns in a schedule group). */
3989 schedule_insn (rtx_insn
*insn
)
3991 sd_iterator_def sd_it
;
3996 if (sched_verbose
>= 1)
3998 struct reg_pressure_data
*pressure_info
;
3999 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
4000 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
4001 str_pattern_slim (PATTERN (insn
)));
4003 if (recog_memoized (insn
) < 0)
4004 fprintf (sched_dump
, "nothing");
4006 print_reservation (sched_dump
, insn
);
4007 pressure_info
= INSN_REG_PRESSURE (insn
);
4008 if (pressure_info
!= NULL
)
4010 fputc (':', sched_dump
);
4011 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4012 fprintf (sched_dump
, "%s%s%+d(%d)",
4013 scheduled_insns
.length () > 1
4015 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4016 reg_class_names
[ira_pressure_classes
[i
]],
4017 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4019 if (sched_pressure
== SCHED_PRESSURE_MODEL
4020 && model_curr_point
< model_num_insns
4021 && model_index (insn
) == model_curr_point
)
4022 fprintf (sched_dump
, ":model %d", model_curr_point
);
4023 fputc ('\n', sched_dump
);
4026 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4027 update_reg_and_insn_max_reg_pressure (insn
);
4029 /* Scheduling instruction should have all its dependencies resolved and
4030 should have been removed from the ready list. */
4031 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4033 /* Reset debug insns invalidated by moving this insn. */
4034 if (MAY_HAVE_DEBUG_INSNS
&& !DEBUG_INSN_P (insn
))
4035 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4036 sd_iterator_cond (&sd_it
, &dep
);)
4038 rtx_insn
*dbg
= DEP_PRO (dep
);
4039 struct reg_use_data
*use
, *next
;
4041 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4043 sd_iterator_next (&sd_it
);
4047 gcc_assert (DEBUG_INSN_P (dbg
));
4049 if (sched_verbose
>= 6)
4050 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4053 /* ??? Rather than resetting the debug insn, we might be able
4054 to emit a debug temp before the just-scheduled insn, but
4055 this would involve checking that the expression at the
4056 point of the debug insn is equivalent to the expression
4057 before the just-scheduled insn. They might not be: the
4058 expression in the debug insn may depend on other insns not
4059 yet scheduled that set MEMs, REGs or even other debug
4060 insns. It's not clear that attempting to preserve debug
4061 information in these cases is worth the effort, given how
4062 uncommon these resets are and the likelihood that the debug
4063 temps introduced won't survive the schedule change. */
4064 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4065 df_insn_rescan (dbg
);
4067 /* Unknown location doesn't use any registers. */
4068 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4070 struct reg_use_data
*prev
= use
;
4072 /* Remove use from the cyclic next_regno_use chain first. */
4073 while (prev
->next_regno_use
!= use
)
4074 prev
= prev
->next_regno_use
;
4075 prev
->next_regno_use
= use
->next_regno_use
;
4076 next
= use
->next_insn_use
;
4079 INSN_REG_USE_LIST (dbg
) = NULL
;
4081 /* We delete rather than resolve these deps, otherwise we
4082 crash in sched_free_deps(), because forward deps are
4083 expected to be released before backward deps. */
4084 sd_delete_dep (sd_it
);
4087 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4088 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4090 if (sched_pressure
== SCHED_PRESSURE_MODEL
4091 && model_curr_point
< model_num_insns
4092 && NONDEBUG_INSN_P (insn
))
4094 if (model_index (insn
) == model_curr_point
)
4097 while (model_curr_point
< model_num_insns
4098 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4099 == QUEUE_SCHEDULED
));
4101 model_recompute (insn
);
4102 model_update_limit_points ();
4103 update_register_pressure (insn
);
4104 if (sched_verbose
>= 2)
4105 print_curr_reg_pressure ();
4108 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4109 if (INSN_TICK (insn
) > clock_var
)
4110 /* INSN has been prematurely moved from the queue to the ready list.
4111 This is possible only if following flags are set. */
4112 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4114 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4115 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4116 INSN_TICK (insn
) = clock_var
;
4118 check_clobbered_conditions (insn
);
4120 /* Update dependent instructions. First, see if by scheduling this insn
4121 now we broke a dependence in a way that requires us to change another
4123 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4124 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4126 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4127 rtx_insn
*pro
= DEP_PRO (dep
);
4128 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4129 && desc
!= NULL
&& desc
->insn
== pro
)
4130 apply_replacement (dep
, false);
4133 /* Go through and resolve forward dependencies. */
4134 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4135 sd_iterator_cond (&sd_it
, &dep
);)
4137 rtx_insn
*next
= DEP_CON (dep
);
4138 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4140 /* Resolve the dependence between INSN and NEXT.
4141 sd_resolve_dep () moves current dep to another list thus
4142 advancing the iterator. */
4143 sd_resolve_dep (sd_it
);
4147 if (must_restore_pattern_p (next
, dep
))
4148 restore_pattern (dep
, false);
4152 /* Don't bother trying to mark next as ready if insn is a debug
4153 insn. If insn is the last hard dependency, it will have
4154 already been discounted. */
4155 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4158 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4162 effective_cost
= try_ready (next
);
4164 if (effective_cost
>= 0
4165 && SCHED_GROUP_P (next
)
4166 && advance
< effective_cost
)
4167 advance
= effective_cost
;
4170 /* Check always has only one forward dependence (to the first insn in
4171 the recovery block), therefore, this will be executed only once. */
4173 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4174 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4178 /* Annotate the instruction with issue information -- TImode
4179 indicates that the instruction is expected not to be able
4180 to issue on the same cycle as the previous insn. A machine
4181 may use this information to decide how the instruction should
4184 && GET_CODE (PATTERN (insn
)) != USE
4185 && GET_CODE (PATTERN (insn
)) != CLOBBER
4186 && !DEBUG_INSN_P (insn
))
4188 if (reload_completed
)
4189 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4190 last_clock_var
= clock_var
;
4193 if (nonscheduled_insns_begin
!= NULL_RTX
)
4194 /* Indicate to debug counters that INSN is scheduled. */
4195 nonscheduled_insns_begin
= insn
;
4200 /* Functions for handling of notes. */
4202 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4204 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4206 rtx_insn
*from_start
;
4208 /* It's easy when have nothing to concat. */
4209 if (from_end
== NULL
)
4212 /* It's also easy when destination is empty. */
4213 if (*to_endp
== NULL
)
4215 *to_endp
= from_end
;
4219 from_start
= from_end
;
4220 while (PREV_INSN (from_start
) != NULL
)
4221 from_start
= PREV_INSN (from_start
);
4223 SET_PREV_INSN (from_start
) = *to_endp
;
4224 SET_NEXT_INSN (*to_endp
) = from_start
;
4225 *to_endp
= from_end
;
4228 /* Delete notes between HEAD and TAIL and put them in the chain
4229 of notes ended by NOTE_LIST. */
4231 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4233 rtx_insn
*next_tail
, *insn
, *next
;
4236 if (head
== tail
&& !INSN_P (head
))
4239 next_tail
= NEXT_INSN (tail
);
4240 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4242 next
= NEXT_INSN (insn
);
4246 switch (NOTE_KIND (insn
))
4248 case NOTE_INSN_BASIC_BLOCK
:
4251 case NOTE_INSN_EPILOGUE_BEG
:
4255 add_reg_note (next
, REG_SAVE_NOTE
,
4256 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4264 /* Add the note to list that ends at NOTE_LIST. */
4265 SET_PREV_INSN (insn
) = note_list
;
4266 SET_NEXT_INSN (insn
) = NULL_RTX
;
4268 SET_NEXT_INSN (note_list
) = insn
;
4273 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4277 /* A structure to record enough data to allow us to backtrack the scheduler to
4278 a previous state. */
4279 struct haifa_saved_data
4281 /* Next entry on the list. */
4282 struct haifa_saved_data
*next
;
4284 /* Backtracking is associated with scheduling insns that have delay slots.
4285 DELAY_PAIR points to the structure that contains the insns involved, and
4286 the number of cycles between them. */
4287 struct delay_pair
*delay_pair
;
4289 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4290 void *fe_saved_data
;
4291 /* Data used by the backend. */
4292 void *be_saved_data
;
4294 /* Copies of global state. */
4295 int clock_var
, last_clock_var
;
4296 struct ready_list ready
;
4299 rtx_insn
*last_scheduled_insn
;
4300 rtx_insn
*last_nondebug_scheduled_insn
;
4301 rtx_insn
*nonscheduled_insns_begin
;
4302 int cycle_issued_insns
;
4304 /* Copies of state used in the inner loop of schedule_block. */
4305 struct sched_block_state sched_block
;
4307 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4308 to 0 when restoring. */
4310 rtx_insn_list
**insn_queue
;
4312 /* Describe pattern replacements that occurred since this backtrack point
4314 vec
<dep_t
> replacement_deps
;
4315 vec
<int> replace_apply
;
4317 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4319 vec
<dep_t
> next_cycle_deps
;
4320 vec
<int> next_cycle_apply
;
4323 /* A record, in reverse order, of all scheduled insns which have delay slots
4324 and may require backtracking. */
4325 static struct haifa_saved_data
*backtrack_queue
;
4327 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4330 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4332 sd_iterator_def sd_it
;
4334 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4336 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4340 /* Save the current scheduler state so that we can backtrack to it
4341 later if necessary. PAIR gives the insns that make it necessary to
4342 save this point. SCHED_BLOCK is the local state of schedule_block
4343 that need to be saved. */
4345 save_backtrack_point (struct delay_pair
*pair
,
4346 struct sched_block_state sched_block
)
4349 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4351 save
->curr_state
= xmalloc (dfa_state_size
);
4352 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4354 save
->ready
.first
= ready
.first
;
4355 save
->ready
.n_ready
= ready
.n_ready
;
4356 save
->ready
.n_debug
= ready
.n_debug
;
4357 save
->ready
.veclen
= ready
.veclen
;
4358 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4359 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4361 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4362 save
->q_size
= q_size
;
4363 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4365 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4366 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4369 save
->clock_var
= clock_var
;
4370 save
->last_clock_var
= last_clock_var
;
4371 save
->cycle_issued_insns
= cycle_issued_insns
;
4372 save
->last_scheduled_insn
= last_scheduled_insn
;
4373 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4374 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4376 save
->sched_block
= sched_block
;
4378 save
->replacement_deps
.create (0);
4379 save
->replace_apply
.create (0);
4380 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4381 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4383 if (current_sched_info
->save_state
)
4384 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4386 if (targetm
.sched
.alloc_sched_context
)
4388 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4389 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4392 save
->be_saved_data
= NULL
;
4394 save
->delay_pair
= pair
;
4396 save
->next
= backtrack_queue
;
4397 backtrack_queue
= save
;
4401 mark_backtrack_feeds (pair
->i2
, 1);
4402 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4403 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4404 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4405 pair
= pair
->next_same_i1
;
4409 /* Walk the ready list and all queues. If any insns have unresolved backwards
4410 dependencies, these must be cancelled deps, broken by predication. Set or
4411 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4414 toggle_cancelled_flags (bool set
)
4417 sd_iterator_def sd_it
;
4420 if (ready
.n_ready
> 0)
4422 rtx_insn
**first
= ready_lastpos (&ready
);
4423 for (i
= 0; i
< ready
.n_ready
; i
++)
4424 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4425 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4428 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4430 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4433 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4435 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4436 rtx_insn_list
*link
;
4437 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4439 rtx_insn
*insn
= link
->insn ();
4440 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4441 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4444 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4446 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4452 /* Undo the replacements that have occurred after backtrack point SAVE
4455 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4457 while (!save
->replacement_deps
.is_empty ())
4459 dep_t dep
= save
->replacement_deps
.pop ();
4460 int apply_p
= save
->replace_apply
.pop ();
4463 restore_pattern (dep
, true);
4465 apply_replacement (dep
, true);
4467 save
->replacement_deps
.release ();
4468 save
->replace_apply
.release ();
4471 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4472 Restore their dependencies to an unresolved state, and mark them as
4476 unschedule_insns_until (rtx_insn
*insn
)
4478 auto_vec
<rtx_insn
*> recompute_vec
;
4480 /* Make two passes over the insns to be unscheduled. First, we clear out
4481 dependencies and other trivial bookkeeping. */
4485 sd_iterator_def sd_it
;
4488 last
= scheduled_insns
.pop ();
4490 /* This will be changed by restore_backtrack_point if the insn is in
4492 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4494 INSN_TICK (last
) = INVALID_TICK
;
4496 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4497 modulo_insns_scheduled
--;
4499 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4500 sd_iterator_cond (&sd_it
, &dep
);)
4502 rtx_insn
*con
= DEP_CON (dep
);
4503 sd_unresolve_dep (sd_it
);
4504 if (!MUST_RECOMPUTE_SPEC_P (con
))
4506 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4507 recompute_vec
.safe_push (con
);
4515 /* A second pass, to update ready and speculation status for insns
4516 depending on the unscheduled ones. The first pass must have
4517 popped the scheduled_insns vector up to the point where we
4518 restart scheduling, as recompute_todo_spec requires it to be
4520 while (!recompute_vec
.is_empty ())
4524 con
= recompute_vec
.pop ();
4525 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4526 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4528 TODO_SPEC (con
) = HARD_DEP
;
4529 INSN_TICK (con
) = INVALID_TICK
;
4530 if (PREDICATED_PAT (con
) != NULL_RTX
)
4531 haifa_change_pattern (con
, ORIG_PAT (con
));
4533 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4534 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4538 /* Restore scheduler state from the topmost entry on the backtracking queue.
4539 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4540 overwrite with the saved data.
4541 The caller must already have called unschedule_insns_until. */
4544 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4547 struct haifa_saved_data
*save
= backtrack_queue
;
4549 backtrack_queue
= save
->next
;
4551 if (current_sched_info
->restore_state
)
4552 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4554 if (targetm
.sched
.alloc_sched_context
)
4556 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4557 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4560 /* Do this first since it clobbers INSN_TICK of the involved
4562 undo_replacements_for_backtrack (save
);
4564 /* Clear the QUEUE_INDEX of everything in the ready list or one
4566 if (ready
.n_ready
> 0)
4568 rtx_insn
**first
= ready_lastpos (&ready
);
4569 for (i
= 0; i
< ready
.n_ready
; i
++)
4571 rtx_insn
*insn
= first
[i
];
4572 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4573 INSN_TICK (insn
) = INVALID_TICK
;
4576 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4578 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4580 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4582 rtx_insn
*x
= link
->insn ();
4583 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4584 INSN_TICK (x
) = INVALID_TICK
;
4586 free_INSN_LIST_list (&insn_queue
[q
]);
4590 ready
= save
->ready
;
4592 if (ready
.n_ready
> 0)
4594 rtx_insn
**first
= ready_lastpos (&ready
);
4595 for (i
= 0; i
< ready
.n_ready
; i
++)
4597 rtx_insn
*insn
= first
[i
];
4598 QUEUE_INDEX (insn
) = QUEUE_READY
;
4599 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4600 INSN_TICK (insn
) = save
->clock_var
;
4605 q_size
= save
->q_size
;
4606 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4608 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4610 insn_queue
[q
] = save
->insn_queue
[q
];
4612 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4614 rtx_insn
*x
= link
->insn ();
4615 QUEUE_INDEX (x
) = i
;
4616 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4617 INSN_TICK (x
) = save
->clock_var
+ i
;
4620 free (save
->insn_queue
);
4622 toggle_cancelled_flags (true);
4624 clock_var
= save
->clock_var
;
4625 last_clock_var
= save
->last_clock_var
;
4626 cycle_issued_insns
= save
->cycle_issued_insns
;
4627 last_scheduled_insn
= save
->last_scheduled_insn
;
4628 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4629 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4631 *psched_block
= save
->sched_block
;
4633 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4634 free (save
->curr_state
);
4636 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4638 gcc_assert (next_cycle_replace_deps
.is_empty ());
4639 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4640 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4644 for (save
= backtrack_queue
; save
; save
= save
->next
)
4646 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4650 /* Discard all data associated with the topmost entry in the backtrack
4651 queue. If RESET_TICK is false, we just want to free the data. If true,
4652 we are doing this because we discovered a reason to backtrack. In the
4653 latter case, also reset the INSN_TICK for the shadow insn. */
4655 free_topmost_backtrack_point (bool reset_tick
)
4657 struct haifa_saved_data
*save
= backtrack_queue
;
4660 backtrack_queue
= save
->next
;
4664 struct delay_pair
*pair
= save
->delay_pair
;
4667 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4668 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4669 pair
= pair
->next_same_i1
;
4671 undo_replacements_for_backtrack (save
);
4675 save
->replacement_deps
.release ();
4676 save
->replace_apply
.release ();
4679 if (targetm
.sched
.free_sched_context
)
4680 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4681 if (current_sched_info
->restore_state
)
4682 free (save
->fe_saved_data
);
4683 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4684 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4685 free (save
->insn_queue
);
4686 free (save
->curr_state
);
4687 free (save
->ready
.vec
);
4691 /* Free the entire backtrack queue. */
4693 free_backtrack_queue (void)
4695 while (backtrack_queue
)
4696 free_topmost_backtrack_point (false);
4699 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4700 may have to postpone the replacement until the start of the next cycle,
4701 at which point we will be called again with IMMEDIATELY true. This is
4702 only done for machines which have instruction packets with explicit
4703 parallelism however. */
4705 apply_replacement (dep_t dep
, bool immediately
)
4707 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4708 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4710 next_cycle_replace_deps
.safe_push (dep
);
4711 next_cycle_apply
.safe_push (1);
4717 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4720 if (sched_verbose
>= 5)
4721 fprintf (sched_dump
, "applying replacement for insn %d\n",
4722 INSN_UID (desc
->insn
));
4724 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4725 gcc_assert (success
);
4727 update_insn_after_change (desc
->insn
);
4728 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4729 fix_tick_ready (desc
->insn
);
4731 if (backtrack_queue
!= NULL
)
4733 backtrack_queue
->replacement_deps
.safe_push (dep
);
4734 backtrack_queue
->replace_apply
.safe_push (1);
4739 /* We have determined that a pattern involved in DEP must be restored.
4740 If IMMEDIATELY is false, we may have to postpone the replacement
4741 until the start of the next cycle, at which point we will be called
4742 again with IMMEDIATELY true. */
4744 restore_pattern (dep_t dep
, bool immediately
)
4746 rtx_insn
*next
= DEP_CON (dep
);
4747 int tick
= INSN_TICK (next
);
4749 /* If we already scheduled the insn, the modified version is
4751 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4754 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4756 next_cycle_replace_deps
.safe_push (dep
);
4757 next_cycle_apply
.safe_push (0);
4762 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4764 if (sched_verbose
>= 5)
4765 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4767 haifa_change_pattern (next
, ORIG_PAT (next
));
4771 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4774 if (sched_verbose
>= 5)
4775 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4776 INSN_UID (desc
->insn
));
4777 tick
= INSN_TICK (desc
->insn
);
4779 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4780 gcc_assert (success
);
4781 update_insn_after_change (desc
->insn
);
4782 if (backtrack_queue
!= NULL
)
4784 backtrack_queue
->replacement_deps
.safe_push (dep
);
4785 backtrack_queue
->replace_apply
.safe_push (0);
4788 INSN_TICK (next
) = tick
;
4789 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4792 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4793 TODO_SPEC (next
) = 0;
4794 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4795 TODO_SPEC (next
) = HARD_DEP
;
4798 /* Perform pattern replacements that were queued up until the next
4801 perform_replacements_new_cycle (void)
4805 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4807 int apply_p
= next_cycle_apply
[i
];
4809 apply_replacement (dep
, true);
4811 restore_pattern (dep
, true);
4813 next_cycle_replace_deps
.truncate (0);
4814 next_cycle_apply
.truncate (0);
4817 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4818 instructions we've previously encountered, a set bit prevents
4819 recursion. BUDGET is a limit on how far ahead we look, it is
4820 reduced on recursive calls. Return true if we produced a good
4821 estimate, or false if we exceeded the budget. */
4823 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4825 sd_iterator_def sd_it
;
4827 int earliest
= INSN_TICK (insn
);
4829 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4831 rtx_insn
*pro
= DEP_PRO (dep
);
4834 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4837 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4838 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4841 int cost
= dep_cost (dep
);
4844 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4846 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4849 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4850 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4851 if (earliest
== INVALID_TICK
|| t
> earliest
)
4855 bitmap_set_bit (processed
, INSN_LUID (insn
));
4856 INSN_TICK_ESTIMATE (insn
) = earliest
;
4860 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4861 infinite resources) the cycle in which the delayed shadow can be issued.
4862 Return the number of cycles that must pass before the real insn can be
4863 issued in order to meet this constraint. */
4865 estimate_shadow_tick (struct delay_pair
*p
)
4867 bitmap_head processed
;
4870 bitmap_initialize (&processed
, 0);
4872 cutoff
= !estimate_insn_tick (&processed
, p
->i2
,
4873 max_insn_queue_index
+ pair_delay (p
));
4874 bitmap_clear (&processed
);
4876 return max_insn_queue_index
;
4877 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4883 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4884 recursively resolve all its forward dependencies. */
4886 resolve_dependencies (rtx_insn
*insn
)
4888 sd_iterator_def sd_it
;
4891 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4892 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4893 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4896 if (sched_verbose
>= 4)
4897 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4899 if (QUEUE_INDEX (insn
) >= 0)
4900 queue_remove (insn
);
4902 scheduled_insns
.safe_push (insn
);
4904 /* Update dependent instructions. */
4905 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4906 sd_iterator_cond (&sd_it
, &dep
);)
4908 rtx_insn
*next
= DEP_CON (dep
);
4910 if (sched_verbose
>= 4)
4911 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4914 /* Resolve the dependence between INSN and NEXT.
4915 sd_resolve_dep () moves current dep to another list thus
4916 advancing the iterator. */
4917 sd_resolve_dep (sd_it
);
4919 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4921 resolve_dependencies (next
);
4924 /* Check always has only one forward dependence (to the first insn in
4925 the recovery block), therefore, this will be executed only once. */
4927 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4933 /* Return the head and tail pointers of ebb starting at BEG and ending
4936 get_ebb_head_tail (basic_block beg
, basic_block end
,
4937 rtx_insn
**headp
, rtx_insn
**tailp
)
4939 rtx_insn
*beg_head
= BB_HEAD (beg
);
4940 rtx_insn
* beg_tail
= BB_END (beg
);
4941 rtx_insn
* end_head
= BB_HEAD (end
);
4942 rtx_insn
* end_tail
= BB_END (end
);
4944 /* Don't include any notes or labels at the beginning of the BEG
4945 basic block, or notes at the end of the END basic blocks. */
4947 if (LABEL_P (beg_head
))
4948 beg_head
= NEXT_INSN (beg_head
);
4950 while (beg_head
!= beg_tail
)
4951 if (NOTE_P (beg_head
))
4952 beg_head
= NEXT_INSN (beg_head
);
4953 else if (DEBUG_INSN_P (beg_head
))
4955 rtx_insn
* note
, *next
;
4957 for (note
= NEXT_INSN (beg_head
);
4961 next
= NEXT_INSN (note
);
4964 if (sched_verbose
>= 9)
4965 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4967 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4969 if (BLOCK_FOR_INSN (note
) != beg
)
4970 df_insn_change_bb (note
, beg
);
4972 else if (!DEBUG_INSN_P (note
))
4984 end_head
= beg_head
;
4985 else if (LABEL_P (end_head
))
4986 end_head
= NEXT_INSN (end_head
);
4988 while (end_head
!= end_tail
)
4989 if (NOTE_P (end_tail
))
4990 end_tail
= PREV_INSN (end_tail
);
4991 else if (DEBUG_INSN_P (end_tail
))
4993 rtx_insn
* note
, *prev
;
4995 for (note
= PREV_INSN (end_tail
);
4999 prev
= PREV_INSN (note
);
5002 if (sched_verbose
>= 9)
5003 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5005 reorder_insns_nobb (note
, note
, end_tail
);
5007 if (end_tail
== BB_END (end
))
5008 BB_END (end
) = note
;
5010 if (BLOCK_FOR_INSN (note
) != end
)
5011 df_insn_change_bb (note
, end
);
5013 else if (!DEBUG_INSN_P (note
))
5025 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5028 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5030 while (head
!= NEXT_INSN (tail
))
5032 if (!NOTE_P (head
) && !LABEL_P (head
))
5034 head
= NEXT_INSN (head
);
5039 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5040 previously found among the insns. Insert them just before HEAD. */
5042 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5046 rtx_insn
*note_head
= note_list
;
5049 head_bb
= BLOCK_FOR_INSN (head
);
5051 head
= NEXT_INSN (bb_note (head_bb
));
5053 while (PREV_INSN (note_head
))
5055 set_block_for_insn (note_head
, head_bb
);
5056 note_head
= PREV_INSN (note_head
);
5058 /* In the above cycle we've missed this note. */
5059 set_block_for_insn (note_head
, head_bb
);
5061 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5062 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5063 SET_PREV_INSN (head
) = note_list
;
5064 SET_NEXT_INSN (note_list
) = head
;
5066 if (BLOCK_FOR_INSN (head
) != head_bb
)
5067 BB_END (head_bb
) = note_list
;
5075 /* When we know we are going to discard the schedule due to a failed attempt
5076 at modulo scheduling, undo all replacements. */
5078 undo_all_replacements (void)
5083 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5085 sd_iterator_def sd_it
;
5088 /* See if we must undo a replacement. */
5089 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5090 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5092 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5094 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5099 /* Return first non-scheduled insn in the current scheduling block.
5100 This is mostly used for debug-counter purposes. */
5102 first_nonscheduled_insn (void)
5104 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5105 ? nonscheduled_insns_begin
5106 : current_sched_info
->prev_head
);
5110 insn
= next_nonnote_nondebug_insn (insn
);
5112 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5117 /* Move insns that became ready to fire from queue to ready list. */
5120 queue_to_ready (struct ready_list
*ready
)
5123 rtx_insn_list
*link
;
5124 rtx_insn
*skip_insn
;
5126 q_ptr
= NEXT_Q (q_ptr
);
5128 if (dbg_cnt (sched_insn
) == false)
5129 /* If debug counter is activated do not requeue the first
5130 nonscheduled insn. */
5131 skip_insn
= first_nonscheduled_insn ();
5135 /* Add all pending insns that can be scheduled without stalls to the
5137 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5139 insn
= link
->insn ();
5142 if (sched_verbose
>= 2)
5143 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5144 (*current_sched_info
->print_insn
) (insn
, 0));
5146 /* If the ready list is full, delay the insn for 1 cycle.
5147 See the comment in schedule_block for the rationale. */
5148 if (!reload_completed
5149 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5150 || (sched_pressure
== SCHED_PRESSURE_MODEL
5151 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5152 instructions too. */
5153 && model_index (insn
) > (model_curr_point
5154 + MAX_SCHED_READY_INSNS
)))
5155 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5156 && model_curr_point
< model_num_insns
5157 /* Always allow the next model instruction to issue. */
5158 && model_index (insn
) == model_curr_point
)
5159 && !SCHED_GROUP_P (insn
)
5160 && insn
!= skip_insn
)
5162 if (sched_verbose
>= 2)
5163 fprintf (sched_dump
, "keeping in queue, ready full\n");
5164 queue_insn (insn
, 1, "ready full");
5168 ready_add (ready
, insn
, false);
5169 if (sched_verbose
>= 2)
5170 fprintf (sched_dump
, "moving to ready without stalls\n");
5173 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5175 /* If there are no ready insns, stall until one is ready and add all
5176 of the pending insns at that point to the ready list. */
5177 if (ready
->n_ready
== 0)
5181 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5183 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5185 for (; link
; link
= link
->next ())
5187 insn
= link
->insn ();
5190 if (sched_verbose
>= 2)
5191 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5192 (*current_sched_info
->print_insn
) (insn
, 0));
5194 ready_add (ready
, insn
, false);
5195 if (sched_verbose
>= 2)
5196 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5198 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5200 advance_one_cycle ();
5205 advance_one_cycle ();
5208 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5209 clock_var
+= stalls
;
5210 if (sched_verbose
>= 2)
5211 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5216 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5217 prematurely move INSN from the queue to the ready list. Currently,
5218 if a target defines the hook 'is_costly_dependence', this function
5219 uses the hook to check whether there exist any dependences which are
5220 considered costly by the target, between INSN and other insns that
5221 have already been scheduled. Dependences are checked up to Y cycles
5222 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5223 controlling this value.
5224 (Other considerations could be taken into account instead (or in
5225 addition) depending on user flags and target hooks. */
5228 ok_for_early_queue_removal (rtx_insn
*insn
)
5230 if (targetm
.sched
.is_costly_dependence
)
5233 int i
= scheduled_insns
.length ();
5234 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5240 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5242 if (!NOTE_P (prev_insn
))
5246 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5250 cost
= dep_cost (dep
);
5252 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5253 flag_sched_stalled_insns_dep
- n_cycles
))
5258 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5271 /* Remove insns from the queue, before they become "ready" with respect
5272 to FU latency considerations. */
5275 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5278 rtx_insn_list
*link
;
5279 rtx_insn_list
*next_link
;
5280 rtx_insn_list
*prev_link
;
5283 state_t temp_state
= alloca (dfa_state_size
);
5285 int insns_removed
= 0;
5288 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5291 X == 0: There is no limit on how many queued insns can be removed
5292 prematurely. (flag_sched_stalled_insns = -1).
5294 X >= 1: Only X queued insns can be removed prematurely in each
5295 invocation. (flag_sched_stalled_insns = X).
5297 Otherwise: Early queue removal is disabled.
5298 (flag_sched_stalled_insns = 0)
5301 if (! flag_sched_stalled_insns
)
5304 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5306 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5308 if (sched_verbose
> 6)
5309 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5314 next_link
= link
->next ();
5315 insn
= link
->insn ();
5316 if (insn
&& sched_verbose
> 6)
5317 print_rtl_single (sched_dump
, insn
);
5319 memcpy (temp_state
, state
, dfa_state_size
);
5320 if (recog_memoized (insn
) < 0)
5321 /* non-negative to indicate that it's not ready
5322 to avoid infinite Q->R->Q->R... */
5325 cost
= state_transition (temp_state
, insn
);
5327 if (sched_verbose
>= 6)
5328 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5330 move_to_ready
= false;
5333 move_to_ready
= ok_for_early_queue_removal (insn
);
5334 if (move_to_ready
== true)
5336 /* move from Q to R */
5338 ready_add (ready
, insn
, false);
5341 XEXP (prev_link
, 1) = next_link
;
5343 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5345 free_INSN_LIST_node (link
);
5347 if (sched_verbose
>= 2)
5348 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5349 (*current_sched_info
->print_insn
) (insn
, 0));
5352 if (insns_removed
== flag_sched_stalled_insns
)
5353 /* Remove no more than flag_sched_stalled_insns insns
5354 from Q at a time. */
5355 return insns_removed
;
5359 if (move_to_ready
== false)
5366 } /* for stalls.. */
5368 return insns_removed
;
5372 /* Print the ready list for debugging purposes.
5373 If READY_TRY is non-zero then only print insns that max_issue
5376 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5381 if (ready
->n_ready
== 0)
5383 fprintf (sched_dump
, "\n");
5387 p
= ready_lastpos (ready
);
5388 for (i
= 0; i
< ready
->n_ready
; i
++)
5390 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5393 fprintf (sched_dump
, " %s:%d",
5394 (*current_sched_info
->print_insn
) (p
[i
], 0),
5396 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5397 fprintf (sched_dump
, "(cost=%d",
5398 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5399 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5400 if (INSN_TICK (p
[i
]) > clock_var
)
5401 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5402 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5403 fprintf (sched_dump
, ":idx=%d",
5404 model_index (p
[i
]));
5405 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5406 fprintf (sched_dump
, ")");
5408 fprintf (sched_dump
, "\n");
5411 /* Print the ready list. Callable from debugger. */
5413 debug_ready_list (struct ready_list
*ready
)
5415 debug_ready_list_1 (ready
, NULL
);
5418 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5419 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5420 replaces the epilogue note in the correct basic block. */
5422 reemit_notes (rtx_insn
*insn
)
5425 rtx_insn
*last
= insn
;
5427 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5429 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5431 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5433 last
= emit_note_before (note_type
, last
);
5434 remove_note (insn
, note
);
5439 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5441 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5443 if (PREV_INSN (insn
) != last
)
5449 bb
= BLOCK_FOR_INSN (insn
);
5451 /* BB_HEAD is either LABEL or NOTE. */
5452 gcc_assert (BB_HEAD (bb
) != insn
);
5454 if (BB_END (bb
) == insn
)
5455 /* If this is last instruction in BB, move end marker one
5458 /* Jumps are always placed at the end of basic block. */
5459 jump_p
= control_flow_insn_p (insn
);
5462 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5463 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5464 || (common_sched_info
->sched_pass_id
5465 == SCHED_EBB_PASS
));
5467 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5469 BB_END (bb
) = PREV_INSN (insn
);
5472 gcc_assert (BB_END (bb
) != last
);
5475 /* We move the block note along with jump. */
5479 note
= NEXT_INSN (insn
);
5480 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5481 note
= NEXT_INSN (note
);
5485 || BARRIER_P (note
)))
5486 note
= NEXT_INSN (note
);
5488 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5493 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5494 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5496 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5497 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5499 SET_NEXT_INSN (last
) = insn
;
5500 SET_PREV_INSN (insn
) = last
;
5502 bb
= BLOCK_FOR_INSN (last
);
5506 fix_jump_move (insn
);
5508 if (BLOCK_FOR_INSN (insn
) != bb
)
5509 move_block_after_check (insn
);
5511 gcc_assert (BB_END (bb
) == last
);
5514 df_insn_change_bb (insn
, bb
);
5516 /* Update BB_END, if needed. */
5517 if (BB_END (bb
) == last
)
5521 SCHED_GROUP_P (insn
) = 0;
5524 /* Return true if scheduling INSN will finish current clock cycle. */
5526 insn_finishes_cycle_p (rtx_insn
*insn
)
5528 if (SCHED_GROUP_P (insn
))
5529 /* After issuing INSN, rest of the sched_group will be forced to issue
5530 in order. Don't make any plans for the rest of cycle. */
5533 /* Finishing the block will, apparently, finish the cycle. */
5534 if (current_sched_info
->insn_finishes_block_p
5535 && current_sched_info
->insn_finishes_block_p (insn
))
5541 /* Functions to model cache auto-prefetcher.
5543 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5544 memory prefetches if it sees instructions with consequitive memory accesses
5545 in the instruction stream. Details of such hardware units are not published,
5546 so we can only guess what exactly is going on there.
5547 In the scheduler, we model abstract auto-prefetcher. If there are memory
5548 insns in the ready list (or the queue) that have same memory base, but
5549 different offsets, then we delay the insns with larger offsets until insns
5550 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5551 is "1", then we look at the ready list; if it is N>1, then we also look
5552 through N-1 queue entries.
5553 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5554 among its heuristics.
5555 Param value of "-1" disables modelling of the auto-prefetcher. */
5557 /* Initialize autoprefetcher model data for INSN. */
5559 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5561 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5563 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5564 data
->base
= NULL_RTX
;
5566 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5567 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5569 rtx set
= single_set (insn
);
5570 if (set
== NULL_RTX
)
5573 rtx mem
= write
? SET_DEST (set
) : SET_SRC (set
);
5577 struct address_info info
;
5578 decompose_mem_address (&info
, mem
);
5580 /* TODO: Currently only (base+const) addressing is supported. */
5581 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5582 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5585 /* This insn is relevant for auto-prefetcher. */
5586 data
->base
= *info
.base
;
5587 data
->offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5588 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5591 /* Helper function for rank_for_schedule sorting. */
5593 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5595 for (int write
= 0; write
< 2; ++write
)
5597 autopref_multipass_data_t data1
5598 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5599 autopref_multipass_data_t data2
5600 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5602 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5603 autopref_multipass_init (insn1
, write
);
5604 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5607 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5608 autopref_multipass_init (insn2
, write
);
5609 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5612 if (!rtx_equal_p (data1
->base
, data2
->base
))
5615 return data1
->offset
- data2
->offset
;
5621 /* True if header of debug dump was printed. */
5622 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5624 /* Helper for autopref_multipass_dfa_lookahead_guard.
5625 Return "1" if INSN1 should be delayed in favor of INSN2. */
5627 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5628 const rtx_insn
*insn2
, int write
)
5630 autopref_multipass_data_t data1
5631 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5632 autopref_multipass_data_t data2
5633 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5635 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5636 autopref_multipass_init (insn2
, write
);
5637 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5640 if (rtx_equal_p (data1
->base
, data2
->base
)
5641 && data1
->offset
> data2
->offset
)
5643 if (sched_verbose
>= 2)
5645 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5647 fprintf (sched_dump
,
5648 ";;\t\tnot trying in max_issue due to autoprefetch "
5650 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5653 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5664 We could have also hooked autoprefetcher model into
5665 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5666 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5667 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5668 unblocked). We don't bother about this yet because target of interest
5669 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5671 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5672 Return "1" if INSN1 should not be considered in max_issue due to
5673 auto-prefetcher considerations. */
5675 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5679 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5682 if (sched_verbose
>= 2 && ready_index
== 0)
5683 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5685 for (int write
= 0; write
< 2; ++write
)
5687 autopref_multipass_data_t data1
5688 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5690 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5691 autopref_multipass_init (insn1
, write
);
5692 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5695 if (ready_index
== 0
5696 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5697 /* We allow only a single delay on priviledged instructions.
5698 Doing otherwise would cause infinite loop. */
5700 if (sched_verbose
>= 2)
5702 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5704 fprintf (sched_dump
,
5705 ";;\t\tnot trying in max_issue due to autoprefetch "
5707 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5710 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5715 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5717 rtx_insn
*insn2
= get_ready_element (i2
);
5720 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5723 if (ready_index
== 0)
5726 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5732 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5735 /* Everything from the current queue slot should have been moved to
5737 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5739 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5740 if (n_stalls
> max_insn_queue_index
)
5741 n_stalls
= max_insn_queue_index
;
5743 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5745 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5747 link
= link
->next ())
5749 rtx_insn
*insn2
= link
->insn ();
5750 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5754 /* Queue INSN1 until INSN2 can issue. */
5756 if (ready_index
== 0)
5757 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5765 if (sched_verbose
>= 2
5766 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5767 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5768 /* This does not /always/ trigger. We don't output EOL if the last
5769 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5770 called. We can live with this. */
5771 fprintf (sched_dump
, "\n");
5776 /* Define type for target data used in multipass scheduling. */
5777 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5778 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5780 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5782 /* The following structure describe an entry of the stack of choices. */
5785 /* Ordinal number of the issued insn in the ready queue. */
5787 /* The number of the rest insns whose issues we should try. */
5789 /* The number of issued essential insns. */
5791 /* State after issuing the insn. */
5793 /* Target-specific data. */
5794 first_cycle_multipass_data_t target_data
;
5797 /* The following array is used to implement a stack of choices used in
5798 function max_issue. */
5799 static struct choice_entry
*choice_stack
;
5801 /* This holds the value of the target dfa_lookahead hook. */
5804 /* The following variable value is maximal number of tries of issuing
5805 insns for the first cycle multipass insn scheduling. We define
5806 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5807 need this constraint if all real insns (with non-negative codes)
5808 had reservations because in this case the algorithm complexity is
5809 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5810 might be incomplete and such insn might occur. For such
5811 descriptions, the complexity of algorithm (without the constraint)
5812 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5813 static int max_lookahead_tries
;
5815 /* The following function returns maximal (or close to maximal) number
5816 of insns which can be issued on the same cycle and one of which
5817 insns is insns with the best rank (the first insn in READY). To
5818 make this function tries different samples of ready insns. READY
5819 is current queue `ready'. Global array READY_TRY reflects what
5820 insns are already issued in this try. The function stops immediately,
5821 if it reached the such a solution, that all instruction can be issued.
5822 INDEX will contain index of the best insn in READY. The following
5823 function is used only for first cycle multipass scheduling.
5827 This function expects recognized insns only. All USEs,
5828 CLOBBERs, etc must be filtered elsewhere. */
5830 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5831 bool first_cycle_insn_p
, int *index
)
5833 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5835 struct choice_entry
*top
;
5841 n_ready
= ready
->n_ready
;
5842 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5843 && privileged_n
<= n_ready
);
5845 /* Init MAX_LOOKAHEAD_TRIES. */
5846 if (max_lookahead_tries
== 0)
5848 max_lookahead_tries
= 100;
5849 for (i
= 0; i
< issue_rate
; i
++)
5850 max_lookahead_tries
*= dfa_lookahead
;
5853 /* Init max_points. */
5854 more_issue
= issue_rate
- cycle_issued_insns
;
5855 gcc_assert (more_issue
>= 0);
5857 /* The number of the issued insns in the best solution. */
5862 /* Set initial state of the search. */
5863 memcpy (top
->state
, state
, dfa_state_size
);
5864 top
->rest
= dfa_lookahead
;
5866 if (targetm
.sched
.first_cycle_multipass_begin
)
5867 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5869 first_cycle_insn_p
);
5871 /* Count the number of the insns to search among. */
5872 for (all
= i
= 0; i
< n_ready
; i
++)
5876 if (sched_verbose
>= 2)
5878 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5879 debug_ready_list_1 (ready
, ready_try
);
5882 /* I is the index of the insn to try next. */
5887 if (/* If we've reached a dead end or searched enough of what we have
5890 /* or have nothing else to try... */
5892 /* or should not issue more. */
5893 || top
->n
>= more_issue
)
5895 /* ??? (... || i == n_ready). */
5896 gcc_assert (i
<= n_ready
);
5898 /* We should not issue more than issue_rate instructions. */
5899 gcc_assert (top
->n
<= more_issue
);
5901 if (top
== choice_stack
)
5904 if (best
< top
- choice_stack
)
5909 /* Try to find issued privileged insn. */
5910 while (n
&& !ready_try
[--n
])
5914 if (/* If all insns are equally good... */
5916 /* Or a privileged insn will be issued. */
5918 /* Then we have a solution. */
5920 best
= top
- choice_stack
;
5921 /* This is the index of the insn issued first in this
5923 *index
= choice_stack
[1].index
;
5924 if (top
->n
== more_issue
|| best
== all
)
5929 /* Set ready-list index to point to the last insn
5930 ('i++' below will advance it to the next insn). */
5936 if (targetm
.sched
.first_cycle_multipass_backtrack
)
5937 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
5938 ready_try
, n_ready
);
5941 memcpy (state
, top
->state
, dfa_state_size
);
5943 else if (!ready_try
[i
])
5946 if (tries_num
> max_lookahead_tries
)
5948 insn
= ready_element (ready
, i
);
5949 delay
= state_transition (state
, insn
);
5952 if (state_dead_lock_p (state
)
5953 || insn_finishes_cycle_p (insn
))
5954 /* We won't issue any more instructions in the next
5961 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
5964 /* Advance to the next choice_entry. */
5966 /* Initialize it. */
5967 top
->rest
= dfa_lookahead
;
5970 memcpy (top
->state
, state
, dfa_state_size
);
5973 if (targetm
.sched
.first_cycle_multipass_issue
)
5974 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
5984 /* Increase ready-list index. */
5988 if (targetm
.sched
.first_cycle_multipass_end
)
5989 targetm
.sched
.first_cycle_multipass_end (best
!= 0
5990 ? &choice_stack
[1].target_data
5993 /* Restore the original state of the DFA. */
5994 memcpy (state
, choice_stack
->state
, dfa_state_size
);
5999 /* The following function chooses insn from READY and modifies
6000 READY. The following function is used only for first
6001 cycle multipass scheduling.
6003 -1 if cycle should be advanced,
6004 0 if INSN_PTR is set to point to the desirable insn,
6005 1 if choose_ready () should be restarted without advancing the cycle. */
6007 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6008 rtx_insn
**insn_ptr
)
6010 if (dbg_cnt (sched_insn
) == false)
6012 if (nonscheduled_insns_begin
== NULL_RTX
)
6013 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6015 rtx_insn
*insn
= first_nonscheduled_insn ();
6017 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6018 /* INSN is in the ready_list. */
6020 ready_remove_insn (insn
);
6025 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6026 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6030 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6031 || DEBUG_INSN_P (ready_element (ready
, 0)))
6033 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6034 *insn_ptr
= ready_remove_first_dispatch (ready
);
6036 *insn_ptr
= ready_remove_first (ready
);
6042 /* Try to choose the best insn. */
6046 insn
= ready_element (ready
, 0);
6047 if (INSN_CODE (insn
) < 0)
6049 *insn_ptr
= ready_remove_first (ready
);
6053 /* Filter the search space. */
6054 for (i
= 0; i
< ready
->n_ready
; i
++)
6058 insn
= ready_element (ready
, i
);
6060 /* If this insn is recognizable we should have already
6061 recognized it earlier.
6062 ??? Not very clear where this is supposed to be done.
6064 gcc_checking_assert (INSN_CODE (insn
) >= 0
6065 || recog_memoized (insn
) < 0);
6066 if (INSN_CODE (insn
) < 0)
6068 /* Non-recognized insns at position 0 are handled above. */
6074 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6077 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6080 if (ready_try
[i
] < 0)
6081 /* Queue instruction for several cycles.
6082 We need to restart choose_ready as we have changed
6085 change_queue_index (insn
, -ready_try
[i
]);
6089 /* Make sure that we didn't end up with 0'th insn filtered out.
6090 Don't be tempted to make life easier for backends and just
6091 requeue 0'th insn if (ready_try[0] == 0) and restart
6092 choose_ready. Backends should be very considerate about
6093 requeueing instructions -- especially the highest priority
6094 one at position 0. */
6095 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6100 gcc_assert (ready_try
[i
] == 0);
6101 /* INSN made it through the scrutiny of filters! */
6104 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6106 *insn_ptr
= ready_remove_first (ready
);
6107 if (sched_verbose
>= 4)
6108 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6109 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6114 if (sched_verbose
>= 4)
6115 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6116 (*current_sched_info
->print_insn
)
6117 (ready_element (ready
, index
), 0));
6119 *insn_ptr
= ready_remove (ready
, index
);
6125 /* This function is called when we have successfully scheduled a
6126 block. It uses the schedule stored in the scheduled_insns vector
6127 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6128 append the scheduled insns; TAIL is the insn after the scheduled
6129 block. TARGET_BB is the argument passed to schedule_block. */
6132 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6137 last_scheduled_insn
= prev_head
;
6139 scheduled_insns
.iterate (i
, &insn
);
6142 if (control_flow_insn_p (last_scheduled_insn
)
6143 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6145 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6151 x
= next_real_insn (last_scheduled_insn
);
6153 dump_new_block_header (1, *target_bb
, x
, tail
);
6156 last_scheduled_insn
= bb_note (*target_bb
);
6159 if (current_sched_info
->begin_move_insn
)
6160 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6161 move_insn (insn
, last_scheduled_insn
,
6162 current_sched_info
->next_tail
);
6163 if (!DEBUG_INSN_P (insn
))
6164 reemit_notes (insn
);
6165 last_scheduled_insn
= insn
;
6168 scheduled_insns
.truncate (0);
6171 /* Examine all insns on the ready list and queue those which can't be
6172 issued in this cycle. TEMP_STATE is temporary scheduler state we
6173 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6174 have been issued for the current cycle, which means it is valid to
6175 issue an asm statement.
6177 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6178 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6179 we only leave insns which have an INSN_EXACT_TICK. */
6182 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6183 bool shadows_only_p
, bool modulo_epilogue_p
)
6186 bool sched_group_found
= false;
6187 int min_cost_group
= 1;
6192 for (i
= 0; i
< ready
.n_ready
; i
++)
6194 rtx_insn
*insn
= ready_element (&ready
, i
);
6195 if (SCHED_GROUP_P (insn
))
6197 sched_group_found
= true;
6202 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6203 such an insn first and note its cost, then schedule all other insns
6204 for one cycle later. */
6205 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6207 int n
= ready
.n_ready
;
6208 for (i
= 0; i
< n
; i
++)
6210 rtx_insn
*insn
= ready_element (&ready
, i
);
6212 const char *reason
= "resource conflict";
6214 if (DEBUG_INSN_P (insn
))
6217 if (sched_group_found
&& !SCHED_GROUP_P (insn
))
6221 cost
= min_cost_group
;
6222 reason
= "not in sched group";
6224 else if (modulo_epilogue_p
6225 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6227 cost
= max_insn_queue_index
;
6228 reason
= "not an epilogue insn";
6230 else if (shadows_only_p
&& !SHADOW_P (insn
))
6233 reason
= "not a shadow";
6235 else if (recog_memoized (insn
) < 0)
6237 if (!first_cycle_insn_p
6238 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6239 || asm_noperands (PATTERN (insn
)) >= 0))
6243 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6245 if (sched_pressure
== SCHED_PRESSURE_MODEL
6246 && INSN_TICK (insn
) <= clock_var
)
6248 memcpy (temp_state
, curr_state
, dfa_state_size
);
6249 if (state_transition (temp_state
, insn
) >= 0)
6250 INSN_TICK (insn
) = clock_var
+ 1;
6260 struct delay_pair
*delay_entry
;
6262 = delay_htab
->find_with_hash (insn
,
6263 htab_hash_pointer (insn
));
6264 while (delay_entry
&& delay_cost
== 0)
6266 delay_cost
= estimate_shadow_tick (delay_entry
);
6267 if (delay_cost
> max_insn_queue_index
)
6268 delay_cost
= max_insn_queue_index
;
6269 delay_entry
= delay_entry
->next_same_i1
;
6273 memcpy (temp_state
, curr_state
, dfa_state_size
);
6274 cost
= state_transition (temp_state
, insn
);
6279 if (cost
< delay_cost
)
6282 reason
= "shadow tick";
6287 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6288 min_cost_group
= cost
;
6289 ready_remove (&ready
, i
);
6290 /* Normally we'd want to queue INSN for COST cycles. However,
6291 if SCHED_GROUP_P is set, then we must ensure that nothing
6292 else comes between INSN and its predecessor. If there is
6293 some other insn ready to fire on the next cycle, then that
6294 invariant would be broken.
6296 So when SCHED_GROUP_P is set, just queue this insn for a
6298 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6308 /* Called when we detect that the schedule is impossible. We examine the
6309 backtrack queue to find the earliest insn that caused this condition. */
6311 static struct haifa_saved_data
*
6312 verify_shadows (void)
6314 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6315 for (save
= backtrack_queue
; save
; save
= save
->next
)
6318 struct delay_pair
*pair
= save
->delay_pair
;
6319 rtx_insn
*i1
= pair
->i1
;
6321 for (; pair
; pair
= pair
->next_same_i1
)
6323 rtx_insn
*i2
= pair
->i2
;
6325 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6328 t
= INSN_TICK (i1
) + pair_delay (pair
);
6331 if (sched_verbose
>= 2)
6332 fprintf (sched_dump
,
6333 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6335 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6336 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6337 earliest_fail
= save
;
6340 if (QUEUE_INDEX (i2
) >= 0)
6342 int queued_for
= INSN_TICK (i2
);
6346 if (sched_verbose
>= 2)
6347 fprintf (sched_dump
,
6348 ";;\t\tfailed delay requirements for %d/%d"
6349 " (%d->%d), queued too late\n",
6350 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6351 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6352 earliest_fail
= save
;
6359 return earliest_fail
;
6362 /* Print instructions together with useful scheduling information between
6363 HEAD and TAIL (inclusive). */
6365 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6367 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6369 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6370 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6372 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6373 const char *pattern
= (NOTE_P (insn
)
6375 : str_pattern_slim (PATTERN (insn
)));
6377 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6378 INSN_UID (insn
), priority
, pattern
);
6380 if (sched_verbose
>= 4)
6382 if (NOTE_P (insn
) || recog_memoized (insn
) < 0)
6383 fprintf (sched_dump
, "nothing");
6385 print_reservation (sched_dump
, insn
);
6387 fprintf (sched_dump
, "\n");
6391 /* Use forward list scheduling to rearrange insns of block pointed to by
6392 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6396 schedule_block (basic_block
*target_bb
, state_t init_state
)
6399 bool success
= modulo_ii
== 0;
6400 struct sched_block_state ls
;
6401 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6402 int sort_p
, advance
, start_clock_var
;
6404 /* Head/tail info for this block. */
6405 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6406 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6407 rtx_insn
*head
= NEXT_INSN (prev_head
);
6408 rtx_insn
*tail
= PREV_INSN (next_tail
);
6410 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6411 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6412 find_modifiable_mems (head
, tail
);
6414 /* We used to have code to avoid getting parameters moved from hard
6415 argument registers into pseudos.
6417 However, it was removed when it proved to be of marginal benefit
6418 and caused problems because schedule_block and compute_forward_dependences
6419 had different notions of what the "head" insn was. */
6421 gcc_assert (head
!= tail
|| INSN_P (head
));
6423 haifa_recovery_bb_recently_added_p
= false;
6425 backtrack_queue
= NULL
;
6430 dump_new_block_header (0, *target_bb
, head
, tail
);
6432 if (sched_verbose
>= 2)
6434 dump_insn_stream (head
, tail
);
6435 memset (&rank_for_schedule_stats
, 0,
6436 sizeof (rank_for_schedule_stats
));
6440 if (init_state
== NULL
)
6441 state_reset (curr_state
);
6443 memcpy (curr_state
, init_state
, dfa_state_size
);
6445 /* Clear the ready list. */
6446 ready
.first
= ready
.veclen
- 1;
6450 /* It is used for first cycle multipass scheduling. */
6451 temp_state
= alloca (dfa_state_size
);
6453 if (targetm
.sched
.init
)
6454 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6456 /* We start inserting insns after PREV_HEAD. */
6457 last_scheduled_insn
= prev_head
;
6458 last_nondebug_scheduled_insn
= NULL
;
6459 nonscheduled_insns_begin
= NULL
;
6461 gcc_assert ((NOTE_P (last_scheduled_insn
)
6462 || DEBUG_INSN_P (last_scheduled_insn
))
6463 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6465 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6470 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6471 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6473 /* Start just before the beginning of time. */
6476 /* We need queue and ready lists and clock_var be initialized
6477 in try_ready () (which is called through init_ready_list ()). */
6478 (*current_sched_info
->init_ready_list
) ();
6481 sched_pressure_start_bb (*target_bb
);
6483 /* The algorithm is O(n^2) in the number of ready insns at any given
6484 time in the worst case. Before reload we are more likely to have
6485 big lists so truncate them to a reasonable size. */
6486 if (!reload_completed
6487 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6489 ready_sort_debug (&ready
);
6490 ready_sort_real (&ready
);
6492 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6493 If there are debug insns, we know they're first. */
6494 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6495 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6498 if (sched_verbose
>= 2)
6500 fprintf (sched_dump
,
6501 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6502 debug_ready_list (&ready
);
6503 fprintf (sched_dump
,
6504 ";;\t\t before reload => truncated to %d insns\n", i
);
6507 /* Delay all insns past it for 1 cycle. If debug counter is
6508 activated make an exception for the insn right after
6509 nonscheduled_insns_begin. */
6511 rtx_insn
*skip_insn
;
6513 if (dbg_cnt (sched_insn
) == false)
6514 skip_insn
= first_nonscheduled_insn ();
6518 while (i
< ready
.n_ready
)
6522 insn
= ready_remove (&ready
, i
);
6524 if (insn
!= skip_insn
)
6525 queue_insn (insn
, 1, "list truncated");
6528 ready_add (&ready
, skip_insn
, true);
6532 /* Now we can restore basic block notes and maintain precise cfg. */
6533 restore_bb_notes (*target_bb
);
6535 last_clock_var
= -1;
6539 gcc_assert (scheduled_insns
.length () == 0);
6541 must_backtrack
= false;
6542 modulo_insns_scheduled
= 0;
6544 ls
.modulo_epilogue
= false;
6545 ls
.first_cycle_insn_p
= true;
6547 /* Loop until all the insns in BB are scheduled. */
6548 while ((*current_sched_info
->schedule_more_p
) ())
6550 perform_replacements_new_cycle ();
6553 start_clock_var
= clock_var
;
6557 advance_one_cycle ();
6559 /* Add to the ready list all pending insns that can be issued now.
6560 If there are no ready insns, increment clock until one
6561 is ready and add all pending insns at that point to the ready
6563 queue_to_ready (&ready
);
6565 gcc_assert (ready
.n_ready
);
6567 if (sched_verbose
>= 2)
6569 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6570 debug_ready_list (&ready
);
6572 advance
-= clock_var
- start_clock_var
;
6574 while (advance
> 0);
6576 if (ls
.modulo_epilogue
)
6578 int stage
= clock_var
/ modulo_ii
;
6579 if (stage
> modulo_last_stage
* 2 + 2)
6581 if (sched_verbose
>= 2)
6582 fprintf (sched_dump
,
6583 ";;\t\tmodulo scheduled succeeded at II %d\n",
6589 else if (modulo_ii
> 0)
6591 int stage
= clock_var
/ modulo_ii
;
6592 if (stage
> modulo_max_stages
)
6594 if (sched_verbose
>= 2)
6595 fprintf (sched_dump
,
6596 ";;\t\tfailing schedule due to excessive stages\n");
6599 if (modulo_n_insns
== modulo_insns_scheduled
6600 && stage
> modulo_last_stage
)
6602 if (sched_verbose
>= 2)
6603 fprintf (sched_dump
,
6604 ";;\t\tfound kernel after %d stages, II %d\n",
6606 ls
.modulo_epilogue
= true;
6610 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6611 if (ready
.n_ready
== 0)
6616 ls
.shadows_only_p
= false;
6617 cycle_issued_insns
= 0;
6618 ls
.can_issue_more
= issue_rate
;
6625 if (sort_p
&& ready
.n_ready
> 0)
6627 /* Sort the ready list based on priority. This must be
6628 done every iteration through the loop, as schedule_insn
6629 may have readied additional insns that will not be
6630 sorted correctly. */
6631 ready_sort (&ready
);
6633 if (sched_verbose
>= 2)
6635 fprintf (sched_dump
,
6636 ";;\t\tReady list after ready_sort: ");
6637 debug_ready_list (&ready
);
6641 /* We don't want md sched reorder to even see debug isns, so put
6642 them out right away. */
6643 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6644 && (*current_sched_info
->schedule_more_p
) ())
6646 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6648 rtx_insn
*insn
= ready_remove_first (&ready
);
6649 gcc_assert (DEBUG_INSN_P (insn
));
6650 (*current_sched_info
->begin_schedule_ready
) (insn
);
6651 scheduled_insns
.safe_push (insn
);
6652 last_scheduled_insn
= insn
;
6653 advance
= schedule_insn (insn
);
6654 gcc_assert (advance
== 0);
6655 if (ready
.n_ready
> 0)
6656 ready_sort (&ready
);
6660 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6663 resume_after_backtrack
:
6664 /* Allow the target to reorder the list, typically for
6665 better instruction bundling. */
6667 && (ready
.n_ready
== 0
6668 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6670 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6672 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6673 ready_lastpos (&ready
),
6674 &ready
.n_ready
, clock_var
);
6675 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6677 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6679 ? ready_lastpos (&ready
) : NULL
,
6680 &ready
.n_ready
, clock_var
);
6683 restart_choose_ready
:
6684 if (sched_verbose
>= 2)
6686 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6688 debug_ready_list (&ready
);
6689 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6690 print_curr_reg_pressure ();
6693 if (ready
.n_ready
== 0
6694 && ls
.can_issue_more
6695 && reload_completed
)
6697 /* Allow scheduling insns directly from the queue in case
6698 there's nothing better to do (ready list is empty) but
6699 there are still vacant dispatch slots in the current cycle. */
6700 if (sched_verbose
>= 6)
6701 fprintf (sched_dump
,";;\t\tSecond chance\n");
6702 memcpy (temp_state
, curr_state
, dfa_state_size
);
6703 if (early_queue_to_ready (temp_state
, &ready
))
6704 ready_sort (&ready
);
6707 if (ready
.n_ready
== 0
6708 || !ls
.can_issue_more
6709 || state_dead_lock_p (curr_state
)
6710 || !(*current_sched_info
->schedule_more_p
) ())
6713 /* Select and remove the insn from the ready list. */
6719 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6725 goto restart_choose_ready
;
6727 gcc_assert (insn
!= NULL_RTX
);
6730 insn
= ready_remove_first (&ready
);
6732 if (sched_pressure
!= SCHED_PRESSURE_NONE
6733 && INSN_TICK (insn
) > clock_var
)
6735 ready_add (&ready
, insn
, true);
6740 if (targetm
.sched
.dfa_new_cycle
6741 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6742 insn
, last_clock_var
,
6743 clock_var
, &sort_p
))
6744 /* SORT_P is used by the target to override sorting
6745 of the ready list. This is needed when the target
6746 has modified its internal structures expecting that
6747 the insn will be issued next. As we need the insn
6748 to have the highest priority (so it will be returned by
6749 the ready_remove_first call above), we invoke
6750 ready_add (&ready, insn, true).
6751 But, still, there is one issue: INSN can be later
6752 discarded by scheduler's front end through
6753 current_sched_info->can_schedule_ready_p, hence, won't
6756 ready_add (&ready
, insn
, true);
6762 if (current_sched_info
->can_schedule_ready_p
6763 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6764 /* We normally get here only if we don't want to move
6765 insn from the split block. */
6767 TODO_SPEC (insn
) = DEP_POSTPONED
;
6768 goto restart_choose_ready
;
6773 /* If this insn is the first part of a delay-slot pair, record a
6775 struct delay_pair
*delay_entry
;
6777 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6780 save_backtrack_point (delay_entry
, ls
);
6781 if (sched_verbose
>= 2)
6782 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6786 /* DECISION is made. */
6788 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6790 modulo_insns_scheduled
++;
6791 modulo_last_stage
= clock_var
/ modulo_ii
;
6793 if (TODO_SPEC (insn
) & SPECULATIVE
)
6794 generate_recovery_code (insn
);
6796 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6797 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6799 /* Update counters, etc in the scheduler's front end. */
6800 (*current_sched_info
->begin_schedule_ready
) (insn
);
6801 scheduled_insns
.safe_push (insn
);
6802 gcc_assert (NONDEBUG_INSN_P (insn
));
6803 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6805 if (recog_memoized (insn
) >= 0)
6807 memcpy (temp_state
, curr_state
, dfa_state_size
);
6808 cost
= state_transition (curr_state
, insn
);
6809 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6810 gcc_assert (cost
< 0);
6811 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6812 cycle_issued_insns
++;
6816 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6817 || asm_noperands (PATTERN (insn
)) >= 0);
6819 if (targetm
.sched
.variable_issue
)
6821 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6822 insn
, ls
.can_issue_more
);
6823 /* A naked CLOBBER or USE generates no instruction, so do
6824 not count them against the issue rate. */
6825 else if (GET_CODE (PATTERN (insn
)) != USE
6826 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6827 ls
.can_issue_more
--;
6828 advance
= schedule_insn (insn
);
6830 if (SHADOW_P (insn
))
6831 ls
.shadows_only_p
= true;
6833 /* After issuing an asm insn we should start a new cycle. */
6834 if (advance
== 0 && asm_p
)
6843 ls
.first_cycle_insn_p
= false;
6844 if (ready
.n_ready
> 0)
6845 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6846 ls
.modulo_epilogue
);
6850 if (!must_backtrack
)
6851 for (i
= 0; i
< ready
.n_ready
; i
++)
6853 rtx_insn
*insn
= ready_element (&ready
, i
);
6854 if (INSN_EXACT_TICK (insn
) == clock_var
)
6856 must_backtrack
= true;
6861 if (must_backtrack
&& modulo_ii
> 0)
6863 if (modulo_backtracks_left
== 0)
6865 modulo_backtracks_left
--;
6867 while (must_backtrack
)
6869 struct haifa_saved_data
*failed
;
6870 rtx_insn
*failed_insn
;
6872 must_backtrack
= false;
6873 failed
= verify_shadows ();
6874 gcc_assert (failed
);
6876 failed_insn
= failed
->delay_pair
->i1
;
6877 /* Clear these queues. */
6878 perform_replacements_new_cycle ();
6879 toggle_cancelled_flags (false);
6880 unschedule_insns_until (failed_insn
);
6881 while (failed
!= backtrack_queue
)
6882 free_topmost_backtrack_point (true);
6883 restore_last_backtrack_point (&ls
);
6884 if (sched_verbose
>= 2)
6885 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6886 /* Delay by at least a cycle. This could cause additional
6888 queue_insn (failed_insn
, 1, "backtracked");
6892 if (ready
.n_ready
> 0)
6893 goto resume_after_backtrack
;
6896 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6902 ls
.first_cycle_insn_p
= true;
6904 if (ls
.modulo_epilogue
)
6907 if (!ls
.first_cycle_insn_p
|| advance
)
6908 advance_one_cycle ();
6909 perform_replacements_new_cycle ();
6912 /* Once again, debug insn suckiness: they can be on the ready list
6913 even if they have unresolved dependencies. To make our view
6914 of the world consistent, remove such "ready" insns. */
6915 restart_debug_insn_loop
:
6916 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6920 x
= ready_element (&ready
, i
);
6921 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
6922 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
6924 ready_remove (&ready
, i
);
6925 goto restart_debug_insn_loop
;
6928 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6932 x
= ready_element (&ready
, i
);
6933 resolve_dependencies (x
);
6935 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6937 rtx_insn_list
*link
;
6938 while ((link
= insn_queue
[i
]) != NULL
)
6940 rtx_insn
*x
= link
->insn ();
6941 insn_queue
[i
] = link
->next ();
6942 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6943 free_INSN_LIST_node (link
);
6944 resolve_dependencies (x
);
6950 undo_all_replacements ();
6955 fprintf (sched_dump
, ";;\tReady list (final): ");
6956 debug_ready_list (&ready
);
6959 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
6960 /* Sanity check -- queue must be empty now. Meaningless if region has
6962 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
6963 else if (modulo_ii
== 0)
6965 /* We must maintain QUEUE_INDEX between blocks in region. */
6966 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6970 x
= ready_element (&ready
, i
);
6971 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6972 TODO_SPEC (x
) = HARD_DEP
;
6976 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6978 rtx_insn_list
*link
;
6979 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
6984 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6985 TODO_SPEC (x
) = HARD_DEP
;
6987 free_INSN_LIST_list (&insn_queue
[i
]);
6991 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
6992 model_end_schedule ();
6996 commit_schedule (prev_head
, tail
, target_bb
);
6998 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7001 last_scheduled_insn
= tail
;
7003 scheduled_insns
.truncate (0);
7005 if (!current_sched_info
->queue_must_finish_empty
7006 || haifa_recovery_bb_recently_added_p
)
7008 /* INSN_TICK (minimum clock tick at which the insn becomes
7009 ready) may be not correct for the insn in the subsequent
7010 blocks of the region. We should use a correct value of
7011 `clock_var' or modify INSN_TICK. It is better to keep
7012 clock_var value equal to 0 at the start of a basic block.
7013 Therefore we modify INSN_TICK here. */
7014 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7017 if (targetm
.sched
.finish
)
7019 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7020 /* Target might have added some instructions to the scheduled block
7021 in its md_finish () hook. These new insns don't have any data
7022 initialized and to identify them we extend h_i_d so that they'll
7024 sched_extend_luids ();
7027 /* Update head/tail boundaries. */
7028 head
= NEXT_INSN (prev_head
);
7029 tail
= last_scheduled_insn
;
7033 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7034 INSN_UID (head
), INSN_UID (tail
));
7036 if (sched_verbose
>= 2)
7038 dump_insn_stream (head
, tail
);
7039 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7043 fprintf (sched_dump
, "\n");
7046 head
= restore_other_notes (head
, NULL
);
7048 current_sched_info
->head
= head
;
7049 current_sched_info
->tail
= tail
;
7051 free_backtrack_queue ();
7056 /* Set_priorities: compute priority of each insn in the block. */
7059 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7063 int sched_max_insns_priority
=
7064 current_sched_info
->sched_max_insns_priority
;
7065 rtx_insn
*prev_head
;
7067 if (head
== tail
&& ! INSN_P (head
))
7072 prev_head
= PREV_INSN (head
);
7073 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7079 (void) priority (insn
);
7081 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7083 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7084 INSN_PRIORITY (insn
));
7087 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7092 /* Set dump and sched_verbose for the desired debugging output. If no
7093 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7094 For -fsched-verbose=N, N>=10, print everything to stderr. */
7096 setup_sched_dump (void)
7098 sched_verbose
= sched_verbose_param
;
7099 if (sched_verbose_param
== 0 && dump_file
)
7101 sched_dump
= ((sched_verbose_param
>= 10 || !dump_file
)
7102 ? stderr
: dump_file
);
7105 /* Allocate data for register pressure sensitive scheduling. */
7107 alloc_global_sched_pressure_data (void)
7109 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7111 int i
, max_regno
= max_reg_num ();
7113 if (sched_dump
!= NULL
)
7114 /* We need info about pseudos for rtl dumps about pseudo
7115 classes and costs. */
7116 regstat_init_n_sets_and_refs ();
7117 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7118 sched_regno_pressure_class
7119 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7120 for (i
= 0; i
< max_regno
; i
++)
7121 sched_regno_pressure_class
[i
]
7122 = (i
< FIRST_PSEUDO_REGISTER
7123 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7124 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7125 curr_reg_live
= BITMAP_ALLOC (NULL
);
7126 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7128 saved_reg_live
= BITMAP_ALLOC (NULL
);
7129 region_ref_regs
= BITMAP_ALLOC (NULL
);
7132 /* Calculate number of CALL_USED_REGS in register classes that
7133 we calculate register pressure for. */
7134 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7136 enum reg_class cl
= ira_pressure_classes
[c
];
7138 call_used_regs_num
[cl
] = 0;
7140 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7141 if (call_used_regs
[ira_class_hard_regs
[cl
][i
]])
7142 ++call_used_regs_num
[cl
];
7147 /* Free data for register pressure sensitive scheduling. Also called
7148 from schedule_region when stopping sched-pressure early. */
7150 free_global_sched_pressure_data (void)
7152 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7154 if (regstat_n_sets_and_refs
!= NULL
)
7155 regstat_free_n_sets_and_refs ();
7156 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7158 BITMAP_FREE (region_ref_regs
);
7159 BITMAP_FREE (saved_reg_live
);
7161 BITMAP_FREE (curr_reg_live
);
7162 free (sched_regno_pressure_class
);
7166 /* Initialize some global state for the scheduler. This function works
7167 with the common data shared between all the schedulers. It is called
7168 from the scheduler specific initialization routine. */
7173 /* Disable speculative loads in their presence if cc0 defined. */
7175 flag_schedule_speculative_load
= 0;
7177 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7178 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7180 if (live_range_shrinkage_p
)
7181 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7182 else if (flag_sched_pressure
7183 && !reload_completed
7184 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7185 sched_pressure
= ((enum sched_pressure_algorithm
)
7186 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7188 sched_pressure
= SCHED_PRESSURE_NONE
;
7190 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7191 ira_setup_eliminable_regset ();
7193 /* Initialize SPEC_INFO. */
7194 if (targetm
.sched
.set_sched_flags
)
7196 spec_info
= &spec_info_var
;
7197 targetm
.sched
.set_sched_flags (spec_info
);
7199 if (spec_info
->mask
!= 0)
7201 spec_info
->data_weakness_cutoff
=
7202 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7203 spec_info
->control_weakness_cutoff
=
7204 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7205 * REG_BR_PROB_BASE
) / 100;
7208 /* So we won't read anything accidentally. */
7213 /* So we won't read anything accidentally. */
7216 /* Initialize issue_rate. */
7217 if (targetm
.sched
.issue_rate
)
7218 issue_rate
= targetm
.sched
.issue_rate ();
7222 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7223 /* Don't use max_issue with reg_pressure scheduling. Multipass
7224 scheduling and reg_pressure scheduling undo each other's decisions. */
7225 && sched_pressure
== SCHED_PRESSURE_NONE
)
7226 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7230 /* Set to "0" so that we recalculate. */
7231 max_lookahead_tries
= 0;
7233 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7234 targetm
.sched
.init_dfa_pre_cycle_insn ();
7236 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7237 targetm
.sched
.init_dfa_post_cycle_insn ();
7240 dfa_state_size
= state_size ();
7242 init_alias_analysis ();
7245 df_set_flags (DF_LR_RUN_DCE
);
7246 df_note_add_problem ();
7248 /* More problems needed for interloop dep calculation in SMS. */
7249 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7251 df_rd_add_problem ();
7252 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7257 /* Do not run DCE after reload, as this can kill nops inserted
7259 if (reload_completed
)
7260 df_clear_flags (DF_LR_RUN_DCE
);
7262 regstat_compute_calls_crossed ();
7264 if (targetm
.sched
.init_global
)
7265 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7267 alloc_global_sched_pressure_data ();
7269 curr_state
= xmalloc (dfa_state_size
);
7272 static void haifa_init_only_bb (basic_block
, basic_block
);
7274 /* Initialize data structures specific to the Haifa scheduler. */
7276 haifa_sched_init (void)
7278 setup_sched_dump ();
7281 scheduled_insns
.create (0);
7283 if (spec_info
!= NULL
)
7285 sched_deps_info
->use_deps_list
= 1;
7286 sched_deps_info
->generate_spec_deps
= 1;
7289 /* Initialize luids, dependency caches, target and h_i_d for the
7293 bbs
.create (n_basic_blocks_for_fn (cfun
));
7298 FOR_EACH_BB_FN (bb
, cfun
)
7299 bbs
.quick_push (bb
);
7300 sched_init_luids (bbs
);
7301 sched_deps_init (true);
7302 sched_extend_target ();
7303 haifa_init_h_i_d (bbs
);
7308 sched_init_only_bb
= haifa_init_only_bb
;
7309 sched_split_block
= sched_split_block_1
;
7310 sched_create_empty_bb
= sched_create_empty_bb_1
;
7311 haifa_recovery_bb_ever_added_p
= false;
7313 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7314 before_recovery
= 0;
7320 /* Finish work with the data specific to the Haifa scheduler. */
7322 haifa_sched_finish (void)
7324 sched_create_empty_bb
= NULL
;
7325 sched_split_block
= NULL
;
7326 sched_init_only_bb
= NULL
;
7328 if (spec_info
&& spec_info
->dump
)
7330 char c
= reload_completed
? 'a' : 'b';
7332 fprintf (spec_info
->dump
,
7333 ";; %s:\n", current_function_name ());
7335 fprintf (spec_info
->dump
,
7336 ";; Procedure %cr-begin-data-spec motions == %d\n",
7338 fprintf (spec_info
->dump
,
7339 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7341 fprintf (spec_info
->dump
,
7342 ";; Procedure %cr-begin-control-spec motions == %d\n",
7343 c
, nr_begin_control
);
7344 fprintf (spec_info
->dump
,
7345 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7346 c
, nr_be_in_control
);
7349 scheduled_insns
.release ();
7351 /* Finalize h_i_d, dependency caches, and luids for the whole
7352 function. Target will be finalized in md_global_finish (). */
7353 sched_deps_finish ();
7354 sched_finish_luids ();
7355 current_sched_info
= NULL
;
7359 /* Free global data used during insn scheduling. This function works with
7360 the common data shared between the schedulers. */
7365 haifa_finish_h_i_d ();
7366 free_global_sched_pressure_data ();
7369 if (targetm
.sched
.finish_global
)
7370 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7372 end_alias_analysis ();
7374 regstat_free_calls_crossed ();
7379 /* Free all delay_pair structures that were recorded. */
7381 free_delay_pairs (void)
7385 delay_htab
->empty ();
7386 delay_htab_i2
->empty ();
7390 /* Fix INSN_TICKs of the instructions in the current block as well as
7391 INSN_TICKs of their dependents.
7392 HEAD and TAIL are the begin and the end of the current scheduled block. */
7394 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7396 /* Set of instructions with corrected INSN_TICK. */
7397 bitmap_head processed
;
7398 /* ??? It is doubtful if we should assume that cycle advance happens on
7399 basic block boundaries. Basically insns that are unconditionally ready
7400 on the start of the block are more preferable then those which have
7401 a one cycle dependency over insn from the previous block. */
7402 int next_clock
= clock_var
+ 1;
7404 bitmap_initialize (&processed
, 0);
7406 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7407 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7408 across different blocks. */
7409 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7414 sd_iterator_def sd_it
;
7417 tick
= INSN_TICK (head
);
7418 gcc_assert (tick
>= MIN_TICK
);
7420 /* Fix INSN_TICK of instruction from just scheduled block. */
7421 if (bitmap_set_bit (&processed
, INSN_LUID (head
)))
7425 if (tick
< MIN_TICK
)
7428 INSN_TICK (head
) = tick
;
7431 if (DEBUG_INSN_P (head
))
7434 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7438 next
= DEP_CON (dep
);
7439 tick
= INSN_TICK (next
);
7441 if (tick
!= INVALID_TICK
7442 /* If NEXT has its INSN_TICK calculated, fix it.
7443 If not - it will be properly calculated from
7444 scratch later in fix_tick_ready. */
7445 && bitmap_set_bit (&processed
, INSN_LUID (next
)))
7449 if (tick
< MIN_TICK
)
7452 if (tick
> INTER_TICK (next
))
7453 INTER_TICK (next
) = tick
;
7455 tick
= INTER_TICK (next
);
7457 INSN_TICK (next
) = tick
;
7462 bitmap_clear (&processed
);
7465 /* Check if NEXT is ready to be added to the ready or queue list.
7466 If "yes", add it to the proper list.
7468 -1 - is not ready yet,
7469 0 - added to the ready list,
7470 0 < N - queued for N cycles. */
7472 try_ready (rtx_insn
*next
)
7474 ds_t old_ts
, new_ts
;
7476 old_ts
= TODO_SPEC (next
);
7478 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7479 && (old_ts
== HARD_DEP
7480 || old_ts
== DEP_POSTPONED
7481 || (old_ts
& SPECULATIVE
)
7482 || old_ts
== DEP_CONTROL
));
7484 new_ts
= recompute_todo_spec (next
, false);
7486 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7487 gcc_assert (new_ts
== old_ts
7488 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7489 else if (current_sched_info
->new_ready
)
7490 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7492 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7493 have its original pattern or changed (speculative) one. This is due
7494 to changing ebb in region scheduling.
7495 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7496 has speculative pattern.
7498 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7499 control-speculative NEXT could have been discarded by sched-rgn.c
7500 (the same case as when discarded by can_schedule_ready_p ()). */
7502 if ((new_ts
& SPECULATIVE
)
7503 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7504 need to change anything. */
7505 && new_ts
!= old_ts
)
7510 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7512 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7517 /* It would be nice to change DEP_STATUS of all dependences,
7518 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7519 so we won't reanalyze anything. */
7524 /* We follow the rule, that every speculative insn
7525 has non-null ORIG_PAT. */
7526 if (!ORIG_PAT (next
))
7527 ORIG_PAT (next
) = PATTERN (next
);
7531 if (!ORIG_PAT (next
))
7532 /* If we gonna to overwrite the original pattern of insn,
7534 ORIG_PAT (next
) = PATTERN (next
);
7536 res
= haifa_change_pattern (next
, new_pat
);
7545 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7546 either correct (new_ts & SPECULATIVE),
7547 or we simply don't care (new_ts & HARD_DEP). */
7549 gcc_assert (!ORIG_PAT (next
)
7550 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7552 TODO_SPEC (next
) = new_ts
;
7554 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7556 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7557 control-speculative NEXT could have been discarded by sched-rgn.c
7558 (the same case as when discarded by can_schedule_ready_p ()). */
7559 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7561 change_queue_index (next
, QUEUE_NOWHERE
);
7565 else if (!(new_ts
& BEGIN_SPEC
)
7566 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7567 && !IS_SPECULATION_CHECK_P (next
))
7568 /* We should change pattern of every previously speculative
7569 instruction - and we determine if NEXT was speculative by using
7570 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7571 pat too, so skip them. */
7573 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7574 gcc_assert (success
);
7575 ORIG_PAT (next
) = 0;
7578 if (sched_verbose
>= 2)
7580 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7581 (*current_sched_info
->print_insn
) (next
, 0));
7583 if (spec_info
&& spec_info
->dump
)
7585 if (new_ts
& BEGIN_DATA
)
7586 fprintf (spec_info
->dump
, "; data-spec;");
7587 if (new_ts
& BEGIN_CONTROL
)
7588 fprintf (spec_info
->dump
, "; control-spec;");
7589 if (new_ts
& BE_IN_CONTROL
)
7590 fprintf (spec_info
->dump
, "; in-control-spec;");
7592 if (TODO_SPEC (next
) & DEP_CONTROL
)
7593 fprintf (sched_dump
, " predicated");
7594 fprintf (sched_dump
, "\n");
7597 adjust_priority (next
);
7599 return fix_tick_ready (next
);
7602 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7604 fix_tick_ready (rtx_insn
*next
)
7608 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7611 sd_iterator_def sd_it
;
7614 tick
= INSN_TICK (next
);
7615 /* if tick is not equal to INVALID_TICK, then update
7616 INSN_TICK of NEXT with the most recent resolved dependence
7617 cost. Otherwise, recalculate from scratch. */
7618 full_p
= (tick
== INVALID_TICK
);
7620 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7622 rtx_insn
*pro
= DEP_PRO (dep
);
7625 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7627 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7638 INSN_TICK (next
) = tick
;
7640 delay
= tick
- clock_var
;
7641 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7642 delay
= QUEUE_READY
;
7644 change_queue_index (next
, delay
);
7649 /* Move NEXT to the proper queue list with (DELAY >= 1),
7650 or add it to the ready list (DELAY == QUEUE_READY),
7651 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7653 change_queue_index (rtx_insn
*next
, int delay
)
7655 int i
= QUEUE_INDEX (next
);
7657 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7659 gcc_assert (i
!= QUEUE_SCHEDULED
);
7661 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7662 || (delay
< 0 && delay
== i
))
7663 /* We have nothing to do. */
7666 /* Remove NEXT from wherever it is now. */
7667 if (i
== QUEUE_READY
)
7668 ready_remove_insn (next
);
7670 queue_remove (next
);
7672 /* Add it to the proper place. */
7673 if (delay
== QUEUE_READY
)
7674 ready_add (readyp
, next
, false);
7675 else if (delay
>= 1)
7676 queue_insn (next
, delay
, "change queue index");
7678 if (sched_verbose
>= 2)
7680 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7681 (*current_sched_info
->print_insn
) (next
, 0));
7683 if (delay
== QUEUE_READY
)
7684 fprintf (sched_dump
, " into ready\n");
7685 else if (delay
>= 1)
7686 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7688 fprintf (sched_dump
, " removed from ready or queue lists\n");
7692 static int sched_ready_n_insns
= -1;
7694 /* Initialize per region data structures. */
7696 sched_extend_ready_list (int new_sched_ready_n_insns
)
7700 if (sched_ready_n_insns
== -1)
7701 /* At the first call we need to initialize one more choice_stack
7705 sched_ready_n_insns
= 0;
7706 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7709 i
= sched_ready_n_insns
+ 1;
7711 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7712 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7714 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7716 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7717 sched_ready_n_insns
,
7718 sizeof (*ready_try
));
7720 /* We allocate +1 element to save initial state in the choice_stack[0]
7722 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7723 new_sched_ready_n_insns
+ 1);
7725 for (; i
<= new_sched_ready_n_insns
; i
++)
7727 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7729 if (targetm
.sched
.first_cycle_multipass_init
)
7730 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7734 sched_ready_n_insns
= new_sched_ready_n_insns
;
7737 /* Free per region data structures. */
7739 sched_finish_ready_list (void)
7750 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7752 if (targetm
.sched
.first_cycle_multipass_fini
)
7753 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7756 free (choice_stack
[i
].state
);
7758 free (choice_stack
);
7759 choice_stack
= NULL
;
7761 sched_ready_n_insns
= -1;
7765 haifa_luid_for_non_insn (rtx x
)
7767 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7772 /* Generates recovery code for INSN. */
7774 generate_recovery_code (rtx_insn
*insn
)
7776 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7777 begin_speculative_block (insn
);
7779 /* Here we have insn with no dependencies to
7780 instructions other then CHECK_SPEC ones. */
7782 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7783 add_to_speculative_block (insn
);
7787 Tries to add speculative dependencies of type FS between instructions
7788 in deps_list L and TWIN. */
7790 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7792 sd_iterator_def sd_it
;
7795 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7800 consumer
= DEP_CON (dep
);
7802 ds
= DEP_STATUS (dep
);
7804 if (/* If we want to create speculative dep. */
7806 /* And we can do that because this is a true dep. */
7807 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7809 gcc_assert (!(ds
& BE_IN_SPEC
));
7811 if (/* If this dep can be overcome with 'begin speculation'. */
7813 /* Then we have a choice: keep the dep 'begin speculative'
7814 or transform it into 'be in speculative'. */
7816 if (/* In try_ready we assert that if insn once became ready
7817 it can be removed from the ready (or queue) list only
7818 due to backend decision. Hence we can't let the
7819 probability of the speculative dep to decrease. */
7820 ds_weak (ds
) <= ds_weak (fs
))
7824 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7826 if (/* consumer can 'be in speculative'. */
7827 sched_insn_is_legitimate_for_speculation_p (consumer
,
7829 /* Transform it to be in speculative. */
7834 /* Mark the dep as 'be in speculative'. */
7839 dep_def _new_dep
, *new_dep
= &_new_dep
;
7841 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7842 sd_add_dep (new_dep
, false);
7847 /* Generates recovery code for BEGIN speculative INSN. */
7849 begin_speculative_block (rtx_insn
*insn
)
7851 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7853 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7856 create_check_block_twin (insn
, false);
7858 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7861 static void haifa_init_insn (rtx_insn
*);
7863 /* Generates recovery code for BE_IN speculative INSN. */
7865 add_to_speculative_block (rtx_insn
*insn
)
7868 sd_iterator_def sd_it
;
7870 rtx_insn_list
*twins
= NULL
;
7871 rtx_vec_t priorities_roots
;
7873 ts
= TODO_SPEC (insn
);
7874 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7876 if (ts
& BE_IN_DATA
)
7878 if (ts
& BE_IN_CONTROL
)
7881 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7882 gcc_assert (!TODO_SPEC (insn
));
7884 DONE_SPEC (insn
) |= ts
;
7886 /* First we convert all simple checks to branchy. */
7887 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7888 sd_iterator_cond (&sd_it
, &dep
);)
7890 rtx_insn
*check
= DEP_PRO (dep
);
7892 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7894 create_check_block_twin (check
, true);
7896 /* Restart search. */
7897 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7900 /* Continue search. */
7901 sd_iterator_next (&sd_it
);
7904 priorities_roots
.create (0);
7905 clear_priorities (insn
, &priorities_roots
);
7909 rtx_insn
*check
, *twin
;
7912 /* Get the first backward dependency of INSN. */
7913 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7914 if (!sd_iterator_cond (&sd_it
, &dep
))
7915 /* INSN has no backward dependencies left. */
7918 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
7919 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
7920 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
7922 check
= DEP_PRO (dep
);
7924 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
7925 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
7927 rec
= BLOCK_FOR_INSN (check
);
7929 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
7930 haifa_init_insn (twin
);
7932 sd_copy_back_deps (twin
, insn
, true);
7934 if (sched_verbose
&& spec_info
->dump
)
7935 /* INSN_BB (insn) isn't determined for twin insns yet.
7936 So we can't use current_sched_info->print_insn. */
7937 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
7938 INSN_UID (twin
), rec
->index
);
7940 twins
= alloc_INSN_LIST (twin
, twins
);
7942 /* Add dependences between TWIN and all appropriate
7943 instructions from REC. */
7944 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
7946 rtx_insn
*pro
= DEP_PRO (dep
);
7948 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
7950 /* INSN might have dependencies from the instructions from
7951 several recovery blocks. At this iteration we process those
7952 producers that reside in REC. */
7953 if (BLOCK_FOR_INSN (pro
) == rec
)
7955 dep_def _new_dep
, *new_dep
= &_new_dep
;
7957 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
7958 sd_add_dep (new_dep
, false);
7962 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
7964 /* Remove all dependencies between INSN and insns in REC. */
7965 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7966 sd_iterator_cond (&sd_it
, &dep
);)
7968 rtx_insn
*pro
= DEP_PRO (dep
);
7970 if (BLOCK_FOR_INSN (pro
) == rec
)
7971 sd_delete_dep (sd_it
);
7973 sd_iterator_next (&sd_it
);
7977 /* We couldn't have added the dependencies between INSN and TWINS earlier
7978 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
7982 rtx_insn_list
*next_node
;
7984 twin
= twins
->insn ();
7987 dep_def _new_dep
, *new_dep
= &_new_dep
;
7989 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
7990 sd_add_dep (new_dep
, false);
7993 next_node
= twins
->next ();
7994 free_INSN_LIST_node (twins
);
7998 calc_priorities (priorities_roots
);
7999 priorities_roots
.release ();
8002 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8004 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8006 gcc_assert (new_nmemb
>= old_nmemb
);
8007 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8008 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8013 Find fallthru edge from PRED. */
8015 find_fallthru_edge_from (basic_block pred
)
8020 succ
= pred
->next_bb
;
8021 gcc_assert (succ
->prev_bb
== pred
);
8023 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8025 e
= find_fallthru_edge (pred
->succs
);
8029 gcc_assert (e
->dest
== succ
);
8035 e
= find_fallthru_edge (succ
->preds
);
8039 gcc_assert (e
->src
== pred
);
8047 /* Extend per basic block data structures. */
8049 sched_extend_bb (void)
8051 /* The following is done to keep current_sched_info->next_tail non null. */
8052 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8053 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8054 if (NEXT_INSN (end
) == 0
8057 /* Don't emit a NOTE if it would end up before a BARRIER. */
8058 && !BARRIER_P (NEXT_INSN (end
))))
8060 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8061 /* Make note appear outside BB. */
8062 set_block_for_insn (note
, NULL
);
8063 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8067 /* Init per basic block data structures. */
8069 sched_init_bbs (void)
8074 /* Initialize BEFORE_RECOVERY variable. */
8076 init_before_recovery (basic_block
*before_recovery_ptr
)
8081 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8082 e
= find_fallthru_edge_from (last
);
8086 /* We create two basic blocks:
8087 1. Single instruction block is inserted right after E->SRC
8089 2. Empty block right before EXIT_BLOCK.
8090 Between these two blocks recovery blocks will be emitted. */
8092 basic_block single
, empty
;
8094 /* If the fallthrough edge to exit we've found is from the block we've
8095 created before, don't do anything more. */
8096 if (last
== after_recovery
)
8099 adding_bb_to_current_region_p
= false;
8101 single
= sched_create_empty_bb (last
);
8102 empty
= sched_create_empty_bb (single
);
8104 /* Add new blocks to the root loop. */
8105 if (current_loops
!= NULL
)
8107 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8108 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8111 single
->count
= last
->count
;
8112 empty
->count
= last
->count
;
8113 single
->frequency
= last
->frequency
;
8114 empty
->frequency
= last
->frequency
;
8115 BB_COPY_PARTITION (single
, last
);
8116 BB_COPY_PARTITION (empty
, last
);
8118 redirect_edge_succ (e
, single
);
8119 make_single_succ_edge (single
, empty
, 0);
8120 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8123 rtx_code_label
*label
= block_label (empty
);
8124 rtx_jump_insn
*x
= emit_jump_insn_after (gen_jump (label
),
8126 JUMP_LABEL (x
) = label
;
8127 LABEL_NUSES (label
)++;
8128 haifa_init_insn (x
);
8130 emit_barrier_after (x
);
8132 sched_init_only_bb (empty
, NULL
);
8133 sched_init_only_bb (single
, NULL
);
8136 adding_bb_to_current_region_p
= true;
8137 before_recovery
= single
;
8138 after_recovery
= empty
;
8140 if (before_recovery_ptr
)
8141 *before_recovery_ptr
= before_recovery
;
8143 if (sched_verbose
>= 2 && spec_info
->dump
)
8144 fprintf (spec_info
->dump
,
8145 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8146 last
->index
, single
->index
, empty
->index
);
8149 before_recovery
= last
;
8152 /* Returns new recovery block. */
8154 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8159 haifa_recovery_bb_recently_added_p
= true;
8160 haifa_recovery_bb_ever_added_p
= true;
8162 init_before_recovery (before_recovery_ptr
);
8164 barrier
= get_last_bb_insn (before_recovery
);
8165 gcc_assert (BARRIER_P (barrier
));
8167 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8169 rec
= create_basic_block (label
, label
, before_recovery
);
8171 /* A recovery block always ends with an unconditional jump. */
8172 emit_barrier_after (BB_END (rec
));
8174 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8175 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8177 if (sched_verbose
&& spec_info
->dump
)
8178 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8184 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8185 and emit necessary jumps. */
8187 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8188 basic_block second_bb
)
8192 /* This is fixing of incoming edge. */
8193 /* ??? Which other flags should be specified? */
8194 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8195 /* Partition type is the same, if it is "unpartitioned". */
8196 edge_flags
= EDGE_CROSSING
;
8200 make_edge (first_bb
, rec
, edge_flags
);
8201 rtx_code_label
*label
= block_label (second_bb
);
8202 rtx_jump_insn
*jump
= emit_jump_insn_after (gen_jump (label
), BB_END (rec
));
8203 JUMP_LABEL (jump
) = label
;
8204 LABEL_NUSES (label
)++;
8206 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8207 /* Partition type is the same, if it is "unpartitioned". */
8209 /* Rewritten from cfgrtl.c. */
8210 if (flag_reorder_blocks_and_partition
8211 && targetm_common
.have_named_sections
)
8213 /* We don't need the same note for the check because
8214 any_condjump_p (check) == true. */
8215 CROSSING_JUMP_P (jump
) = 1;
8217 edge_flags
= EDGE_CROSSING
;
8222 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8223 if (dom_info_available_p (CDI_DOMINATORS
))
8224 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8227 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8228 INSN is a simple check, that should be converted to branchy one. */
8230 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8233 rtx_insn
*label
, *check
, *twin
;
8236 sd_iterator_def sd_it
;
8238 dep_def _new_dep
, *new_dep
= &_new_dep
;
8241 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8244 todo_spec
= TODO_SPEC (insn
);
8247 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8248 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8250 todo_spec
= CHECK_SPEC (insn
);
8253 todo_spec
&= SPECULATIVE
;
8255 /* Create recovery block. */
8256 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8258 rec
= sched_create_recovery_block (NULL
);
8259 label
= BB_HEAD (rec
);
8263 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8268 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8270 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8272 /* To have mem_reg alive at the beginning of second_bb,
8273 we emit check BEFORE insn, so insn after splitting
8274 insn will be at the beginning of second_bb, which will
8275 provide us with the correct life information. */
8276 check
= emit_jump_insn_before (check_pat
, insn
);
8277 JUMP_LABEL (check
) = label
;
8278 LABEL_NUSES (label
)++;
8281 check
= emit_insn_before (check_pat
, insn
);
8283 /* Extend data structures. */
8284 haifa_init_insn (check
);
8286 /* CHECK is being added to current region. Extend ready list. */
8287 gcc_assert (sched_ready_n_insns
!= -1);
8288 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8290 if (current_sched_info
->add_remove_insn
)
8291 current_sched_info
->add_remove_insn (insn
, 0);
8293 RECOVERY_BLOCK (check
) = rec
;
8295 if (sched_verbose
&& spec_info
->dump
)
8296 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8297 (*current_sched_info
->print_insn
) (check
, 0));
8299 gcc_assert (ORIG_PAT (insn
));
8301 /* Initialize TWIN (twin is a duplicate of original instruction
8302 in the recovery block). */
8303 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8305 sd_iterator_def sd_it
;
8308 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8309 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8311 struct _dep _dep2
, *dep2
= &_dep2
;
8313 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8315 sd_add_dep (dep2
, true);
8318 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8319 haifa_init_insn (twin
);
8321 if (sched_verbose
&& spec_info
->dump
)
8322 /* INSN_BB (insn) isn't determined for twin insns yet.
8323 So we can't use current_sched_info->print_insn. */
8324 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8325 INSN_UID (twin
), rec
->index
);
8329 ORIG_PAT (check
) = ORIG_PAT (insn
);
8330 HAS_INTERNAL_DEP (check
) = 1;
8332 /* ??? We probably should change all OUTPUT dependencies to
8336 /* Copy all resolved back dependencies of INSN to TWIN. This will
8337 provide correct value for INSN_TICK (TWIN). */
8338 sd_copy_back_deps (twin
, insn
, true);
8340 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8341 /* In case of branchy check, fix CFG. */
8343 basic_block first_bb
, second_bb
;
8346 first_bb
= BLOCK_FOR_INSN (check
);
8347 second_bb
= sched_split_block (first_bb
, check
);
8349 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8351 sched_init_only_bb (second_bb
, first_bb
);
8352 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8354 jump
= BB_END (rec
);
8355 haifa_init_insn (jump
);
8358 /* Move backward dependences from INSN to CHECK and
8359 move forward dependences from INSN to TWIN. */
8361 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8362 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8364 rtx_insn
*pro
= DEP_PRO (dep
);
8367 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8368 check --TRUE--> producer ??? or ANTI ???
8369 twin --TRUE--> producer
8370 twin --ANTI--> check
8372 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8373 check --ANTI--> producer
8374 twin --ANTI--> producer
8375 twin --ANTI--> check
8377 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8378 check ~~TRUE~~> producer
8379 twin ~~TRUE~~> producer
8380 twin --ANTI--> check */
8382 ds
= DEP_STATUS (dep
);
8384 if (ds
& BEGIN_SPEC
)
8386 gcc_assert (!mutate_p
);
8390 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8391 sd_add_dep (new_dep
, false);
8393 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8395 DEP_CON (new_dep
) = twin
;
8396 sd_add_dep (new_dep
, false);
8400 /* Second, remove backward dependencies of INSN. */
8401 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8402 sd_iterator_cond (&sd_it
, &dep
);)
8404 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8406 /* We can delete this dep because we overcome it with
8407 BEGIN_SPECULATION. */
8408 sd_delete_dep (sd_it
);
8410 sd_iterator_next (&sd_it
);
8413 /* Future Speculations. Determine what BE_IN speculations will be like. */
8416 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8419 gcc_assert (!DONE_SPEC (insn
));
8423 ds_t ts
= TODO_SPEC (insn
);
8425 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8426 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8428 /* Luckiness of future speculations solely depends upon initial
8429 BEGIN speculation. */
8430 if (ts
& BEGIN_DATA
)
8431 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8432 if (ts
& BEGIN_CONTROL
)
8433 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8434 get_dep_weak (ts
, BEGIN_CONTROL
));
8437 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8439 /* Future speculations: call the helper. */
8440 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8442 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8444 /* Which types of dependencies should we use here is,
8445 generally, machine-dependent question... But, for now,
8450 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8451 sd_add_dep (new_dep
, false);
8453 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8454 sd_add_dep (new_dep
, false);
8458 if (spec_info
->dump
)
8459 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8460 (*current_sched_info
->print_insn
) (insn
, 0));
8462 /* Remove all dependencies of the INSN. */
8464 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8466 | SD_LIST_RES_BACK
));
8467 while (sd_iterator_cond (&sd_it
, &dep
))
8468 sd_delete_dep (sd_it
);
8471 /* If former check (INSN) already was moved to the ready (or queue)
8472 list, add new check (CHECK) there too. */
8473 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8476 /* Remove old check from instruction stream and free its
8478 sched_remove_insn (insn
);
8481 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8482 sd_add_dep (new_dep
, false);
8486 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8487 sd_add_dep (new_dep
, false);
8491 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8492 because it'll be done later in add_to_speculative_block. */
8494 rtx_vec_t priorities_roots
= rtx_vec_t ();
8496 clear_priorities (twin
, &priorities_roots
);
8497 calc_priorities (priorities_roots
);
8498 priorities_roots
.release ();
8502 /* Removes dependency between instructions in the recovery block REC
8503 and usual region instructions. It keeps inner dependences so it
8504 won't be necessary to recompute them. */
8506 fix_recovery_deps (basic_block rec
)
8508 rtx_insn
*note
, *insn
, *jump
;
8509 rtx_insn_list
*ready_list
= 0;
8510 bitmap_head in_ready
;
8511 rtx_insn_list
*link
;
8513 bitmap_initialize (&in_ready
, 0);
8515 /* NOTE - a basic block note. */
8516 note
= NEXT_INSN (BB_HEAD (rec
));
8517 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8518 insn
= BB_END (rec
);
8519 gcc_assert (JUMP_P (insn
));
8520 insn
= PREV_INSN (insn
);
8524 sd_iterator_def sd_it
;
8527 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8528 sd_iterator_cond (&sd_it
, &dep
);)
8530 rtx_insn
*consumer
= DEP_CON (dep
);
8532 if (BLOCK_FOR_INSN (consumer
) != rec
)
8534 sd_delete_dep (sd_it
);
8536 if (bitmap_set_bit (&in_ready
, INSN_LUID (consumer
)))
8537 ready_list
= alloc_INSN_LIST (consumer
, ready_list
);
8541 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8543 sd_iterator_next (&sd_it
);
8547 insn
= PREV_INSN (insn
);
8549 while (insn
!= note
);
8551 bitmap_clear (&in_ready
);
8553 /* Try to add instructions to the ready or queue list. */
8554 for (link
= ready_list
; link
; link
= link
->next ())
8555 try_ready (link
->insn ());
8556 free_INSN_LIST_list (&ready_list
);
8558 /* Fixing jump's dependences. */
8559 insn
= BB_HEAD (rec
);
8560 jump
= BB_END (rec
);
8562 gcc_assert (LABEL_P (insn
));
8563 insn
= NEXT_INSN (insn
);
8565 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8566 add_jump_dependencies (insn
, jump
);
8569 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8570 instruction data. */
8572 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8576 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8580 update_insn_after_change (insn
);
8584 /* -1 - can't speculate,
8585 0 - for speculation with REQUEST mode it is OK to use
8586 current instruction pattern,
8587 1 - need to change pattern for *NEW_PAT to be speculative. */
8589 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8591 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8592 && (request
& SPECULATIVE
)
8593 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8595 if ((request
& spec_info
->mask
) != request
)
8598 if (request
& BE_IN_SPEC
8599 && !(request
& BEGIN_SPEC
))
8602 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8606 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8608 gcc_assert (sched_deps_info
->generate_spec_deps
8609 && !IS_SPECULATION_CHECK_P (insn
));
8611 if (HAS_INTERNAL_DEP (insn
)
8612 || SCHED_GROUP_P (insn
))
8615 return sched_speculate_insn (insn
, request
, new_pat
);
8618 /* Print some information about block BB, which starts with HEAD and
8619 ends with TAIL, before scheduling it.
8620 I is zero, if scheduler is about to start with the fresh ebb. */
8622 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8625 fprintf (sched_dump
,
8626 ";; ======================================================\n");
8628 fprintf (sched_dump
,
8629 ";; =====================ADVANCING TO=====================\n");
8630 fprintf (sched_dump
,
8631 ";; -- basic block %d from %d to %d -- %s reload\n",
8632 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8633 (reload_completed
? "after" : "before"));
8634 fprintf (sched_dump
,
8635 ";; ======================================================\n");
8636 fprintf (sched_dump
, "\n");
8639 /* Unlink basic block notes and labels and saves them, so they
8640 can be easily restored. We unlink basic block notes in EBB to
8641 provide back-compatibility with the previous code, as target backends
8642 assume, that there'll be only instructions between
8643 current_sched_info->{head and tail}. We restore these notes as soon
8645 FIRST (LAST) is the first (last) basic block in the ebb.
8646 NB: In usual case (FIRST == LAST) nothing is really done. */
8648 unlink_bb_notes (basic_block first
, basic_block last
)
8650 /* We DON'T unlink basic block notes of the first block in the ebb. */
8654 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8656 /* Make a sentinel. */
8657 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8658 bb_header
[last
->next_bb
->index
] = 0;
8660 first
= first
->next_bb
;
8663 rtx_insn
*prev
, *label
, *note
, *next
;
8665 label
= BB_HEAD (last
);
8666 if (LABEL_P (label
))
8667 note
= NEXT_INSN (label
);
8670 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8672 prev
= PREV_INSN (label
);
8673 next
= NEXT_INSN (note
);
8674 gcc_assert (prev
&& next
);
8676 SET_NEXT_INSN (prev
) = next
;
8677 SET_PREV_INSN (next
) = prev
;
8679 bb_header
[last
->index
] = label
;
8684 last
= last
->prev_bb
;
8689 /* Restore basic block notes.
8690 FIRST is the first basic block in the ebb. */
8692 restore_bb_notes (basic_block first
)
8697 /* We DON'T unlink basic block notes of the first block in the ebb. */
8698 first
= first
->next_bb
;
8699 /* Remember: FIRST is actually a second basic block in the ebb. */
8701 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8702 && bb_header
[first
->index
])
8704 rtx_insn
*prev
, *label
, *note
, *next
;
8706 label
= bb_header
[first
->index
];
8707 prev
= PREV_INSN (label
);
8708 next
= NEXT_INSN (prev
);
8710 if (LABEL_P (label
))
8711 note
= NEXT_INSN (label
);
8714 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8716 bb_header
[first
->index
] = 0;
8718 SET_NEXT_INSN (prev
) = label
;
8719 SET_NEXT_INSN (note
) = next
;
8720 SET_PREV_INSN (next
) = note
;
8722 first
= first
->next_bb
;
8730 Fix CFG after both in- and inter-block movement of
8731 control_flow_insn_p JUMP. */
8733 fix_jump_move (rtx_insn
*jump
)
8735 basic_block bb
, jump_bb
, jump_bb_next
;
8737 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8738 jump_bb
= BLOCK_FOR_INSN (jump
);
8739 jump_bb_next
= jump_bb
->next_bb
;
8741 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8742 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8744 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8745 /* if jump_bb_next is not empty. */
8746 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8748 if (BB_END (bb
) != PREV_INSN (jump
))
8749 /* Then there are instruction after jump that should be placed
8751 BB_END (jump_bb_next
) = BB_END (bb
);
8753 /* Otherwise jump_bb_next is empty. */
8754 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8756 /* To make assertion in move_insn happy. */
8757 BB_END (bb
) = PREV_INSN (jump
);
8759 update_bb_for_insn (jump_bb_next
);
8762 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8764 move_block_after_check (rtx_insn
*jump
)
8766 basic_block bb
, jump_bb
, jump_bb_next
;
8767 vec
<edge
, va_gc
> *t
;
8769 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8770 jump_bb
= BLOCK_FOR_INSN (jump
);
8771 jump_bb_next
= jump_bb
->next_bb
;
8773 update_bb_for_insn (jump_bb
);
8775 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8776 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8778 unlink_block (jump_bb_next
);
8779 link_block (jump_bb_next
, bb
);
8783 move_succs (&(jump_bb
->succs
), bb
);
8784 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8785 move_succs (&t
, jump_bb_next
);
8787 df_mark_solutions_dirty ();
8789 common_sched_info
->fix_recovery_cfg
8790 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8793 /* Helper function for move_block_after_check.
8794 This functions attaches edge vector pointed to by SUCCSP to
8797 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8802 gcc_assert (to
->succs
== 0);
8804 to
->succs
= *succsp
;
8806 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8812 /* Remove INSN from the instruction stream.
8813 INSN should have any dependencies. */
8815 sched_remove_insn (rtx_insn
*insn
)
8817 sd_finish_insn (insn
);
8819 change_queue_index (insn
, QUEUE_NOWHERE
);
8820 current_sched_info
->add_remove_insn (insn
, 1);
8824 /* Clear priorities of all instructions, that are forward dependent on INSN.
8825 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8826 be invoked to initialize all cleared priorities. */
8828 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8830 sd_iterator_def sd_it
;
8832 bool insn_is_root_p
= true;
8834 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8836 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8838 rtx_insn
*pro
= DEP_PRO (dep
);
8840 if (INSN_PRIORITY_STATUS (pro
) >= 0
8841 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8843 /* If DEP doesn't contribute to priority then INSN itself should
8844 be added to priority roots. */
8845 if (contributes_to_priority_p (dep
))
8846 insn_is_root_p
= false;
8848 INSN_PRIORITY_STATUS (pro
) = -1;
8849 clear_priorities (pro
, roots_ptr
);
8854 roots_ptr
->safe_push (insn
);
8857 /* Recompute priorities of instructions, whose priorities might have been
8858 changed. ROOTS is a vector of instructions whose priority computation will
8859 trigger initialization of all cleared priorities. */
8861 calc_priorities (rtx_vec_t roots
)
8866 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8871 /* Add dependences between JUMP and other instructions in the recovery
8872 block. INSN is the first insn the recovery block. */
8874 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8878 insn
= NEXT_INSN (insn
);
8882 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8884 dep_def _new_dep
, *new_dep
= &_new_dep
;
8886 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8887 sd_add_dep (new_dep
, false);
8892 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8895 /* Extend data structures for logical insn UID. */
8897 sched_extend_luids (void)
8899 int new_luids_max_uid
= get_max_uid () + 1;
8901 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
8904 /* Initialize LUID for INSN. */
8906 sched_init_insn_luid (rtx_insn
*insn
)
8908 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8913 luid
= sched_max_luid
;
8914 sched_max_luid
+= i
;
8919 SET_INSN_LUID (insn
, luid
);
8922 /* Initialize luids for BBS.
8923 The hook common_sched_info->luid_for_non_insn () is used to determine
8924 if notes, labels, etc. need luids. */
8926 sched_init_luids (bb_vec_t bbs
)
8931 sched_extend_luids ();
8932 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
8936 FOR_BB_INSNS (bb
, insn
)
8937 sched_init_insn_luid (insn
);
8943 sched_finish_luids (void)
8945 sched_luids
.release ();
8949 /* Return logical uid of INSN. Helpful while debugging. */
8951 insn_luid (rtx_insn
*insn
)
8953 return INSN_LUID (insn
);
8956 /* Extend per insn data in the target. */
8958 sched_extend_target (void)
8960 if (targetm
.sched
.h_i_d_extended
)
8961 targetm
.sched
.h_i_d_extended ();
8964 /* Extend global scheduler structures (those, that live across calls to
8965 schedule_block) to include information about just emitted INSN. */
8969 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
8971 && ! h_i_d
.space (reserve
))
8973 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
8974 sched_extend_target ();
8978 /* Initialize h_i_d entry of the INSN with default values.
8979 Values, that are not explicitly initialized here, hold zero. */
8981 init_h_i_d (rtx_insn
*insn
)
8983 if (INSN_LUID (insn
) > 0)
8985 INSN_COST (insn
) = -1;
8986 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
8987 INSN_TICK (insn
) = INVALID_TICK
;
8988 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
8989 INTER_TICK (insn
) = INVALID_TICK
;
8990 TODO_SPEC (insn
) = HARD_DEP
;
8991 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
8992 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
8993 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
8994 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
8998 /* Initialize haifa_insn_data for BBS. */
9000 haifa_init_h_i_d (bb_vec_t bbs
)
9006 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9010 FOR_BB_INSNS (bb
, insn
)
9015 /* Finalize haifa_insn_data. */
9017 haifa_finish_h_i_d (void)
9020 haifa_insn_data_t data
;
9021 struct reg_use_data
*use
, *next
;
9023 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9025 free (data
->max_reg_pressure
);
9026 free (data
->reg_pressure
);
9027 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next
)
9029 next
= use
->next_insn_use
;
9036 /* Init data for the new insn INSN. */
9038 haifa_init_insn (rtx_insn
*insn
)
9040 gcc_assert (insn
!= NULL
);
9042 sched_extend_luids ();
9043 sched_init_insn_luid (insn
);
9044 sched_extend_target ();
9045 sched_deps_init (false);
9049 if (adding_bb_to_current_region_p
)
9051 sd_init_insn (insn
);
9053 /* Extend dependency caches by one element. */
9054 extend_dependency_caches (1, false);
9056 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9057 init_insn_reg_pressure_info (insn
);
9060 /* Init data for the new basic block BB which comes after AFTER. */
9062 haifa_init_only_bb (basic_block bb
, basic_block after
)
9064 gcc_assert (bb
!= NULL
);
9068 if (common_sched_info
->add_block
)
9069 /* This changes only data structures of the front-end. */
9070 common_sched_info
->add_block (bb
, after
);
9073 /* A generic version of sched_split_block (). */
9075 sched_split_block_1 (basic_block first_bb
, rtx after
)
9079 e
= split_block (first_bb
, after
);
9080 gcc_assert (e
->src
== first_bb
);
9082 /* sched_split_block emits note if *check == BB_END. Probably it
9083 is better to rip that note off. */
9088 /* A generic version of sched_create_empty_bb (). */
9090 sched_create_empty_bb_1 (basic_block after
)
9092 return create_empty_bb (after
);
9095 /* Insert PAT as an INSN into the schedule and update the necessary data
9096 structures to account for it. */
9098 sched_emit_insn (rtx pat
)
9100 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9101 haifa_init_insn (insn
);
9103 if (current_sched_info
->add_remove_insn
)
9104 current_sched_info
->add_remove_insn (insn
, 0);
9106 (*current_sched_info
->begin_schedule_ready
) (insn
);
9107 scheduled_insns
.safe_push (insn
);
9109 last_scheduled_insn
= insn
;
9113 /* This function returns a candidate satisfying dispatch constraints from
9117 ready_remove_first_dispatch (struct ready_list
*ready
)
9120 rtx_insn
*insn
= ready_element (ready
, 0);
9122 if (ready
->n_ready
== 1
9124 || INSN_CODE (insn
) < 0
9125 || !active_insn_p (insn
)
9126 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9127 return ready_remove_first (ready
);
9129 for (i
= 1; i
< ready
->n_ready
; i
++)
9131 insn
= ready_element (ready
, i
);
9134 || INSN_CODE (insn
) < 0
9135 || !active_insn_p (insn
))
9138 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9140 /* Return ith element of ready. */
9141 insn
= ready_remove (ready
, i
);
9146 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9147 return ready_remove_first (ready
);
9149 for (i
= 1; i
< ready
->n_ready
; i
++)
9151 insn
= ready_element (ready
, i
);
9154 || INSN_CODE (insn
) < 0
9155 || !active_insn_p (insn
))
9158 /* Return i-th element of ready. */
9159 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9160 return ready_remove (ready
, i
);
9163 return ready_remove_first (ready
);
9166 /* Get number of ready insn in the ready list. */
9169 number_in_ready (void)
9171 return ready
.n_ready
;
9174 /* Get number of ready's in the ready list. */
9177 get_ready_element (int i
)
9179 return ready_element (&ready
, i
);
9182 #endif /* INSN_SCHEDULING */