1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
129 #include "diagnostic-core.h"
130 #include "hard-reg-set.h"
135 #include "hash-set.h"
138 #include "function.h"
140 #include "insn-config.h"
141 #include "insn-attr.h"
144 #include "dominance.h"
147 #include "cfgbuild.h"
149 #include "basic-block.h"
150 #include "sched-int.h"
152 #include "common/common-target.h"
157 #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
158 #include "hash-table.h"
159 #include "dumpfile.h"
161 #ifdef INSN_SCHEDULING
163 /* True if we do register pressure relief through live-range
165 static bool live_range_shrinkage_p
;
167 /* Switch on live range shrinkage. */
169 initialize_live_range_shrinkage (void)
171 live_range_shrinkage_p
= true;
174 /* Switch off live range shrinkage. */
176 finish_live_range_shrinkage (void)
178 live_range_shrinkage_p
= false;
181 /* issue_rate is the number of insns that can be scheduled in the same
182 machine cycle. It can be defined in the config/mach/mach.h file,
183 otherwise we set it to 1. */
187 /* This can be set to true by a backend if the scheduler should not
188 enable a DCE pass. */
191 /* The current initiation interval used when modulo scheduling. */
192 static int modulo_ii
;
194 /* The maximum number of stages we are prepared to handle. */
195 static int modulo_max_stages
;
197 /* The number of insns that exist in each iteration of the loop. We use this
198 to detect when we've scheduled all insns from the first iteration. */
199 static int modulo_n_insns
;
201 /* The current count of insns in the first iteration of the loop that have
202 already been scheduled. */
203 static int modulo_insns_scheduled
;
205 /* The maximum uid of insns from the first iteration of the loop. */
206 static int modulo_iter0_max_uid
;
208 /* The number of times we should attempt to backtrack when modulo scheduling.
209 Decreased each time we have to backtrack. */
210 static int modulo_backtracks_left
;
212 /* The stage in which the last insn from the original loop was
214 static int modulo_last_stage
;
216 /* sched-verbose controls the amount of debugging output the
217 scheduler prints. It is controlled by -fsched-verbose=N:
218 N>0 and no -DSR : the output is directed to stderr.
219 N>=10 will direct the printouts to stderr (regardless of -dSR).
221 N=2: bb's probabilities, detailed ready list info, unit/insn info.
222 N=3: rtl at abort point, control-flow, regions info.
223 N=5: dependences info. */
225 int sched_verbose
= 0;
227 /* Debugging file. All printouts are sent to dump, which is always set,
228 either to stderr, or to the dump listing file (-dRS). */
229 FILE *sched_dump
= 0;
231 /* This is a placeholder for the scheduler parameters common
232 to all schedulers. */
233 struct common_sched_info_def
*common_sched_info
;
235 #define INSN_TICK(INSN) (HID (INSN)->tick)
236 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
237 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
238 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
239 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
240 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
241 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
242 /* Cached cost of the instruction. Use insn_cost to get cost of the
243 insn. -1 here means that the field is not initialized. */
244 #define INSN_COST(INSN) (HID (INSN)->cost)
246 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
247 then it should be recalculated from scratch. */
248 #define INVALID_TICK (-(max_insn_queue_index + 1))
249 /* The minimal value of the INSN_TICK of an instruction. */
250 #define MIN_TICK (-max_insn_queue_index)
252 /* Original order of insns in the ready list.
253 Used to keep order of normal insns while separating DEBUG_INSNs. */
254 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
256 /* The deciding reason for INSN's place in the ready list. */
257 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
259 /* List of important notes we must keep around. This is a pointer to the
260 last element in the list. */
263 static struct spec_info_def spec_info_var
;
264 /* Description of the speculative part of the scheduling.
265 If NULL - no speculation. */
266 spec_info_t spec_info
= NULL
;
268 /* True, if recovery block was added during scheduling of current block.
269 Used to determine, if we need to fix INSN_TICKs. */
270 static bool haifa_recovery_bb_recently_added_p
;
272 /* True, if recovery block was added during this scheduling pass.
273 Used to determine if we should have empty memory pools of dependencies
274 after finishing current region. */
275 bool haifa_recovery_bb_ever_added_p
;
277 /* Counters of different types of speculative instructions. */
278 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
280 /* Array used in {unlink, restore}_bb_notes. */
281 static rtx_insn
**bb_header
= 0;
283 /* Basic block after which recovery blocks will be created. */
284 static basic_block before_recovery
;
286 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
288 basic_block after_recovery
;
290 /* FALSE if we add bb to another region, so we don't need to initialize it. */
291 bool adding_bb_to_current_region_p
= true;
295 /* An instruction is ready to be scheduled when all insns preceding it
296 have already been scheduled. It is important to ensure that all
297 insns which use its result will not be executed until its result
298 has been computed. An insn is maintained in one of four structures:
300 (P) the "Pending" set of insns which cannot be scheduled until
301 their dependencies have been satisfied.
302 (Q) the "Queued" set of insns that can be scheduled when sufficient
304 (R) the "Ready" list of unscheduled, uncommitted insns.
305 (S) the "Scheduled" list of insns.
307 Initially, all insns are either "Pending" or "Ready" depending on
308 whether their dependencies are satisfied.
310 Insns move from the "Ready" list to the "Scheduled" list as they
311 are committed to the schedule. As this occurs, the insns in the
312 "Pending" list have their dependencies satisfied and move to either
313 the "Ready" list or the "Queued" set depending on whether
314 sufficient time has passed to make them ready. As time passes,
315 insns move from the "Queued" set to the "Ready" list.
317 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
318 unscheduled insns, i.e., those that are ready, queued, and pending.
319 The "Queued" set (Q) is implemented by the variable `insn_queue'.
320 The "Ready" list (R) is implemented by the variables `ready' and
322 The "Scheduled" list (S) is the new insn chain built by this pass.
324 The transition (R->S) is implemented in the scheduling loop in
325 `schedule_block' when the best insn to schedule is chosen.
326 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
327 insns move from the ready list to the scheduled list.
328 The transition (Q->R) is implemented in 'queue_to_insn' as time
329 passes or stalls are introduced. */
331 /* Implement a circular buffer to delay instructions until sufficient
332 time has passed. For the new pipeline description interface,
333 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
334 than maximal time of instruction execution computed by genattr.c on
335 the base maximal time of functional unit reservations and getting a
336 result. This is the longest time an insn may be queued. */
338 static rtx_insn_list
**insn_queue
;
339 static int q_ptr
= 0;
340 static int q_size
= 0;
341 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
342 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
344 #define QUEUE_SCHEDULED (-3)
345 #define QUEUE_NOWHERE (-2)
346 #define QUEUE_READY (-1)
347 /* QUEUE_SCHEDULED - INSN is scheduled.
348 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
350 QUEUE_READY - INSN is in ready list.
351 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
353 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
355 /* The following variable value refers for all current and future
356 reservations of the processor units. */
359 /* The following variable value is size of memory representing all
360 current and future reservations of the processor units. */
361 size_t dfa_state_size
;
363 /* The following array is used to find the best insn from ready when
364 the automaton pipeline interface is used. */
365 signed char *ready_try
= NULL
;
367 /* The ready list. */
368 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
370 /* The pointer to the ready list (to be removed). */
371 static struct ready_list
*readyp
= &ready
;
373 /* Scheduling clock. */
374 static int clock_var
;
376 /* Clock at which the previous instruction was issued. */
377 static int last_clock_var
;
379 /* Set to true if, when queuing a shadow insn, we discover that it would be
380 scheduled too late. */
381 static bool must_backtrack
;
383 /* The following variable value is number of essential insns issued on
384 the current cycle. An insn is essential one if it changes the
386 int cycle_issued_insns
;
388 /* This records the actual schedule. It is built up during the main phase
389 of schedule_block, and afterwards used to reorder the insns in the RTL. */
390 static vec
<rtx_insn
*> scheduled_insns
;
392 static int may_trap_exp (const_rtx
, int);
394 /* Nonzero iff the address is comprised from at most 1 register. */
395 #define CONST_BASED_ADDRESS_P(x) \
397 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
398 || (GET_CODE (x) == LO_SUM)) \
399 && (CONSTANT_P (XEXP (x, 0)) \
400 || CONSTANT_P (XEXP (x, 1)))))
402 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
403 as found by analyzing insn's expression. */
406 static int haifa_luid_for_non_insn (rtx x
);
408 /* Haifa version of sched_info hooks common to all headers. */
409 const struct common_sched_info_def haifa_common_sched_info
=
411 NULL
, /* fix_recovery_cfg */
412 NULL
, /* add_block */
413 NULL
, /* estimate_number_of_insns */
414 haifa_luid_for_non_insn
, /* luid_for_non_insn */
415 SCHED_PASS_UNKNOWN
/* sched_pass_id */
418 /* Mapping from instruction UID to its Logical UID. */
419 vec
<int> sched_luids
= vNULL
;
421 /* Next LUID to assign to an instruction. */
422 int sched_max_luid
= 1;
424 /* Haifa Instruction Data. */
425 vec
<haifa_insn_data_def
> h_i_d
= vNULL
;
427 void (* sched_init_only_bb
) (basic_block
, basic_block
);
429 /* Split block function. Different schedulers might use different functions
430 to handle their internal data consistent. */
431 basic_block (* sched_split_block
) (basic_block
, rtx
);
433 /* Create empty basic block after the specified block. */
434 basic_block (* sched_create_empty_bb
) (basic_block
);
436 /* Return the number of cycles until INSN is expected to be ready.
437 Return zero if it already is. */
439 insn_delay (rtx_insn
*insn
)
441 return MAX (INSN_TICK (insn
) - clock_var
, 0);
445 may_trap_exp (const_rtx x
, int is_store
)
454 if (code
== MEM
&& may_trap_p (x
))
461 /* The insn uses memory: a volatile load. */
462 if (MEM_VOLATILE_P (x
))
464 /* An exception-free load. */
467 /* A load with 1 base register, to be further checked. */
468 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
469 return PFREE_CANDIDATE
;
470 /* No info on the load, to be further checked. */
471 return PRISKY_CANDIDATE
;
476 int i
, insn_class
= TRAP_FREE
;
478 /* Neither store nor load, check if it may cause a trap. */
481 /* Recursive step: walk the insn... */
482 fmt
= GET_RTX_FORMAT (code
);
483 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
487 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
488 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
490 else if (fmt
[i
] == 'E')
493 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
495 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
496 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
497 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
501 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
508 /* Classifies rtx X of an insn for the purpose of verifying that X can be
509 executed speculatively (and consequently the insn can be moved
510 speculatively), by examining X, returning:
511 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
512 TRAP_FREE: non-load insn.
513 IFREE: load from a globally safe location.
514 IRISKY: volatile load.
515 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
516 being either PFREE or PRISKY. */
519 haifa_classify_rtx (const_rtx x
)
521 int tmp_class
= TRAP_FREE
;
522 int insn_class
= TRAP_FREE
;
525 if (GET_CODE (x
) == PARALLEL
)
527 int i
, len
= XVECLEN (x
, 0);
529 for (i
= len
- 1; i
>= 0; i
--)
531 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
532 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
533 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
543 /* Test if it is a 'store'. */
544 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
547 /* Test if it is a store. */
548 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
549 if (tmp_class
== TRAP_RISKY
)
551 /* Test if it is a load. */
553 WORST_CLASS (tmp_class
,
554 may_trap_exp (SET_SRC (x
), 0));
557 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
558 if (tmp_class
== TRAP_RISKY
)
560 tmp_class
= WORST_CLASS (tmp_class
,
561 may_trap_exp (COND_EXEC_TEST (x
), 0));
564 tmp_class
= TRAP_RISKY
;
568 insn_class
= tmp_class
;
575 haifa_classify_insn (const_rtx insn
)
577 return haifa_classify_rtx (PATTERN (insn
));
580 /* After the scheduler initialization function has been called, this function
581 can be called to enable modulo scheduling. II is the initiation interval
582 we should use, it affects the delays for delay_pairs that were recorded as
583 separated by a given number of stages.
585 MAX_STAGES provides us with a limit
586 after which we give up scheduling; the caller must have unrolled at least
587 as many copies of the loop body and recorded delay_pairs for them.
589 INSNS is the number of real (non-debug) insns in one iteration of
590 the loop. MAX_UID can be used to test whether an insn belongs to
591 the first iteration of the loop; all of them have a uid lower than
594 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
597 modulo_max_stages
= max_stages
;
598 modulo_n_insns
= insns
;
599 modulo_iter0_max_uid
= max_uid
;
600 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
603 /* A structure to record a pair of insns where the first one is a real
604 insn that has delay slots, and the second is its delayed shadow.
605 I1 is scheduled normally and will emit an assembly instruction,
606 while I2 describes the side effect that takes place at the
607 transition between cycles CYCLES and (CYCLES + 1) after I1. */
610 struct delay_pair
*next_same_i1
;
613 /* When doing modulo scheduling, we a delay_pair can also be used to
614 show that I1 and I2 are the same insn in a different stage. If that
615 is the case, STAGES will be nonzero. */
619 /* Helpers for delay hashing. */
621 struct delay_i1_hasher
: typed_noop_remove
<delay_pair
>
623 typedef delay_pair
*value_type
;
624 typedef void *compare_type
;
625 static inline hashval_t
hash (const delay_pair
*);
626 static inline bool equal (const delay_pair
*, const void *);
629 /* Returns a hash value for X, based on hashing just I1. */
632 delay_i1_hasher::hash (const delay_pair
*x
)
634 return htab_hash_pointer (x
->i1
);
637 /* Return true if I1 of pair X is the same as that of pair Y. */
640 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
645 struct delay_i2_hasher
: typed_free_remove
<delay_pair
>
647 typedef delay_pair
*value_type
;
648 typedef void *compare_type
;
649 static inline hashval_t
hash (const delay_pair
*);
650 static inline bool equal (const delay_pair
*, const void *);
653 /* Returns a hash value for X, based on hashing just I2. */
656 delay_i2_hasher::hash (const delay_pair
*x
)
658 return htab_hash_pointer (x
->i2
);
661 /* Return true if I2 of pair X is the same as that of pair Y. */
664 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
669 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
671 static hash_table
<delay_i1_hasher
> *delay_htab
;
672 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
674 /* Called through htab_traverse. Walk the hashtable using I2 as
675 index, and delete all elements involving an UID higher than
676 that pointed to by *DATA. */
678 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
681 struct delay_pair
*p
= *slot
;
682 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
684 delay_htab_i2
->clear_slot (slot
);
689 /* Called through htab_traverse. Walk the hashtable using I2 as
690 index, and delete all elements involving an UID higher than
691 that pointed to by *DATA. */
693 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
696 struct delay_pair
*p
, *first
, **pprev
;
698 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
700 delay_htab
->clear_slot (pslot
);
704 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
706 if (INSN_UID (p
->i2
) < maxuid
)
709 pprev
= &p
->next_same_i1
;
714 delay_htab
->clear_slot (pslot
);
720 /* Discard all delay pairs which involve an insn with an UID higher
723 discard_delay_pairs_above (int max_uid
)
725 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
726 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
729 /* This function can be called by a port just before it starts the final
730 scheduling pass. It records the fact that an instruction with delay
731 slots has been split into two insns, I1 and I2. The first one will be
732 scheduled normally and initiates the operation. The second one is a
733 shadow which must follow a specific number of cycles after I1; its only
734 purpose is to show the side effect that occurs at that cycle in the RTL.
735 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
736 while I2 retains the original insn type.
738 There are two ways in which the number of cycles can be specified,
739 involving the CYCLES and STAGES arguments to this function. If STAGES
740 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
741 which is multiplied by MODULO_II to give the number of cycles. This is
742 only useful if the caller also calls set_modulo_params to enable modulo
746 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
748 struct delay_pair
*p
= XNEW (struct delay_pair
);
749 struct delay_pair
**slot
;
758 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
759 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
761 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
762 p
->next_same_i1
= *slot
;
764 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
768 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
769 and return the other insn if so. Return NULL otherwise. */
771 real_insn_for_shadow (rtx_insn
*insn
)
773 struct delay_pair
*pair
;
778 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
779 if (!pair
|| pair
->stages
> 0)
784 /* For a pair P of insns, return the fixed distance in cycles from the first
785 insn after which the second must be scheduled. */
787 pair_delay (struct delay_pair
*p
)
792 return p
->stages
* modulo_ii
;
795 /* Given an insn INSN, add a dependence on its delayed shadow if it
796 has one. Also try to find situations where shadows depend on each other
797 and add dependencies to the real insns to limit the amount of backtracking
800 add_delay_dependencies (rtx_insn
*insn
)
802 struct delay_pair
*pair
;
803 sd_iterator_def sd_it
;
809 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
812 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
816 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
818 rtx_insn
*pro
= DEP_PRO (dep
);
819 struct delay_pair
*other_pair
820 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
821 if (!other_pair
|| other_pair
->stages
)
823 if (pair_delay (other_pair
) >= pair_delay (pair
))
825 if (sched_verbose
>= 4)
827 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
828 INSN_UID (other_pair
->i1
),
829 INSN_UID (pair
->i1
));
830 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
834 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
835 INSN_UID (other_pair
->i1
),
836 INSN_UID (other_pair
->i2
),
837 pair_delay (other_pair
));
839 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
844 /* Forward declarations. */
846 static int priority (rtx_insn
*);
847 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
848 static int rank_for_schedule (const void *, const void *);
849 static void swap_sort (rtx_insn
**, int);
850 static void queue_insn (rtx_insn
*, int, const char *);
851 static int schedule_insn (rtx_insn
*);
852 static void adjust_priority (rtx_insn
*);
853 static void advance_one_cycle (void);
854 static void extend_h_i_d (void);
857 /* Notes handling mechanism:
858 =========================
859 Generally, NOTES are saved before scheduling and restored after scheduling.
860 The scheduler distinguishes between two types of notes:
862 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
863 Before scheduling a region, a pointer to the note is added to the insn
864 that follows or precedes it. (This happens as part of the data dependence
865 computation). After scheduling an insn, the pointer contained in it is
866 used for regenerating the corresponding note (in reemit_notes).
868 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
869 these notes are put in a list (in rm_other_notes() and
870 unlink_other_notes ()). After scheduling the block, these notes are
871 inserted at the beginning of the block (in schedule_block()). */
873 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
874 static rtx_insn
*ready_remove_first (struct ready_list
*);
875 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
877 static void queue_to_ready (struct ready_list
*);
878 static int early_queue_to_ready (state_t
, struct ready_list
*);
880 /* The following functions are used to implement multi-pass scheduling
881 on the first cycle. */
882 static rtx_insn
*ready_remove (struct ready_list
*, int);
883 static void ready_remove_insn (rtx_insn
*);
885 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
886 static int fix_tick_ready (rtx_insn
*);
887 static void change_queue_index (rtx_insn
*, int);
889 /* The following functions are used to implement scheduling of data/control
890 speculative instructions. */
892 static void extend_h_i_d (void);
893 static void init_h_i_d (rtx_insn
*);
894 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
895 static void generate_recovery_code (rtx_insn
*);
896 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
897 static void begin_speculative_block (rtx_insn
*);
898 static void add_to_speculative_block (rtx_insn
*);
899 static void init_before_recovery (basic_block
*);
900 static void create_check_block_twin (rtx_insn
*, bool);
901 static void fix_recovery_deps (basic_block
);
902 static bool haifa_change_pattern (rtx_insn
*, rtx
);
903 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
904 static void restore_bb_notes (basic_block
);
905 static void fix_jump_move (rtx_insn
*);
906 static void move_block_after_check (rtx_insn
*);
907 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
908 static void sched_remove_insn (rtx_insn
*);
909 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
910 static void calc_priorities (rtx_vec_t
);
911 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
913 #endif /* INSN_SCHEDULING */
915 /* Point to state used for the current scheduling pass. */
916 struct haifa_sched_info
*current_sched_info
;
918 #ifndef INSN_SCHEDULING
920 schedule_insns (void)
925 /* Do register pressure sensitive insn scheduling if the flag is set
927 enum sched_pressure_algorithm sched_pressure
;
929 /* Map regno -> its pressure class. The map defined only when
930 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
931 enum reg_class
*sched_regno_pressure_class
;
933 /* The current register pressure. Only elements corresponding pressure
934 classes are defined. */
935 static int curr_reg_pressure
[N_REG_CLASSES
];
937 /* Saved value of the previous array. */
938 static int saved_reg_pressure
[N_REG_CLASSES
];
940 /* Register living at given scheduling point. */
941 static bitmap curr_reg_live
;
943 /* Saved value of the previous array. */
944 static bitmap saved_reg_live
;
946 /* Registers mentioned in the current region. */
947 static bitmap region_ref_regs
;
949 /* Effective number of available registers of a given class (see comment
950 in sched_pressure_start_bb). */
951 static int sched_class_regs_num
[N_REG_CLASSES
];
952 /* Number of call_used_regs. This is a helper for calculating of
953 sched_class_regs_num. */
954 static int call_used_regs_num
[N_REG_CLASSES
];
956 /* Initiate register pressure relative info for scheduling the current
957 region. Currently it is only clearing register mentioned in the
960 sched_init_region_reg_pressure_info (void)
962 bitmap_clear (region_ref_regs
);
965 /* PRESSURE[CL] describes the pressure on register class CL. Update it
966 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
967 LIVE tracks the set of live registers; if it is null, assume that
968 every birth or death is genuine. */
970 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
972 enum reg_class pressure_class
;
974 pressure_class
= sched_regno_pressure_class
[regno
];
975 if (regno
>= FIRST_PSEUDO_REGISTER
)
977 if (pressure_class
!= NO_REGS
)
981 if (!live
|| bitmap_set_bit (live
, regno
))
982 pressure
[pressure_class
]
983 += (ira_reg_class_max_nregs
984 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
988 if (!live
|| bitmap_clear_bit (live
, regno
))
989 pressure
[pressure_class
]
990 -= (ira_reg_class_max_nregs
991 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
995 else if (pressure_class
!= NO_REGS
996 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
1000 if (!live
|| bitmap_set_bit (live
, regno
))
1001 pressure
[pressure_class
]++;
1005 if (!live
|| bitmap_clear_bit (live
, regno
))
1006 pressure
[pressure_class
]--;
1011 /* Initiate current register pressure related info from living
1012 registers given by LIVE. */
1014 initiate_reg_pressure_info (bitmap live
)
1020 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1021 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1022 bitmap_clear (curr_reg_live
);
1023 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1024 if (sched_pressure
== SCHED_PRESSURE_MODEL
1025 || current_nr_blocks
== 1
1026 || bitmap_bit_p (region_ref_regs
, j
))
1027 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1030 /* Mark registers in X as mentioned in the current region. */
1032 setup_ref_regs (rtx x
)
1035 const RTX_CODE code
= GET_CODE (x
);
1040 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1043 fmt
= GET_RTX_FORMAT (code
);
1044 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1046 setup_ref_regs (XEXP (x
, i
));
1047 else if (fmt
[i
] == 'E')
1049 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1050 setup_ref_regs (XVECEXP (x
, i
, j
));
1054 /* Initiate current register pressure related info at the start of
1057 initiate_bb_reg_pressure_info (basic_block bb
)
1059 unsigned int i ATTRIBUTE_UNUSED
;
1062 if (current_nr_blocks
> 1)
1063 FOR_BB_INSNS (bb
, insn
)
1064 if (NONDEBUG_INSN_P (insn
))
1065 setup_ref_regs (PATTERN (insn
));
1066 initiate_reg_pressure_info (df_get_live_in (bb
));
1067 if (bb_has_eh_pred (bb
))
1070 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1072 if (regno
== INVALID_REGNUM
)
1074 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1075 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1080 /* Save current register pressure related info. */
1082 save_reg_pressure (void)
1086 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1087 saved_reg_pressure
[ira_pressure_classes
[i
]]
1088 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1089 bitmap_copy (saved_reg_live
, curr_reg_live
);
1092 /* Restore saved register pressure related info. */
1094 restore_reg_pressure (void)
1098 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1099 curr_reg_pressure
[ira_pressure_classes
[i
]]
1100 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1101 bitmap_copy (curr_reg_live
, saved_reg_live
);
1104 /* Return TRUE if the register is dying after its USE. */
1106 dying_use_p (struct reg_use_data
*use
)
1108 struct reg_use_data
*next
;
1110 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1111 if (NONDEBUG_INSN_P (next
->insn
)
1112 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1117 /* Print info about the current register pressure and its excess for
1118 each pressure class. */
1120 print_curr_reg_pressure (void)
1125 fprintf (sched_dump
, ";;\t");
1126 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1128 cl
= ira_pressure_classes
[i
];
1129 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1130 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1131 curr_reg_pressure
[cl
],
1132 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1134 fprintf (sched_dump
, "\n");
1137 /* Determine if INSN has a condition that is clobbered if a register
1138 in SET_REGS is modified. */
1140 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1142 rtx pat
= PATTERN (insn
);
1143 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1144 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1146 sd_iterator_def sd_it
;
1148 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1149 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1150 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1151 TODO_SPEC (insn
) = HARD_DEP
;
1152 if (sched_verbose
>= 2)
1153 fprintf (sched_dump
,
1154 ";;\t\tdequeue insn %s because of clobbered condition\n",
1155 (*current_sched_info
->print_insn
) (insn
, 0));
1162 /* This function should be called after modifying the pattern of INSN,
1163 to update scheduler data structures as needed. */
1165 update_insn_after_change (rtx_insn
*insn
)
1167 sd_iterator_def sd_it
;
1170 dfa_clear_single_insn_cache (insn
);
1172 sd_it
= sd_iterator_start (insn
,
1173 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1174 while (sd_iterator_cond (&sd_it
, &dep
))
1176 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1177 sd_iterator_next (&sd_it
);
1180 /* Invalidate INSN_COST, so it'll be recalculated. */
1181 INSN_COST (insn
) = -1;
1182 /* Invalidate INSN_TICK, so it'll be recalculated. */
1183 INSN_TICK (insn
) = INVALID_TICK
;
1185 /* Invalidate autoprefetch data entry. */
1186 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1187 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1188 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1189 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1193 /* Two VECs, one to hold dependencies for which pattern replacements
1194 need to be applied or restored at the start of the next cycle, and
1195 another to hold an integer that is either one, to apply the
1196 corresponding replacement, or zero to restore it. */
1197 static vec
<dep_t
> next_cycle_replace_deps
;
1198 static vec
<int> next_cycle_apply
;
1200 static void apply_replacement (dep_t
, bool);
1201 static void restore_pattern (dep_t
, bool);
1203 /* Look at the remaining dependencies for insn NEXT, and compute and return
1204 the TODO_SPEC value we should use for it. This is called after one of
1205 NEXT's dependencies has been resolved.
1206 We also perform pattern replacements for predication, and for broken
1207 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1211 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1214 sd_iterator_def sd_it
;
1215 dep_t dep
, modify_dep
= NULL
;
1219 bool first_p
= true;
1221 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1222 /* NEXT has all its dependencies resolved. */
1225 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1228 /* If NEXT is intended to sit adjacent to this instruction, we don't
1229 want to try to break any dependencies. Treat it as a HARD_DEP. */
1230 if (SCHED_GROUP_P (next
))
1233 /* Now we've got NEXT with speculative deps only.
1234 1. Look at the deps to see what we have to do.
1235 2. Check if we can do 'todo'. */
1238 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1240 rtx_insn
*pro
= DEP_PRO (dep
);
1241 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1243 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1256 new_ds
= ds_merge (new_ds
, ds
);
1258 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1260 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1265 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1267 else if (DEP_REPLACE (dep
) != NULL
)
1269 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1274 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1278 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1280 if (!dbg_cnt (sched_breakdep
))
1282 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1284 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1287 if (desc
->insn
== next
&& !for_backtrack
)
1289 gcc_assert (n_replace
== 1);
1290 apply_replacement (dep
, true);
1292 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1298 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1300 rtx_insn
*pro
, *other
;
1302 rtx cond
= NULL_RTX
;
1304 rtx_insn
*prev
= NULL
;
1308 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1309 || (ORIG_PAT (next
) != NULL_RTX
1310 && PREDICATED_PAT (next
) == NULL_RTX
))
1313 pro
= DEP_PRO (modify_dep
);
1314 other
= real_insn_for_shadow (pro
);
1315 if (other
!= NULL_RTX
)
1318 cond
= sched_get_reverse_condition_uncached (pro
);
1319 regno
= REGNO (XEXP (cond
, 0));
1321 /* Find the last scheduled insn that modifies the condition register.
1322 We can stop looking once we find the insn we depend on through the
1323 REG_DEP_CONTROL; if the condition register isn't modified after it,
1324 we know that it still has the right value. */
1325 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1326 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1330 find_all_hard_reg_sets (prev
, &t
, true);
1331 if (TEST_HARD_REG_BIT (t
, regno
))
1336 if (ORIG_PAT (next
) == NULL_RTX
)
1338 ORIG_PAT (next
) = PATTERN (next
);
1340 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1341 success
= haifa_change_pattern (next
, new_pat
);
1344 PREDICATED_PAT (next
) = new_pat
;
1346 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1348 bool success
= haifa_change_pattern (next
,
1349 PREDICATED_PAT (next
));
1350 gcc_assert (success
);
1352 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1356 if (PREDICATED_PAT (next
) != NULL_RTX
)
1358 int tick
= INSN_TICK (next
);
1359 bool success
= haifa_change_pattern (next
,
1361 INSN_TICK (next
) = tick
;
1362 gcc_assert (success
);
1365 /* We can't handle the case where there are both speculative and control
1366 dependencies, so we return HARD_DEP in such a case. Also fail if
1367 we have speculative dependencies with not enough points, or more than
1368 one control dependency. */
1369 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1371 /* Too few points? */
1372 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1380 /* Pointer to the last instruction scheduled. */
1381 static rtx_insn
*last_scheduled_insn
;
1383 /* Pointer to the last nondebug instruction scheduled within the
1384 block, or the prev_head of the scheduling block. Used by
1385 rank_for_schedule, so that insns independent of the last scheduled
1386 insn will be preferred over dependent instructions. */
1387 static rtx_insn
*last_nondebug_scheduled_insn
;
1389 /* Pointer that iterates through the list of unscheduled insns if we
1390 have a dbg_cnt enabled. It always points at an insn prior to the
1391 first unscheduled one. */
1392 static rtx_insn
*nonscheduled_insns_begin
;
1394 /* Compute cost of executing INSN.
1395 This is the number of cycles between instruction issue and
1396 instruction results. */
1398 insn_cost (rtx_insn
*insn
)
1407 if (recog_memoized (insn
) < 0)
1410 cost
= insn_default_latency (insn
);
1417 cost
= INSN_COST (insn
);
1421 /* A USE insn, or something else we don't need to
1422 understand. We can't pass these directly to
1423 result_ready_cost or insn_default_latency because it will
1424 trigger a fatal error for unrecognizable insns. */
1425 if (recog_memoized (insn
) < 0)
1427 INSN_COST (insn
) = 0;
1432 cost
= insn_default_latency (insn
);
1436 INSN_COST (insn
) = cost
;
1443 /* Compute cost of dependence LINK.
1444 This is the number of cycles between instruction issue and
1445 instruction results.
1446 ??? We also use this function to call recog_memoized on all insns. */
1448 dep_cost_1 (dep_t link
, dw_t dw
)
1450 rtx_insn
*insn
= DEP_PRO (link
);
1451 rtx_insn
*used
= DEP_CON (link
);
1454 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1455 return DEP_COST (link
);
1459 struct delay_pair
*delay_entry
;
1461 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1464 if (delay_entry
->i1
== insn
)
1466 DEP_COST (link
) = pair_delay (delay_entry
);
1467 return DEP_COST (link
);
1472 /* A USE insn should never require the value used to be computed.
1473 This allows the computation of a function's result and parameter
1474 values to overlap the return and call. We don't care about the
1475 dependence cost when only decreasing register pressure. */
1476 if (recog_memoized (used
) < 0)
1479 recog_memoized (insn
);
1483 enum reg_note dep_type
= DEP_TYPE (link
);
1485 cost
= insn_cost (insn
);
1487 if (INSN_CODE (insn
) >= 0)
1489 if (dep_type
== REG_DEP_ANTI
)
1491 else if (dep_type
== REG_DEP_OUTPUT
)
1493 cost
= (insn_default_latency (insn
)
1494 - insn_default_latency (used
));
1498 else if (bypass_p (insn
))
1499 cost
= insn_latency (insn
, used
);
1503 if (targetm
.sched
.adjust_cost_2
)
1504 cost
= targetm
.sched
.adjust_cost_2 (used
, (int) dep_type
, insn
, cost
,
1506 else if (targetm
.sched
.adjust_cost
!= NULL
)
1508 /* This variable is used for backward compatibility with the
1510 rtx_insn_list
*dep_cost_rtx_link
=
1511 alloc_INSN_LIST (NULL_RTX
, NULL
);
1513 /* Make it self-cycled, so that if some tries to walk over this
1514 incomplete list he/she will be caught in an endless loop. */
1515 XEXP (dep_cost_rtx_link
, 1) = dep_cost_rtx_link
;
1517 /* Targets use only REG_NOTE_KIND of the link. */
1518 PUT_REG_NOTE_KIND (dep_cost_rtx_link
, DEP_TYPE (link
));
1520 cost
= targetm
.sched
.adjust_cost (used
, dep_cost_rtx_link
,
1523 free_INSN_LIST_node (dep_cost_rtx_link
);
1530 DEP_COST (link
) = cost
;
1534 /* Compute cost of dependence LINK.
1535 This is the number of cycles between instruction issue and
1536 instruction results. */
1538 dep_cost (dep_t link
)
1540 return dep_cost_1 (link
, 0);
1543 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1544 INSN_PRIORITY explicitly. */
1546 increase_insn_priority (rtx_insn
*insn
, int amount
)
1548 if (!sel_sched_p ())
1550 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1551 if (INSN_PRIORITY_KNOWN (insn
))
1552 INSN_PRIORITY (insn
) += amount
;
1556 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1557 Use EXPR_PRIORITY instead. */
1558 sel_add_to_insn_priority (insn
, amount
);
1562 /* Return 'true' if DEP should be included in priority calculations. */
1564 contributes_to_priority_p (dep_t dep
)
1566 if (DEBUG_INSN_P (DEP_CON (dep
))
1567 || DEBUG_INSN_P (DEP_PRO (dep
)))
1570 /* Critical path is meaningful in block boundaries only. */
1571 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1575 if (DEP_REPLACE (dep
) != NULL
)
1578 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1579 then speculative instructions will less likely be
1580 scheduled. That is because the priority of
1581 their producers will increase, and, thus, the
1582 producers will more likely be scheduled, thus,
1583 resolving the dependence. */
1584 if (sched_deps_info
->generate_spec_deps
1585 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1586 && (DEP_STATUS (dep
) & SPECULATIVE
))
1592 /* Compute the number of nondebug deps in list LIST for INSN. */
1595 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1597 sd_iterator_def sd_it
;
1599 int dbgcount
= 0, nodbgcount
= 0;
1601 if (!MAY_HAVE_DEBUG_INSNS
)
1602 return sd_lists_size (insn
, list
);
1604 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1606 if (DEBUG_INSN_P (DEP_CON (dep
)))
1608 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1612 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1619 /* Compute the priority number for INSN. */
1621 priority (rtx_insn
*insn
)
1623 if (! INSN_P (insn
))
1626 /* We should not be interested in priority of an already scheduled insn. */
1627 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1629 if (!INSN_PRIORITY_KNOWN (insn
))
1631 int this_priority
= -1;
1635 int this_fusion_priority
;
1637 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1638 &this_fusion_priority
, &this_priority
);
1639 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1641 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1642 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1643 some forward deps but all of them are ignored by
1644 contributes_to_priority hook. At the moment we set priority of
1646 this_priority
= insn_cost (insn
);
1649 rtx_insn
*prev_first
, *twin
;
1652 /* For recovery check instructions we calculate priority slightly
1653 different than that of normal instructions. Instead of walking
1654 through INSN_FORW_DEPS (check) list, we walk through
1655 INSN_FORW_DEPS list of each instruction in the corresponding
1658 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1659 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1660 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1662 prev_first
= PREV_INSN (insn
);
1667 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1668 twin
= PREV_INSN (BB_END (rec
));
1673 sd_iterator_def sd_it
;
1676 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1681 next
= DEP_CON (dep
);
1683 if (BLOCK_FOR_INSN (next
) != rec
)
1687 if (!contributes_to_priority_p (dep
))
1691 cost
= dep_cost (dep
);
1694 struct _dep _dep1
, *dep1
= &_dep1
;
1696 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1698 cost
= dep_cost (dep1
);
1701 next_priority
= cost
+ priority (next
);
1703 if (next_priority
> this_priority
)
1704 this_priority
= next_priority
;
1708 twin
= PREV_INSN (twin
);
1710 while (twin
!= prev_first
);
1713 if (this_priority
< 0)
1715 gcc_assert (this_priority
== -1);
1717 this_priority
= insn_cost (insn
);
1720 INSN_PRIORITY (insn
) = this_priority
;
1721 INSN_PRIORITY_STATUS (insn
) = 1;
1724 return INSN_PRIORITY (insn
);
1727 /* Macros and functions for keeping the priority queue sorted, and
1728 dealing with queuing and dequeuing of instructions. */
1730 /* For each pressure class CL, set DEATH[CL] to the number of registers
1731 in that class that die in INSN. */
1734 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1737 struct reg_use_data
*use
;
1739 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1740 death
[ira_pressure_classes
[i
]] = 0;
1741 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1742 if (dying_use_p (use
))
1743 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1746 /* Setup info about the current register pressure impact of scheduling
1747 INSN at the current scheduling point. */
1749 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1751 int i
, change
, before
, after
, hard_regno
;
1752 int excess_cost_change
;
1755 struct reg_pressure_data
*pressure_info
;
1756 int *max_reg_pressure
;
1757 static int death
[N_REG_CLASSES
];
1759 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1761 excess_cost_change
= 0;
1762 calculate_reg_deaths (insn
, death
);
1763 pressure_info
= INSN_REG_PRESSURE (insn
);
1764 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1765 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1766 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1768 cl
= ira_pressure_classes
[i
];
1769 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1770 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1771 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1772 after
= MAX (0, max_reg_pressure
[i
] + change
1773 - sched_class_regs_num
[cl
]);
1774 hard_regno
= ira_class_hard_regs
[cl
][0];
1775 gcc_assert (hard_regno
>= 0);
1776 mode
= reg_raw_mode
[hard_regno
];
1777 excess_cost_change
+= ((after
- before
)
1778 * (ira_memory_move_cost
[mode
][cl
][0]
1779 + ira_memory_move_cost
[mode
][cl
][1]));
1781 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1784 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1785 It tries to make the scheduler take register pressure into account
1786 without introducing too many unnecessary stalls. It hooks into the
1787 main scheduling algorithm at several points:
1789 - Before scheduling starts, model_start_schedule constructs a
1790 "model schedule" for the current block. This model schedule is
1791 chosen solely to keep register pressure down. It does not take the
1792 target's pipeline or the original instruction order into account,
1793 except as a tie-breaker. It also doesn't work to a particular
1796 This model schedule gives us an idea of what pressure can be
1797 achieved for the block and gives us an example of a schedule that
1798 keeps to that pressure. It also makes the final schedule less
1799 dependent on the original instruction order. This is important
1800 because the original order can either be "wide" (many values live
1801 at once, such as in user-scheduled code) or "narrow" (few values
1802 live at once, such as after loop unrolling, where several
1803 iterations are executed sequentially).
1805 We do not apply this model schedule to the rtx stream. We simply
1806 record it in model_schedule. We also compute the maximum pressure,
1807 MP, that was seen during this schedule.
1809 - Instructions are added to the ready queue even if they require
1810 a stall. The length of the stall is instead computed as:
1812 MAX (INSN_TICK (INSN) - clock_var, 0)
1814 (= insn_delay). This allows rank_for_schedule to choose between
1815 introducing a deliberate stall or increasing pressure.
1817 - Before sorting the ready queue, model_set_excess_costs assigns
1818 a pressure-based cost to each ready instruction in the queue.
1819 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1820 (ECC for short) and is effectively measured in cycles.
1822 - rank_for_schedule ranks instructions based on:
1824 ECC (insn) + insn_delay (insn)
1830 So, for example, an instruction X1 with an ECC of 1 that can issue
1831 now will win over an instruction X0 with an ECC of zero that would
1832 introduce a stall of one cycle. However, an instruction X2 with an
1833 ECC of 2 that can issue now will lose to both X0 and X1.
1835 - When an instruction is scheduled, model_recompute updates the model
1836 schedule with the new pressures (some of which might now exceed the
1837 original maximum pressure MP). model_update_limit_points then searches
1838 for the new point of maximum pressure, if not already known. */
1840 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1841 from surrounding debug information. */
1843 ";;\t\t+------------------------------------------------------\n"
1845 /* Information about the pressure on a particular register class at a
1846 particular point of the model schedule. */
1847 struct model_pressure_data
{
1848 /* The pressure at this point of the model schedule, or -1 if the
1849 point is associated with an instruction that has already been
1853 /* The maximum pressure during or after this point of the model schedule. */
1857 /* Per-instruction information that is used while building the model
1858 schedule. Here, "schedule" refers to the model schedule rather
1859 than the main schedule. */
1860 struct model_insn_info
{
1861 /* The instruction itself. */
1864 /* If this instruction is in model_worklist, these fields link to the
1865 previous (higher-priority) and next (lower-priority) instructions
1867 struct model_insn_info
*prev
;
1868 struct model_insn_info
*next
;
1870 /* While constructing the schedule, QUEUE_INDEX describes whether an
1871 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1872 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1873 old_queue records the value that QUEUE_INDEX had before scheduling
1874 started, so that we can restore it once the schedule is complete. */
1877 /* The relative importance of an unscheduled instruction. Higher
1878 values indicate greater importance. */
1879 unsigned int model_priority
;
1881 /* The length of the longest path of satisfied true dependencies
1882 that leads to this instruction. */
1885 /* The length of the longest path of dependencies of any kind
1886 that leads from this instruction. */
1889 /* The number of predecessor nodes that must still be scheduled. */
1890 int unscheduled_preds
;
1893 /* Information about the pressure limit for a particular register class.
1894 This structure is used when applying a model schedule to the main
1896 struct model_pressure_limit
{
1897 /* The maximum register pressure seen in the original model schedule. */
1900 /* The maximum register pressure seen in the current model schedule
1901 (which excludes instructions that have already been scheduled). */
1904 /* The point of the current model schedule at which PRESSURE is first
1905 reached. It is set to -1 if the value needs to be recomputed. */
1909 /* Describes a particular way of measuring register pressure. */
1910 struct model_pressure_group
{
1911 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1912 struct model_pressure_limit limits
[N_REG_CLASSES
];
1914 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1915 on register class ira_pressure_classes[PCI] at point POINT of the
1916 current model schedule. A POINT of model_num_insns describes the
1917 pressure at the end of the schedule. */
1918 struct model_pressure_data
*model
;
1921 /* Index POINT gives the instruction at point POINT of the model schedule.
1922 This array doesn't change during main scheduling. */
1923 static vec
<rtx_insn
*> model_schedule
;
1925 /* The list of instructions in the model worklist, sorted in order of
1926 decreasing priority. */
1927 static struct model_insn_info
*model_worklist
;
1929 /* Index I describes the instruction with INSN_LUID I. */
1930 static struct model_insn_info
*model_insns
;
1932 /* The number of instructions in the model schedule. */
1933 static int model_num_insns
;
1935 /* The index of the first instruction in model_schedule that hasn't yet been
1936 added to the main schedule, or model_num_insns if all of them have. */
1937 static int model_curr_point
;
1939 /* Describes the pressure before each instruction in the model schedule. */
1940 static struct model_pressure_group model_before_pressure
;
1942 /* The first unused model_priority value (as used in model_insn_info). */
1943 static unsigned int model_next_priority
;
1946 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1947 at point POINT of the model schedule. */
1948 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1949 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1951 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1952 after point POINT of the model schedule. */
1953 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1954 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1956 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1957 of the model schedule. */
1958 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1959 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1961 /* Information about INSN that is used when creating the model schedule. */
1962 #define MODEL_INSN_INFO(INSN) \
1963 (&model_insns[INSN_LUID (INSN)])
1965 /* The instruction at point POINT of the model schedule. */
1966 #define MODEL_INSN(POINT) \
1967 (model_schedule[POINT])
1970 /* Return INSN's index in the model schedule, or model_num_insns if it
1971 doesn't belong to that schedule. */
1974 model_index (rtx_insn
*insn
)
1976 if (INSN_MODEL_INDEX (insn
) == 0)
1977 return model_num_insns
;
1978 return INSN_MODEL_INDEX (insn
) - 1;
1981 /* Make sure that GROUP->limits is up-to-date for the current point
1982 of the model schedule. */
1985 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1987 int pci
, max_pressure
, point
;
1989 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1991 /* We may have passed the final point at which the pressure in
1992 group->limits[pci].pressure was reached. Update the limit if so. */
1993 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1994 group
->limits
[pci
].pressure
= max_pressure
;
1996 /* Find the point at which MAX_PRESSURE is first reached. We need
1997 to search in three cases:
1999 - We've already moved past the previous pressure point.
2000 In this case we search forward from model_curr_point.
2002 - We scheduled the previous point of maximum pressure ahead of
2003 its position in the model schedule, but doing so didn't bring
2004 the pressure point earlier. In this case we search forward
2005 from that previous pressure point.
2007 - Scheduling an instruction early caused the maximum pressure
2008 to decrease. In this case we will have set the pressure
2009 point to -1, and we search forward from model_curr_point. */
2010 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
2011 while (point
< model_num_insns
2012 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
2014 group
->limits
[pci
].point
= point
;
2016 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
2017 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
2021 /* Make sure that all register-pressure limits are up-to-date for the
2022 current position in the model schedule. */
2025 model_update_limit_points (void)
2027 model_update_limit_points_in_group (&model_before_pressure
);
2030 /* Return the model_index of the last unscheduled use in chain USE
2031 outside of USE's instruction. Return -1 if there are no other uses,
2032 or model_num_insns if the register is live at the end of the block. */
2035 model_last_use_except (struct reg_use_data
*use
)
2037 struct reg_use_data
*next
;
2041 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2042 if (NONDEBUG_INSN_P (next
->insn
)
2043 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2045 index
= model_index (next
->insn
);
2046 if (index
== model_num_insns
)
2047 return model_num_insns
;
2054 /* An instruction with model_index POINT has just been scheduled, and it
2055 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2056 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2057 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2060 model_start_update_pressure (struct model_pressure_group
*group
,
2061 int point
, int pci
, int delta
)
2063 int next_max_pressure
;
2065 if (point
== model_num_insns
)
2067 /* The instruction wasn't part of the model schedule; it was moved
2068 from a different block. Update the pressure for the end of
2069 the model schedule. */
2070 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2071 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2075 /* Record that this instruction has been scheduled. Nothing now
2076 changes between POINT and POINT + 1, so get the maximum pressure
2077 from the latter. If the maximum pressure decreases, the new
2078 pressure point may be before POINT. */
2079 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2080 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2081 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2083 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2084 if (group
->limits
[pci
].point
== point
)
2085 group
->limits
[pci
].point
= -1;
2090 /* Record that scheduling a later instruction has changed the pressure
2091 at point POINT of the model schedule by DELTA (which might be 0).
2092 Update GROUP accordingly. Return nonzero if these changes might
2093 trigger changes to previous points as well. */
2096 model_update_pressure (struct model_pressure_group
*group
,
2097 int point
, int pci
, int delta
)
2099 int ref_pressure
, max_pressure
, next_max_pressure
;
2101 /* If POINT hasn't yet been scheduled, update its pressure. */
2102 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2103 if (ref_pressure
>= 0 && delta
!= 0)
2105 ref_pressure
+= delta
;
2106 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2108 /* Check whether the maximum pressure in the overall schedule
2109 has increased. (This means that the MODEL_MAX_PRESSURE of
2110 every point <= POINT will need to increase too; see below.) */
2111 if (group
->limits
[pci
].pressure
< ref_pressure
)
2112 group
->limits
[pci
].pressure
= ref_pressure
;
2114 /* If we are at maximum pressure, and the maximum pressure
2115 point was previously unknown or later than POINT,
2116 bring it forward. */
2117 if (group
->limits
[pci
].pressure
== ref_pressure
2118 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2119 group
->limits
[pci
].point
= point
;
2121 /* If POINT used to be the point of maximum pressure, but isn't
2122 any longer, we need to recalculate it using a forward walk. */
2123 if (group
->limits
[pci
].pressure
> ref_pressure
2124 && group
->limits
[pci
].point
== point
)
2125 group
->limits
[pci
].point
= -1;
2128 /* Update the maximum pressure at POINT. Changes here might also
2129 affect the maximum pressure at POINT - 1. */
2130 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2131 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2132 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2134 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2140 /* INSN has just been scheduled. Update the model schedule accordingly. */
2143 model_recompute (rtx_insn
*insn
)
2148 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2149 struct reg_use_data
*use
;
2150 struct reg_pressure_data
*reg_pressure
;
2151 int delta
[N_REG_CLASSES
];
2152 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2153 unsigned int i
, num_uses
, num_pending_births
;
2156 /* The destinations of INSN were previously live from POINT onwards, but are
2157 now live from model_curr_point onwards. Set up DELTA accordingly. */
2158 point
= model_index (insn
);
2159 reg_pressure
= INSN_REG_PRESSURE (insn
);
2160 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2162 cl
= ira_pressure_classes
[pci
];
2163 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2166 /* Record which registers previously died at POINT, but which now die
2167 before POINT. Adjust DELTA so that it represents the effect of
2168 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2169 registers that will be born in the range [model_curr_point, POINT). */
2171 num_pending_births
= 0;
2172 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2174 new_last
= model_last_use_except (use
);
2175 if (new_last
< point
)
2177 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2178 uses
[num_uses
].last_use
= new_last
;
2179 uses
[num_uses
].regno
= use
->regno
;
2180 /* This register is no longer live after POINT - 1. */
2181 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2184 num_pending_births
++;
2188 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2189 Also set each group pressure limit for POINT. */
2190 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2192 cl
= ira_pressure_classes
[pci
];
2193 model_start_update_pressure (&model_before_pressure
,
2194 point
, pci
, delta
[cl
]);
2197 /* Walk the model schedule backwards, starting immediately before POINT. */
2199 if (point
!= model_curr_point
)
2203 insn
= MODEL_INSN (point
);
2204 queue
= QUEUE_INDEX (insn
);
2206 if (queue
!= QUEUE_SCHEDULED
)
2208 /* DELTA describes the effect of the move on the register pressure
2209 after POINT. Make it describe the effect on the pressure
2212 while (i
< num_uses
)
2214 if (uses
[i
].last_use
== point
)
2216 /* This register is now live again. */
2217 mark_regno_birth_or_death (NULL
, delta
,
2218 uses
[i
].regno
, true);
2220 /* Remove this use from the array. */
2221 uses
[i
] = uses
[num_uses
- 1];
2223 num_pending_births
--;
2229 if (sched_verbose
>= 5)
2233 fprintf (sched_dump
, MODEL_BAR
);
2234 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2236 fprintf (sched_dump
, MODEL_BAR
);
2240 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2241 point
, INSN_UID (insn
),
2242 str_pattern_slim (PATTERN (insn
)));
2243 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2245 cl
= ira_pressure_classes
[pci
];
2246 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2248 fprintf (sched_dump
, " %s:[%d->%d]",
2249 reg_class_names
[ira_pressure_classes
[pci
]],
2250 ref_pressure
, ref_pressure
+ delta
[cl
]);
2252 fprintf (sched_dump
, "\n");
2256 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2257 might have changed as well. */
2258 mix
= num_pending_births
;
2259 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2261 cl
= ira_pressure_classes
[pci
];
2263 mix
|= model_update_pressure (&model_before_pressure
,
2264 point
, pci
, delta
[cl
]);
2267 while (mix
&& point
> model_curr_point
);
2270 fprintf (sched_dump
, MODEL_BAR
);
2273 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2274 check whether the insn's pattern needs restoring. */
2276 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2278 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2281 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2283 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2284 gcc_assert (next
== DEP_CON (dep
));
2288 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2289 if (desc
->insn
!= next
)
2291 gcc_assert (*desc
->loc
== desc
->orig
);
2298 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2299 pressure on CL from P to P'. We use this to calculate a "base ECC",
2300 baseECC (CL, X), for each pressure class CL and each instruction X.
2301 Supposing X changes the pressure on CL from P to P', and that the
2302 maximum pressure on CL in the current model schedule is MP', then:
2304 * if X occurs before or at the next point of maximum pressure in
2305 the model schedule and P' > MP', then:
2307 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2309 The idea is that the pressure after scheduling a fixed set of
2310 instructions -- in this case, the set up to and including the
2311 next maximum pressure point -- is going to be the same regardless
2312 of the order; we simply want to keep the intermediate pressure
2313 under control. Thus X has a cost of zero unless scheduling it
2314 now would exceed MP'.
2316 If all increases in the set are by the same amount, no zero-cost
2317 instruction will ever cause the pressure to exceed MP'. However,
2318 if X is instead moved past an instruction X' with pressure in the
2319 range (MP' - (P' - P), MP'), the pressure at X' will increase
2320 beyond MP'. Since baseECC is very much a heuristic anyway,
2321 it doesn't seem worth the overhead of tracking cases like these.
2323 The cost of exceeding MP' is always based on the original maximum
2324 pressure MP. This is so that going 2 registers over the original
2325 limit has the same cost regardless of whether it comes from two
2326 separate +1 deltas or from a single +2 delta.
2328 * if X occurs after the next point of maximum pressure in the model
2329 schedule and P' > P, then:
2331 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2333 That is, if we move X forward across a point of maximum pressure,
2334 and if X increases the pressure by P' - P, then we conservatively
2335 assume that scheduling X next would increase the maximum pressure
2336 by P' - P. Again, the cost of doing this is based on the original
2337 maximum pressure MP, for the same reason as above.
2339 * if P' < P, P > MP, and X occurs at or after the next point of
2340 maximum pressure, then:
2342 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2344 That is, if we have already exceeded the original maximum pressure MP,
2345 and if X might reduce the maximum pressure again -- or at least push
2346 it further back, and thus allow more scheduling freedom -- it is given
2347 a negative cost to reflect the improvement.
2353 In this case, X is not expected to affect the maximum pressure MP',
2354 so it has zero cost.
2356 We then create a combined value baseECC (X) that is the sum of
2357 baseECC (CL, X) for each pressure class CL.
2359 baseECC (X) could itself be used as the ECC value described above.
2360 However, this is often too conservative, in the sense that it
2361 tends to make high-priority instructions that increase pressure
2362 wait too long in cases where introducing a spill would be better.
2363 For this reason the final ECC is a priority-adjusted form of
2364 baseECC (X). Specifically, we calculate:
2366 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2367 baseP = MAX { P (X) | baseECC (X) <= 0 }
2371 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2373 Thus an instruction's effect on pressure is ignored if it has a high
2374 enough priority relative to the ones that don't increase pressure.
2375 Negative values of baseECC (X) do not increase the priority of X
2376 itself, but they do make it harder for other instructions to
2377 increase the pressure further.
2379 This pressure cost is deliberately timid. The intention has been
2380 to choose a heuristic that rarely interferes with the normal list
2381 scheduler in cases where that scheduler would produce good code.
2382 We simply want to curb some of its worst excesses. */
2384 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2386 Here we use the very simplistic cost model that every register above
2387 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2388 measures instead, such as one based on MEMORY_MOVE_COST. However:
2390 (1) In order for an instruction to be scheduled, the higher cost
2391 would need to be justified in a single saving of that many stalls.
2392 This is overly pessimistic, because the benefit of spilling is
2393 often to avoid a sequence of several short stalls rather than
2396 (2) The cost is still arbitrary. Because we are not allocating
2397 registers during scheduling, we have no way of knowing for
2398 sure how many memory accesses will be required by each spill,
2399 where the spills will be placed within the block, or even
2400 which block(s) will contain the spills.
2402 So a higher cost than 1 is often too conservative in practice,
2403 forcing blocks to contain unnecessary stalls instead of spill code.
2404 The simple cost below seems to be the best compromise. It reduces
2405 the interference with the normal list scheduler, which helps make
2406 it more suitable for a default-on option. */
2409 model_spill_cost (int cl
, int from
, int to
)
2411 from
= MAX (from
, sched_class_regs_num
[cl
]);
2412 return MAX (to
, from
) - from
;
2415 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2416 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2420 model_excess_group_cost (struct model_pressure_group
*group
,
2421 int point
, int pci
, int delta
)
2425 cl
= ira_pressure_classes
[pci
];
2426 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2428 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2429 curr_reg_pressure
[cl
] + delta
);
2430 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2435 if (point
> group
->limits
[pci
].point
)
2436 pressure
= group
->limits
[pci
].pressure
+ delta
;
2438 pressure
= curr_reg_pressure
[cl
] + delta
;
2440 if (pressure
> group
->limits
[pci
].pressure
)
2441 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2448 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2452 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2454 int point
, pci
, cl
, cost
, this_cost
, delta
;
2455 struct reg_pressure_data
*insn_reg_pressure
;
2456 int insn_death
[N_REG_CLASSES
];
2458 calculate_reg_deaths (insn
, insn_death
);
2459 point
= model_index (insn
);
2460 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2464 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2465 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2467 /* Sum up the individual costs for each register class. */
2468 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2470 cl
= ira_pressure_classes
[pci
];
2471 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2472 this_cost
= model_excess_group_cost (&model_before_pressure
,
2476 fprintf (sched_dump
, " %s:[%d base cost %d]",
2477 reg_class_names
[cl
], delta
, this_cost
);
2481 fprintf (sched_dump
, "\n");
2486 /* Dump the next points of maximum pressure for GROUP. */
2489 model_dump_pressure_points (struct model_pressure_group
*group
)
2493 fprintf (sched_dump
, ";;\t\t| pressure points");
2494 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2496 cl
= ira_pressure_classes
[pci
];
2497 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2498 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2499 if (group
->limits
[pci
].point
< model_num_insns
)
2500 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2501 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2503 fprintf (sched_dump
, "end]");
2505 fprintf (sched_dump
, "\n");
2508 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2511 model_set_excess_costs (rtx_insn
**insns
, int count
)
2513 int i
, cost
, priority_base
, priority
;
2516 /* Record the baseECC value for each instruction in the model schedule,
2517 except that negative costs are converted to zero ones now rather than
2518 later. Do not assign a cost to debug instructions, since they must
2519 not change code-generation decisions. Experiments suggest we also
2520 get better results by not assigning a cost to instructions from
2523 Set PRIORITY_BASE to baseP in the block comment above. This is the
2524 maximum priority of the "cheap" instructions, which should always
2525 include the next model instruction. */
2528 for (i
= 0; i
< count
; i
++)
2529 if (INSN_MODEL_INDEX (insns
[i
]))
2531 if (sched_verbose
>= 6 && !print_p
)
2533 fprintf (sched_dump
, MODEL_BAR
);
2534 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2535 model_dump_pressure_points (&model_before_pressure
);
2536 fprintf (sched_dump
, MODEL_BAR
);
2539 cost
= model_excess_cost (insns
[i
], print_p
);
2542 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2543 priority_base
= MAX (priority_base
, priority
);
2546 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2549 fprintf (sched_dump
, MODEL_BAR
);
2551 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2553 for (i
= 0; i
< count
; i
++)
2555 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2556 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2557 if (cost
> 0 && priority
> priority_base
)
2559 cost
+= priority_base
- priority
;
2560 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2566 /* Enum of rank_for_schedule heuristic decisions. */
2568 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2569 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2570 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2571 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2572 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_N
};
2574 /* Corresponding strings for print outs. */
2575 static const char *rfs_str
[RFS_N
] = {
2576 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2577 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2578 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2579 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2580 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2582 /* Statistical breakdown of rank_for_schedule decisions. */
2583 typedef struct { unsigned stats
[RFS_N
]; } rank_for_schedule_stats_t
;
2584 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2586 /* Return the result of comparing insns TMP and TMP2 and update
2587 Rank_For_Schedule statistics. */
2589 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2591 ++rank_for_schedule_stats
.stats
[decision
];
2593 INSN_LAST_RFS_WIN (tmp
) = decision
;
2594 else if (result
> 0)
2595 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2601 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2602 keeping normal insns in original order. */
2605 rank_for_schedule_debug (const void *x
, const void *y
)
2607 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2608 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2610 /* Schedule debug insns as early as possible. */
2611 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2613 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2615 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2616 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2618 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2621 /* Returns a positive value if x is preferred; returns a negative value if
2622 y is preferred. Should never return 0, since that will make the sort
2626 rank_for_schedule (const void *x
, const void *y
)
2628 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2629 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2630 int tmp_class
, tmp2_class
;
2631 int val
, priority_val
, info_val
, diff
;
2633 if (live_range_shrinkage_p
)
2635 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2637 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2638 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2639 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2640 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2641 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2642 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2643 /* Sort by INSN_LUID (original insn order), so that we make the
2644 sort stable. This minimizes instruction movement, thus
2645 minimizing sched's effect on debugging and cross-jumping. */
2646 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2647 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2650 /* The insn in a schedule group should be issued the first. */
2651 if (flag_sched_group_heuristic
&&
2652 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2653 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2656 /* Make sure that priority of TMP and TMP2 are initialized. */
2657 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2661 /* The instruction that has the same fusion priority as the last
2662 instruction is the instruction we picked next. If that is not
2663 the case, we sort ready list firstly by fusion priority, then
2664 by priority, and at last by INSN_LUID. */
2665 int a
= INSN_FUSION_PRIORITY (tmp
);
2666 int b
= INSN_FUSION_PRIORITY (tmp2
);
2669 if (last_nondebug_scheduled_insn
2670 && !NOTE_P (last_nondebug_scheduled_insn
)
2671 && BLOCK_FOR_INSN (tmp
)
2672 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2673 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2675 if (a
!= last
&& b
!= last
)
2679 a
= INSN_PRIORITY (tmp
);
2680 b
= INSN_PRIORITY (tmp2
);
2683 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2685 return rfs_result (RFS_FUSION
,
2686 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2690 gcc_assert (last_nondebug_scheduled_insn
2691 && !NOTE_P (last_nondebug_scheduled_insn
));
2692 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2694 a
= abs (INSN_PRIORITY (tmp
) - last
);
2695 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2697 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2699 return rfs_result (RFS_FUSION
,
2700 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2703 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2705 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2708 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2710 /* Prefer insn whose scheduling results in the smallest register
2712 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2714 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2715 - insn_delay (tmp2
))))
2716 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2719 if (sched_pressure
!= SCHED_PRESSURE_NONE
2720 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2721 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2723 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2724 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2727 /* If we are doing backtracking in this schedule, prefer insns that
2728 have forward dependencies with negative cost against an insn that
2729 was already scheduled. */
2730 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2732 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2734 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2737 /* Prefer insn with higher priority. */
2738 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2740 if (flag_sched_critical_path_heuristic
&& priority_val
)
2741 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2743 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2745 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2750 /* Prefer speculative insn with greater dependencies weakness. */
2751 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2757 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2759 dw1
= ds_weak (ds1
);
2763 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2765 dw2
= ds_weak (ds2
);
2770 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2771 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2774 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2775 if (flag_sched_rank_heuristic
&& info_val
)
2776 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2778 /* Compare insns based on their relation to the last scheduled
2780 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2784 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2786 /* Classify the instructions into three classes:
2787 1) Data dependent on last schedule insn.
2788 2) Anti/Output dependent on last scheduled insn.
2789 3) Independent of last scheduled insn, or has latency of one.
2790 Choose the insn from the highest numbered class if different. */
2791 dep1
= sd_find_dep_between (last
, tmp
, true);
2793 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2795 else if (/* Data dependence. */
2796 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2801 dep2
= sd_find_dep_between (last
, tmp2
, true);
2803 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2805 else if (/* Data dependence. */
2806 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2811 if ((val
= tmp2_class
- tmp_class
))
2812 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2815 /* Prefer instructions that occur earlier in the model schedule. */
2816 if (sched_pressure
== SCHED_PRESSURE_MODEL
2817 && INSN_BB (tmp
) == target_bb
&& INSN_BB (tmp2
) == target_bb
)
2819 diff
= model_index (tmp
) - model_index (tmp2
);
2820 gcc_assert (diff
!= 0);
2821 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2824 /* Prefer the insn which has more later insns that depend on it.
2825 This gives the scheduler more freedom when scheduling later
2826 instructions at the expense of added register pressure. */
2828 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2829 - dep_list_size (tmp
, SD_LIST_FORW
));
2831 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2832 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2834 /* If insns are equally good, sort by INSN_LUID (original insn order),
2835 so that we make the sort stable. This minimizes instruction movement,
2836 thus minimizing sched's effect on debugging and cross-jumping. */
2837 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2840 /* Resort the array A in which only element at index N may be out of order. */
2842 HAIFA_INLINE
static void
2843 swap_sort (rtx_insn
**a
, int n
)
2845 rtx_insn
*insn
= a
[n
- 1];
2848 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2856 /* Add INSN to the insn queue so that it can be executed at least
2857 N_CYCLES after the currently executing insn. Preserve insns
2858 chain for debugging purposes. REASON will be printed in debugging
2861 HAIFA_INLINE
static void
2862 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2864 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2865 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2868 gcc_assert (n_cycles
<= max_insn_queue_index
);
2869 gcc_assert (!DEBUG_INSN_P (insn
));
2871 insn_queue
[next_q
] = link
;
2874 if (sched_verbose
>= 2)
2876 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2877 (*current_sched_info
->print_insn
) (insn
, 0));
2879 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2882 QUEUE_INDEX (insn
) = next_q
;
2884 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2886 new_tick
= clock_var
+ n_cycles
;
2887 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2888 INSN_TICK (insn
) = new_tick
;
2890 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2891 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2893 must_backtrack
= true;
2894 if (sched_verbose
>= 2)
2895 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2900 /* Remove INSN from queue. */
2902 queue_remove (rtx_insn
*insn
)
2904 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2905 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2907 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2910 /* Return a pointer to the bottom of the ready list, i.e. the insn
2911 with the lowest priority. */
2914 ready_lastpos (struct ready_list
*ready
)
2916 gcc_assert (ready
->n_ready
>= 1);
2917 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2920 /* Add an element INSN to the ready list so that it ends up with the
2921 lowest/highest priority depending on FIRST_P. */
2923 HAIFA_INLINE
static void
2924 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2928 if (ready
->first
== ready
->n_ready
)
2930 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2931 ready_lastpos (ready
),
2932 ready
->n_ready
* sizeof (rtx
));
2933 ready
->first
= ready
->veclen
- 1;
2935 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2939 if (ready
->first
== ready
->veclen
- 1)
2942 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2943 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2944 ready_lastpos (ready
),
2945 ready
->n_ready
* sizeof (rtx
));
2946 ready
->first
= ready
->veclen
- 2;
2948 ready
->vec
[++(ready
->first
)] = insn
;
2952 if (DEBUG_INSN_P (insn
))
2955 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2956 QUEUE_INDEX (insn
) = QUEUE_READY
;
2958 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2959 && INSN_EXACT_TICK (insn
) < clock_var
)
2961 must_backtrack
= true;
2965 /* Remove the element with the highest priority from the ready list and
2968 HAIFA_INLINE
static rtx_insn
*
2969 ready_remove_first (struct ready_list
*ready
)
2973 gcc_assert (ready
->n_ready
);
2974 t
= ready
->vec
[ready
->first
--];
2976 if (DEBUG_INSN_P (t
))
2978 /* If the queue becomes empty, reset it. */
2979 if (ready
->n_ready
== 0)
2980 ready
->first
= ready
->veclen
- 1;
2982 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2983 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2988 /* The following code implements multi-pass scheduling for the first
2989 cycle. In other words, we will try to choose ready insn which
2990 permits to start maximum number of insns on the same cycle. */
2992 /* Return a pointer to the element INDEX from the ready. INDEX for
2993 insn with the highest priority is 0, and the lowest priority has
2997 ready_element (struct ready_list
*ready
, int index
)
2999 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3001 return ready
->vec
[ready
->first
- index
];
3004 /* Remove the element INDEX from the ready list and return it. INDEX
3005 for insn with the highest priority is 0, and the lowest priority
3008 HAIFA_INLINE
static rtx_insn
*
3009 ready_remove (struct ready_list
*ready
, int index
)
3015 return ready_remove_first (ready
);
3016 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3017 t
= ready
->vec
[ready
->first
- index
];
3019 if (DEBUG_INSN_P (t
))
3021 for (i
= index
; i
< ready
->n_ready
; i
++)
3022 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3023 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3027 /* Remove INSN from the ready list. */
3029 ready_remove_insn (rtx_insn
*insn
)
3033 for (i
= 0; i
< readyp
->n_ready
; i
++)
3034 if (ready_element (readyp
, i
) == insn
)
3036 ready_remove (readyp
, i
);
3042 /* Calculate difference of two statistics set WAS and NOW.
3043 Result returned in WAS. */
3045 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3046 const rank_for_schedule_stats_t
*now
)
3048 for (int i
= 0; i
< RFS_N
; ++i
)
3049 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3052 /* Print rank_for_schedule statistics. */
3054 print_rank_for_schedule_stats (const char *prefix
,
3055 const rank_for_schedule_stats_t
*stats
,
3056 struct ready_list
*ready
)
3058 for (int i
= 0; i
< RFS_N
; ++i
)
3059 if (stats
->stats
[i
])
3061 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3064 /* Print out insns that won due to RFS_<I>. */
3066 rtx_insn
**p
= ready_lastpos (ready
);
3068 fprintf (sched_dump
, ":");
3069 /* Start with 1 since least-priority insn didn't have any wins. */
3070 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3071 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3072 fprintf (sched_dump
, " %s",
3073 (*current_sched_info
->print_insn
) (p
[j
], 0));
3075 fprintf (sched_dump
, "\n");
3079 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3082 ready_sort_debug (struct ready_list
*ready
)
3085 rtx_insn
**first
= ready_lastpos (ready
);
3087 for (i
= 0; i
< ready
->n_ready
; ++i
)
3088 if (!DEBUG_INSN_P (first
[i
]))
3089 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3091 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3094 /* Sort non-debug insns in the ready list READY by ascending priority.
3095 Assumes that all debug insns are separated from the real insns. */
3097 ready_sort_real (struct ready_list
*ready
)
3100 rtx_insn
**first
= ready_lastpos (ready
);
3101 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3103 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3104 for (i
= 0; i
< n_ready_real
; ++i
)
3105 setup_insn_reg_pressure_info (first
[i
]);
3106 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3107 && model_curr_point
< model_num_insns
)
3108 model_set_excess_costs (first
, n_ready_real
);
3110 rank_for_schedule_stats_t stats1
;
3111 if (sched_verbose
>= 4)
3112 stats1
= rank_for_schedule_stats
;
3114 if (n_ready_real
== 2)
3115 swap_sort (first
, n_ready_real
);
3116 else if (n_ready_real
> 2)
3117 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3119 if (sched_verbose
>= 4)
3121 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3122 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3126 /* Sort the ready list READY by ascending priority. */
3128 ready_sort (struct ready_list
*ready
)
3130 if (ready
->n_debug
> 0)
3131 ready_sort_debug (ready
);
3133 ready_sort_real (ready
);
3136 /* PREV is an insn that is ready to execute. Adjust its priority if that
3137 will help shorten or lengthen register lifetimes as appropriate. Also
3138 provide a hook for the target to tweak itself. */
3140 HAIFA_INLINE
static void
3141 adjust_priority (rtx_insn
*prev
)
3143 /* ??? There used to be code here to try and estimate how an insn
3144 affected register lifetimes, but it did it by looking at REG_DEAD
3145 notes, which we removed in schedule_region. Nor did it try to
3146 take into account register pressure or anything useful like that.
3148 Revisit when we have a machine model to work with and not before. */
3150 if (targetm
.sched
.adjust_priority
)
3151 INSN_PRIORITY (prev
) =
3152 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3155 /* Advance DFA state STATE on one cycle. */
3157 advance_state (state_t state
)
3159 if (targetm
.sched
.dfa_pre_advance_cycle
)
3160 targetm
.sched
.dfa_pre_advance_cycle ();
3162 if (targetm
.sched
.dfa_pre_cycle_insn
)
3163 state_transition (state
,
3164 targetm
.sched
.dfa_pre_cycle_insn ());
3166 state_transition (state
, NULL
);
3168 if (targetm
.sched
.dfa_post_cycle_insn
)
3169 state_transition (state
,
3170 targetm
.sched
.dfa_post_cycle_insn ());
3172 if (targetm
.sched
.dfa_post_advance_cycle
)
3173 targetm
.sched
.dfa_post_advance_cycle ();
3176 /* Advance time on one cycle. */
3177 HAIFA_INLINE
static void
3178 advance_one_cycle (void)
3180 advance_state (curr_state
);
3181 if (sched_verbose
>= 4)
3182 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3185 /* Update register pressure after scheduling INSN. */
3187 update_register_pressure (rtx_insn
*insn
)
3189 struct reg_use_data
*use
;
3190 struct reg_set_data
*set
;
3192 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3194 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3195 if (dying_use_p (use
))
3196 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3198 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3199 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3203 /* Set up or update (if UPDATE_P) max register pressure (see its
3204 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3205 after insn AFTER. */
3207 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3212 static int max_reg_pressure
[N_REG_CLASSES
];
3214 save_reg_pressure ();
3215 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3216 max_reg_pressure
[ira_pressure_classes
[i
]]
3217 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3218 for (insn
= NEXT_INSN (after
);
3219 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3220 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3221 insn
= NEXT_INSN (insn
))
3222 if (NONDEBUG_INSN_P (insn
))
3225 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3227 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3228 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3231 INSN_MAX_REG_PRESSURE (insn
)[i
]
3232 = max_reg_pressure
[ira_pressure_classes
[i
]];
3235 if (update_p
&& eq_p
)
3237 update_register_pressure (insn
);
3238 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3239 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3240 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3241 max_reg_pressure
[ira_pressure_classes
[i
]]
3242 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3244 restore_reg_pressure ();
3247 /* Update the current register pressure after scheduling INSN. Update
3248 also max register pressure for unscheduled insns of the current
3251 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3254 int before
[N_REG_CLASSES
];
3256 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3257 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3258 update_register_pressure (insn
);
3259 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3260 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3262 if (i
< ira_pressure_classes_num
)
3263 setup_insn_max_reg_pressure (insn
, true);
3266 /* Set up register pressure at the beginning of basic block BB whose
3267 insns starting after insn AFTER. Set up also max register pressure
3268 for all insns of the basic block. */
3270 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3272 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3273 initiate_bb_reg_pressure_info (bb
);
3274 setup_insn_max_reg_pressure (after
, false);
3277 /* If doing predication while scheduling, verify whether INSN, which
3278 has just been scheduled, clobbers the conditions of any
3279 instructions that must be predicated in order to break their
3280 dependencies. If so, remove them from the queues so that they will
3281 only be scheduled once their control dependency is resolved. */
3284 check_clobbered_conditions (rtx_insn
*insn
)
3289 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3292 find_all_hard_reg_sets (insn
, &t
, true);
3295 for (i
= 0; i
< ready
.n_ready
; i
++)
3297 rtx_insn
*x
= ready_element (&ready
, i
);
3298 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3300 ready_remove_insn (x
);
3304 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3306 rtx_insn_list
*link
;
3307 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3310 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3312 rtx_insn
*x
= link
->insn ();
3313 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3322 /* Return (in order):
3324 - positive if INSN adversely affects the pressure on one
3327 - negative if INSN reduces the pressure on one register class
3329 - 0 if INSN doesn't affect the pressure on any register class. */
3332 model_classify_pressure (struct model_insn_info
*insn
)
3334 struct reg_pressure_data
*reg_pressure
;
3335 int death
[N_REG_CLASSES
];
3338 calculate_reg_deaths (insn
->insn
, death
);
3339 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3341 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3343 cl
= ira_pressure_classes
[pci
];
3344 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3346 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3351 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3354 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3356 unsigned int height1
, height2
;
3357 unsigned int priority1
, priority2
;
3359 /* Prefer instructions with a higher model priority. */
3360 if (insn1
->model_priority
!= insn2
->model_priority
)
3361 return insn1
->model_priority
> insn2
->model_priority
;
3363 /* Combine the length of the longest path of satisfied true dependencies
3364 that leads to each instruction (depth) with the length of the longest
3365 path of any dependencies that leads from the instruction (alap).
3366 Prefer instructions with the greatest combined length. If the combined
3367 lengths are equal, prefer instructions with the greatest depth.
3369 The idea is that, if we have a set S of "equal" instructions that each
3370 have ALAP value X, and we pick one such instruction I, any true-dependent
3371 successors of I that have ALAP value X - 1 should be preferred over S.
3372 This encourages the schedule to be "narrow" rather than "wide".
3373 However, if I is a low-priority instruction that we decided to
3374 schedule because of its model_classify_pressure, and if there
3375 is a set of higher-priority instructions T, the aforementioned
3376 successors of I should not have the edge over T. */
3377 height1
= insn1
->depth
+ insn1
->alap
;
3378 height2
= insn2
->depth
+ insn2
->alap
;
3379 if (height1
!= height2
)
3380 return height1
> height2
;
3381 if (insn1
->depth
!= insn2
->depth
)
3382 return insn1
->depth
> insn2
->depth
;
3384 /* We have no real preference between INSN1 an INSN2 as far as attempts
3385 to reduce pressure go. Prefer instructions with higher priorities. */
3386 priority1
= INSN_PRIORITY (insn1
->insn
);
3387 priority2
= INSN_PRIORITY (insn2
->insn
);
3388 if (priority1
!= priority2
)
3389 return priority1
> priority2
;
3391 /* Use the original rtl sequence as a tie-breaker. */
3392 return insn1
< insn2
;
3395 /* Add INSN to the model worklist immediately after PREV. Add it to the
3396 beginning of the list if PREV is null. */
3399 model_add_to_worklist_at (struct model_insn_info
*insn
,
3400 struct model_insn_info
*prev
)
3402 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3403 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3408 insn
->next
= prev
->next
;
3413 insn
->next
= model_worklist
;
3414 model_worklist
= insn
;
3417 insn
->next
->prev
= insn
;
3420 /* Remove INSN from the model worklist. */
3423 model_remove_from_worklist (struct model_insn_info
*insn
)
3425 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3426 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3429 insn
->prev
->next
= insn
->next
;
3431 model_worklist
= insn
->next
;
3433 insn
->next
->prev
= insn
->prev
;
3436 /* Add INSN to the model worklist. Start looking for a suitable position
3437 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3438 insns either side. A null PREV indicates the beginning of the list and
3439 a null NEXT indicates the end. */
3442 model_add_to_worklist (struct model_insn_info
*insn
,
3443 struct model_insn_info
*prev
,
3444 struct model_insn_info
*next
)
3448 count
= MAX_SCHED_READY_INSNS
;
3449 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3455 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3457 while (count
> 0 && next
&& model_order_p (next
, insn
))
3463 model_add_to_worklist_at (insn
, prev
);
3466 /* INSN may now have a higher priority (in the model_order_p sense)
3467 than before. Move it up the worklist if necessary. */
3470 model_promote_insn (struct model_insn_info
*insn
)
3472 struct model_insn_info
*prev
;
3476 count
= MAX_SCHED_READY_INSNS
;
3477 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3482 if (prev
!= insn
->prev
)
3484 model_remove_from_worklist (insn
);
3485 model_add_to_worklist_at (insn
, prev
);
3489 /* Add INSN to the end of the model schedule. */
3492 model_add_to_schedule (rtx_insn
*insn
)
3496 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3497 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3499 point
= model_schedule
.length ();
3500 model_schedule
.quick_push (insn
);
3501 INSN_MODEL_INDEX (insn
) = point
+ 1;
3504 /* Analyze the instructions that are to be scheduled, setting up
3505 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3506 instructions to model_worklist. */
3509 model_analyze_insns (void)
3511 rtx_insn
*start
, *end
, *iter
;
3512 sd_iterator_def sd_it
;
3514 struct model_insn_info
*insn
, *con
;
3516 model_num_insns
= 0;
3517 start
= PREV_INSN (current_sched_info
->next_tail
);
3518 end
= current_sched_info
->prev_head
;
3519 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3520 if (NONDEBUG_INSN_P (iter
))
3522 insn
= MODEL_INSN_INFO (iter
);
3524 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3526 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3527 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3528 insn
->alap
= con
->alap
+ 1;
3531 insn
->old_queue
= QUEUE_INDEX (iter
);
3532 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3534 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3535 if (insn
->unscheduled_preds
== 0)
3536 model_add_to_worklist (insn
, NULL
, model_worklist
);
3542 /* The global state describes the register pressure at the start of the
3543 model schedule. Initialize GROUP accordingly. */
3546 model_init_pressure_group (struct model_pressure_group
*group
)
3550 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3552 cl
= ira_pressure_classes
[pci
];
3553 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3554 group
->limits
[pci
].point
= 0;
3556 /* Use index model_num_insns to record the state after the last
3557 instruction in the model schedule. */
3558 group
->model
= XNEWVEC (struct model_pressure_data
,
3559 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3562 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3563 Update the maximum pressure for the whole schedule. */
3566 model_record_pressure (struct model_pressure_group
*group
,
3567 int point
, int pci
, int pressure
)
3569 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3570 if (group
->limits
[pci
].pressure
< pressure
)
3572 group
->limits
[pci
].pressure
= pressure
;
3573 group
->limits
[pci
].point
= point
;
3577 /* INSN has just been added to the end of the model schedule. Record its
3578 register-pressure information. */
3581 model_record_pressures (struct model_insn_info
*insn
)
3583 struct reg_pressure_data
*reg_pressure
;
3584 int point
, pci
, cl
, delta
;
3585 int death
[N_REG_CLASSES
];
3587 point
= model_index (insn
->insn
);
3588 if (sched_verbose
>= 2)
3592 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3593 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3595 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3596 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3597 insn
->depth
+ insn
->alap
, insn
->depth
,
3598 INSN_PRIORITY (insn
->insn
),
3599 str_pattern_slim (PATTERN (insn
->insn
)));
3601 calculate_reg_deaths (insn
->insn
, death
);
3602 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3603 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3605 cl
= ira_pressure_classes
[pci
];
3606 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3607 if (sched_verbose
>= 2)
3608 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3609 curr_reg_pressure
[cl
], delta
);
3610 model_record_pressure (&model_before_pressure
, point
, pci
,
3611 curr_reg_pressure
[cl
]);
3613 if (sched_verbose
>= 2)
3614 fprintf (sched_dump
, "\n");
3617 /* All instructions have been added to the model schedule. Record the
3618 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3621 model_record_final_pressures (struct model_pressure_group
*group
)
3623 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3625 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3627 /* Record the final pressure for this class. */
3628 cl
= ira_pressure_classes
[pci
];
3629 point
= model_num_insns
;
3630 ref_pressure
= curr_reg_pressure
[cl
];
3631 model_record_pressure (group
, point
, pci
, ref_pressure
);
3633 /* Record the original maximum pressure. */
3634 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3636 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3637 max_pressure
= ref_pressure
;
3638 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3642 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3643 max_pressure
= MAX (max_pressure
, ref_pressure
);
3644 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3649 /* Update all successors of INSN, given that INSN has just been scheduled. */
3652 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3654 sd_iterator_def sd_it
;
3655 struct model_insn_info
*con
;
3658 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3660 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3661 /* Ignore debug instructions, and instructions from other blocks. */
3664 con
->unscheduled_preds
--;
3666 /* Update the depth field of each true-dependent successor.
3667 Increasing the depth gives them a higher priority than
3669 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3671 con
->depth
= insn
->depth
+ 1;
3672 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3673 model_promote_insn (con
);
3676 /* If this is a true dependency, or if there are no remaining
3677 dependencies for CON (meaning that CON only had non-true
3678 dependencies), make sure that CON is on the worklist.
3679 We don't bother otherwise because it would tend to fill the
3680 worklist with a lot of low-priority instructions that are not
3681 yet ready to issue. */
3682 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3683 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3684 model_add_to_worklist (con
, insn
, insn
->next
);
3689 /* Give INSN a higher priority than any current instruction, then give
3690 unscheduled predecessors of INSN a higher priority still. If any of
3691 those predecessors are not on the model worklist, do the same for its
3692 predecessors, and so on. */
3695 model_promote_predecessors (struct model_insn_info
*insn
)
3697 struct model_insn_info
*pro
, *first
;
3698 sd_iterator_def sd_it
;
3701 if (sched_verbose
>= 7)
3702 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3703 INSN_UID (insn
->insn
), model_next_priority
);
3704 insn
->model_priority
= model_next_priority
++;
3705 model_remove_from_worklist (insn
);
3706 model_add_to_worklist_at (insn
, NULL
);
3711 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3713 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3714 /* The first test is to ignore debug instructions, and instructions
3715 from other blocks. */
3717 && pro
->model_priority
!= model_next_priority
3718 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3720 pro
->model_priority
= model_next_priority
;
3721 if (sched_verbose
>= 7)
3722 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3723 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3725 /* PRO is already in the worklist, but it now has
3726 a higher priority than before. Move it at the
3727 appropriate place. */
3728 model_remove_from_worklist (pro
);
3729 model_add_to_worklist (pro
, NULL
, model_worklist
);
3733 /* PRO isn't in the worklist. Recursively process
3734 its predecessors until we find one that is. */
3745 if (sched_verbose
>= 7)
3746 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3747 model_next_priority
++;
3750 /* Pick one instruction from model_worklist and process it. */
3753 model_choose_insn (void)
3755 struct model_insn_info
*insn
, *fallback
;
3758 if (sched_verbose
>= 7)
3760 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3761 insn
= model_worklist
;
3762 count
= MAX_SCHED_READY_INSNS
;
3763 while (count
> 0 && insn
)
3765 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3766 INSN_UID (insn
->insn
), insn
->model_priority
,
3767 insn
->depth
+ insn
->alap
, insn
->depth
,
3768 INSN_PRIORITY (insn
->insn
));
3774 /* Look for a ready instruction whose model_classify_priority is zero
3775 or negative, picking the highest-priority one. Adding such an
3776 instruction to the schedule now should do no harm, and may actually
3779 Failing that, see whether there is an instruction with the highest
3780 extant model_priority that is not yet ready, but which would reduce
3781 pressure if it became ready. This is designed to catch cases like:
3783 (set (mem (reg R1)) (reg R2))
3785 where the instruction is the last remaining use of R1 and where the
3786 value of R2 is not yet available (or vice versa). The death of R1
3787 means that this instruction already reduces pressure. It is of
3788 course possible that the computation of R2 involves other registers
3789 that are hard to kill, but such cases are rare enough for this
3790 heuristic to be a win in general.
3792 Failing that, just pick the highest-priority instruction in the
3794 count
= MAX_SCHED_READY_INSNS
;
3795 insn
= model_worklist
;
3799 if (count
== 0 || !insn
)
3801 insn
= fallback
? fallback
: model_worklist
;
3804 if (insn
->unscheduled_preds
)
3806 if (model_worklist
->model_priority
== insn
->model_priority
3808 && model_classify_pressure (insn
) < 0)
3813 if (model_classify_pressure (insn
) <= 0)
3820 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3822 if (insn
->unscheduled_preds
)
3823 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3824 INSN_UID (insn
->insn
));
3826 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3827 INSN_UID (insn
->insn
));
3829 if (insn
->unscheduled_preds
)
3830 /* INSN isn't yet ready to issue. Give all its predecessors the
3831 highest priority. */
3832 model_promote_predecessors (insn
);
3835 /* INSN is ready. Add it to the end of model_schedule and
3836 process its successors. */
3837 model_add_successors_to_worklist (insn
);
3838 model_remove_from_worklist (insn
);
3839 model_add_to_schedule (insn
->insn
);
3840 model_record_pressures (insn
);
3841 update_register_pressure (insn
->insn
);
3845 /* Restore all QUEUE_INDEXs to the values that they had before
3846 model_start_schedule was called. */
3849 model_reset_queue_indices (void)
3854 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3855 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3858 /* We have calculated the model schedule and spill costs. Print a summary
3862 model_dump_pressure_summary (void)
3866 fprintf (sched_dump
, ";; Pressure summary:");
3867 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3869 cl
= ira_pressure_classes
[pci
];
3870 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3871 model_before_pressure
.limits
[pci
].pressure
);
3873 fprintf (sched_dump
, "\n\n");
3876 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3877 scheduling region. */
3880 model_start_schedule (basic_block bb
)
3882 model_next_priority
= 1;
3883 model_schedule
.create (sched_max_luid
);
3884 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3886 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3887 initiate_reg_pressure_info (df_get_live_in (bb
));
3889 model_analyze_insns ();
3890 model_init_pressure_group (&model_before_pressure
);
3891 while (model_worklist
)
3892 model_choose_insn ();
3893 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3894 if (sched_verbose
>= 2)
3895 fprintf (sched_dump
, "\n");
3897 model_record_final_pressures (&model_before_pressure
);
3898 model_reset_queue_indices ();
3900 XDELETEVEC (model_insns
);
3902 model_curr_point
= 0;
3903 initiate_reg_pressure_info (df_get_live_in (bb
));
3904 if (sched_verbose
>= 1)
3905 model_dump_pressure_summary ();
3908 /* Free the information associated with GROUP. */
3911 model_finalize_pressure_group (struct model_pressure_group
*group
)
3913 XDELETEVEC (group
->model
);
3916 /* Free the information created by model_start_schedule. */
3919 model_end_schedule (void)
3921 model_finalize_pressure_group (&model_before_pressure
);
3922 model_schedule
.release ();
3925 /* Prepare reg pressure scheduling for basic block BB. */
3927 sched_pressure_start_bb (basic_block bb
)
3929 /* Set the number of available registers for each class taking into account
3930 relative probability of current basic block versus function prologue and
3932 * If the basic block executes much more often than the prologue/epilogue
3933 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3934 nil, so the effective number of available registers is
3935 (ira_class_hard_regs_num[cl] - 0).
3936 * If the basic block executes as often as the prologue/epilogue,
3937 then spill in the block is as costly as in the prologue, so the effective
3938 number of available registers is
3939 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3940 Note that all-else-equal, we prefer to spill in the prologue, since that
3941 allows "extra" registers for other basic blocks of the function.
3942 * If the basic block is on the cold path of the function and executes
3943 rarely, then we should always prefer to spill in the block, rather than
3944 in the prologue/epilogue. The effective number of available register is
3945 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3948 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
3949 int bb_freq
= bb
->frequency
;
3953 if (entry_freq
== 0)
3954 entry_freq
= bb_freq
= 1;
3956 if (bb_freq
< entry_freq
)
3957 bb_freq
= entry_freq
;
3959 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3961 enum reg_class cl
= ira_pressure_classes
[i
];
3962 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
];
3963 sched_class_regs_num
[cl
]
3964 -= (call_used_regs_num
[cl
] * entry_freq
) / bb_freq
;
3968 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3969 model_start_schedule (bb
);
3972 /* A structure that holds local state for the loop in schedule_block. */
3973 struct sched_block_state
3975 /* True if no real insns have been scheduled in the current cycle. */
3976 bool first_cycle_insn_p
;
3977 /* True if a shadow insn has been scheduled in the current cycle, which
3978 means that no more normal insns can be issued. */
3979 bool shadows_only_p
;
3980 /* True if we're winding down a modulo schedule, which means that we only
3981 issue insns with INSN_EXACT_TICK set. */
3982 bool modulo_epilogue
;
3983 /* Initialized with the machine's issue rate every cycle, and updated
3984 by calls to the variable_issue hook. */
3988 /* INSN is the "currently executing insn". Launch each insn which was
3989 waiting on INSN. READY is the ready list which contains the insns
3990 that are ready to fire. CLOCK is the current cycle. The function
3991 returns necessary cycle advance after issuing the insn (it is not
3992 zero for insns in a schedule group). */
3995 schedule_insn (rtx_insn
*insn
)
3997 sd_iterator_def sd_it
;
4002 if (sched_verbose
>= 1)
4004 struct reg_pressure_data
*pressure_info
;
4005 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
4006 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
4007 str_pattern_slim (PATTERN (insn
)));
4009 if (recog_memoized (insn
) < 0)
4010 fprintf (sched_dump
, "nothing");
4012 print_reservation (sched_dump
, insn
);
4013 pressure_info
= INSN_REG_PRESSURE (insn
);
4014 if (pressure_info
!= NULL
)
4016 fputc (':', sched_dump
);
4017 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4018 fprintf (sched_dump
, "%s%s%+d(%d)",
4019 scheduled_insns
.length () > 1
4021 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4022 reg_class_names
[ira_pressure_classes
[i
]],
4023 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4025 if (sched_pressure
== SCHED_PRESSURE_MODEL
4026 && model_curr_point
< model_num_insns
4027 && model_index (insn
) == model_curr_point
)
4028 fprintf (sched_dump
, ":model %d", model_curr_point
);
4029 fputc ('\n', sched_dump
);
4032 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4033 update_reg_and_insn_max_reg_pressure (insn
);
4035 /* Scheduling instruction should have all its dependencies resolved and
4036 should have been removed from the ready list. */
4037 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4039 /* Reset debug insns invalidated by moving this insn. */
4040 if (MAY_HAVE_DEBUG_INSNS
&& !DEBUG_INSN_P (insn
))
4041 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4042 sd_iterator_cond (&sd_it
, &dep
);)
4044 rtx_insn
*dbg
= DEP_PRO (dep
);
4045 struct reg_use_data
*use
, *next
;
4047 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4049 sd_iterator_next (&sd_it
);
4053 gcc_assert (DEBUG_INSN_P (dbg
));
4055 if (sched_verbose
>= 6)
4056 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4059 /* ??? Rather than resetting the debug insn, we might be able
4060 to emit a debug temp before the just-scheduled insn, but
4061 this would involve checking that the expression at the
4062 point of the debug insn is equivalent to the expression
4063 before the just-scheduled insn. They might not be: the
4064 expression in the debug insn may depend on other insns not
4065 yet scheduled that set MEMs, REGs or even other debug
4066 insns. It's not clear that attempting to preserve debug
4067 information in these cases is worth the effort, given how
4068 uncommon these resets are and the likelihood that the debug
4069 temps introduced won't survive the schedule change. */
4070 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4071 df_insn_rescan (dbg
);
4073 /* Unknown location doesn't use any registers. */
4074 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4076 struct reg_use_data
*prev
= use
;
4078 /* Remove use from the cyclic next_regno_use chain first. */
4079 while (prev
->next_regno_use
!= use
)
4080 prev
= prev
->next_regno_use
;
4081 prev
->next_regno_use
= use
->next_regno_use
;
4082 next
= use
->next_insn_use
;
4085 INSN_REG_USE_LIST (dbg
) = NULL
;
4087 /* We delete rather than resolve these deps, otherwise we
4088 crash in sched_free_deps(), because forward deps are
4089 expected to be released before backward deps. */
4090 sd_delete_dep (sd_it
);
4093 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4094 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4096 if (sched_pressure
== SCHED_PRESSURE_MODEL
4097 && model_curr_point
< model_num_insns
4098 && NONDEBUG_INSN_P (insn
))
4100 if (model_index (insn
) == model_curr_point
)
4103 while (model_curr_point
< model_num_insns
4104 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4105 == QUEUE_SCHEDULED
));
4107 model_recompute (insn
);
4108 model_update_limit_points ();
4109 update_register_pressure (insn
);
4110 if (sched_verbose
>= 2)
4111 print_curr_reg_pressure ();
4114 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4115 if (INSN_TICK (insn
) > clock_var
)
4116 /* INSN has been prematurely moved from the queue to the ready list.
4117 This is possible only if following flags are set. */
4118 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4120 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4121 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4122 INSN_TICK (insn
) = clock_var
;
4124 check_clobbered_conditions (insn
);
4126 /* Update dependent instructions. First, see if by scheduling this insn
4127 now we broke a dependence in a way that requires us to change another
4129 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4130 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4132 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4133 rtx_insn
*pro
= DEP_PRO (dep
);
4134 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4135 && desc
!= NULL
&& desc
->insn
== pro
)
4136 apply_replacement (dep
, false);
4139 /* Go through and resolve forward dependencies. */
4140 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4141 sd_iterator_cond (&sd_it
, &dep
);)
4143 rtx_insn
*next
= DEP_CON (dep
);
4144 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4146 /* Resolve the dependence between INSN and NEXT.
4147 sd_resolve_dep () moves current dep to another list thus
4148 advancing the iterator. */
4149 sd_resolve_dep (sd_it
);
4153 if (must_restore_pattern_p (next
, dep
))
4154 restore_pattern (dep
, false);
4158 /* Don't bother trying to mark next as ready if insn is a debug
4159 insn. If insn is the last hard dependency, it will have
4160 already been discounted. */
4161 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4164 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4168 effective_cost
= try_ready (next
);
4170 if (effective_cost
>= 0
4171 && SCHED_GROUP_P (next
)
4172 && advance
< effective_cost
)
4173 advance
= effective_cost
;
4176 /* Check always has only one forward dependence (to the first insn in
4177 the recovery block), therefore, this will be executed only once. */
4179 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4180 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4184 /* Annotate the instruction with issue information -- TImode
4185 indicates that the instruction is expected not to be able
4186 to issue on the same cycle as the previous insn. A machine
4187 may use this information to decide how the instruction should
4190 && GET_CODE (PATTERN (insn
)) != USE
4191 && GET_CODE (PATTERN (insn
)) != CLOBBER
4192 && !DEBUG_INSN_P (insn
))
4194 if (reload_completed
)
4195 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4196 last_clock_var
= clock_var
;
4199 if (nonscheduled_insns_begin
!= NULL_RTX
)
4200 /* Indicate to debug counters that INSN is scheduled. */
4201 nonscheduled_insns_begin
= insn
;
4206 /* Functions for handling of notes. */
4208 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4210 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4212 rtx_insn
*from_start
;
4214 /* It's easy when have nothing to concat. */
4215 if (from_end
== NULL
)
4218 /* It's also easy when destination is empty. */
4219 if (*to_endp
== NULL
)
4221 *to_endp
= from_end
;
4225 from_start
= from_end
;
4226 while (PREV_INSN (from_start
) != NULL
)
4227 from_start
= PREV_INSN (from_start
);
4229 SET_PREV_INSN (from_start
) = *to_endp
;
4230 SET_NEXT_INSN (*to_endp
) = from_start
;
4231 *to_endp
= from_end
;
4234 /* Delete notes between HEAD and TAIL and put them in the chain
4235 of notes ended by NOTE_LIST. */
4237 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4239 rtx_insn
*next_tail
, *insn
, *next
;
4242 if (head
== tail
&& !INSN_P (head
))
4245 next_tail
= NEXT_INSN (tail
);
4246 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4248 next
= NEXT_INSN (insn
);
4252 switch (NOTE_KIND (insn
))
4254 case NOTE_INSN_BASIC_BLOCK
:
4257 case NOTE_INSN_EPILOGUE_BEG
:
4261 add_reg_note (next
, REG_SAVE_NOTE
,
4262 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4270 /* Add the note to list that ends at NOTE_LIST. */
4271 SET_PREV_INSN (insn
) = note_list
;
4272 SET_NEXT_INSN (insn
) = NULL_RTX
;
4274 SET_NEXT_INSN (note_list
) = insn
;
4279 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4283 /* A structure to record enough data to allow us to backtrack the scheduler to
4284 a previous state. */
4285 struct haifa_saved_data
4287 /* Next entry on the list. */
4288 struct haifa_saved_data
*next
;
4290 /* Backtracking is associated with scheduling insns that have delay slots.
4291 DELAY_PAIR points to the structure that contains the insns involved, and
4292 the number of cycles between them. */
4293 struct delay_pair
*delay_pair
;
4295 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4296 void *fe_saved_data
;
4297 /* Data used by the backend. */
4298 void *be_saved_data
;
4300 /* Copies of global state. */
4301 int clock_var
, last_clock_var
;
4302 struct ready_list ready
;
4305 rtx_insn
*last_scheduled_insn
;
4306 rtx_insn
*last_nondebug_scheduled_insn
;
4307 rtx_insn
*nonscheduled_insns_begin
;
4308 int cycle_issued_insns
;
4310 /* Copies of state used in the inner loop of schedule_block. */
4311 struct sched_block_state sched_block
;
4313 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4314 to 0 when restoring. */
4316 rtx_insn_list
**insn_queue
;
4318 /* Describe pattern replacements that occurred since this backtrack point
4320 vec
<dep_t
> replacement_deps
;
4321 vec
<int> replace_apply
;
4323 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4325 vec
<dep_t
> next_cycle_deps
;
4326 vec
<int> next_cycle_apply
;
4329 /* A record, in reverse order, of all scheduled insns which have delay slots
4330 and may require backtracking. */
4331 static struct haifa_saved_data
*backtrack_queue
;
4333 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4336 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4338 sd_iterator_def sd_it
;
4340 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4342 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4346 /* Save the current scheduler state so that we can backtrack to it
4347 later if necessary. PAIR gives the insns that make it necessary to
4348 save this point. SCHED_BLOCK is the local state of schedule_block
4349 that need to be saved. */
4351 save_backtrack_point (struct delay_pair
*pair
,
4352 struct sched_block_state sched_block
)
4355 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4357 save
->curr_state
= xmalloc (dfa_state_size
);
4358 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4360 save
->ready
.first
= ready
.first
;
4361 save
->ready
.n_ready
= ready
.n_ready
;
4362 save
->ready
.n_debug
= ready
.n_debug
;
4363 save
->ready
.veclen
= ready
.veclen
;
4364 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4365 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4367 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4368 save
->q_size
= q_size
;
4369 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4371 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4372 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4375 save
->clock_var
= clock_var
;
4376 save
->last_clock_var
= last_clock_var
;
4377 save
->cycle_issued_insns
= cycle_issued_insns
;
4378 save
->last_scheduled_insn
= last_scheduled_insn
;
4379 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4380 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4382 save
->sched_block
= sched_block
;
4384 save
->replacement_deps
.create (0);
4385 save
->replace_apply
.create (0);
4386 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4387 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4389 if (current_sched_info
->save_state
)
4390 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4392 if (targetm
.sched
.alloc_sched_context
)
4394 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4395 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4398 save
->be_saved_data
= NULL
;
4400 save
->delay_pair
= pair
;
4402 save
->next
= backtrack_queue
;
4403 backtrack_queue
= save
;
4407 mark_backtrack_feeds (pair
->i2
, 1);
4408 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4409 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4410 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4411 pair
= pair
->next_same_i1
;
4415 /* Walk the ready list and all queues. If any insns have unresolved backwards
4416 dependencies, these must be cancelled deps, broken by predication. Set or
4417 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4420 toggle_cancelled_flags (bool set
)
4423 sd_iterator_def sd_it
;
4426 if (ready
.n_ready
> 0)
4428 rtx_insn
**first
= ready_lastpos (&ready
);
4429 for (i
= 0; i
< ready
.n_ready
; i
++)
4430 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4431 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4434 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4436 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4439 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4441 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4442 rtx_insn_list
*link
;
4443 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4445 rtx_insn
*insn
= link
->insn ();
4446 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4447 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4450 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4452 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4458 /* Undo the replacements that have occurred after backtrack point SAVE
4461 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4463 while (!save
->replacement_deps
.is_empty ())
4465 dep_t dep
= save
->replacement_deps
.pop ();
4466 int apply_p
= save
->replace_apply
.pop ();
4469 restore_pattern (dep
, true);
4471 apply_replacement (dep
, true);
4473 save
->replacement_deps
.release ();
4474 save
->replace_apply
.release ();
4477 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4478 Restore their dependencies to an unresolved state, and mark them as
4482 unschedule_insns_until (rtx_insn
*insn
)
4484 auto_vec
<rtx_insn
*> recompute_vec
;
4486 /* Make two passes over the insns to be unscheduled. First, we clear out
4487 dependencies and other trivial bookkeeping. */
4491 sd_iterator_def sd_it
;
4494 last
= scheduled_insns
.pop ();
4496 /* This will be changed by restore_backtrack_point if the insn is in
4498 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4500 INSN_TICK (last
) = INVALID_TICK
;
4502 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4503 modulo_insns_scheduled
--;
4505 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4506 sd_iterator_cond (&sd_it
, &dep
);)
4508 rtx_insn
*con
= DEP_CON (dep
);
4509 sd_unresolve_dep (sd_it
);
4510 if (!MUST_RECOMPUTE_SPEC_P (con
))
4512 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4513 recompute_vec
.safe_push (con
);
4521 /* A second pass, to update ready and speculation status for insns
4522 depending on the unscheduled ones. The first pass must have
4523 popped the scheduled_insns vector up to the point where we
4524 restart scheduling, as recompute_todo_spec requires it to be
4526 while (!recompute_vec
.is_empty ())
4530 con
= recompute_vec
.pop ();
4531 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4532 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4534 TODO_SPEC (con
) = HARD_DEP
;
4535 INSN_TICK (con
) = INVALID_TICK
;
4536 if (PREDICATED_PAT (con
) != NULL_RTX
)
4537 haifa_change_pattern (con
, ORIG_PAT (con
));
4539 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4540 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4544 /* Restore scheduler state from the topmost entry on the backtracking queue.
4545 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4546 overwrite with the saved data.
4547 The caller must already have called unschedule_insns_until. */
4550 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4553 struct haifa_saved_data
*save
= backtrack_queue
;
4555 backtrack_queue
= save
->next
;
4557 if (current_sched_info
->restore_state
)
4558 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4560 if (targetm
.sched
.alloc_sched_context
)
4562 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4563 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4566 /* Do this first since it clobbers INSN_TICK of the involved
4568 undo_replacements_for_backtrack (save
);
4570 /* Clear the QUEUE_INDEX of everything in the ready list or one
4572 if (ready
.n_ready
> 0)
4574 rtx_insn
**first
= ready_lastpos (&ready
);
4575 for (i
= 0; i
< ready
.n_ready
; i
++)
4577 rtx_insn
*insn
= first
[i
];
4578 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4579 INSN_TICK (insn
) = INVALID_TICK
;
4582 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4584 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4586 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4588 rtx_insn
*x
= link
->insn ();
4589 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4590 INSN_TICK (x
) = INVALID_TICK
;
4592 free_INSN_LIST_list (&insn_queue
[q
]);
4596 ready
= save
->ready
;
4598 if (ready
.n_ready
> 0)
4600 rtx_insn
**first
= ready_lastpos (&ready
);
4601 for (i
= 0; i
< ready
.n_ready
; i
++)
4603 rtx_insn
*insn
= first
[i
];
4604 QUEUE_INDEX (insn
) = QUEUE_READY
;
4605 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4606 INSN_TICK (insn
) = save
->clock_var
;
4611 q_size
= save
->q_size
;
4612 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4614 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4616 insn_queue
[q
] = save
->insn_queue
[q
];
4618 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4620 rtx_insn
*x
= link
->insn ();
4621 QUEUE_INDEX (x
) = i
;
4622 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4623 INSN_TICK (x
) = save
->clock_var
+ i
;
4626 free (save
->insn_queue
);
4628 toggle_cancelled_flags (true);
4630 clock_var
= save
->clock_var
;
4631 last_clock_var
= save
->last_clock_var
;
4632 cycle_issued_insns
= save
->cycle_issued_insns
;
4633 last_scheduled_insn
= save
->last_scheduled_insn
;
4634 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4635 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4637 *psched_block
= save
->sched_block
;
4639 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4640 free (save
->curr_state
);
4642 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4644 gcc_assert (next_cycle_replace_deps
.is_empty ());
4645 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4646 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4650 for (save
= backtrack_queue
; save
; save
= save
->next
)
4652 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4656 /* Discard all data associated with the topmost entry in the backtrack
4657 queue. If RESET_TICK is false, we just want to free the data. If true,
4658 we are doing this because we discovered a reason to backtrack. In the
4659 latter case, also reset the INSN_TICK for the shadow insn. */
4661 free_topmost_backtrack_point (bool reset_tick
)
4663 struct haifa_saved_data
*save
= backtrack_queue
;
4666 backtrack_queue
= save
->next
;
4670 struct delay_pair
*pair
= save
->delay_pair
;
4673 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4674 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4675 pair
= pair
->next_same_i1
;
4677 undo_replacements_for_backtrack (save
);
4681 save
->replacement_deps
.release ();
4682 save
->replace_apply
.release ();
4685 if (targetm
.sched
.free_sched_context
)
4686 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4687 if (current_sched_info
->restore_state
)
4688 free (save
->fe_saved_data
);
4689 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4690 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4691 free (save
->insn_queue
);
4692 free (save
->curr_state
);
4693 free (save
->ready
.vec
);
4697 /* Free the entire backtrack queue. */
4699 free_backtrack_queue (void)
4701 while (backtrack_queue
)
4702 free_topmost_backtrack_point (false);
4705 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4706 may have to postpone the replacement until the start of the next cycle,
4707 at which point we will be called again with IMMEDIATELY true. This is
4708 only done for machines which have instruction packets with explicit
4709 parallelism however. */
4711 apply_replacement (dep_t dep
, bool immediately
)
4713 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4714 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4716 next_cycle_replace_deps
.safe_push (dep
);
4717 next_cycle_apply
.safe_push (1);
4723 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4726 if (sched_verbose
>= 5)
4727 fprintf (sched_dump
, "applying replacement for insn %d\n",
4728 INSN_UID (desc
->insn
));
4730 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4731 gcc_assert (success
);
4733 update_insn_after_change (desc
->insn
);
4734 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4735 fix_tick_ready (desc
->insn
);
4737 if (backtrack_queue
!= NULL
)
4739 backtrack_queue
->replacement_deps
.safe_push (dep
);
4740 backtrack_queue
->replace_apply
.safe_push (1);
4745 /* We have determined that a pattern involved in DEP must be restored.
4746 If IMMEDIATELY is false, we may have to postpone the replacement
4747 until the start of the next cycle, at which point we will be called
4748 again with IMMEDIATELY true. */
4750 restore_pattern (dep_t dep
, bool immediately
)
4752 rtx_insn
*next
= DEP_CON (dep
);
4753 int tick
= INSN_TICK (next
);
4755 /* If we already scheduled the insn, the modified version is
4757 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4760 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4762 next_cycle_replace_deps
.safe_push (dep
);
4763 next_cycle_apply
.safe_push (0);
4768 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4770 if (sched_verbose
>= 5)
4771 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4773 haifa_change_pattern (next
, ORIG_PAT (next
));
4777 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4780 if (sched_verbose
>= 5)
4781 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4782 INSN_UID (desc
->insn
));
4783 tick
= INSN_TICK (desc
->insn
);
4785 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4786 gcc_assert (success
);
4787 update_insn_after_change (desc
->insn
);
4788 if (backtrack_queue
!= NULL
)
4790 backtrack_queue
->replacement_deps
.safe_push (dep
);
4791 backtrack_queue
->replace_apply
.safe_push (0);
4794 INSN_TICK (next
) = tick
;
4795 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4798 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4799 TODO_SPEC (next
) = 0;
4800 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4801 TODO_SPEC (next
) = HARD_DEP
;
4804 /* Perform pattern replacements that were queued up until the next
4807 perform_replacements_new_cycle (void)
4811 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4813 int apply_p
= next_cycle_apply
[i
];
4815 apply_replacement (dep
, true);
4817 restore_pattern (dep
, true);
4819 next_cycle_replace_deps
.truncate (0);
4820 next_cycle_apply
.truncate (0);
4823 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4824 instructions we've previously encountered, a set bit prevents
4825 recursion. BUDGET is a limit on how far ahead we look, it is
4826 reduced on recursive calls. Return true if we produced a good
4827 estimate, or false if we exceeded the budget. */
4829 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4831 sd_iterator_def sd_it
;
4833 int earliest
= INSN_TICK (insn
);
4835 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4837 rtx_insn
*pro
= DEP_PRO (dep
);
4840 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4843 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4844 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4847 int cost
= dep_cost (dep
);
4850 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4852 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4855 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4856 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4857 if (earliest
== INVALID_TICK
|| t
> earliest
)
4861 bitmap_set_bit (processed
, INSN_LUID (insn
));
4862 INSN_TICK_ESTIMATE (insn
) = earliest
;
4866 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4867 infinite resources) the cycle in which the delayed shadow can be issued.
4868 Return the number of cycles that must pass before the real insn can be
4869 issued in order to meet this constraint. */
4871 estimate_shadow_tick (struct delay_pair
*p
)
4873 bitmap_head processed
;
4876 bitmap_initialize (&processed
, 0);
4878 cutoff
= !estimate_insn_tick (&processed
, p
->i2
,
4879 max_insn_queue_index
+ pair_delay (p
));
4880 bitmap_clear (&processed
);
4882 return max_insn_queue_index
;
4883 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4889 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4890 recursively resolve all its forward dependencies. */
4892 resolve_dependencies (rtx_insn
*insn
)
4894 sd_iterator_def sd_it
;
4897 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4898 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4899 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4902 if (sched_verbose
>= 4)
4903 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4905 if (QUEUE_INDEX (insn
) >= 0)
4906 queue_remove (insn
);
4908 scheduled_insns
.safe_push (insn
);
4910 /* Update dependent instructions. */
4911 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4912 sd_iterator_cond (&sd_it
, &dep
);)
4914 rtx_insn
*next
= DEP_CON (dep
);
4916 if (sched_verbose
>= 4)
4917 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4920 /* Resolve the dependence between INSN and NEXT.
4921 sd_resolve_dep () moves current dep to another list thus
4922 advancing the iterator. */
4923 sd_resolve_dep (sd_it
);
4925 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4927 resolve_dependencies (next
);
4930 /* Check always has only one forward dependence (to the first insn in
4931 the recovery block), therefore, this will be executed only once. */
4933 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4939 /* Return the head and tail pointers of ebb starting at BEG and ending
4942 get_ebb_head_tail (basic_block beg
, basic_block end
,
4943 rtx_insn
**headp
, rtx_insn
**tailp
)
4945 rtx_insn
*beg_head
= BB_HEAD (beg
);
4946 rtx_insn
* beg_tail
= BB_END (beg
);
4947 rtx_insn
* end_head
= BB_HEAD (end
);
4948 rtx_insn
* end_tail
= BB_END (end
);
4950 /* Don't include any notes or labels at the beginning of the BEG
4951 basic block, or notes at the end of the END basic blocks. */
4953 if (LABEL_P (beg_head
))
4954 beg_head
= NEXT_INSN (beg_head
);
4956 while (beg_head
!= beg_tail
)
4957 if (NOTE_P (beg_head
))
4958 beg_head
= NEXT_INSN (beg_head
);
4959 else if (DEBUG_INSN_P (beg_head
))
4961 rtx_insn
* note
, *next
;
4963 for (note
= NEXT_INSN (beg_head
);
4967 next
= NEXT_INSN (note
);
4970 if (sched_verbose
>= 9)
4971 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4973 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4975 if (BLOCK_FOR_INSN (note
) != beg
)
4976 df_insn_change_bb (note
, beg
);
4978 else if (!DEBUG_INSN_P (note
))
4990 end_head
= beg_head
;
4991 else if (LABEL_P (end_head
))
4992 end_head
= NEXT_INSN (end_head
);
4994 while (end_head
!= end_tail
)
4995 if (NOTE_P (end_tail
))
4996 end_tail
= PREV_INSN (end_tail
);
4997 else if (DEBUG_INSN_P (end_tail
))
4999 rtx_insn
* note
, *prev
;
5001 for (note
= PREV_INSN (end_tail
);
5005 prev
= PREV_INSN (note
);
5008 if (sched_verbose
>= 9)
5009 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5011 reorder_insns_nobb (note
, note
, end_tail
);
5013 if (end_tail
== BB_END (end
))
5014 BB_END (end
) = note
;
5016 if (BLOCK_FOR_INSN (note
) != end
)
5017 df_insn_change_bb (note
, end
);
5019 else if (!DEBUG_INSN_P (note
))
5031 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5034 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5036 while (head
!= NEXT_INSN (tail
))
5038 if (!NOTE_P (head
) && !LABEL_P (head
))
5040 head
= NEXT_INSN (head
);
5045 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5046 previously found among the insns. Insert them just before HEAD. */
5048 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5052 rtx_insn
*note_head
= note_list
;
5055 head_bb
= BLOCK_FOR_INSN (head
);
5057 head
= NEXT_INSN (bb_note (head_bb
));
5059 while (PREV_INSN (note_head
))
5061 set_block_for_insn (note_head
, head_bb
);
5062 note_head
= PREV_INSN (note_head
);
5064 /* In the above cycle we've missed this note. */
5065 set_block_for_insn (note_head
, head_bb
);
5067 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5068 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5069 SET_PREV_INSN (head
) = note_list
;
5070 SET_NEXT_INSN (note_list
) = head
;
5072 if (BLOCK_FOR_INSN (head
) != head_bb
)
5073 BB_END (head_bb
) = note_list
;
5081 /* When we know we are going to discard the schedule due to a failed attempt
5082 at modulo scheduling, undo all replacements. */
5084 undo_all_replacements (void)
5089 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5091 sd_iterator_def sd_it
;
5094 /* See if we must undo a replacement. */
5095 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5096 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5098 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5100 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5105 /* Return first non-scheduled insn in the current scheduling block.
5106 This is mostly used for debug-counter purposes. */
5108 first_nonscheduled_insn (void)
5110 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5111 ? nonscheduled_insns_begin
5112 : current_sched_info
->prev_head
);
5116 insn
= next_nonnote_nondebug_insn (insn
);
5118 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5123 /* Move insns that became ready to fire from queue to ready list. */
5126 queue_to_ready (struct ready_list
*ready
)
5129 rtx_insn_list
*link
;
5130 rtx_insn
*skip_insn
;
5132 q_ptr
= NEXT_Q (q_ptr
);
5134 if (dbg_cnt (sched_insn
) == false)
5135 /* If debug counter is activated do not requeue the first
5136 nonscheduled insn. */
5137 skip_insn
= first_nonscheduled_insn ();
5141 /* Add all pending insns that can be scheduled without stalls to the
5143 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5145 insn
= link
->insn ();
5148 if (sched_verbose
>= 2)
5149 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5150 (*current_sched_info
->print_insn
) (insn
, 0));
5152 /* If the ready list is full, delay the insn for 1 cycle.
5153 See the comment in schedule_block for the rationale. */
5154 if (!reload_completed
5155 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5156 || (sched_pressure
== SCHED_PRESSURE_MODEL
5157 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5158 instructions too. */
5159 && model_index (insn
) > (model_curr_point
5160 + MAX_SCHED_READY_INSNS
)))
5161 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5162 && model_curr_point
< model_num_insns
5163 /* Always allow the next model instruction to issue. */
5164 && model_index (insn
) == model_curr_point
)
5165 && !SCHED_GROUP_P (insn
)
5166 && insn
!= skip_insn
)
5168 if (sched_verbose
>= 2)
5169 fprintf (sched_dump
, "keeping in queue, ready full\n");
5170 queue_insn (insn
, 1, "ready full");
5174 ready_add (ready
, insn
, false);
5175 if (sched_verbose
>= 2)
5176 fprintf (sched_dump
, "moving to ready without stalls\n");
5179 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5181 /* If there are no ready insns, stall until one is ready and add all
5182 of the pending insns at that point to the ready list. */
5183 if (ready
->n_ready
== 0)
5187 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5189 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5191 for (; link
; link
= link
->next ())
5193 insn
= link
->insn ();
5196 if (sched_verbose
>= 2)
5197 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5198 (*current_sched_info
->print_insn
) (insn
, 0));
5200 ready_add (ready
, insn
, false);
5201 if (sched_verbose
>= 2)
5202 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5204 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5206 advance_one_cycle ();
5211 advance_one_cycle ();
5214 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5215 clock_var
+= stalls
;
5216 if (sched_verbose
>= 2)
5217 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5222 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5223 prematurely move INSN from the queue to the ready list. Currently,
5224 if a target defines the hook 'is_costly_dependence', this function
5225 uses the hook to check whether there exist any dependences which are
5226 considered costly by the target, between INSN and other insns that
5227 have already been scheduled. Dependences are checked up to Y cycles
5228 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5229 controlling this value.
5230 (Other considerations could be taken into account instead (or in
5231 addition) depending on user flags and target hooks. */
5234 ok_for_early_queue_removal (rtx_insn
*insn
)
5236 if (targetm
.sched
.is_costly_dependence
)
5239 int i
= scheduled_insns
.length ();
5240 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5246 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5248 if (!NOTE_P (prev_insn
))
5252 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5256 cost
= dep_cost (dep
);
5258 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5259 flag_sched_stalled_insns_dep
- n_cycles
))
5264 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5277 /* Remove insns from the queue, before they become "ready" with respect
5278 to FU latency considerations. */
5281 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5284 rtx_insn_list
*link
;
5285 rtx_insn_list
*next_link
;
5286 rtx_insn_list
*prev_link
;
5289 state_t temp_state
= alloca (dfa_state_size
);
5291 int insns_removed
= 0;
5294 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5297 X == 0: There is no limit on how many queued insns can be removed
5298 prematurely. (flag_sched_stalled_insns = -1).
5300 X >= 1: Only X queued insns can be removed prematurely in each
5301 invocation. (flag_sched_stalled_insns = X).
5303 Otherwise: Early queue removal is disabled.
5304 (flag_sched_stalled_insns = 0)
5307 if (! flag_sched_stalled_insns
)
5310 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5312 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5314 if (sched_verbose
> 6)
5315 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5320 next_link
= link
->next ();
5321 insn
= link
->insn ();
5322 if (insn
&& sched_verbose
> 6)
5323 print_rtl_single (sched_dump
, insn
);
5325 memcpy (temp_state
, state
, dfa_state_size
);
5326 if (recog_memoized (insn
) < 0)
5327 /* non-negative to indicate that it's not ready
5328 to avoid infinite Q->R->Q->R... */
5331 cost
= state_transition (temp_state
, insn
);
5333 if (sched_verbose
>= 6)
5334 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5336 move_to_ready
= false;
5339 move_to_ready
= ok_for_early_queue_removal (insn
);
5340 if (move_to_ready
== true)
5342 /* move from Q to R */
5344 ready_add (ready
, insn
, false);
5347 XEXP (prev_link
, 1) = next_link
;
5349 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5351 free_INSN_LIST_node (link
);
5353 if (sched_verbose
>= 2)
5354 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5355 (*current_sched_info
->print_insn
) (insn
, 0));
5358 if (insns_removed
== flag_sched_stalled_insns
)
5359 /* Remove no more than flag_sched_stalled_insns insns
5360 from Q at a time. */
5361 return insns_removed
;
5365 if (move_to_ready
== false)
5372 } /* for stalls.. */
5374 return insns_removed
;
5378 /* Print the ready list for debugging purposes.
5379 If READY_TRY is non-zero then only print insns that max_issue
5382 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5387 if (ready
->n_ready
== 0)
5389 fprintf (sched_dump
, "\n");
5393 p
= ready_lastpos (ready
);
5394 for (i
= 0; i
< ready
->n_ready
; i
++)
5396 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5399 fprintf (sched_dump
, " %s:%d",
5400 (*current_sched_info
->print_insn
) (p
[i
], 0),
5402 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5403 fprintf (sched_dump
, "(cost=%d",
5404 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5405 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5406 if (INSN_TICK (p
[i
]) > clock_var
)
5407 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5408 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5409 fprintf (sched_dump
, ":idx=%d",
5410 model_index (p
[i
]));
5411 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5412 fprintf (sched_dump
, ")");
5414 fprintf (sched_dump
, "\n");
5417 /* Print the ready list. Callable from debugger. */
5419 debug_ready_list (struct ready_list
*ready
)
5421 debug_ready_list_1 (ready
, NULL
);
5424 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5425 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5426 replaces the epilogue note in the correct basic block. */
5428 reemit_notes (rtx_insn
*insn
)
5431 rtx_insn
*last
= insn
;
5433 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5435 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5437 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5439 last
= emit_note_before (note_type
, last
);
5440 remove_note (insn
, note
);
5445 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5447 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5449 if (PREV_INSN (insn
) != last
)
5455 bb
= BLOCK_FOR_INSN (insn
);
5457 /* BB_HEAD is either LABEL or NOTE. */
5458 gcc_assert (BB_HEAD (bb
) != insn
);
5460 if (BB_END (bb
) == insn
)
5461 /* If this is last instruction in BB, move end marker one
5464 /* Jumps are always placed at the end of basic block. */
5465 jump_p
= control_flow_insn_p (insn
);
5468 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5469 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5470 || (common_sched_info
->sched_pass_id
5471 == SCHED_EBB_PASS
));
5473 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5475 BB_END (bb
) = PREV_INSN (insn
);
5478 gcc_assert (BB_END (bb
) != last
);
5481 /* We move the block note along with jump. */
5485 note
= NEXT_INSN (insn
);
5486 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5487 note
= NEXT_INSN (note
);
5491 || BARRIER_P (note
)))
5492 note
= NEXT_INSN (note
);
5494 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5499 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5500 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5502 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5503 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5505 SET_NEXT_INSN (last
) = insn
;
5506 SET_PREV_INSN (insn
) = last
;
5508 bb
= BLOCK_FOR_INSN (last
);
5512 fix_jump_move (insn
);
5514 if (BLOCK_FOR_INSN (insn
) != bb
)
5515 move_block_after_check (insn
);
5517 gcc_assert (BB_END (bb
) == last
);
5520 df_insn_change_bb (insn
, bb
);
5522 /* Update BB_END, if needed. */
5523 if (BB_END (bb
) == last
)
5527 SCHED_GROUP_P (insn
) = 0;
5530 /* Return true if scheduling INSN will finish current clock cycle. */
5532 insn_finishes_cycle_p (rtx_insn
*insn
)
5534 if (SCHED_GROUP_P (insn
))
5535 /* After issuing INSN, rest of the sched_group will be forced to issue
5536 in order. Don't make any plans for the rest of cycle. */
5539 /* Finishing the block will, apparently, finish the cycle. */
5540 if (current_sched_info
->insn_finishes_block_p
5541 && current_sched_info
->insn_finishes_block_p (insn
))
5547 /* Functions to model cache auto-prefetcher.
5549 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5550 memory prefetches if it sees instructions with consequitive memory accesses
5551 in the instruction stream. Details of such hardware units are not published,
5552 so we can only guess what exactly is going on there.
5553 In the scheduler, we model abstract auto-prefetcher. If there are memory
5554 insns in the ready list (or the queue) that have same memory base, but
5555 different offsets, then we delay the insns with larger offsets until insns
5556 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5557 is "1", then we look at the ready list; if it is N>1, then we also look
5558 through N-1 queue entries.
5559 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5560 among its heuristics.
5561 Param value of "-1" disables modelling of the auto-prefetcher. */
5563 /* Initialize autoprefetcher model data for INSN. */
5565 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5567 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5569 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5570 data
->base
= NULL_RTX
;
5572 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5573 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5575 rtx set
= single_set (insn
);
5576 if (set
== NULL_RTX
)
5579 rtx mem
= write
? SET_DEST (set
) : SET_SRC (set
);
5583 struct address_info info
;
5584 decompose_mem_address (&info
, mem
);
5586 /* TODO: Currently only (base+const) addressing is supported. */
5587 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5588 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5591 /* This insn is relevant for auto-prefetcher. */
5592 data
->base
= *info
.base
;
5593 data
->offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5594 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5597 /* Helper function for rank_for_schedule sorting. */
5599 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5601 for (int write
= 0; write
< 2; ++write
)
5603 autopref_multipass_data_t data1
5604 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5605 autopref_multipass_data_t data2
5606 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5608 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5609 autopref_multipass_init (insn1
, write
);
5610 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5613 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5614 autopref_multipass_init (insn2
, write
);
5615 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5618 if (!rtx_equal_p (data1
->base
, data2
->base
))
5621 return data1
->offset
- data2
->offset
;
5627 /* True if header of debug dump was printed. */
5628 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5630 /* Helper for autopref_multipass_dfa_lookahead_guard.
5631 Return "1" if INSN1 should be delayed in favor of INSN2. */
5633 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5634 const rtx_insn
*insn2
, int write
)
5636 autopref_multipass_data_t data1
5637 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5638 autopref_multipass_data_t data2
5639 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5641 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5642 autopref_multipass_init (insn2
, write
);
5643 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5646 if (rtx_equal_p (data1
->base
, data2
->base
)
5647 && data1
->offset
> data2
->offset
)
5649 if (sched_verbose
>= 2)
5651 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5653 fprintf (sched_dump
,
5654 ";;\t\tnot trying in max_issue due to autoprefetch "
5656 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5659 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5670 We could have also hooked autoprefetcher model into
5671 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5672 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5673 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5674 unblocked). We don't bother about this yet because target of interest
5675 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5677 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5678 Return "1" if INSN1 should not be considered in max_issue due to
5679 auto-prefetcher considerations. */
5681 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5685 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5688 if (sched_verbose
>= 2 && ready_index
== 0)
5689 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5691 for (int write
= 0; write
< 2; ++write
)
5693 autopref_multipass_data_t data1
5694 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5696 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5697 autopref_multipass_init (insn1
, write
);
5698 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5701 if (ready_index
== 0
5702 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5703 /* We allow only a single delay on priviledged instructions.
5704 Doing otherwise would cause infinite loop. */
5706 if (sched_verbose
>= 2)
5708 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5710 fprintf (sched_dump
,
5711 ";;\t\tnot trying in max_issue due to autoprefetch "
5713 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5716 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5721 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5723 rtx_insn
*insn2
= get_ready_element (i2
);
5726 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5729 if (ready_index
== 0)
5732 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5738 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5741 /* Everything from the current queue slot should have been moved to
5743 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5745 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5746 if (n_stalls
> max_insn_queue_index
)
5747 n_stalls
= max_insn_queue_index
;
5749 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5751 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5753 link
= link
->next ())
5755 rtx_insn
*insn2
= link
->insn ();
5756 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5760 /* Queue INSN1 until INSN2 can issue. */
5762 if (ready_index
== 0)
5763 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5771 if (sched_verbose
>= 2
5772 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5773 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5774 /* This does not /always/ trigger. We don't output EOL if the last
5775 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5776 called. We can live with this. */
5777 fprintf (sched_dump
, "\n");
5782 /* Define type for target data used in multipass scheduling. */
5783 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5784 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5786 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5788 /* The following structure describe an entry of the stack of choices. */
5791 /* Ordinal number of the issued insn in the ready queue. */
5793 /* The number of the rest insns whose issues we should try. */
5795 /* The number of issued essential insns. */
5797 /* State after issuing the insn. */
5799 /* Target-specific data. */
5800 first_cycle_multipass_data_t target_data
;
5803 /* The following array is used to implement a stack of choices used in
5804 function max_issue. */
5805 static struct choice_entry
*choice_stack
;
5807 /* This holds the value of the target dfa_lookahead hook. */
5810 /* The following variable value is maximal number of tries of issuing
5811 insns for the first cycle multipass insn scheduling. We define
5812 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5813 need this constraint if all real insns (with non-negative codes)
5814 had reservations because in this case the algorithm complexity is
5815 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5816 might be incomplete and such insn might occur. For such
5817 descriptions, the complexity of algorithm (without the constraint)
5818 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5819 static int max_lookahead_tries
;
5821 /* The following function returns maximal (or close to maximal) number
5822 of insns which can be issued on the same cycle and one of which
5823 insns is insns with the best rank (the first insn in READY). To
5824 make this function tries different samples of ready insns. READY
5825 is current queue `ready'. Global array READY_TRY reflects what
5826 insns are already issued in this try. The function stops immediately,
5827 if it reached the such a solution, that all instruction can be issued.
5828 INDEX will contain index of the best insn in READY. The following
5829 function is used only for first cycle multipass scheduling.
5833 This function expects recognized insns only. All USEs,
5834 CLOBBERs, etc must be filtered elsewhere. */
5836 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5837 bool first_cycle_insn_p
, int *index
)
5839 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5841 struct choice_entry
*top
;
5847 n_ready
= ready
->n_ready
;
5848 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5849 && privileged_n
<= n_ready
);
5851 /* Init MAX_LOOKAHEAD_TRIES. */
5852 if (max_lookahead_tries
== 0)
5854 max_lookahead_tries
= 100;
5855 for (i
= 0; i
< issue_rate
; i
++)
5856 max_lookahead_tries
*= dfa_lookahead
;
5859 /* Init max_points. */
5860 more_issue
= issue_rate
- cycle_issued_insns
;
5861 gcc_assert (more_issue
>= 0);
5863 /* The number of the issued insns in the best solution. */
5868 /* Set initial state of the search. */
5869 memcpy (top
->state
, state
, dfa_state_size
);
5870 top
->rest
= dfa_lookahead
;
5872 if (targetm
.sched
.first_cycle_multipass_begin
)
5873 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5875 first_cycle_insn_p
);
5877 /* Count the number of the insns to search among. */
5878 for (all
= i
= 0; i
< n_ready
; i
++)
5882 if (sched_verbose
>= 2)
5884 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5885 debug_ready_list_1 (ready
, ready_try
);
5888 /* I is the index of the insn to try next. */
5893 if (/* If we've reached a dead end or searched enough of what we have
5896 /* or have nothing else to try... */
5898 /* or should not issue more. */
5899 || top
->n
>= more_issue
)
5901 /* ??? (... || i == n_ready). */
5902 gcc_assert (i
<= n_ready
);
5904 /* We should not issue more than issue_rate instructions. */
5905 gcc_assert (top
->n
<= more_issue
);
5907 if (top
== choice_stack
)
5910 if (best
< top
- choice_stack
)
5915 /* Try to find issued privileged insn. */
5916 while (n
&& !ready_try
[--n
])
5920 if (/* If all insns are equally good... */
5922 /* Or a privileged insn will be issued. */
5924 /* Then we have a solution. */
5926 best
= top
- choice_stack
;
5927 /* This is the index of the insn issued first in this
5929 *index
= choice_stack
[1].index
;
5930 if (top
->n
== more_issue
|| best
== all
)
5935 /* Set ready-list index to point to the last insn
5936 ('i++' below will advance it to the next insn). */
5942 if (targetm
.sched
.first_cycle_multipass_backtrack
)
5943 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
5944 ready_try
, n_ready
);
5947 memcpy (state
, top
->state
, dfa_state_size
);
5949 else if (!ready_try
[i
])
5952 if (tries_num
> max_lookahead_tries
)
5954 insn
= ready_element (ready
, i
);
5955 delay
= state_transition (state
, insn
);
5958 if (state_dead_lock_p (state
)
5959 || insn_finishes_cycle_p (insn
))
5960 /* We won't issue any more instructions in the next
5967 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
5970 /* Advance to the next choice_entry. */
5972 /* Initialize it. */
5973 top
->rest
= dfa_lookahead
;
5976 memcpy (top
->state
, state
, dfa_state_size
);
5979 if (targetm
.sched
.first_cycle_multipass_issue
)
5980 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
5990 /* Increase ready-list index. */
5994 if (targetm
.sched
.first_cycle_multipass_end
)
5995 targetm
.sched
.first_cycle_multipass_end (best
!= 0
5996 ? &choice_stack
[1].target_data
5999 /* Restore the original state of the DFA. */
6000 memcpy (state
, choice_stack
->state
, dfa_state_size
);
6005 /* The following function chooses insn from READY and modifies
6006 READY. The following function is used only for first
6007 cycle multipass scheduling.
6009 -1 if cycle should be advanced,
6010 0 if INSN_PTR is set to point to the desirable insn,
6011 1 if choose_ready () should be restarted without advancing the cycle. */
6013 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6014 rtx_insn
**insn_ptr
)
6016 if (dbg_cnt (sched_insn
) == false)
6018 if (nonscheduled_insns_begin
== NULL_RTX
)
6019 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6021 rtx_insn
*insn
= first_nonscheduled_insn ();
6023 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6024 /* INSN is in the ready_list. */
6026 ready_remove_insn (insn
);
6031 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6032 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6036 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6037 || DEBUG_INSN_P (ready_element (ready
, 0)))
6039 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6040 *insn_ptr
= ready_remove_first_dispatch (ready
);
6042 *insn_ptr
= ready_remove_first (ready
);
6048 /* Try to choose the best insn. */
6052 insn
= ready_element (ready
, 0);
6053 if (INSN_CODE (insn
) < 0)
6055 *insn_ptr
= ready_remove_first (ready
);
6059 /* Filter the search space. */
6060 for (i
= 0; i
< ready
->n_ready
; i
++)
6064 insn
= ready_element (ready
, i
);
6066 /* If this insn is recognizable we should have already
6067 recognized it earlier.
6068 ??? Not very clear where this is supposed to be done.
6070 gcc_checking_assert (INSN_CODE (insn
) >= 0
6071 || recog_memoized (insn
) < 0);
6072 if (INSN_CODE (insn
) < 0)
6074 /* Non-recognized insns at position 0 are handled above. */
6080 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6083 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6086 if (ready_try
[i
] < 0)
6087 /* Queue instruction for several cycles.
6088 We need to restart choose_ready as we have changed
6091 change_queue_index (insn
, -ready_try
[i
]);
6095 /* Make sure that we didn't end up with 0'th insn filtered out.
6096 Don't be tempted to make life easier for backends and just
6097 requeue 0'th insn if (ready_try[0] == 0) and restart
6098 choose_ready. Backends should be very considerate about
6099 requeueing instructions -- especially the highest priority
6100 one at position 0. */
6101 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6106 gcc_assert (ready_try
[i
] == 0);
6107 /* INSN made it through the scrutiny of filters! */
6110 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6112 *insn_ptr
= ready_remove_first (ready
);
6113 if (sched_verbose
>= 4)
6114 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6115 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6120 if (sched_verbose
>= 4)
6121 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6122 (*current_sched_info
->print_insn
)
6123 (ready_element (ready
, index
), 0));
6125 *insn_ptr
= ready_remove (ready
, index
);
6131 /* This function is called when we have successfully scheduled a
6132 block. It uses the schedule stored in the scheduled_insns vector
6133 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6134 append the scheduled insns; TAIL is the insn after the scheduled
6135 block. TARGET_BB is the argument passed to schedule_block. */
6138 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6143 last_scheduled_insn
= prev_head
;
6145 scheduled_insns
.iterate (i
, &insn
);
6148 if (control_flow_insn_p (last_scheduled_insn
)
6149 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6151 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6157 x
= next_real_insn (last_scheduled_insn
);
6159 dump_new_block_header (1, *target_bb
, x
, tail
);
6162 last_scheduled_insn
= bb_note (*target_bb
);
6165 if (current_sched_info
->begin_move_insn
)
6166 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6167 move_insn (insn
, last_scheduled_insn
,
6168 current_sched_info
->next_tail
);
6169 if (!DEBUG_INSN_P (insn
))
6170 reemit_notes (insn
);
6171 last_scheduled_insn
= insn
;
6174 scheduled_insns
.truncate (0);
6177 /* Examine all insns on the ready list and queue those which can't be
6178 issued in this cycle. TEMP_STATE is temporary scheduler state we
6179 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6180 have been issued for the current cycle, which means it is valid to
6181 issue an asm statement.
6183 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6184 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6185 we only leave insns which have an INSN_EXACT_TICK. */
6188 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6189 bool shadows_only_p
, bool modulo_epilogue_p
)
6192 bool sched_group_found
= false;
6193 int min_cost_group
= 1;
6198 for (i
= 0; i
< ready
.n_ready
; i
++)
6200 rtx_insn
*insn
= ready_element (&ready
, i
);
6201 if (SCHED_GROUP_P (insn
))
6203 sched_group_found
= true;
6208 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6209 such an insn first and note its cost, then schedule all other insns
6210 for one cycle later. */
6211 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6213 int n
= ready
.n_ready
;
6214 for (i
= 0; i
< n
; i
++)
6216 rtx_insn
*insn
= ready_element (&ready
, i
);
6218 const char *reason
= "resource conflict";
6220 if (DEBUG_INSN_P (insn
))
6223 if (sched_group_found
&& !SCHED_GROUP_P (insn
))
6227 cost
= min_cost_group
;
6228 reason
= "not in sched group";
6230 else if (modulo_epilogue_p
6231 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6233 cost
= max_insn_queue_index
;
6234 reason
= "not an epilogue insn";
6236 else if (shadows_only_p
&& !SHADOW_P (insn
))
6239 reason
= "not a shadow";
6241 else if (recog_memoized (insn
) < 0)
6243 if (!first_cycle_insn_p
6244 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6245 || asm_noperands (PATTERN (insn
)) >= 0))
6249 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6251 if (sched_pressure
== SCHED_PRESSURE_MODEL
6252 && INSN_TICK (insn
) <= clock_var
)
6254 memcpy (temp_state
, curr_state
, dfa_state_size
);
6255 if (state_transition (temp_state
, insn
) >= 0)
6256 INSN_TICK (insn
) = clock_var
+ 1;
6266 struct delay_pair
*delay_entry
;
6268 = delay_htab
->find_with_hash (insn
,
6269 htab_hash_pointer (insn
));
6270 while (delay_entry
&& delay_cost
== 0)
6272 delay_cost
= estimate_shadow_tick (delay_entry
);
6273 if (delay_cost
> max_insn_queue_index
)
6274 delay_cost
= max_insn_queue_index
;
6275 delay_entry
= delay_entry
->next_same_i1
;
6279 memcpy (temp_state
, curr_state
, dfa_state_size
);
6280 cost
= state_transition (temp_state
, insn
);
6285 if (cost
< delay_cost
)
6288 reason
= "shadow tick";
6293 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6294 min_cost_group
= cost
;
6295 ready_remove (&ready
, i
);
6296 /* Normally we'd want to queue INSN for COST cycles. However,
6297 if SCHED_GROUP_P is set, then we must ensure that nothing
6298 else comes between INSN and its predecessor. If there is
6299 some other insn ready to fire on the next cycle, then that
6300 invariant would be broken.
6302 So when SCHED_GROUP_P is set, just queue this insn for a
6304 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6314 /* Called when we detect that the schedule is impossible. We examine the
6315 backtrack queue to find the earliest insn that caused this condition. */
6317 static struct haifa_saved_data
*
6318 verify_shadows (void)
6320 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6321 for (save
= backtrack_queue
; save
; save
= save
->next
)
6324 struct delay_pair
*pair
= save
->delay_pair
;
6325 rtx_insn
*i1
= pair
->i1
;
6327 for (; pair
; pair
= pair
->next_same_i1
)
6329 rtx_insn
*i2
= pair
->i2
;
6331 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6334 t
= INSN_TICK (i1
) + pair_delay (pair
);
6337 if (sched_verbose
>= 2)
6338 fprintf (sched_dump
,
6339 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6341 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6342 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6343 earliest_fail
= save
;
6346 if (QUEUE_INDEX (i2
) >= 0)
6348 int queued_for
= INSN_TICK (i2
);
6352 if (sched_verbose
>= 2)
6353 fprintf (sched_dump
,
6354 ";;\t\tfailed delay requirements for %d/%d"
6355 " (%d->%d), queued too late\n",
6356 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6357 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6358 earliest_fail
= save
;
6365 return earliest_fail
;
6368 /* Print instructions together with useful scheduling information between
6369 HEAD and TAIL (inclusive). */
6371 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6373 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6375 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6376 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6378 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6379 const char *pattern
= (NOTE_P (insn
)
6381 : str_pattern_slim (PATTERN (insn
)));
6383 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6384 INSN_UID (insn
), priority
, pattern
);
6386 if (sched_verbose
>= 4)
6388 if (NOTE_P (insn
) || recog_memoized (insn
) < 0)
6389 fprintf (sched_dump
, "nothing");
6391 print_reservation (sched_dump
, insn
);
6393 fprintf (sched_dump
, "\n");
6397 /* Use forward list scheduling to rearrange insns of block pointed to by
6398 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6402 schedule_block (basic_block
*target_bb
, state_t init_state
)
6405 bool success
= modulo_ii
== 0;
6406 struct sched_block_state ls
;
6407 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6408 int sort_p
, advance
, start_clock_var
;
6410 /* Head/tail info for this block. */
6411 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6412 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6413 rtx_insn
*head
= NEXT_INSN (prev_head
);
6414 rtx_insn
*tail
= PREV_INSN (next_tail
);
6416 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6417 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6418 find_modifiable_mems (head
, tail
);
6420 /* We used to have code to avoid getting parameters moved from hard
6421 argument registers into pseudos.
6423 However, it was removed when it proved to be of marginal benefit
6424 and caused problems because schedule_block and compute_forward_dependences
6425 had different notions of what the "head" insn was. */
6427 gcc_assert (head
!= tail
|| INSN_P (head
));
6429 haifa_recovery_bb_recently_added_p
= false;
6431 backtrack_queue
= NULL
;
6436 dump_new_block_header (0, *target_bb
, head
, tail
);
6438 if (sched_verbose
>= 2)
6440 dump_insn_stream (head
, tail
);
6441 memset (&rank_for_schedule_stats
, 0,
6442 sizeof (rank_for_schedule_stats
));
6446 if (init_state
== NULL
)
6447 state_reset (curr_state
);
6449 memcpy (curr_state
, init_state
, dfa_state_size
);
6451 /* Clear the ready list. */
6452 ready
.first
= ready
.veclen
- 1;
6456 /* It is used for first cycle multipass scheduling. */
6457 temp_state
= alloca (dfa_state_size
);
6459 if (targetm
.sched
.init
)
6460 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6462 /* We start inserting insns after PREV_HEAD. */
6463 last_scheduled_insn
= prev_head
;
6464 last_nondebug_scheduled_insn
= NULL
;
6465 nonscheduled_insns_begin
= NULL
;
6467 gcc_assert ((NOTE_P (last_scheduled_insn
)
6468 || DEBUG_INSN_P (last_scheduled_insn
))
6469 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6471 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6476 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6477 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6479 /* Start just before the beginning of time. */
6482 /* We need queue and ready lists and clock_var be initialized
6483 in try_ready () (which is called through init_ready_list ()). */
6484 (*current_sched_info
->init_ready_list
) ();
6487 sched_pressure_start_bb (*target_bb
);
6489 /* The algorithm is O(n^2) in the number of ready insns at any given
6490 time in the worst case. Before reload we are more likely to have
6491 big lists so truncate them to a reasonable size. */
6492 if (!reload_completed
6493 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6495 ready_sort_debug (&ready
);
6496 ready_sort_real (&ready
);
6498 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6499 If there are debug insns, we know they're first. */
6500 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6501 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6504 if (sched_verbose
>= 2)
6506 fprintf (sched_dump
,
6507 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6508 debug_ready_list (&ready
);
6509 fprintf (sched_dump
,
6510 ";;\t\t before reload => truncated to %d insns\n", i
);
6513 /* Delay all insns past it for 1 cycle. If debug counter is
6514 activated make an exception for the insn right after
6515 nonscheduled_insns_begin. */
6517 rtx_insn
*skip_insn
;
6519 if (dbg_cnt (sched_insn
) == false)
6520 skip_insn
= first_nonscheduled_insn ();
6524 while (i
< ready
.n_ready
)
6528 insn
= ready_remove (&ready
, i
);
6530 if (insn
!= skip_insn
)
6531 queue_insn (insn
, 1, "list truncated");
6534 ready_add (&ready
, skip_insn
, true);
6538 /* Now we can restore basic block notes and maintain precise cfg. */
6539 restore_bb_notes (*target_bb
);
6541 last_clock_var
= -1;
6545 gcc_assert (scheduled_insns
.length () == 0);
6547 must_backtrack
= false;
6548 modulo_insns_scheduled
= 0;
6550 ls
.modulo_epilogue
= false;
6551 ls
.first_cycle_insn_p
= true;
6553 /* Loop until all the insns in BB are scheduled. */
6554 while ((*current_sched_info
->schedule_more_p
) ())
6556 perform_replacements_new_cycle ();
6559 start_clock_var
= clock_var
;
6563 advance_one_cycle ();
6565 /* Add to the ready list all pending insns that can be issued now.
6566 If there are no ready insns, increment clock until one
6567 is ready and add all pending insns at that point to the ready
6569 queue_to_ready (&ready
);
6571 gcc_assert (ready
.n_ready
);
6573 if (sched_verbose
>= 2)
6575 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6576 debug_ready_list (&ready
);
6578 advance
-= clock_var
- start_clock_var
;
6580 while (advance
> 0);
6582 if (ls
.modulo_epilogue
)
6584 int stage
= clock_var
/ modulo_ii
;
6585 if (stage
> modulo_last_stage
* 2 + 2)
6587 if (sched_verbose
>= 2)
6588 fprintf (sched_dump
,
6589 ";;\t\tmodulo scheduled succeeded at II %d\n",
6595 else if (modulo_ii
> 0)
6597 int stage
= clock_var
/ modulo_ii
;
6598 if (stage
> modulo_max_stages
)
6600 if (sched_verbose
>= 2)
6601 fprintf (sched_dump
,
6602 ";;\t\tfailing schedule due to excessive stages\n");
6605 if (modulo_n_insns
== modulo_insns_scheduled
6606 && stage
> modulo_last_stage
)
6608 if (sched_verbose
>= 2)
6609 fprintf (sched_dump
,
6610 ";;\t\tfound kernel after %d stages, II %d\n",
6612 ls
.modulo_epilogue
= true;
6616 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6617 if (ready
.n_ready
== 0)
6622 ls
.shadows_only_p
= false;
6623 cycle_issued_insns
= 0;
6624 ls
.can_issue_more
= issue_rate
;
6631 if (sort_p
&& ready
.n_ready
> 0)
6633 /* Sort the ready list based on priority. This must be
6634 done every iteration through the loop, as schedule_insn
6635 may have readied additional insns that will not be
6636 sorted correctly. */
6637 ready_sort (&ready
);
6639 if (sched_verbose
>= 2)
6641 fprintf (sched_dump
,
6642 ";;\t\tReady list after ready_sort: ");
6643 debug_ready_list (&ready
);
6647 /* We don't want md sched reorder to even see debug isns, so put
6648 them out right away. */
6649 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6650 && (*current_sched_info
->schedule_more_p
) ())
6652 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6654 rtx_insn
*insn
= ready_remove_first (&ready
);
6655 gcc_assert (DEBUG_INSN_P (insn
));
6656 (*current_sched_info
->begin_schedule_ready
) (insn
);
6657 scheduled_insns
.safe_push (insn
);
6658 last_scheduled_insn
= insn
;
6659 advance
= schedule_insn (insn
);
6660 gcc_assert (advance
== 0);
6661 if (ready
.n_ready
> 0)
6662 ready_sort (&ready
);
6666 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6669 resume_after_backtrack
:
6670 /* Allow the target to reorder the list, typically for
6671 better instruction bundling. */
6673 && (ready
.n_ready
== 0
6674 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6676 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6678 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6679 ready_lastpos (&ready
),
6680 &ready
.n_ready
, clock_var
);
6681 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6683 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6685 ? ready_lastpos (&ready
) : NULL
,
6686 &ready
.n_ready
, clock_var
);
6689 restart_choose_ready
:
6690 if (sched_verbose
>= 2)
6692 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6694 debug_ready_list (&ready
);
6695 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6696 print_curr_reg_pressure ();
6699 if (ready
.n_ready
== 0
6700 && ls
.can_issue_more
6701 && reload_completed
)
6703 /* Allow scheduling insns directly from the queue in case
6704 there's nothing better to do (ready list is empty) but
6705 there are still vacant dispatch slots in the current cycle. */
6706 if (sched_verbose
>= 6)
6707 fprintf (sched_dump
,";;\t\tSecond chance\n");
6708 memcpy (temp_state
, curr_state
, dfa_state_size
);
6709 if (early_queue_to_ready (temp_state
, &ready
))
6710 ready_sort (&ready
);
6713 if (ready
.n_ready
== 0
6714 || !ls
.can_issue_more
6715 || state_dead_lock_p (curr_state
)
6716 || !(*current_sched_info
->schedule_more_p
) ())
6719 /* Select and remove the insn from the ready list. */
6725 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6731 goto restart_choose_ready
;
6733 gcc_assert (insn
!= NULL_RTX
);
6736 insn
= ready_remove_first (&ready
);
6738 if (sched_pressure
!= SCHED_PRESSURE_NONE
6739 && INSN_TICK (insn
) > clock_var
)
6741 ready_add (&ready
, insn
, true);
6746 if (targetm
.sched
.dfa_new_cycle
6747 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6748 insn
, last_clock_var
,
6749 clock_var
, &sort_p
))
6750 /* SORT_P is used by the target to override sorting
6751 of the ready list. This is needed when the target
6752 has modified its internal structures expecting that
6753 the insn will be issued next. As we need the insn
6754 to have the highest priority (so it will be returned by
6755 the ready_remove_first call above), we invoke
6756 ready_add (&ready, insn, true).
6757 But, still, there is one issue: INSN can be later
6758 discarded by scheduler's front end through
6759 current_sched_info->can_schedule_ready_p, hence, won't
6762 ready_add (&ready
, insn
, true);
6768 if (current_sched_info
->can_schedule_ready_p
6769 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6770 /* We normally get here only if we don't want to move
6771 insn from the split block. */
6773 TODO_SPEC (insn
) = DEP_POSTPONED
;
6774 goto restart_choose_ready
;
6779 /* If this insn is the first part of a delay-slot pair, record a
6781 struct delay_pair
*delay_entry
;
6783 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6786 save_backtrack_point (delay_entry
, ls
);
6787 if (sched_verbose
>= 2)
6788 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6792 /* DECISION is made. */
6794 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6796 modulo_insns_scheduled
++;
6797 modulo_last_stage
= clock_var
/ modulo_ii
;
6799 if (TODO_SPEC (insn
) & SPECULATIVE
)
6800 generate_recovery_code (insn
);
6802 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6803 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6805 /* Update counters, etc in the scheduler's front end. */
6806 (*current_sched_info
->begin_schedule_ready
) (insn
);
6807 scheduled_insns
.safe_push (insn
);
6808 gcc_assert (NONDEBUG_INSN_P (insn
));
6809 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6811 if (recog_memoized (insn
) >= 0)
6813 memcpy (temp_state
, curr_state
, dfa_state_size
);
6814 cost
= state_transition (curr_state
, insn
);
6815 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6816 gcc_assert (cost
< 0);
6817 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6818 cycle_issued_insns
++;
6822 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6823 || asm_noperands (PATTERN (insn
)) >= 0);
6825 if (targetm
.sched
.variable_issue
)
6827 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6828 insn
, ls
.can_issue_more
);
6829 /* A naked CLOBBER or USE generates no instruction, so do
6830 not count them against the issue rate. */
6831 else if (GET_CODE (PATTERN (insn
)) != USE
6832 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6833 ls
.can_issue_more
--;
6834 advance
= schedule_insn (insn
);
6836 if (SHADOW_P (insn
))
6837 ls
.shadows_only_p
= true;
6839 /* After issuing an asm insn we should start a new cycle. */
6840 if (advance
== 0 && asm_p
)
6849 ls
.first_cycle_insn_p
= false;
6850 if (ready
.n_ready
> 0)
6851 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6852 ls
.modulo_epilogue
);
6856 if (!must_backtrack
)
6857 for (i
= 0; i
< ready
.n_ready
; i
++)
6859 rtx_insn
*insn
= ready_element (&ready
, i
);
6860 if (INSN_EXACT_TICK (insn
) == clock_var
)
6862 must_backtrack
= true;
6867 if (must_backtrack
&& modulo_ii
> 0)
6869 if (modulo_backtracks_left
== 0)
6871 modulo_backtracks_left
--;
6873 while (must_backtrack
)
6875 struct haifa_saved_data
*failed
;
6876 rtx_insn
*failed_insn
;
6878 must_backtrack
= false;
6879 failed
= verify_shadows ();
6880 gcc_assert (failed
);
6882 failed_insn
= failed
->delay_pair
->i1
;
6883 /* Clear these queues. */
6884 perform_replacements_new_cycle ();
6885 toggle_cancelled_flags (false);
6886 unschedule_insns_until (failed_insn
);
6887 while (failed
!= backtrack_queue
)
6888 free_topmost_backtrack_point (true);
6889 restore_last_backtrack_point (&ls
);
6890 if (sched_verbose
>= 2)
6891 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6892 /* Delay by at least a cycle. This could cause additional
6894 queue_insn (failed_insn
, 1, "backtracked");
6898 if (ready
.n_ready
> 0)
6899 goto resume_after_backtrack
;
6902 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6908 ls
.first_cycle_insn_p
= true;
6910 if (ls
.modulo_epilogue
)
6913 if (!ls
.first_cycle_insn_p
|| advance
)
6914 advance_one_cycle ();
6915 perform_replacements_new_cycle ();
6918 /* Once again, debug insn suckiness: they can be on the ready list
6919 even if they have unresolved dependencies. To make our view
6920 of the world consistent, remove such "ready" insns. */
6921 restart_debug_insn_loop
:
6922 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6926 x
= ready_element (&ready
, i
);
6927 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
6928 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
6930 ready_remove (&ready
, i
);
6931 goto restart_debug_insn_loop
;
6934 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6938 x
= ready_element (&ready
, i
);
6939 resolve_dependencies (x
);
6941 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6943 rtx_insn_list
*link
;
6944 while ((link
= insn_queue
[i
]) != NULL
)
6946 rtx_insn
*x
= link
->insn ();
6947 insn_queue
[i
] = link
->next ();
6948 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6949 free_INSN_LIST_node (link
);
6950 resolve_dependencies (x
);
6956 undo_all_replacements ();
6961 fprintf (sched_dump
, ";;\tReady list (final): ");
6962 debug_ready_list (&ready
);
6965 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
6966 /* Sanity check -- queue must be empty now. Meaningless if region has
6968 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
6969 else if (modulo_ii
== 0)
6971 /* We must maintain QUEUE_INDEX between blocks in region. */
6972 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6976 x
= ready_element (&ready
, i
);
6977 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6978 TODO_SPEC (x
) = HARD_DEP
;
6982 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6984 rtx_insn_list
*link
;
6985 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
6990 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6991 TODO_SPEC (x
) = HARD_DEP
;
6993 free_INSN_LIST_list (&insn_queue
[i
]);
6997 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
6998 model_end_schedule ();
7002 commit_schedule (prev_head
, tail
, target_bb
);
7004 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7007 last_scheduled_insn
= tail
;
7009 scheduled_insns
.truncate (0);
7011 if (!current_sched_info
->queue_must_finish_empty
7012 || haifa_recovery_bb_recently_added_p
)
7014 /* INSN_TICK (minimum clock tick at which the insn becomes
7015 ready) may be not correct for the insn in the subsequent
7016 blocks of the region. We should use a correct value of
7017 `clock_var' or modify INSN_TICK. It is better to keep
7018 clock_var value equal to 0 at the start of a basic block.
7019 Therefore we modify INSN_TICK here. */
7020 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7023 if (targetm
.sched
.finish
)
7025 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7026 /* Target might have added some instructions to the scheduled block
7027 in its md_finish () hook. These new insns don't have any data
7028 initialized and to identify them we extend h_i_d so that they'll
7030 sched_extend_luids ();
7033 /* Update head/tail boundaries. */
7034 head
= NEXT_INSN (prev_head
);
7035 tail
= last_scheduled_insn
;
7039 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7040 INSN_UID (head
), INSN_UID (tail
));
7042 if (sched_verbose
>= 2)
7044 dump_insn_stream (head
, tail
);
7045 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7049 fprintf (sched_dump
, "\n");
7052 head
= restore_other_notes (head
, NULL
);
7054 current_sched_info
->head
= head
;
7055 current_sched_info
->tail
= tail
;
7057 free_backtrack_queue ();
7062 /* Set_priorities: compute priority of each insn in the block. */
7065 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7069 int sched_max_insns_priority
=
7070 current_sched_info
->sched_max_insns_priority
;
7071 rtx_insn
*prev_head
;
7073 if (head
== tail
&& ! INSN_P (head
))
7078 prev_head
= PREV_INSN (head
);
7079 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7085 (void) priority (insn
);
7087 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7089 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7090 INSN_PRIORITY (insn
));
7093 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7098 /* Set dump and sched_verbose for the desired debugging output. If no
7099 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7100 For -fsched-verbose=N, N>=10, print everything to stderr. */
7102 setup_sched_dump (void)
7104 sched_verbose
= sched_verbose_param
;
7105 if (sched_verbose_param
== 0 && dump_file
)
7107 sched_dump
= ((sched_verbose_param
>= 10 || !dump_file
)
7108 ? stderr
: dump_file
);
7111 /* Allocate data for register pressure sensitive scheduling. */
7113 alloc_global_sched_pressure_data (void)
7115 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7117 int i
, max_regno
= max_reg_num ();
7119 if (sched_dump
!= NULL
)
7120 /* We need info about pseudos for rtl dumps about pseudo
7121 classes and costs. */
7122 regstat_init_n_sets_and_refs ();
7123 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7124 sched_regno_pressure_class
7125 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7126 for (i
= 0; i
< max_regno
; i
++)
7127 sched_regno_pressure_class
[i
]
7128 = (i
< FIRST_PSEUDO_REGISTER
7129 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7130 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7131 curr_reg_live
= BITMAP_ALLOC (NULL
);
7132 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7134 saved_reg_live
= BITMAP_ALLOC (NULL
);
7135 region_ref_regs
= BITMAP_ALLOC (NULL
);
7138 /* Calculate number of CALL_USED_REGS in register classes that
7139 we calculate register pressure for. */
7140 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7142 enum reg_class cl
= ira_pressure_classes
[c
];
7144 call_used_regs_num
[cl
] = 0;
7146 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7147 if (call_used_regs
[ira_class_hard_regs
[cl
][i
]])
7148 ++call_used_regs_num
[cl
];
7153 /* Free data for register pressure sensitive scheduling. Also called
7154 from schedule_region when stopping sched-pressure early. */
7156 free_global_sched_pressure_data (void)
7158 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7160 if (regstat_n_sets_and_refs
!= NULL
)
7161 regstat_free_n_sets_and_refs ();
7162 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7164 BITMAP_FREE (region_ref_regs
);
7165 BITMAP_FREE (saved_reg_live
);
7167 BITMAP_FREE (curr_reg_live
);
7168 free (sched_regno_pressure_class
);
7172 /* Initialize some global state for the scheduler. This function works
7173 with the common data shared between all the schedulers. It is called
7174 from the scheduler specific initialization routine. */
7179 /* Disable speculative loads in their presence if cc0 defined. */
7181 flag_schedule_speculative_load
= 0;
7183 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7184 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7186 if (live_range_shrinkage_p
)
7187 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7188 else if (flag_sched_pressure
7189 && !reload_completed
7190 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7191 sched_pressure
= ((enum sched_pressure_algorithm
)
7192 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7194 sched_pressure
= SCHED_PRESSURE_NONE
;
7196 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7197 ira_setup_eliminable_regset ();
7199 /* Initialize SPEC_INFO. */
7200 if (targetm
.sched
.set_sched_flags
)
7202 spec_info
= &spec_info_var
;
7203 targetm
.sched
.set_sched_flags (spec_info
);
7205 if (spec_info
->mask
!= 0)
7207 spec_info
->data_weakness_cutoff
=
7208 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7209 spec_info
->control_weakness_cutoff
=
7210 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7211 * REG_BR_PROB_BASE
) / 100;
7214 /* So we won't read anything accidentally. */
7219 /* So we won't read anything accidentally. */
7222 /* Initialize issue_rate. */
7223 if (targetm
.sched
.issue_rate
)
7224 issue_rate
= targetm
.sched
.issue_rate ();
7228 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7229 /* Don't use max_issue with reg_pressure scheduling. Multipass
7230 scheduling and reg_pressure scheduling undo each other's decisions. */
7231 && sched_pressure
== SCHED_PRESSURE_NONE
)
7232 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7236 /* Set to "0" so that we recalculate. */
7237 max_lookahead_tries
= 0;
7239 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7240 targetm
.sched
.init_dfa_pre_cycle_insn ();
7242 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7243 targetm
.sched
.init_dfa_post_cycle_insn ();
7246 dfa_state_size
= state_size ();
7248 init_alias_analysis ();
7251 df_set_flags (DF_LR_RUN_DCE
);
7252 df_note_add_problem ();
7254 /* More problems needed for interloop dep calculation in SMS. */
7255 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7257 df_rd_add_problem ();
7258 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7263 /* Do not run DCE after reload, as this can kill nops inserted
7265 if (reload_completed
)
7266 df_clear_flags (DF_LR_RUN_DCE
);
7268 regstat_compute_calls_crossed ();
7270 if (targetm
.sched
.init_global
)
7271 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7273 alloc_global_sched_pressure_data ();
7275 curr_state
= xmalloc (dfa_state_size
);
7278 static void haifa_init_only_bb (basic_block
, basic_block
);
7280 /* Initialize data structures specific to the Haifa scheduler. */
7282 haifa_sched_init (void)
7284 setup_sched_dump ();
7287 scheduled_insns
.create (0);
7289 if (spec_info
!= NULL
)
7291 sched_deps_info
->use_deps_list
= 1;
7292 sched_deps_info
->generate_spec_deps
= 1;
7295 /* Initialize luids, dependency caches, target and h_i_d for the
7299 bbs
.create (n_basic_blocks_for_fn (cfun
));
7304 FOR_EACH_BB_FN (bb
, cfun
)
7305 bbs
.quick_push (bb
);
7306 sched_init_luids (bbs
);
7307 sched_deps_init (true);
7308 sched_extend_target ();
7309 haifa_init_h_i_d (bbs
);
7314 sched_init_only_bb
= haifa_init_only_bb
;
7315 sched_split_block
= sched_split_block_1
;
7316 sched_create_empty_bb
= sched_create_empty_bb_1
;
7317 haifa_recovery_bb_ever_added_p
= false;
7319 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7320 before_recovery
= 0;
7326 /* Finish work with the data specific to the Haifa scheduler. */
7328 haifa_sched_finish (void)
7330 sched_create_empty_bb
= NULL
;
7331 sched_split_block
= NULL
;
7332 sched_init_only_bb
= NULL
;
7334 if (spec_info
&& spec_info
->dump
)
7336 char c
= reload_completed
? 'a' : 'b';
7338 fprintf (spec_info
->dump
,
7339 ";; %s:\n", current_function_name ());
7341 fprintf (spec_info
->dump
,
7342 ";; Procedure %cr-begin-data-spec motions == %d\n",
7344 fprintf (spec_info
->dump
,
7345 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7347 fprintf (spec_info
->dump
,
7348 ";; Procedure %cr-begin-control-spec motions == %d\n",
7349 c
, nr_begin_control
);
7350 fprintf (spec_info
->dump
,
7351 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7352 c
, nr_be_in_control
);
7355 scheduled_insns
.release ();
7357 /* Finalize h_i_d, dependency caches, and luids for the whole
7358 function. Target will be finalized in md_global_finish (). */
7359 sched_deps_finish ();
7360 sched_finish_luids ();
7361 current_sched_info
= NULL
;
7365 /* Free global data used during insn scheduling. This function works with
7366 the common data shared between the schedulers. */
7371 haifa_finish_h_i_d ();
7372 free_global_sched_pressure_data ();
7375 if (targetm
.sched
.finish_global
)
7376 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7378 end_alias_analysis ();
7380 regstat_free_calls_crossed ();
7385 /* Free all delay_pair structures that were recorded. */
7387 free_delay_pairs (void)
7391 delay_htab
->empty ();
7392 delay_htab_i2
->empty ();
7396 /* Fix INSN_TICKs of the instructions in the current block as well as
7397 INSN_TICKs of their dependents.
7398 HEAD and TAIL are the begin and the end of the current scheduled block. */
7400 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7402 /* Set of instructions with corrected INSN_TICK. */
7403 bitmap_head processed
;
7404 /* ??? It is doubtful if we should assume that cycle advance happens on
7405 basic block boundaries. Basically insns that are unconditionally ready
7406 on the start of the block are more preferable then those which have
7407 a one cycle dependency over insn from the previous block. */
7408 int next_clock
= clock_var
+ 1;
7410 bitmap_initialize (&processed
, 0);
7412 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7413 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7414 across different blocks. */
7415 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7420 sd_iterator_def sd_it
;
7423 tick
= INSN_TICK (head
);
7424 gcc_assert (tick
>= MIN_TICK
);
7426 /* Fix INSN_TICK of instruction from just scheduled block. */
7427 if (bitmap_set_bit (&processed
, INSN_LUID (head
)))
7431 if (tick
< MIN_TICK
)
7434 INSN_TICK (head
) = tick
;
7437 if (DEBUG_INSN_P (head
))
7440 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7444 next
= DEP_CON (dep
);
7445 tick
= INSN_TICK (next
);
7447 if (tick
!= INVALID_TICK
7448 /* If NEXT has its INSN_TICK calculated, fix it.
7449 If not - it will be properly calculated from
7450 scratch later in fix_tick_ready. */
7451 && bitmap_set_bit (&processed
, INSN_LUID (next
)))
7455 if (tick
< MIN_TICK
)
7458 if (tick
> INTER_TICK (next
))
7459 INTER_TICK (next
) = tick
;
7461 tick
= INTER_TICK (next
);
7463 INSN_TICK (next
) = tick
;
7468 bitmap_clear (&processed
);
7471 /* Check if NEXT is ready to be added to the ready or queue list.
7472 If "yes", add it to the proper list.
7474 -1 - is not ready yet,
7475 0 - added to the ready list,
7476 0 < N - queued for N cycles. */
7478 try_ready (rtx_insn
*next
)
7480 ds_t old_ts
, new_ts
;
7482 old_ts
= TODO_SPEC (next
);
7484 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7485 && (old_ts
== HARD_DEP
7486 || old_ts
== DEP_POSTPONED
7487 || (old_ts
& SPECULATIVE
)
7488 || old_ts
== DEP_CONTROL
));
7490 new_ts
= recompute_todo_spec (next
, false);
7492 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7493 gcc_assert (new_ts
== old_ts
7494 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7495 else if (current_sched_info
->new_ready
)
7496 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7498 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7499 have its original pattern or changed (speculative) one. This is due
7500 to changing ebb in region scheduling.
7501 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7502 has speculative pattern.
7504 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7505 control-speculative NEXT could have been discarded by sched-rgn.c
7506 (the same case as when discarded by can_schedule_ready_p ()). */
7508 if ((new_ts
& SPECULATIVE
)
7509 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7510 need to change anything. */
7511 && new_ts
!= old_ts
)
7516 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7518 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7523 /* It would be nice to change DEP_STATUS of all dependences,
7524 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7525 so we won't reanalyze anything. */
7530 /* We follow the rule, that every speculative insn
7531 has non-null ORIG_PAT. */
7532 if (!ORIG_PAT (next
))
7533 ORIG_PAT (next
) = PATTERN (next
);
7537 if (!ORIG_PAT (next
))
7538 /* If we gonna to overwrite the original pattern of insn,
7540 ORIG_PAT (next
) = PATTERN (next
);
7542 res
= haifa_change_pattern (next
, new_pat
);
7551 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7552 either correct (new_ts & SPECULATIVE),
7553 or we simply don't care (new_ts & HARD_DEP). */
7555 gcc_assert (!ORIG_PAT (next
)
7556 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7558 TODO_SPEC (next
) = new_ts
;
7560 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7562 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7563 control-speculative NEXT could have been discarded by sched-rgn.c
7564 (the same case as when discarded by can_schedule_ready_p ()). */
7565 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7567 change_queue_index (next
, QUEUE_NOWHERE
);
7571 else if (!(new_ts
& BEGIN_SPEC
)
7572 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7573 && !IS_SPECULATION_CHECK_P (next
))
7574 /* We should change pattern of every previously speculative
7575 instruction - and we determine if NEXT was speculative by using
7576 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7577 pat too, so skip them. */
7579 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7580 gcc_assert (success
);
7581 ORIG_PAT (next
) = 0;
7584 if (sched_verbose
>= 2)
7586 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7587 (*current_sched_info
->print_insn
) (next
, 0));
7589 if (spec_info
&& spec_info
->dump
)
7591 if (new_ts
& BEGIN_DATA
)
7592 fprintf (spec_info
->dump
, "; data-spec;");
7593 if (new_ts
& BEGIN_CONTROL
)
7594 fprintf (spec_info
->dump
, "; control-spec;");
7595 if (new_ts
& BE_IN_CONTROL
)
7596 fprintf (spec_info
->dump
, "; in-control-spec;");
7598 if (TODO_SPEC (next
) & DEP_CONTROL
)
7599 fprintf (sched_dump
, " predicated");
7600 fprintf (sched_dump
, "\n");
7603 adjust_priority (next
);
7605 return fix_tick_ready (next
);
7608 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7610 fix_tick_ready (rtx_insn
*next
)
7614 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7617 sd_iterator_def sd_it
;
7620 tick
= INSN_TICK (next
);
7621 /* if tick is not equal to INVALID_TICK, then update
7622 INSN_TICK of NEXT with the most recent resolved dependence
7623 cost. Otherwise, recalculate from scratch. */
7624 full_p
= (tick
== INVALID_TICK
);
7626 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7628 rtx_insn
*pro
= DEP_PRO (dep
);
7631 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7633 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7644 INSN_TICK (next
) = tick
;
7646 delay
= tick
- clock_var
;
7647 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7648 delay
= QUEUE_READY
;
7650 change_queue_index (next
, delay
);
7655 /* Move NEXT to the proper queue list with (DELAY >= 1),
7656 or add it to the ready list (DELAY == QUEUE_READY),
7657 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7659 change_queue_index (rtx_insn
*next
, int delay
)
7661 int i
= QUEUE_INDEX (next
);
7663 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7665 gcc_assert (i
!= QUEUE_SCHEDULED
);
7667 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7668 || (delay
< 0 && delay
== i
))
7669 /* We have nothing to do. */
7672 /* Remove NEXT from wherever it is now. */
7673 if (i
== QUEUE_READY
)
7674 ready_remove_insn (next
);
7676 queue_remove (next
);
7678 /* Add it to the proper place. */
7679 if (delay
== QUEUE_READY
)
7680 ready_add (readyp
, next
, false);
7681 else if (delay
>= 1)
7682 queue_insn (next
, delay
, "change queue index");
7684 if (sched_verbose
>= 2)
7686 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7687 (*current_sched_info
->print_insn
) (next
, 0));
7689 if (delay
== QUEUE_READY
)
7690 fprintf (sched_dump
, " into ready\n");
7691 else if (delay
>= 1)
7692 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7694 fprintf (sched_dump
, " removed from ready or queue lists\n");
7698 static int sched_ready_n_insns
= -1;
7700 /* Initialize per region data structures. */
7702 sched_extend_ready_list (int new_sched_ready_n_insns
)
7706 if (sched_ready_n_insns
== -1)
7707 /* At the first call we need to initialize one more choice_stack
7711 sched_ready_n_insns
= 0;
7712 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7715 i
= sched_ready_n_insns
+ 1;
7717 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7718 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7720 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7722 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7723 sched_ready_n_insns
,
7724 sizeof (*ready_try
));
7726 /* We allocate +1 element to save initial state in the choice_stack[0]
7728 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7729 new_sched_ready_n_insns
+ 1);
7731 for (; i
<= new_sched_ready_n_insns
; i
++)
7733 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7735 if (targetm
.sched
.first_cycle_multipass_init
)
7736 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7740 sched_ready_n_insns
= new_sched_ready_n_insns
;
7743 /* Free per region data structures. */
7745 sched_finish_ready_list (void)
7756 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7758 if (targetm
.sched
.first_cycle_multipass_fini
)
7759 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7762 free (choice_stack
[i
].state
);
7764 free (choice_stack
);
7765 choice_stack
= NULL
;
7767 sched_ready_n_insns
= -1;
7771 haifa_luid_for_non_insn (rtx x
)
7773 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7778 /* Generates recovery code for INSN. */
7780 generate_recovery_code (rtx_insn
*insn
)
7782 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7783 begin_speculative_block (insn
);
7785 /* Here we have insn with no dependencies to
7786 instructions other then CHECK_SPEC ones. */
7788 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7789 add_to_speculative_block (insn
);
7793 Tries to add speculative dependencies of type FS between instructions
7794 in deps_list L and TWIN. */
7796 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7798 sd_iterator_def sd_it
;
7801 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7806 consumer
= DEP_CON (dep
);
7808 ds
= DEP_STATUS (dep
);
7810 if (/* If we want to create speculative dep. */
7812 /* And we can do that because this is a true dep. */
7813 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7815 gcc_assert (!(ds
& BE_IN_SPEC
));
7817 if (/* If this dep can be overcome with 'begin speculation'. */
7819 /* Then we have a choice: keep the dep 'begin speculative'
7820 or transform it into 'be in speculative'. */
7822 if (/* In try_ready we assert that if insn once became ready
7823 it can be removed from the ready (or queue) list only
7824 due to backend decision. Hence we can't let the
7825 probability of the speculative dep to decrease. */
7826 ds_weak (ds
) <= ds_weak (fs
))
7830 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7832 if (/* consumer can 'be in speculative'. */
7833 sched_insn_is_legitimate_for_speculation_p (consumer
,
7835 /* Transform it to be in speculative. */
7840 /* Mark the dep as 'be in speculative'. */
7845 dep_def _new_dep
, *new_dep
= &_new_dep
;
7847 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7848 sd_add_dep (new_dep
, false);
7853 /* Generates recovery code for BEGIN speculative INSN. */
7855 begin_speculative_block (rtx_insn
*insn
)
7857 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7859 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7862 create_check_block_twin (insn
, false);
7864 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7867 static void haifa_init_insn (rtx_insn
*);
7869 /* Generates recovery code for BE_IN speculative INSN. */
7871 add_to_speculative_block (rtx_insn
*insn
)
7874 sd_iterator_def sd_it
;
7876 rtx_insn_list
*twins
= NULL
;
7877 rtx_vec_t priorities_roots
;
7879 ts
= TODO_SPEC (insn
);
7880 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7882 if (ts
& BE_IN_DATA
)
7884 if (ts
& BE_IN_CONTROL
)
7887 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7888 gcc_assert (!TODO_SPEC (insn
));
7890 DONE_SPEC (insn
) |= ts
;
7892 /* First we convert all simple checks to branchy. */
7893 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7894 sd_iterator_cond (&sd_it
, &dep
);)
7896 rtx_insn
*check
= DEP_PRO (dep
);
7898 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7900 create_check_block_twin (check
, true);
7902 /* Restart search. */
7903 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7906 /* Continue search. */
7907 sd_iterator_next (&sd_it
);
7910 priorities_roots
.create (0);
7911 clear_priorities (insn
, &priorities_roots
);
7915 rtx_insn
*check
, *twin
;
7918 /* Get the first backward dependency of INSN. */
7919 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7920 if (!sd_iterator_cond (&sd_it
, &dep
))
7921 /* INSN has no backward dependencies left. */
7924 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
7925 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
7926 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
7928 check
= DEP_PRO (dep
);
7930 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
7931 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
7933 rec
= BLOCK_FOR_INSN (check
);
7935 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
7936 haifa_init_insn (twin
);
7938 sd_copy_back_deps (twin
, insn
, true);
7940 if (sched_verbose
&& spec_info
->dump
)
7941 /* INSN_BB (insn) isn't determined for twin insns yet.
7942 So we can't use current_sched_info->print_insn. */
7943 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
7944 INSN_UID (twin
), rec
->index
);
7946 twins
= alloc_INSN_LIST (twin
, twins
);
7948 /* Add dependences between TWIN and all appropriate
7949 instructions from REC. */
7950 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
7952 rtx_insn
*pro
= DEP_PRO (dep
);
7954 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
7956 /* INSN might have dependencies from the instructions from
7957 several recovery blocks. At this iteration we process those
7958 producers that reside in REC. */
7959 if (BLOCK_FOR_INSN (pro
) == rec
)
7961 dep_def _new_dep
, *new_dep
= &_new_dep
;
7963 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
7964 sd_add_dep (new_dep
, false);
7968 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
7970 /* Remove all dependencies between INSN and insns in REC. */
7971 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7972 sd_iterator_cond (&sd_it
, &dep
);)
7974 rtx_insn
*pro
= DEP_PRO (dep
);
7976 if (BLOCK_FOR_INSN (pro
) == rec
)
7977 sd_delete_dep (sd_it
);
7979 sd_iterator_next (&sd_it
);
7983 /* We couldn't have added the dependencies between INSN and TWINS earlier
7984 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
7988 rtx_insn_list
*next_node
;
7990 twin
= twins
->insn ();
7993 dep_def _new_dep
, *new_dep
= &_new_dep
;
7995 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
7996 sd_add_dep (new_dep
, false);
7999 next_node
= twins
->next ();
8000 free_INSN_LIST_node (twins
);
8004 calc_priorities (priorities_roots
);
8005 priorities_roots
.release ();
8008 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8010 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8012 gcc_assert (new_nmemb
>= old_nmemb
);
8013 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8014 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8019 Find fallthru edge from PRED. */
8021 find_fallthru_edge_from (basic_block pred
)
8026 succ
= pred
->next_bb
;
8027 gcc_assert (succ
->prev_bb
== pred
);
8029 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8031 e
= find_fallthru_edge (pred
->succs
);
8035 gcc_assert (e
->dest
== succ
);
8041 e
= find_fallthru_edge (succ
->preds
);
8045 gcc_assert (e
->src
== pred
);
8053 /* Extend per basic block data structures. */
8055 sched_extend_bb (void)
8057 /* The following is done to keep current_sched_info->next_tail non null. */
8058 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8059 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8060 if (NEXT_INSN (end
) == 0
8063 /* Don't emit a NOTE if it would end up before a BARRIER. */
8064 && !BARRIER_P (NEXT_INSN (end
))))
8066 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8067 /* Make note appear outside BB. */
8068 set_block_for_insn (note
, NULL
);
8069 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8073 /* Init per basic block data structures. */
8075 sched_init_bbs (void)
8080 /* Initialize BEFORE_RECOVERY variable. */
8082 init_before_recovery (basic_block
*before_recovery_ptr
)
8087 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8088 e
= find_fallthru_edge_from (last
);
8092 /* We create two basic blocks:
8093 1. Single instruction block is inserted right after E->SRC
8095 2. Empty block right before EXIT_BLOCK.
8096 Between these two blocks recovery blocks will be emitted. */
8098 basic_block single
, empty
;
8100 /* If the fallthrough edge to exit we've found is from the block we've
8101 created before, don't do anything more. */
8102 if (last
== after_recovery
)
8105 adding_bb_to_current_region_p
= false;
8107 single
= sched_create_empty_bb (last
);
8108 empty
= sched_create_empty_bb (single
);
8110 /* Add new blocks to the root loop. */
8111 if (current_loops
!= NULL
)
8113 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8114 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8117 single
->count
= last
->count
;
8118 empty
->count
= last
->count
;
8119 single
->frequency
= last
->frequency
;
8120 empty
->frequency
= last
->frequency
;
8121 BB_COPY_PARTITION (single
, last
);
8122 BB_COPY_PARTITION (empty
, last
);
8124 redirect_edge_succ (e
, single
);
8125 make_single_succ_edge (single
, empty
, 0);
8126 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8129 rtx_code_label
*label
= block_label (empty
);
8130 rtx_jump_insn
*x
= emit_jump_insn_after (gen_jump (label
),
8132 JUMP_LABEL (x
) = label
;
8133 LABEL_NUSES (label
)++;
8134 haifa_init_insn (x
);
8136 emit_barrier_after (x
);
8138 sched_init_only_bb (empty
, NULL
);
8139 sched_init_only_bb (single
, NULL
);
8142 adding_bb_to_current_region_p
= true;
8143 before_recovery
= single
;
8144 after_recovery
= empty
;
8146 if (before_recovery_ptr
)
8147 *before_recovery_ptr
= before_recovery
;
8149 if (sched_verbose
>= 2 && spec_info
->dump
)
8150 fprintf (spec_info
->dump
,
8151 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8152 last
->index
, single
->index
, empty
->index
);
8155 before_recovery
= last
;
8158 /* Returns new recovery block. */
8160 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8165 haifa_recovery_bb_recently_added_p
= true;
8166 haifa_recovery_bb_ever_added_p
= true;
8168 init_before_recovery (before_recovery_ptr
);
8170 barrier
= get_last_bb_insn (before_recovery
);
8171 gcc_assert (BARRIER_P (barrier
));
8173 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8175 rec
= create_basic_block (label
, label
, before_recovery
);
8177 /* A recovery block always ends with an unconditional jump. */
8178 emit_barrier_after (BB_END (rec
));
8180 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8181 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8183 if (sched_verbose
&& spec_info
->dump
)
8184 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8190 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8191 and emit necessary jumps. */
8193 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8194 basic_block second_bb
)
8198 /* This is fixing of incoming edge. */
8199 /* ??? Which other flags should be specified? */
8200 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8201 /* Partition type is the same, if it is "unpartitioned". */
8202 edge_flags
= EDGE_CROSSING
;
8206 make_edge (first_bb
, rec
, edge_flags
);
8207 rtx_code_label
*label
= block_label (second_bb
);
8208 rtx_jump_insn
*jump
= emit_jump_insn_after (gen_jump (label
), BB_END (rec
));
8209 JUMP_LABEL (jump
) = label
;
8210 LABEL_NUSES (label
)++;
8212 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8213 /* Partition type is the same, if it is "unpartitioned". */
8215 /* Rewritten from cfgrtl.c. */
8216 if (flag_reorder_blocks_and_partition
8217 && targetm_common
.have_named_sections
)
8219 /* We don't need the same note for the check because
8220 any_condjump_p (check) == true. */
8221 CROSSING_JUMP_P (jump
) = 1;
8223 edge_flags
= EDGE_CROSSING
;
8228 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8229 if (dom_info_available_p (CDI_DOMINATORS
))
8230 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8233 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8234 INSN is a simple check, that should be converted to branchy one. */
8236 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8239 rtx_insn
*label
, *check
, *twin
;
8242 sd_iterator_def sd_it
;
8244 dep_def _new_dep
, *new_dep
= &_new_dep
;
8247 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8250 todo_spec
= TODO_SPEC (insn
);
8253 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8254 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8256 todo_spec
= CHECK_SPEC (insn
);
8259 todo_spec
&= SPECULATIVE
;
8261 /* Create recovery block. */
8262 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8264 rec
= sched_create_recovery_block (NULL
);
8265 label
= BB_HEAD (rec
);
8269 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8274 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8276 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8278 /* To have mem_reg alive at the beginning of second_bb,
8279 we emit check BEFORE insn, so insn after splitting
8280 insn will be at the beginning of second_bb, which will
8281 provide us with the correct life information. */
8282 check
= emit_jump_insn_before (check_pat
, insn
);
8283 JUMP_LABEL (check
) = label
;
8284 LABEL_NUSES (label
)++;
8287 check
= emit_insn_before (check_pat
, insn
);
8289 /* Extend data structures. */
8290 haifa_init_insn (check
);
8292 /* CHECK is being added to current region. Extend ready list. */
8293 gcc_assert (sched_ready_n_insns
!= -1);
8294 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8296 if (current_sched_info
->add_remove_insn
)
8297 current_sched_info
->add_remove_insn (insn
, 0);
8299 RECOVERY_BLOCK (check
) = rec
;
8301 if (sched_verbose
&& spec_info
->dump
)
8302 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8303 (*current_sched_info
->print_insn
) (check
, 0));
8305 gcc_assert (ORIG_PAT (insn
));
8307 /* Initialize TWIN (twin is a duplicate of original instruction
8308 in the recovery block). */
8309 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8311 sd_iterator_def sd_it
;
8314 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8315 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8317 struct _dep _dep2
, *dep2
= &_dep2
;
8319 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8321 sd_add_dep (dep2
, true);
8324 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8325 haifa_init_insn (twin
);
8327 if (sched_verbose
&& spec_info
->dump
)
8328 /* INSN_BB (insn) isn't determined for twin insns yet.
8329 So we can't use current_sched_info->print_insn. */
8330 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8331 INSN_UID (twin
), rec
->index
);
8335 ORIG_PAT (check
) = ORIG_PAT (insn
);
8336 HAS_INTERNAL_DEP (check
) = 1;
8338 /* ??? We probably should change all OUTPUT dependencies to
8342 /* Copy all resolved back dependencies of INSN to TWIN. This will
8343 provide correct value for INSN_TICK (TWIN). */
8344 sd_copy_back_deps (twin
, insn
, true);
8346 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8347 /* In case of branchy check, fix CFG. */
8349 basic_block first_bb
, second_bb
;
8352 first_bb
= BLOCK_FOR_INSN (check
);
8353 second_bb
= sched_split_block (first_bb
, check
);
8355 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8357 sched_init_only_bb (second_bb
, first_bb
);
8358 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8360 jump
= BB_END (rec
);
8361 haifa_init_insn (jump
);
8364 /* Move backward dependences from INSN to CHECK and
8365 move forward dependences from INSN to TWIN. */
8367 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8368 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8370 rtx_insn
*pro
= DEP_PRO (dep
);
8373 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8374 check --TRUE--> producer ??? or ANTI ???
8375 twin --TRUE--> producer
8376 twin --ANTI--> check
8378 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8379 check --ANTI--> producer
8380 twin --ANTI--> producer
8381 twin --ANTI--> check
8383 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8384 check ~~TRUE~~> producer
8385 twin ~~TRUE~~> producer
8386 twin --ANTI--> check */
8388 ds
= DEP_STATUS (dep
);
8390 if (ds
& BEGIN_SPEC
)
8392 gcc_assert (!mutate_p
);
8396 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8397 sd_add_dep (new_dep
, false);
8399 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8401 DEP_CON (new_dep
) = twin
;
8402 sd_add_dep (new_dep
, false);
8406 /* Second, remove backward dependencies of INSN. */
8407 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8408 sd_iterator_cond (&sd_it
, &dep
);)
8410 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8412 /* We can delete this dep because we overcome it with
8413 BEGIN_SPECULATION. */
8414 sd_delete_dep (sd_it
);
8416 sd_iterator_next (&sd_it
);
8419 /* Future Speculations. Determine what BE_IN speculations will be like. */
8422 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8425 gcc_assert (!DONE_SPEC (insn
));
8429 ds_t ts
= TODO_SPEC (insn
);
8431 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8432 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8434 /* Luckiness of future speculations solely depends upon initial
8435 BEGIN speculation. */
8436 if (ts
& BEGIN_DATA
)
8437 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8438 if (ts
& BEGIN_CONTROL
)
8439 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8440 get_dep_weak (ts
, BEGIN_CONTROL
));
8443 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8445 /* Future speculations: call the helper. */
8446 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8448 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8450 /* Which types of dependencies should we use here is,
8451 generally, machine-dependent question... But, for now,
8456 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8457 sd_add_dep (new_dep
, false);
8459 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8460 sd_add_dep (new_dep
, false);
8464 if (spec_info
->dump
)
8465 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8466 (*current_sched_info
->print_insn
) (insn
, 0));
8468 /* Remove all dependencies of the INSN. */
8470 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8472 | SD_LIST_RES_BACK
));
8473 while (sd_iterator_cond (&sd_it
, &dep
))
8474 sd_delete_dep (sd_it
);
8477 /* If former check (INSN) already was moved to the ready (or queue)
8478 list, add new check (CHECK) there too. */
8479 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8482 /* Remove old check from instruction stream and free its
8484 sched_remove_insn (insn
);
8487 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8488 sd_add_dep (new_dep
, false);
8492 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8493 sd_add_dep (new_dep
, false);
8497 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8498 because it'll be done later in add_to_speculative_block. */
8500 rtx_vec_t priorities_roots
= rtx_vec_t ();
8502 clear_priorities (twin
, &priorities_roots
);
8503 calc_priorities (priorities_roots
);
8504 priorities_roots
.release ();
8508 /* Removes dependency between instructions in the recovery block REC
8509 and usual region instructions. It keeps inner dependences so it
8510 won't be necessary to recompute them. */
8512 fix_recovery_deps (basic_block rec
)
8514 rtx_insn
*note
, *insn
, *jump
;
8515 rtx_insn_list
*ready_list
= 0;
8516 bitmap_head in_ready
;
8517 rtx_insn_list
*link
;
8519 bitmap_initialize (&in_ready
, 0);
8521 /* NOTE - a basic block note. */
8522 note
= NEXT_INSN (BB_HEAD (rec
));
8523 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8524 insn
= BB_END (rec
);
8525 gcc_assert (JUMP_P (insn
));
8526 insn
= PREV_INSN (insn
);
8530 sd_iterator_def sd_it
;
8533 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8534 sd_iterator_cond (&sd_it
, &dep
);)
8536 rtx_insn
*consumer
= DEP_CON (dep
);
8538 if (BLOCK_FOR_INSN (consumer
) != rec
)
8540 sd_delete_dep (sd_it
);
8542 if (bitmap_set_bit (&in_ready
, INSN_LUID (consumer
)))
8543 ready_list
= alloc_INSN_LIST (consumer
, ready_list
);
8547 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8549 sd_iterator_next (&sd_it
);
8553 insn
= PREV_INSN (insn
);
8555 while (insn
!= note
);
8557 bitmap_clear (&in_ready
);
8559 /* Try to add instructions to the ready or queue list. */
8560 for (link
= ready_list
; link
; link
= link
->next ())
8561 try_ready (link
->insn ());
8562 free_INSN_LIST_list (&ready_list
);
8564 /* Fixing jump's dependences. */
8565 insn
= BB_HEAD (rec
);
8566 jump
= BB_END (rec
);
8568 gcc_assert (LABEL_P (insn
));
8569 insn
= NEXT_INSN (insn
);
8571 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8572 add_jump_dependencies (insn
, jump
);
8575 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8576 instruction data. */
8578 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8582 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8586 update_insn_after_change (insn
);
8590 /* -1 - can't speculate,
8591 0 - for speculation with REQUEST mode it is OK to use
8592 current instruction pattern,
8593 1 - need to change pattern for *NEW_PAT to be speculative. */
8595 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8597 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8598 && (request
& SPECULATIVE
)
8599 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8601 if ((request
& spec_info
->mask
) != request
)
8604 if (request
& BE_IN_SPEC
8605 && !(request
& BEGIN_SPEC
))
8608 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8612 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8614 gcc_assert (sched_deps_info
->generate_spec_deps
8615 && !IS_SPECULATION_CHECK_P (insn
));
8617 if (HAS_INTERNAL_DEP (insn
)
8618 || SCHED_GROUP_P (insn
))
8621 return sched_speculate_insn (insn
, request
, new_pat
);
8624 /* Print some information about block BB, which starts with HEAD and
8625 ends with TAIL, before scheduling it.
8626 I is zero, if scheduler is about to start with the fresh ebb. */
8628 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8631 fprintf (sched_dump
,
8632 ";; ======================================================\n");
8634 fprintf (sched_dump
,
8635 ";; =====================ADVANCING TO=====================\n");
8636 fprintf (sched_dump
,
8637 ";; -- basic block %d from %d to %d -- %s reload\n",
8638 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8639 (reload_completed
? "after" : "before"));
8640 fprintf (sched_dump
,
8641 ";; ======================================================\n");
8642 fprintf (sched_dump
, "\n");
8645 /* Unlink basic block notes and labels and saves them, so they
8646 can be easily restored. We unlink basic block notes in EBB to
8647 provide back-compatibility with the previous code, as target backends
8648 assume, that there'll be only instructions between
8649 current_sched_info->{head and tail}. We restore these notes as soon
8651 FIRST (LAST) is the first (last) basic block in the ebb.
8652 NB: In usual case (FIRST == LAST) nothing is really done. */
8654 unlink_bb_notes (basic_block first
, basic_block last
)
8656 /* We DON'T unlink basic block notes of the first block in the ebb. */
8660 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8662 /* Make a sentinel. */
8663 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8664 bb_header
[last
->next_bb
->index
] = 0;
8666 first
= first
->next_bb
;
8669 rtx_insn
*prev
, *label
, *note
, *next
;
8671 label
= BB_HEAD (last
);
8672 if (LABEL_P (label
))
8673 note
= NEXT_INSN (label
);
8676 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8678 prev
= PREV_INSN (label
);
8679 next
= NEXT_INSN (note
);
8680 gcc_assert (prev
&& next
);
8682 SET_NEXT_INSN (prev
) = next
;
8683 SET_PREV_INSN (next
) = prev
;
8685 bb_header
[last
->index
] = label
;
8690 last
= last
->prev_bb
;
8695 /* Restore basic block notes.
8696 FIRST is the first basic block in the ebb. */
8698 restore_bb_notes (basic_block first
)
8703 /* We DON'T unlink basic block notes of the first block in the ebb. */
8704 first
= first
->next_bb
;
8705 /* Remember: FIRST is actually a second basic block in the ebb. */
8707 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8708 && bb_header
[first
->index
])
8710 rtx_insn
*prev
, *label
, *note
, *next
;
8712 label
= bb_header
[first
->index
];
8713 prev
= PREV_INSN (label
);
8714 next
= NEXT_INSN (prev
);
8716 if (LABEL_P (label
))
8717 note
= NEXT_INSN (label
);
8720 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8722 bb_header
[first
->index
] = 0;
8724 SET_NEXT_INSN (prev
) = label
;
8725 SET_NEXT_INSN (note
) = next
;
8726 SET_PREV_INSN (next
) = note
;
8728 first
= first
->next_bb
;
8736 Fix CFG after both in- and inter-block movement of
8737 control_flow_insn_p JUMP. */
8739 fix_jump_move (rtx_insn
*jump
)
8741 basic_block bb
, jump_bb
, jump_bb_next
;
8743 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8744 jump_bb
= BLOCK_FOR_INSN (jump
);
8745 jump_bb_next
= jump_bb
->next_bb
;
8747 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8748 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8750 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8751 /* if jump_bb_next is not empty. */
8752 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8754 if (BB_END (bb
) != PREV_INSN (jump
))
8755 /* Then there are instruction after jump that should be placed
8757 BB_END (jump_bb_next
) = BB_END (bb
);
8759 /* Otherwise jump_bb_next is empty. */
8760 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8762 /* To make assertion in move_insn happy. */
8763 BB_END (bb
) = PREV_INSN (jump
);
8765 update_bb_for_insn (jump_bb_next
);
8768 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8770 move_block_after_check (rtx_insn
*jump
)
8772 basic_block bb
, jump_bb
, jump_bb_next
;
8773 vec
<edge
, va_gc
> *t
;
8775 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8776 jump_bb
= BLOCK_FOR_INSN (jump
);
8777 jump_bb_next
= jump_bb
->next_bb
;
8779 update_bb_for_insn (jump_bb
);
8781 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8782 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8784 unlink_block (jump_bb_next
);
8785 link_block (jump_bb_next
, bb
);
8789 move_succs (&(jump_bb
->succs
), bb
);
8790 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8791 move_succs (&t
, jump_bb_next
);
8793 df_mark_solutions_dirty ();
8795 common_sched_info
->fix_recovery_cfg
8796 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8799 /* Helper function for move_block_after_check.
8800 This functions attaches edge vector pointed to by SUCCSP to
8803 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8808 gcc_assert (to
->succs
== 0);
8810 to
->succs
= *succsp
;
8812 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8818 /* Remove INSN from the instruction stream.
8819 INSN should have any dependencies. */
8821 sched_remove_insn (rtx_insn
*insn
)
8823 sd_finish_insn (insn
);
8825 change_queue_index (insn
, QUEUE_NOWHERE
);
8826 current_sched_info
->add_remove_insn (insn
, 1);
8830 /* Clear priorities of all instructions, that are forward dependent on INSN.
8831 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8832 be invoked to initialize all cleared priorities. */
8834 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8836 sd_iterator_def sd_it
;
8838 bool insn_is_root_p
= true;
8840 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8842 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8844 rtx_insn
*pro
= DEP_PRO (dep
);
8846 if (INSN_PRIORITY_STATUS (pro
) >= 0
8847 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8849 /* If DEP doesn't contribute to priority then INSN itself should
8850 be added to priority roots. */
8851 if (contributes_to_priority_p (dep
))
8852 insn_is_root_p
= false;
8854 INSN_PRIORITY_STATUS (pro
) = -1;
8855 clear_priorities (pro
, roots_ptr
);
8860 roots_ptr
->safe_push (insn
);
8863 /* Recompute priorities of instructions, whose priorities might have been
8864 changed. ROOTS is a vector of instructions whose priority computation will
8865 trigger initialization of all cleared priorities. */
8867 calc_priorities (rtx_vec_t roots
)
8872 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8877 /* Add dependences between JUMP and other instructions in the recovery
8878 block. INSN is the first insn the recovery block. */
8880 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8884 insn
= NEXT_INSN (insn
);
8888 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8890 dep_def _new_dep
, *new_dep
= &_new_dep
;
8892 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8893 sd_add_dep (new_dep
, false);
8898 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8901 /* Extend data structures for logical insn UID. */
8903 sched_extend_luids (void)
8905 int new_luids_max_uid
= get_max_uid () + 1;
8907 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
8910 /* Initialize LUID for INSN. */
8912 sched_init_insn_luid (rtx_insn
*insn
)
8914 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8919 luid
= sched_max_luid
;
8920 sched_max_luid
+= i
;
8925 SET_INSN_LUID (insn
, luid
);
8928 /* Initialize luids for BBS.
8929 The hook common_sched_info->luid_for_non_insn () is used to determine
8930 if notes, labels, etc. need luids. */
8932 sched_init_luids (bb_vec_t bbs
)
8937 sched_extend_luids ();
8938 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
8942 FOR_BB_INSNS (bb
, insn
)
8943 sched_init_insn_luid (insn
);
8949 sched_finish_luids (void)
8951 sched_luids
.release ();
8955 /* Return logical uid of INSN. Helpful while debugging. */
8957 insn_luid (rtx_insn
*insn
)
8959 return INSN_LUID (insn
);
8962 /* Extend per insn data in the target. */
8964 sched_extend_target (void)
8966 if (targetm
.sched
.h_i_d_extended
)
8967 targetm
.sched
.h_i_d_extended ();
8970 /* Extend global scheduler structures (those, that live across calls to
8971 schedule_block) to include information about just emitted INSN. */
8975 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
8977 && ! h_i_d
.space (reserve
))
8979 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
8980 sched_extend_target ();
8984 /* Initialize h_i_d entry of the INSN with default values.
8985 Values, that are not explicitly initialized here, hold zero. */
8987 init_h_i_d (rtx_insn
*insn
)
8989 if (INSN_LUID (insn
) > 0)
8991 INSN_COST (insn
) = -1;
8992 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
8993 INSN_TICK (insn
) = INVALID_TICK
;
8994 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
8995 INTER_TICK (insn
) = INVALID_TICK
;
8996 TODO_SPEC (insn
) = HARD_DEP
;
8997 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
8998 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
8999 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
9000 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9004 /* Initialize haifa_insn_data for BBS. */
9006 haifa_init_h_i_d (bb_vec_t bbs
)
9012 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9016 FOR_BB_INSNS (bb
, insn
)
9021 /* Finalize haifa_insn_data. */
9023 haifa_finish_h_i_d (void)
9026 haifa_insn_data_t data
;
9027 struct reg_use_data
*use
, *next
;
9029 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9031 free (data
->max_reg_pressure
);
9032 free (data
->reg_pressure
);
9033 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next
)
9035 next
= use
->next_insn_use
;
9042 /* Init data for the new insn INSN. */
9044 haifa_init_insn (rtx_insn
*insn
)
9046 gcc_assert (insn
!= NULL
);
9048 sched_extend_luids ();
9049 sched_init_insn_luid (insn
);
9050 sched_extend_target ();
9051 sched_deps_init (false);
9055 if (adding_bb_to_current_region_p
)
9057 sd_init_insn (insn
);
9059 /* Extend dependency caches by one element. */
9060 extend_dependency_caches (1, false);
9062 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9063 init_insn_reg_pressure_info (insn
);
9066 /* Init data for the new basic block BB which comes after AFTER. */
9068 haifa_init_only_bb (basic_block bb
, basic_block after
)
9070 gcc_assert (bb
!= NULL
);
9074 if (common_sched_info
->add_block
)
9075 /* This changes only data structures of the front-end. */
9076 common_sched_info
->add_block (bb
, after
);
9079 /* A generic version of sched_split_block (). */
9081 sched_split_block_1 (basic_block first_bb
, rtx after
)
9085 e
= split_block (first_bb
, after
);
9086 gcc_assert (e
->src
== first_bb
);
9088 /* sched_split_block emits note if *check == BB_END. Probably it
9089 is better to rip that note off. */
9094 /* A generic version of sched_create_empty_bb (). */
9096 sched_create_empty_bb_1 (basic_block after
)
9098 return create_empty_bb (after
);
9101 /* Insert PAT as an INSN into the schedule and update the necessary data
9102 structures to account for it. */
9104 sched_emit_insn (rtx pat
)
9106 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9107 haifa_init_insn (insn
);
9109 if (current_sched_info
->add_remove_insn
)
9110 current_sched_info
->add_remove_insn (insn
, 0);
9112 (*current_sched_info
->begin_schedule_ready
) (insn
);
9113 scheduled_insns
.safe_push (insn
);
9115 last_scheduled_insn
= insn
;
9119 /* This function returns a candidate satisfying dispatch constraints from
9123 ready_remove_first_dispatch (struct ready_list
*ready
)
9126 rtx_insn
*insn
= ready_element (ready
, 0);
9128 if (ready
->n_ready
== 1
9130 || INSN_CODE (insn
) < 0
9131 || !active_insn_p (insn
)
9132 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9133 return ready_remove_first (ready
);
9135 for (i
= 1; i
< ready
->n_ready
; i
++)
9137 insn
= ready_element (ready
, i
);
9140 || INSN_CODE (insn
) < 0
9141 || !active_insn_p (insn
))
9144 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9146 /* Return ith element of ready. */
9147 insn
= ready_remove (ready
, i
);
9152 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9153 return ready_remove_first (ready
);
9155 for (i
= 1; i
< ready
->n_ready
; i
++)
9157 insn
= ready_element (ready
, i
);
9160 || INSN_CODE (insn
) < 0
9161 || !active_insn_p (insn
))
9164 /* Return i-th element of ready. */
9165 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9166 return ready_remove (ready
, i
);
9169 return ready_remove_first (ready
);
9172 /* Get number of ready insn in the ready list. */
9175 number_in_ready (void)
9177 return ready
.n_ready
;
9180 /* Get number of ready's in the ready list. */
9183 get_ready_element (int i
)
9185 return ready_element (&ready
, i
);
9188 #endif /* INSN_SCHEDULING */