1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2015 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
129 #include "diagnostic-core.h"
130 #include "hard-reg-set.h"
135 #include "hash-set.h"
137 #include "machmode.h"
139 #include "function.h"
141 #include "insn-config.h"
142 #include "insn-attr.h"
145 #include "dominance.h"
148 #include "cfgbuild.h"
150 #include "basic-block.h"
151 #include "sched-int.h"
153 #include "common/common-target.h"
158 #include "emit-rtl.h" /* FIXME: Can go away once crtl is moved to rtl.h. */
159 #include "hash-table.h"
160 #include "dumpfile.h"
162 #ifdef INSN_SCHEDULING
164 /* True if we do register pressure relief through live-range
166 static bool live_range_shrinkage_p
;
168 /* Switch on live range shrinkage. */
170 initialize_live_range_shrinkage (void)
172 live_range_shrinkage_p
= true;
175 /* Switch off live range shrinkage. */
177 finish_live_range_shrinkage (void)
179 live_range_shrinkage_p
= false;
182 /* issue_rate is the number of insns that can be scheduled in the same
183 machine cycle. It can be defined in the config/mach/mach.h file,
184 otherwise we set it to 1. */
188 /* This can be set to true by a backend if the scheduler should not
189 enable a DCE pass. */
192 /* The current initiation interval used when modulo scheduling. */
193 static int modulo_ii
;
195 /* The maximum number of stages we are prepared to handle. */
196 static int modulo_max_stages
;
198 /* The number of insns that exist in each iteration of the loop. We use this
199 to detect when we've scheduled all insns from the first iteration. */
200 static int modulo_n_insns
;
202 /* The current count of insns in the first iteration of the loop that have
203 already been scheduled. */
204 static int modulo_insns_scheduled
;
206 /* The maximum uid of insns from the first iteration of the loop. */
207 static int modulo_iter0_max_uid
;
209 /* The number of times we should attempt to backtrack when modulo scheduling.
210 Decreased each time we have to backtrack. */
211 static int modulo_backtracks_left
;
213 /* The stage in which the last insn from the original loop was
215 static int modulo_last_stage
;
217 /* sched-verbose controls the amount of debugging output the
218 scheduler prints. It is controlled by -fsched-verbose=N:
219 N>0 and no -DSR : the output is directed to stderr.
220 N>=10 will direct the printouts to stderr (regardless of -dSR).
222 N=2: bb's probabilities, detailed ready list info, unit/insn info.
223 N=3: rtl at abort point, control-flow, regions info.
224 N=5: dependences info. */
226 int sched_verbose
= 0;
228 /* Debugging file. All printouts are sent to dump, which is always set,
229 either to stderr, or to the dump listing file (-dRS). */
230 FILE *sched_dump
= 0;
232 /* This is a placeholder for the scheduler parameters common
233 to all schedulers. */
234 struct common_sched_info_def
*common_sched_info
;
236 #define INSN_TICK(INSN) (HID (INSN)->tick)
237 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
238 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
239 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
240 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
241 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
242 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
243 /* Cached cost of the instruction. Use insn_cost to get cost of the
244 insn. -1 here means that the field is not initialized. */
245 #define INSN_COST(INSN) (HID (INSN)->cost)
247 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
248 then it should be recalculated from scratch. */
249 #define INVALID_TICK (-(max_insn_queue_index + 1))
250 /* The minimal value of the INSN_TICK of an instruction. */
251 #define MIN_TICK (-max_insn_queue_index)
253 /* Original order of insns in the ready list.
254 Used to keep order of normal insns while separating DEBUG_INSNs. */
255 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
257 /* The deciding reason for INSN's place in the ready list. */
258 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
260 /* List of important notes we must keep around. This is a pointer to the
261 last element in the list. */
264 static struct spec_info_def spec_info_var
;
265 /* Description of the speculative part of the scheduling.
266 If NULL - no speculation. */
267 spec_info_t spec_info
= NULL
;
269 /* True, if recovery block was added during scheduling of current block.
270 Used to determine, if we need to fix INSN_TICKs. */
271 static bool haifa_recovery_bb_recently_added_p
;
273 /* True, if recovery block was added during this scheduling pass.
274 Used to determine if we should have empty memory pools of dependencies
275 after finishing current region. */
276 bool haifa_recovery_bb_ever_added_p
;
278 /* Counters of different types of speculative instructions. */
279 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
281 /* Array used in {unlink, restore}_bb_notes. */
282 static rtx_insn
**bb_header
= 0;
284 /* Basic block after which recovery blocks will be created. */
285 static basic_block before_recovery
;
287 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
289 basic_block after_recovery
;
291 /* FALSE if we add bb to another region, so we don't need to initialize it. */
292 bool adding_bb_to_current_region_p
= true;
296 /* An instruction is ready to be scheduled when all insns preceding it
297 have already been scheduled. It is important to ensure that all
298 insns which use its result will not be executed until its result
299 has been computed. An insn is maintained in one of four structures:
301 (P) the "Pending" set of insns which cannot be scheduled until
302 their dependencies have been satisfied.
303 (Q) the "Queued" set of insns that can be scheduled when sufficient
305 (R) the "Ready" list of unscheduled, uncommitted insns.
306 (S) the "Scheduled" list of insns.
308 Initially, all insns are either "Pending" or "Ready" depending on
309 whether their dependencies are satisfied.
311 Insns move from the "Ready" list to the "Scheduled" list as they
312 are committed to the schedule. As this occurs, the insns in the
313 "Pending" list have their dependencies satisfied and move to either
314 the "Ready" list or the "Queued" set depending on whether
315 sufficient time has passed to make them ready. As time passes,
316 insns move from the "Queued" set to the "Ready" list.
318 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
319 unscheduled insns, i.e., those that are ready, queued, and pending.
320 The "Queued" set (Q) is implemented by the variable `insn_queue'.
321 The "Ready" list (R) is implemented by the variables `ready' and
323 The "Scheduled" list (S) is the new insn chain built by this pass.
325 The transition (R->S) is implemented in the scheduling loop in
326 `schedule_block' when the best insn to schedule is chosen.
327 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
328 insns move from the ready list to the scheduled list.
329 The transition (Q->R) is implemented in 'queue_to_insn' as time
330 passes or stalls are introduced. */
332 /* Implement a circular buffer to delay instructions until sufficient
333 time has passed. For the new pipeline description interface,
334 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
335 than maximal time of instruction execution computed by genattr.c on
336 the base maximal time of functional unit reservations and getting a
337 result. This is the longest time an insn may be queued. */
339 static rtx_insn_list
**insn_queue
;
340 static int q_ptr
= 0;
341 static int q_size
= 0;
342 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
343 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
345 #define QUEUE_SCHEDULED (-3)
346 #define QUEUE_NOWHERE (-2)
347 #define QUEUE_READY (-1)
348 /* QUEUE_SCHEDULED - INSN is scheduled.
349 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
351 QUEUE_READY - INSN is in ready list.
352 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
354 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
356 /* The following variable value refers for all current and future
357 reservations of the processor units. */
360 /* The following variable value is size of memory representing all
361 current and future reservations of the processor units. */
362 size_t dfa_state_size
;
364 /* The following array is used to find the best insn from ready when
365 the automaton pipeline interface is used. */
366 signed char *ready_try
= NULL
;
368 /* The ready list. */
369 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
371 /* The pointer to the ready list (to be removed). */
372 static struct ready_list
*readyp
= &ready
;
374 /* Scheduling clock. */
375 static int clock_var
;
377 /* Clock at which the previous instruction was issued. */
378 static int last_clock_var
;
380 /* Set to true if, when queuing a shadow insn, we discover that it would be
381 scheduled too late. */
382 static bool must_backtrack
;
384 /* The following variable value is number of essential insns issued on
385 the current cycle. An insn is essential one if it changes the
387 int cycle_issued_insns
;
389 /* This records the actual schedule. It is built up during the main phase
390 of schedule_block, and afterwards used to reorder the insns in the RTL. */
391 static vec
<rtx_insn
*> scheduled_insns
;
393 static int may_trap_exp (const_rtx
, int);
395 /* Nonzero iff the address is comprised from at most 1 register. */
396 #define CONST_BASED_ADDRESS_P(x) \
398 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
399 || (GET_CODE (x) == LO_SUM)) \
400 && (CONSTANT_P (XEXP (x, 0)) \
401 || CONSTANT_P (XEXP (x, 1)))))
403 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
404 as found by analyzing insn's expression. */
407 static int haifa_luid_for_non_insn (rtx x
);
409 /* Haifa version of sched_info hooks common to all headers. */
410 const struct common_sched_info_def haifa_common_sched_info
=
412 NULL
, /* fix_recovery_cfg */
413 NULL
, /* add_block */
414 NULL
, /* estimate_number_of_insns */
415 haifa_luid_for_non_insn
, /* luid_for_non_insn */
416 SCHED_PASS_UNKNOWN
/* sched_pass_id */
419 /* Mapping from instruction UID to its Logical UID. */
420 vec
<int> sched_luids
= vNULL
;
422 /* Next LUID to assign to an instruction. */
423 int sched_max_luid
= 1;
425 /* Haifa Instruction Data. */
426 vec
<haifa_insn_data_def
> h_i_d
= vNULL
;
428 void (* sched_init_only_bb
) (basic_block
, basic_block
);
430 /* Split block function. Different schedulers might use different functions
431 to handle their internal data consistent. */
432 basic_block (* sched_split_block
) (basic_block
, rtx
);
434 /* Create empty basic block after the specified block. */
435 basic_block (* sched_create_empty_bb
) (basic_block
);
437 /* Return the number of cycles until INSN is expected to be ready.
438 Return zero if it already is. */
440 insn_delay (rtx_insn
*insn
)
442 return MAX (INSN_TICK (insn
) - clock_var
, 0);
446 may_trap_exp (const_rtx x
, int is_store
)
455 if (code
== MEM
&& may_trap_p (x
))
462 /* The insn uses memory: a volatile load. */
463 if (MEM_VOLATILE_P (x
))
465 /* An exception-free load. */
468 /* A load with 1 base register, to be further checked. */
469 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
470 return PFREE_CANDIDATE
;
471 /* No info on the load, to be further checked. */
472 return PRISKY_CANDIDATE
;
477 int i
, insn_class
= TRAP_FREE
;
479 /* Neither store nor load, check if it may cause a trap. */
482 /* Recursive step: walk the insn... */
483 fmt
= GET_RTX_FORMAT (code
);
484 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
488 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
489 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
491 else if (fmt
[i
] == 'E')
494 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
496 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
497 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
498 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
502 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
509 /* Classifies rtx X of an insn for the purpose of verifying that X can be
510 executed speculatively (and consequently the insn can be moved
511 speculatively), by examining X, returning:
512 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
513 TRAP_FREE: non-load insn.
514 IFREE: load from a globally safe location.
515 IRISKY: volatile load.
516 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
517 being either PFREE or PRISKY. */
520 haifa_classify_rtx (const_rtx x
)
522 int tmp_class
= TRAP_FREE
;
523 int insn_class
= TRAP_FREE
;
526 if (GET_CODE (x
) == PARALLEL
)
528 int i
, len
= XVECLEN (x
, 0);
530 for (i
= len
- 1; i
>= 0; i
--)
532 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
533 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
534 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
544 /* Test if it is a 'store'. */
545 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
548 /* Test if it is a store. */
549 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
550 if (tmp_class
== TRAP_RISKY
)
552 /* Test if it is a load. */
554 WORST_CLASS (tmp_class
,
555 may_trap_exp (SET_SRC (x
), 0));
558 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
559 if (tmp_class
== TRAP_RISKY
)
561 tmp_class
= WORST_CLASS (tmp_class
,
562 may_trap_exp (COND_EXEC_TEST (x
), 0));
565 tmp_class
= TRAP_RISKY
;
569 insn_class
= tmp_class
;
576 haifa_classify_insn (const_rtx insn
)
578 return haifa_classify_rtx (PATTERN (insn
));
581 /* After the scheduler initialization function has been called, this function
582 can be called to enable modulo scheduling. II is the initiation interval
583 we should use, it affects the delays for delay_pairs that were recorded as
584 separated by a given number of stages.
586 MAX_STAGES provides us with a limit
587 after which we give up scheduling; the caller must have unrolled at least
588 as many copies of the loop body and recorded delay_pairs for them.
590 INSNS is the number of real (non-debug) insns in one iteration of
591 the loop. MAX_UID can be used to test whether an insn belongs to
592 the first iteration of the loop; all of them have a uid lower than
595 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
598 modulo_max_stages
= max_stages
;
599 modulo_n_insns
= insns
;
600 modulo_iter0_max_uid
= max_uid
;
601 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
604 /* A structure to record a pair of insns where the first one is a real
605 insn that has delay slots, and the second is its delayed shadow.
606 I1 is scheduled normally and will emit an assembly instruction,
607 while I2 describes the side effect that takes place at the
608 transition between cycles CYCLES and (CYCLES + 1) after I1. */
611 struct delay_pair
*next_same_i1
;
614 /* When doing modulo scheduling, we a delay_pair can also be used to
615 show that I1 and I2 are the same insn in a different stage. If that
616 is the case, STAGES will be nonzero. */
620 /* Helpers for delay hashing. */
622 struct delay_i1_hasher
: typed_noop_remove
<delay_pair
>
624 typedef delay_pair value_type
;
625 typedef void compare_type
;
626 static inline hashval_t
hash (const value_type
*);
627 static inline bool equal (const value_type
*, const compare_type
*);
630 /* Returns a hash value for X, based on hashing just I1. */
633 delay_i1_hasher::hash (const value_type
*x
)
635 return htab_hash_pointer (x
->i1
);
638 /* Return true if I1 of pair X is the same as that of pair Y. */
641 delay_i1_hasher::equal (const value_type
*x
, const compare_type
*y
)
646 struct delay_i2_hasher
: typed_free_remove
<delay_pair
>
648 typedef delay_pair value_type
;
649 typedef void compare_type
;
650 static inline hashval_t
hash (const value_type
*);
651 static inline bool equal (const value_type
*, const compare_type
*);
654 /* Returns a hash value for X, based on hashing just I2. */
657 delay_i2_hasher::hash (const value_type
*x
)
659 return htab_hash_pointer (x
->i2
);
662 /* Return true if I2 of pair X is the same as that of pair Y. */
665 delay_i2_hasher::equal (const value_type
*x
, const compare_type
*y
)
670 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
672 static hash_table
<delay_i1_hasher
> *delay_htab
;
673 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
675 /* Called through htab_traverse. Walk the hashtable using I2 as
676 index, and delete all elements involving an UID higher than
677 that pointed to by *DATA. */
679 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
682 struct delay_pair
*p
= *slot
;
683 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
685 delay_htab_i2
->clear_slot (slot
);
690 /* Called through htab_traverse. Walk the hashtable using I2 as
691 index, and delete all elements involving an UID higher than
692 that pointed to by *DATA. */
694 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
697 struct delay_pair
*p
, *first
, **pprev
;
699 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
701 delay_htab
->clear_slot (pslot
);
705 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
707 if (INSN_UID (p
->i2
) < maxuid
)
710 pprev
= &p
->next_same_i1
;
715 delay_htab
->clear_slot (pslot
);
721 /* Discard all delay pairs which involve an insn with an UID higher
724 discard_delay_pairs_above (int max_uid
)
726 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
727 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
730 /* This function can be called by a port just before it starts the final
731 scheduling pass. It records the fact that an instruction with delay
732 slots has been split into two insns, I1 and I2. The first one will be
733 scheduled normally and initiates the operation. The second one is a
734 shadow which must follow a specific number of cycles after I1; its only
735 purpose is to show the side effect that occurs at that cycle in the RTL.
736 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
737 while I2 retains the original insn type.
739 There are two ways in which the number of cycles can be specified,
740 involving the CYCLES and STAGES arguments to this function. If STAGES
741 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
742 which is multiplied by MODULO_II to give the number of cycles. This is
743 only useful if the caller also calls set_modulo_params to enable modulo
747 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
749 struct delay_pair
*p
= XNEW (struct delay_pair
);
750 struct delay_pair
**slot
;
759 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
760 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
762 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
763 p
->next_same_i1
= *slot
;
765 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
769 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
770 and return the other insn if so. Return NULL otherwise. */
772 real_insn_for_shadow (rtx_insn
*insn
)
774 struct delay_pair
*pair
;
779 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
780 if (!pair
|| pair
->stages
> 0)
785 /* For a pair P of insns, return the fixed distance in cycles from the first
786 insn after which the second must be scheduled. */
788 pair_delay (struct delay_pair
*p
)
793 return p
->stages
* modulo_ii
;
796 /* Given an insn INSN, add a dependence on its delayed shadow if it
797 has one. Also try to find situations where shadows depend on each other
798 and add dependencies to the real insns to limit the amount of backtracking
801 add_delay_dependencies (rtx_insn
*insn
)
803 struct delay_pair
*pair
;
804 sd_iterator_def sd_it
;
810 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
813 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
817 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
819 rtx_insn
*pro
= DEP_PRO (dep
);
820 struct delay_pair
*other_pair
821 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
822 if (!other_pair
|| other_pair
->stages
)
824 if (pair_delay (other_pair
) >= pair_delay (pair
))
826 if (sched_verbose
>= 4)
828 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
829 INSN_UID (other_pair
->i1
),
830 INSN_UID (pair
->i1
));
831 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
835 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
836 INSN_UID (other_pair
->i1
),
837 INSN_UID (other_pair
->i2
),
838 pair_delay (other_pair
));
840 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
845 /* Forward declarations. */
847 static int priority (rtx_insn
*);
848 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
849 static int rank_for_schedule (const void *, const void *);
850 static void swap_sort (rtx_insn
**, int);
851 static void queue_insn (rtx_insn
*, int, const char *);
852 static int schedule_insn (rtx_insn
*);
853 static void adjust_priority (rtx_insn
*);
854 static void advance_one_cycle (void);
855 static void extend_h_i_d (void);
858 /* Notes handling mechanism:
859 =========================
860 Generally, NOTES are saved before scheduling and restored after scheduling.
861 The scheduler distinguishes between two types of notes:
863 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
864 Before scheduling a region, a pointer to the note is added to the insn
865 that follows or precedes it. (This happens as part of the data dependence
866 computation). After scheduling an insn, the pointer contained in it is
867 used for regenerating the corresponding note (in reemit_notes).
869 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
870 these notes are put in a list (in rm_other_notes() and
871 unlink_other_notes ()). After scheduling the block, these notes are
872 inserted at the beginning of the block (in schedule_block()). */
874 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
875 static rtx_insn
*ready_remove_first (struct ready_list
*);
876 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
878 static void queue_to_ready (struct ready_list
*);
879 static int early_queue_to_ready (state_t
, struct ready_list
*);
881 /* The following functions are used to implement multi-pass scheduling
882 on the first cycle. */
883 static rtx_insn
*ready_remove (struct ready_list
*, int);
884 static void ready_remove_insn (rtx
);
886 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
887 static int fix_tick_ready (rtx_insn
*);
888 static void change_queue_index (rtx_insn
*, int);
890 /* The following functions are used to implement scheduling of data/control
891 speculative instructions. */
893 static void extend_h_i_d (void);
894 static void init_h_i_d (rtx_insn
*);
895 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
896 static void generate_recovery_code (rtx_insn
*);
897 static void process_insn_forw_deps_be_in_spec (rtx
, rtx_insn
*, ds_t
);
898 static void begin_speculative_block (rtx_insn
*);
899 static void add_to_speculative_block (rtx_insn
*);
900 static void init_before_recovery (basic_block
*);
901 static void create_check_block_twin (rtx_insn
*, bool);
902 static void fix_recovery_deps (basic_block
);
903 static bool haifa_change_pattern (rtx_insn
*, rtx
);
904 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
905 static void restore_bb_notes (basic_block
);
906 static void fix_jump_move (rtx_insn
*);
907 static void move_block_after_check (rtx_insn
*);
908 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
909 static void sched_remove_insn (rtx_insn
*);
910 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
911 static void calc_priorities (rtx_vec_t
);
912 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
914 #endif /* INSN_SCHEDULING */
916 /* Point to state used for the current scheduling pass. */
917 struct haifa_sched_info
*current_sched_info
;
919 #ifndef INSN_SCHEDULING
921 schedule_insns (void)
926 /* Do register pressure sensitive insn scheduling if the flag is set
928 enum sched_pressure_algorithm sched_pressure
;
930 /* Map regno -> its pressure class. The map defined only when
931 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
932 enum reg_class
*sched_regno_pressure_class
;
934 /* The current register pressure. Only elements corresponding pressure
935 classes are defined. */
936 static int curr_reg_pressure
[N_REG_CLASSES
];
938 /* Saved value of the previous array. */
939 static int saved_reg_pressure
[N_REG_CLASSES
];
941 /* Register living at given scheduling point. */
942 static bitmap curr_reg_live
;
944 /* Saved value of the previous array. */
945 static bitmap saved_reg_live
;
947 /* Registers mentioned in the current region. */
948 static bitmap region_ref_regs
;
950 /* Effective number of available registers of a given class (see comment
951 in sched_pressure_start_bb). */
952 static int sched_class_regs_num
[N_REG_CLASSES
];
953 /* Number of call_used_regs. This is a helper for calculating of
954 sched_class_regs_num. */
955 static int call_used_regs_num
[N_REG_CLASSES
];
957 /* Initiate register pressure relative info for scheduling the current
958 region. Currently it is only clearing register mentioned in the
961 sched_init_region_reg_pressure_info (void)
963 bitmap_clear (region_ref_regs
);
966 /* PRESSURE[CL] describes the pressure on register class CL. Update it
967 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
968 LIVE tracks the set of live registers; if it is null, assume that
969 every birth or death is genuine. */
971 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
973 enum reg_class pressure_class
;
975 pressure_class
= sched_regno_pressure_class
[regno
];
976 if (regno
>= FIRST_PSEUDO_REGISTER
)
978 if (pressure_class
!= NO_REGS
)
982 if (!live
|| bitmap_set_bit (live
, regno
))
983 pressure
[pressure_class
]
984 += (ira_reg_class_max_nregs
985 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
989 if (!live
|| bitmap_clear_bit (live
, regno
))
990 pressure
[pressure_class
]
991 -= (ira_reg_class_max_nregs
992 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
996 else if (pressure_class
!= NO_REGS
997 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
1001 if (!live
|| bitmap_set_bit (live
, regno
))
1002 pressure
[pressure_class
]++;
1006 if (!live
|| bitmap_clear_bit (live
, regno
))
1007 pressure
[pressure_class
]--;
1012 /* Initiate current register pressure related info from living
1013 registers given by LIVE. */
1015 initiate_reg_pressure_info (bitmap live
)
1021 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1022 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1023 bitmap_clear (curr_reg_live
);
1024 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1025 if (sched_pressure
== SCHED_PRESSURE_MODEL
1026 || current_nr_blocks
== 1
1027 || bitmap_bit_p (region_ref_regs
, j
))
1028 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1031 /* Mark registers in X as mentioned in the current region. */
1033 setup_ref_regs (rtx x
)
1036 const RTX_CODE code
= GET_CODE (x
);
1042 if (HARD_REGISTER_NUM_P (regno
))
1043 bitmap_set_range (region_ref_regs
, regno
,
1044 hard_regno_nregs
[regno
][GET_MODE (x
)]);
1046 bitmap_set_bit (region_ref_regs
, REGNO (x
));
1049 fmt
= GET_RTX_FORMAT (code
);
1050 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1052 setup_ref_regs (XEXP (x
, i
));
1053 else if (fmt
[i
] == 'E')
1055 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1056 setup_ref_regs (XVECEXP (x
, i
, j
));
1060 /* Initiate current register pressure related info at the start of
1063 initiate_bb_reg_pressure_info (basic_block bb
)
1065 unsigned int i ATTRIBUTE_UNUSED
;
1068 if (current_nr_blocks
> 1)
1069 FOR_BB_INSNS (bb
, insn
)
1070 if (NONDEBUG_INSN_P (insn
))
1071 setup_ref_regs (PATTERN (insn
));
1072 initiate_reg_pressure_info (df_get_live_in (bb
));
1073 #ifdef EH_RETURN_DATA_REGNO
1074 if (bb_has_eh_pred (bb
))
1077 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1079 if (regno
== INVALID_REGNUM
)
1081 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1082 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1088 /* Save current register pressure related info. */
1090 save_reg_pressure (void)
1094 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1095 saved_reg_pressure
[ira_pressure_classes
[i
]]
1096 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1097 bitmap_copy (saved_reg_live
, curr_reg_live
);
1100 /* Restore saved register pressure related info. */
1102 restore_reg_pressure (void)
1106 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1107 curr_reg_pressure
[ira_pressure_classes
[i
]]
1108 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1109 bitmap_copy (curr_reg_live
, saved_reg_live
);
1112 /* Return TRUE if the register is dying after its USE. */
1114 dying_use_p (struct reg_use_data
*use
)
1116 struct reg_use_data
*next
;
1118 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1119 if (NONDEBUG_INSN_P (next
->insn
)
1120 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1125 /* Print info about the current register pressure and its excess for
1126 each pressure class. */
1128 print_curr_reg_pressure (void)
1133 fprintf (sched_dump
, ";;\t");
1134 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1136 cl
= ira_pressure_classes
[i
];
1137 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1138 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1139 curr_reg_pressure
[cl
],
1140 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1142 fprintf (sched_dump
, "\n");
1145 /* Determine if INSN has a condition that is clobbered if a register
1146 in SET_REGS is modified. */
1148 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1150 rtx pat
= PATTERN (insn
);
1151 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1152 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1154 sd_iterator_def sd_it
;
1156 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1157 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1158 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1159 TODO_SPEC (insn
) = HARD_DEP
;
1160 if (sched_verbose
>= 2)
1161 fprintf (sched_dump
,
1162 ";;\t\tdequeue insn %s because of clobbered condition\n",
1163 (*current_sched_info
->print_insn
) (insn
, 0));
1170 /* This function should be called after modifying the pattern of INSN,
1171 to update scheduler data structures as needed. */
1173 update_insn_after_change (rtx_insn
*insn
)
1175 sd_iterator_def sd_it
;
1178 dfa_clear_single_insn_cache (insn
);
1180 sd_it
= sd_iterator_start (insn
,
1181 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1182 while (sd_iterator_cond (&sd_it
, &dep
))
1184 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1185 sd_iterator_next (&sd_it
);
1188 /* Invalidate INSN_COST, so it'll be recalculated. */
1189 INSN_COST (insn
) = -1;
1190 /* Invalidate INSN_TICK, so it'll be recalculated. */
1191 INSN_TICK (insn
) = INVALID_TICK
;
1193 /* Invalidate autoprefetch data entry. */
1194 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1195 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1196 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1197 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1201 /* Two VECs, one to hold dependencies for which pattern replacements
1202 need to be applied or restored at the start of the next cycle, and
1203 another to hold an integer that is either one, to apply the
1204 corresponding replacement, or zero to restore it. */
1205 static vec
<dep_t
> next_cycle_replace_deps
;
1206 static vec
<int> next_cycle_apply
;
1208 static void apply_replacement (dep_t
, bool);
1209 static void restore_pattern (dep_t
, bool);
1211 /* Look at the remaining dependencies for insn NEXT, and compute and return
1212 the TODO_SPEC value we should use for it. This is called after one of
1213 NEXT's dependencies has been resolved.
1214 We also perform pattern replacements for predication, and for broken
1215 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1219 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1222 sd_iterator_def sd_it
;
1223 dep_t dep
, modify_dep
= NULL
;
1227 bool first_p
= true;
1229 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1230 /* NEXT has all its dependencies resolved. */
1233 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1236 /* If NEXT is intended to sit adjacent to this instruction, we don't
1237 want to try to break any dependencies. Treat it as a HARD_DEP. */
1238 if (SCHED_GROUP_P (next
))
1241 /* Now we've got NEXT with speculative deps only.
1242 1. Look at the deps to see what we have to do.
1243 2. Check if we can do 'todo'. */
1246 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1248 rtx_insn
*pro
= DEP_PRO (dep
);
1249 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1251 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1264 new_ds
= ds_merge (new_ds
, ds
);
1266 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1268 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1273 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1275 else if (DEP_REPLACE (dep
) != NULL
)
1277 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1282 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1286 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1288 if (!dbg_cnt (sched_breakdep
))
1290 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1292 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1295 if (desc
->insn
== next
&& !for_backtrack
)
1297 gcc_assert (n_replace
== 1);
1298 apply_replacement (dep
, true);
1300 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1306 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1308 rtx_insn
*pro
, *other
;
1310 rtx cond
= NULL_RTX
;
1312 rtx_insn
*prev
= NULL
;
1316 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1317 || (ORIG_PAT (next
) != NULL_RTX
1318 && PREDICATED_PAT (next
) == NULL_RTX
))
1321 pro
= DEP_PRO (modify_dep
);
1322 other
= real_insn_for_shadow (pro
);
1323 if (other
!= NULL_RTX
)
1326 cond
= sched_get_reverse_condition_uncached (pro
);
1327 regno
= REGNO (XEXP (cond
, 0));
1329 /* Find the last scheduled insn that modifies the condition register.
1330 We can stop looking once we find the insn we depend on through the
1331 REG_DEP_CONTROL; if the condition register isn't modified after it,
1332 we know that it still has the right value. */
1333 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1334 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1338 find_all_hard_reg_sets (prev
, &t
, true);
1339 if (TEST_HARD_REG_BIT (t
, regno
))
1344 if (ORIG_PAT (next
) == NULL_RTX
)
1346 ORIG_PAT (next
) = PATTERN (next
);
1348 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1349 success
= haifa_change_pattern (next
, new_pat
);
1352 PREDICATED_PAT (next
) = new_pat
;
1354 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1356 bool success
= haifa_change_pattern (next
,
1357 PREDICATED_PAT (next
));
1358 gcc_assert (success
);
1360 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1364 if (PREDICATED_PAT (next
) != NULL_RTX
)
1366 int tick
= INSN_TICK (next
);
1367 bool success
= haifa_change_pattern (next
,
1369 INSN_TICK (next
) = tick
;
1370 gcc_assert (success
);
1373 /* We can't handle the case where there are both speculative and control
1374 dependencies, so we return HARD_DEP in such a case. Also fail if
1375 we have speculative dependencies with not enough points, or more than
1376 one control dependency. */
1377 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1379 /* Too few points? */
1380 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1388 /* Pointer to the last instruction scheduled. */
1389 static rtx_insn
*last_scheduled_insn
;
1391 /* Pointer to the last nondebug instruction scheduled within the
1392 block, or the prev_head of the scheduling block. Used by
1393 rank_for_schedule, so that insns independent of the last scheduled
1394 insn will be preferred over dependent instructions. */
1395 static rtx last_nondebug_scheduled_insn
;
1397 /* Pointer that iterates through the list of unscheduled insns if we
1398 have a dbg_cnt enabled. It always points at an insn prior to the
1399 first unscheduled one. */
1400 static rtx_insn
*nonscheduled_insns_begin
;
1402 /* Compute cost of executing INSN.
1403 This is the number of cycles between instruction issue and
1404 instruction results. */
1406 insn_cost (rtx_insn
*insn
)
1415 if (recog_memoized (insn
) < 0)
1418 cost
= insn_default_latency (insn
);
1425 cost
= INSN_COST (insn
);
1429 /* A USE insn, or something else we don't need to
1430 understand. We can't pass these directly to
1431 result_ready_cost or insn_default_latency because it will
1432 trigger a fatal error for unrecognizable insns. */
1433 if (recog_memoized (insn
) < 0)
1435 INSN_COST (insn
) = 0;
1440 cost
= insn_default_latency (insn
);
1444 INSN_COST (insn
) = cost
;
1451 /* Compute cost of dependence LINK.
1452 This is the number of cycles between instruction issue and
1453 instruction results.
1454 ??? We also use this function to call recog_memoized on all insns. */
1456 dep_cost_1 (dep_t link
, dw_t dw
)
1458 rtx_insn
*insn
= DEP_PRO (link
);
1459 rtx_insn
*used
= DEP_CON (link
);
1462 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1463 return DEP_COST (link
);
1467 struct delay_pair
*delay_entry
;
1469 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1472 if (delay_entry
->i1
== insn
)
1474 DEP_COST (link
) = pair_delay (delay_entry
);
1475 return DEP_COST (link
);
1480 /* A USE insn should never require the value used to be computed.
1481 This allows the computation of a function's result and parameter
1482 values to overlap the return and call. We don't care about the
1483 dependence cost when only decreasing register pressure. */
1484 if (recog_memoized (used
) < 0)
1487 recog_memoized (insn
);
1491 enum reg_note dep_type
= DEP_TYPE (link
);
1493 cost
= insn_cost (insn
);
1495 if (INSN_CODE (insn
) >= 0)
1497 if (dep_type
== REG_DEP_ANTI
)
1499 else if (dep_type
== REG_DEP_OUTPUT
)
1501 cost
= (insn_default_latency (insn
)
1502 - insn_default_latency (used
));
1506 else if (bypass_p (insn
))
1507 cost
= insn_latency (insn
, used
);
1511 if (targetm
.sched
.adjust_cost_2
)
1512 cost
= targetm
.sched
.adjust_cost_2 (used
, (int) dep_type
, insn
, cost
,
1514 else if (targetm
.sched
.adjust_cost
!= NULL
)
1516 /* This variable is used for backward compatibility with the
1518 rtx_insn_list
*dep_cost_rtx_link
=
1519 alloc_INSN_LIST (NULL_RTX
, NULL
);
1521 /* Make it self-cycled, so that if some tries to walk over this
1522 incomplete list he/she will be caught in an endless loop. */
1523 XEXP (dep_cost_rtx_link
, 1) = dep_cost_rtx_link
;
1525 /* Targets use only REG_NOTE_KIND of the link. */
1526 PUT_REG_NOTE_KIND (dep_cost_rtx_link
, DEP_TYPE (link
));
1528 cost
= targetm
.sched
.adjust_cost (used
, dep_cost_rtx_link
,
1531 free_INSN_LIST_node (dep_cost_rtx_link
);
1538 DEP_COST (link
) = cost
;
1542 /* Compute cost of dependence LINK.
1543 This is the number of cycles between instruction issue and
1544 instruction results. */
1546 dep_cost (dep_t link
)
1548 return dep_cost_1 (link
, 0);
1551 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1552 INSN_PRIORITY explicitly. */
1554 increase_insn_priority (rtx_insn
*insn
, int amount
)
1556 if (!sel_sched_p ())
1558 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1559 if (INSN_PRIORITY_KNOWN (insn
))
1560 INSN_PRIORITY (insn
) += amount
;
1564 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1565 Use EXPR_PRIORITY instead. */
1566 sel_add_to_insn_priority (insn
, amount
);
1570 /* Return 'true' if DEP should be included in priority calculations. */
1572 contributes_to_priority_p (dep_t dep
)
1574 if (DEBUG_INSN_P (DEP_CON (dep
))
1575 || DEBUG_INSN_P (DEP_PRO (dep
)))
1578 /* Critical path is meaningful in block boundaries only. */
1579 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1583 if (DEP_REPLACE (dep
) != NULL
)
1586 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1587 then speculative instructions will less likely be
1588 scheduled. That is because the priority of
1589 their producers will increase, and, thus, the
1590 producers will more likely be scheduled, thus,
1591 resolving the dependence. */
1592 if (sched_deps_info
->generate_spec_deps
1593 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1594 && (DEP_STATUS (dep
) & SPECULATIVE
))
1600 /* Compute the number of nondebug deps in list LIST for INSN. */
1603 dep_list_size (rtx insn
, sd_list_types_def list
)
1605 sd_iterator_def sd_it
;
1607 int dbgcount
= 0, nodbgcount
= 0;
1609 if (!MAY_HAVE_DEBUG_INSNS
)
1610 return sd_lists_size (insn
, list
);
1612 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1614 if (DEBUG_INSN_P (DEP_CON (dep
)))
1616 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1620 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1627 /* Compute the priority number for INSN. */
1629 priority (rtx_insn
*insn
)
1631 if (! INSN_P (insn
))
1634 /* We should not be interested in priority of an already scheduled insn. */
1635 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1637 if (!INSN_PRIORITY_KNOWN (insn
))
1639 int this_priority
= -1;
1643 int this_fusion_priority
;
1645 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1646 &this_fusion_priority
, &this_priority
);
1647 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1649 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1650 /* ??? We should set INSN_PRIORITY to insn_cost when and insn has
1651 some forward deps but all of them are ignored by
1652 contributes_to_priority hook. At the moment we set priority of
1654 this_priority
= insn_cost (insn
);
1657 rtx_insn
*prev_first
, *twin
;
1660 /* For recovery check instructions we calculate priority slightly
1661 different than that of normal instructions. Instead of walking
1662 through INSN_FORW_DEPS (check) list, we walk through
1663 INSN_FORW_DEPS list of each instruction in the corresponding
1666 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1667 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1668 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1670 prev_first
= PREV_INSN (insn
);
1675 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1676 twin
= PREV_INSN (BB_END (rec
));
1681 sd_iterator_def sd_it
;
1684 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1689 next
= DEP_CON (dep
);
1691 if (BLOCK_FOR_INSN (next
) != rec
)
1695 if (!contributes_to_priority_p (dep
))
1699 cost
= dep_cost (dep
);
1702 struct _dep _dep1
, *dep1
= &_dep1
;
1704 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1706 cost
= dep_cost (dep1
);
1709 next_priority
= cost
+ priority (next
);
1711 if (next_priority
> this_priority
)
1712 this_priority
= next_priority
;
1716 twin
= PREV_INSN (twin
);
1718 while (twin
!= prev_first
);
1721 if (this_priority
< 0)
1723 gcc_assert (this_priority
== -1);
1725 this_priority
= insn_cost (insn
);
1728 INSN_PRIORITY (insn
) = this_priority
;
1729 INSN_PRIORITY_STATUS (insn
) = 1;
1732 return INSN_PRIORITY (insn
);
1735 /* Macros and functions for keeping the priority queue sorted, and
1736 dealing with queuing and dequeuing of instructions. */
1738 /* For each pressure class CL, set DEATH[CL] to the number of registers
1739 in that class that die in INSN. */
1742 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1745 struct reg_use_data
*use
;
1747 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1748 death
[ira_pressure_classes
[i
]] = 0;
1749 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1750 if (dying_use_p (use
))
1751 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1754 /* Setup info about the current register pressure impact of scheduling
1755 INSN at the current scheduling point. */
1757 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1759 int i
, change
, before
, after
, hard_regno
;
1760 int excess_cost_change
;
1763 struct reg_pressure_data
*pressure_info
;
1764 int *max_reg_pressure
;
1765 static int death
[N_REG_CLASSES
];
1767 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1769 excess_cost_change
= 0;
1770 calculate_reg_deaths (insn
, death
);
1771 pressure_info
= INSN_REG_PRESSURE (insn
);
1772 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1773 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1774 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1776 cl
= ira_pressure_classes
[i
];
1777 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1778 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1779 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1780 after
= MAX (0, max_reg_pressure
[i
] + change
1781 - sched_class_regs_num
[cl
]);
1782 hard_regno
= ira_class_hard_regs
[cl
][0];
1783 gcc_assert (hard_regno
>= 0);
1784 mode
= reg_raw_mode
[hard_regno
];
1785 excess_cost_change
+= ((after
- before
)
1786 * (ira_memory_move_cost
[mode
][cl
][0]
1787 + ira_memory_move_cost
[mode
][cl
][1]));
1789 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1792 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1793 It tries to make the scheduler take register pressure into account
1794 without introducing too many unnecessary stalls. It hooks into the
1795 main scheduling algorithm at several points:
1797 - Before scheduling starts, model_start_schedule constructs a
1798 "model schedule" for the current block. This model schedule is
1799 chosen solely to keep register pressure down. It does not take the
1800 target's pipeline or the original instruction order into account,
1801 except as a tie-breaker. It also doesn't work to a particular
1804 This model schedule gives us an idea of what pressure can be
1805 achieved for the block and gives us an example of a schedule that
1806 keeps to that pressure. It also makes the final schedule less
1807 dependent on the original instruction order. This is important
1808 because the original order can either be "wide" (many values live
1809 at once, such as in user-scheduled code) or "narrow" (few values
1810 live at once, such as after loop unrolling, where several
1811 iterations are executed sequentially).
1813 We do not apply this model schedule to the rtx stream. We simply
1814 record it in model_schedule. We also compute the maximum pressure,
1815 MP, that was seen during this schedule.
1817 - Instructions are added to the ready queue even if they require
1818 a stall. The length of the stall is instead computed as:
1820 MAX (INSN_TICK (INSN) - clock_var, 0)
1822 (= insn_delay). This allows rank_for_schedule to choose between
1823 introducing a deliberate stall or increasing pressure.
1825 - Before sorting the ready queue, model_set_excess_costs assigns
1826 a pressure-based cost to each ready instruction in the queue.
1827 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1828 (ECC for short) and is effectively measured in cycles.
1830 - rank_for_schedule ranks instructions based on:
1832 ECC (insn) + insn_delay (insn)
1838 So, for example, an instruction X1 with an ECC of 1 that can issue
1839 now will win over an instruction X0 with an ECC of zero that would
1840 introduce a stall of one cycle. However, an instruction X2 with an
1841 ECC of 2 that can issue now will lose to both X0 and X1.
1843 - When an instruction is scheduled, model_recompute updates the model
1844 schedule with the new pressures (some of which might now exceed the
1845 original maximum pressure MP). model_update_limit_points then searches
1846 for the new point of maximum pressure, if not already known. */
1848 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1849 from surrounding debug information. */
1851 ";;\t\t+------------------------------------------------------\n"
1853 /* Information about the pressure on a particular register class at a
1854 particular point of the model schedule. */
1855 struct model_pressure_data
{
1856 /* The pressure at this point of the model schedule, or -1 if the
1857 point is associated with an instruction that has already been
1861 /* The maximum pressure during or after this point of the model schedule. */
1865 /* Per-instruction information that is used while building the model
1866 schedule. Here, "schedule" refers to the model schedule rather
1867 than the main schedule. */
1868 struct model_insn_info
{
1869 /* The instruction itself. */
1872 /* If this instruction is in model_worklist, these fields link to the
1873 previous (higher-priority) and next (lower-priority) instructions
1875 struct model_insn_info
*prev
;
1876 struct model_insn_info
*next
;
1878 /* While constructing the schedule, QUEUE_INDEX describes whether an
1879 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1880 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1881 old_queue records the value that QUEUE_INDEX had before scheduling
1882 started, so that we can restore it once the schedule is complete. */
1885 /* The relative importance of an unscheduled instruction. Higher
1886 values indicate greater importance. */
1887 unsigned int model_priority
;
1889 /* The length of the longest path of satisfied true dependencies
1890 that leads to this instruction. */
1893 /* The length of the longest path of dependencies of any kind
1894 that leads from this instruction. */
1897 /* The number of predecessor nodes that must still be scheduled. */
1898 int unscheduled_preds
;
1901 /* Information about the pressure limit for a particular register class.
1902 This structure is used when applying a model schedule to the main
1904 struct model_pressure_limit
{
1905 /* The maximum register pressure seen in the original model schedule. */
1908 /* The maximum register pressure seen in the current model schedule
1909 (which excludes instructions that have already been scheduled). */
1912 /* The point of the current model schedule at which PRESSURE is first
1913 reached. It is set to -1 if the value needs to be recomputed. */
1917 /* Describes a particular way of measuring register pressure. */
1918 struct model_pressure_group
{
1919 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1920 struct model_pressure_limit limits
[N_REG_CLASSES
];
1922 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1923 on register class ira_pressure_classes[PCI] at point POINT of the
1924 current model schedule. A POINT of model_num_insns describes the
1925 pressure at the end of the schedule. */
1926 struct model_pressure_data
*model
;
1929 /* Index POINT gives the instruction at point POINT of the model schedule.
1930 This array doesn't change during main scheduling. */
1931 static vec
<rtx_insn
*> model_schedule
;
1933 /* The list of instructions in the model worklist, sorted in order of
1934 decreasing priority. */
1935 static struct model_insn_info
*model_worklist
;
1937 /* Index I describes the instruction with INSN_LUID I. */
1938 static struct model_insn_info
*model_insns
;
1940 /* The number of instructions in the model schedule. */
1941 static int model_num_insns
;
1943 /* The index of the first instruction in model_schedule that hasn't yet been
1944 added to the main schedule, or model_num_insns if all of them have. */
1945 static int model_curr_point
;
1947 /* Describes the pressure before each instruction in the model schedule. */
1948 static struct model_pressure_group model_before_pressure
;
1950 /* The first unused model_priority value (as used in model_insn_info). */
1951 static unsigned int model_next_priority
;
1954 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1955 at point POINT of the model schedule. */
1956 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1957 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1959 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1960 after point POINT of the model schedule. */
1961 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1962 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1964 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1965 of the model schedule. */
1966 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1967 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1969 /* Information about INSN that is used when creating the model schedule. */
1970 #define MODEL_INSN_INFO(INSN) \
1971 (&model_insns[INSN_LUID (INSN)])
1973 /* The instruction at point POINT of the model schedule. */
1974 #define MODEL_INSN(POINT) \
1975 (model_schedule[POINT])
1978 /* Return INSN's index in the model schedule, or model_num_insns if it
1979 doesn't belong to that schedule. */
1982 model_index (rtx_insn
*insn
)
1984 if (INSN_MODEL_INDEX (insn
) == 0)
1985 return model_num_insns
;
1986 return INSN_MODEL_INDEX (insn
) - 1;
1989 /* Make sure that GROUP->limits is up-to-date for the current point
1990 of the model schedule. */
1993 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1995 int pci
, max_pressure
, point
;
1997 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1999 /* We may have passed the final point at which the pressure in
2000 group->limits[pci].pressure was reached. Update the limit if so. */
2001 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
2002 group
->limits
[pci
].pressure
= max_pressure
;
2004 /* Find the point at which MAX_PRESSURE is first reached. We need
2005 to search in three cases:
2007 - We've already moved past the previous pressure point.
2008 In this case we search forward from model_curr_point.
2010 - We scheduled the previous point of maximum pressure ahead of
2011 its position in the model schedule, but doing so didn't bring
2012 the pressure point earlier. In this case we search forward
2013 from that previous pressure point.
2015 - Scheduling an instruction early caused the maximum pressure
2016 to decrease. In this case we will have set the pressure
2017 point to -1, and we search forward from model_curr_point. */
2018 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
2019 while (point
< model_num_insns
2020 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
2022 group
->limits
[pci
].point
= point
;
2024 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
2025 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
2029 /* Make sure that all register-pressure limits are up-to-date for the
2030 current position in the model schedule. */
2033 model_update_limit_points (void)
2035 model_update_limit_points_in_group (&model_before_pressure
);
2038 /* Return the model_index of the last unscheduled use in chain USE
2039 outside of USE's instruction. Return -1 if there are no other uses,
2040 or model_num_insns if the register is live at the end of the block. */
2043 model_last_use_except (struct reg_use_data
*use
)
2045 struct reg_use_data
*next
;
2049 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2050 if (NONDEBUG_INSN_P (next
->insn
)
2051 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2053 index
= model_index (next
->insn
);
2054 if (index
== model_num_insns
)
2055 return model_num_insns
;
2062 /* An instruction with model_index POINT has just been scheduled, and it
2063 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2064 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2065 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2068 model_start_update_pressure (struct model_pressure_group
*group
,
2069 int point
, int pci
, int delta
)
2071 int next_max_pressure
;
2073 if (point
== model_num_insns
)
2075 /* The instruction wasn't part of the model schedule; it was moved
2076 from a different block. Update the pressure for the end of
2077 the model schedule. */
2078 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2079 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2083 /* Record that this instruction has been scheduled. Nothing now
2084 changes between POINT and POINT + 1, so get the maximum pressure
2085 from the latter. If the maximum pressure decreases, the new
2086 pressure point may be before POINT. */
2087 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2088 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2089 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2091 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2092 if (group
->limits
[pci
].point
== point
)
2093 group
->limits
[pci
].point
= -1;
2098 /* Record that scheduling a later instruction has changed the pressure
2099 at point POINT of the model schedule by DELTA (which might be 0).
2100 Update GROUP accordingly. Return nonzero if these changes might
2101 trigger changes to previous points as well. */
2104 model_update_pressure (struct model_pressure_group
*group
,
2105 int point
, int pci
, int delta
)
2107 int ref_pressure
, max_pressure
, next_max_pressure
;
2109 /* If POINT hasn't yet been scheduled, update its pressure. */
2110 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2111 if (ref_pressure
>= 0 && delta
!= 0)
2113 ref_pressure
+= delta
;
2114 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2116 /* Check whether the maximum pressure in the overall schedule
2117 has increased. (This means that the MODEL_MAX_PRESSURE of
2118 every point <= POINT will need to increase too; see below.) */
2119 if (group
->limits
[pci
].pressure
< ref_pressure
)
2120 group
->limits
[pci
].pressure
= ref_pressure
;
2122 /* If we are at maximum pressure, and the maximum pressure
2123 point was previously unknown or later than POINT,
2124 bring it forward. */
2125 if (group
->limits
[pci
].pressure
== ref_pressure
2126 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2127 group
->limits
[pci
].point
= point
;
2129 /* If POINT used to be the point of maximum pressure, but isn't
2130 any longer, we need to recalculate it using a forward walk. */
2131 if (group
->limits
[pci
].pressure
> ref_pressure
2132 && group
->limits
[pci
].point
== point
)
2133 group
->limits
[pci
].point
= -1;
2136 /* Update the maximum pressure at POINT. Changes here might also
2137 affect the maximum pressure at POINT - 1. */
2138 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2139 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2140 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2142 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2148 /* INSN has just been scheduled. Update the model schedule accordingly. */
2151 model_recompute (rtx_insn
*insn
)
2156 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2157 struct reg_use_data
*use
;
2158 struct reg_pressure_data
*reg_pressure
;
2159 int delta
[N_REG_CLASSES
];
2160 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2161 unsigned int i
, num_uses
, num_pending_births
;
2164 /* The destinations of INSN were previously live from POINT onwards, but are
2165 now live from model_curr_point onwards. Set up DELTA accordingly. */
2166 point
= model_index (insn
);
2167 reg_pressure
= INSN_REG_PRESSURE (insn
);
2168 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2170 cl
= ira_pressure_classes
[pci
];
2171 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2174 /* Record which registers previously died at POINT, but which now die
2175 before POINT. Adjust DELTA so that it represents the effect of
2176 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2177 registers that will be born in the range [model_curr_point, POINT). */
2179 num_pending_births
= 0;
2180 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2182 new_last
= model_last_use_except (use
);
2183 if (new_last
< point
)
2185 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2186 uses
[num_uses
].last_use
= new_last
;
2187 uses
[num_uses
].regno
= use
->regno
;
2188 /* This register is no longer live after POINT - 1. */
2189 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2192 num_pending_births
++;
2196 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2197 Also set each group pressure limit for POINT. */
2198 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2200 cl
= ira_pressure_classes
[pci
];
2201 model_start_update_pressure (&model_before_pressure
,
2202 point
, pci
, delta
[cl
]);
2205 /* Walk the model schedule backwards, starting immediately before POINT. */
2207 if (point
!= model_curr_point
)
2211 insn
= MODEL_INSN (point
);
2212 queue
= QUEUE_INDEX (insn
);
2214 if (queue
!= QUEUE_SCHEDULED
)
2216 /* DELTA describes the effect of the move on the register pressure
2217 after POINT. Make it describe the effect on the pressure
2220 while (i
< num_uses
)
2222 if (uses
[i
].last_use
== point
)
2224 /* This register is now live again. */
2225 mark_regno_birth_or_death (NULL
, delta
,
2226 uses
[i
].regno
, true);
2228 /* Remove this use from the array. */
2229 uses
[i
] = uses
[num_uses
- 1];
2231 num_pending_births
--;
2237 if (sched_verbose
>= 5)
2241 fprintf (sched_dump
, MODEL_BAR
);
2242 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2244 fprintf (sched_dump
, MODEL_BAR
);
2248 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2249 point
, INSN_UID (insn
),
2250 str_pattern_slim (PATTERN (insn
)));
2251 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2253 cl
= ira_pressure_classes
[pci
];
2254 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2256 fprintf (sched_dump
, " %s:[%d->%d]",
2257 reg_class_names
[ira_pressure_classes
[pci
]],
2258 ref_pressure
, ref_pressure
+ delta
[cl
]);
2260 fprintf (sched_dump
, "\n");
2264 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2265 might have changed as well. */
2266 mix
= num_pending_births
;
2267 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2269 cl
= ira_pressure_classes
[pci
];
2271 mix
|= model_update_pressure (&model_before_pressure
,
2272 point
, pci
, delta
[cl
]);
2275 while (mix
&& point
> model_curr_point
);
2278 fprintf (sched_dump
, MODEL_BAR
);
2281 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2282 check whether the insn's pattern needs restoring. */
2284 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2286 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2289 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2291 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2292 gcc_assert (next
== DEP_CON (dep
));
2296 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2297 if (desc
->insn
!= next
)
2299 gcc_assert (*desc
->loc
== desc
->orig
);
2306 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2307 pressure on CL from P to P'. We use this to calculate a "base ECC",
2308 baseECC (CL, X), for each pressure class CL and each instruction X.
2309 Supposing X changes the pressure on CL from P to P', and that the
2310 maximum pressure on CL in the current model schedule is MP', then:
2312 * if X occurs before or at the next point of maximum pressure in
2313 the model schedule and P' > MP', then:
2315 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2317 The idea is that the pressure after scheduling a fixed set of
2318 instructions -- in this case, the set up to and including the
2319 next maximum pressure point -- is going to be the same regardless
2320 of the order; we simply want to keep the intermediate pressure
2321 under control. Thus X has a cost of zero unless scheduling it
2322 now would exceed MP'.
2324 If all increases in the set are by the same amount, no zero-cost
2325 instruction will ever cause the pressure to exceed MP'. However,
2326 if X is instead moved past an instruction X' with pressure in the
2327 range (MP' - (P' - P), MP'), the pressure at X' will increase
2328 beyond MP'. Since baseECC is very much a heuristic anyway,
2329 it doesn't seem worth the overhead of tracking cases like these.
2331 The cost of exceeding MP' is always based on the original maximum
2332 pressure MP. This is so that going 2 registers over the original
2333 limit has the same cost regardless of whether it comes from two
2334 separate +1 deltas or from a single +2 delta.
2336 * if X occurs after the next point of maximum pressure in the model
2337 schedule and P' > P, then:
2339 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2341 That is, if we move X forward across a point of maximum pressure,
2342 and if X increases the pressure by P' - P, then we conservatively
2343 assume that scheduling X next would increase the maximum pressure
2344 by P' - P. Again, the cost of doing this is based on the original
2345 maximum pressure MP, for the same reason as above.
2347 * if P' < P, P > MP, and X occurs at or after the next point of
2348 maximum pressure, then:
2350 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2352 That is, if we have already exceeded the original maximum pressure MP,
2353 and if X might reduce the maximum pressure again -- or at least push
2354 it further back, and thus allow more scheduling freedom -- it is given
2355 a negative cost to reflect the improvement.
2361 In this case, X is not expected to affect the maximum pressure MP',
2362 so it has zero cost.
2364 We then create a combined value baseECC (X) that is the sum of
2365 baseECC (CL, X) for each pressure class CL.
2367 baseECC (X) could itself be used as the ECC value described above.
2368 However, this is often too conservative, in the sense that it
2369 tends to make high-priority instructions that increase pressure
2370 wait too long in cases where introducing a spill would be better.
2371 For this reason the final ECC is a priority-adjusted form of
2372 baseECC (X). Specifically, we calculate:
2374 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2375 baseP = MAX { P (X) | baseECC (X) <= 0 }
2379 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2381 Thus an instruction's effect on pressure is ignored if it has a high
2382 enough priority relative to the ones that don't increase pressure.
2383 Negative values of baseECC (X) do not increase the priority of X
2384 itself, but they do make it harder for other instructions to
2385 increase the pressure further.
2387 This pressure cost is deliberately timid. The intention has been
2388 to choose a heuristic that rarely interferes with the normal list
2389 scheduler in cases where that scheduler would produce good code.
2390 We simply want to curb some of its worst excesses. */
2392 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2394 Here we use the very simplistic cost model that every register above
2395 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2396 measures instead, such as one based on MEMORY_MOVE_COST. However:
2398 (1) In order for an instruction to be scheduled, the higher cost
2399 would need to be justified in a single saving of that many stalls.
2400 This is overly pessimistic, because the benefit of spilling is
2401 often to avoid a sequence of several short stalls rather than
2404 (2) The cost is still arbitrary. Because we are not allocating
2405 registers during scheduling, we have no way of knowing for
2406 sure how many memory accesses will be required by each spill,
2407 where the spills will be placed within the block, or even
2408 which block(s) will contain the spills.
2410 So a higher cost than 1 is often too conservative in practice,
2411 forcing blocks to contain unnecessary stalls instead of spill code.
2412 The simple cost below seems to be the best compromise. It reduces
2413 the interference with the normal list scheduler, which helps make
2414 it more suitable for a default-on option. */
2417 model_spill_cost (int cl
, int from
, int to
)
2419 from
= MAX (from
, sched_class_regs_num
[cl
]);
2420 return MAX (to
, from
) - from
;
2423 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2424 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2428 model_excess_group_cost (struct model_pressure_group
*group
,
2429 int point
, int pci
, int delta
)
2433 cl
= ira_pressure_classes
[pci
];
2434 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2436 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2437 curr_reg_pressure
[cl
] + delta
);
2438 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2443 if (point
> group
->limits
[pci
].point
)
2444 pressure
= group
->limits
[pci
].pressure
+ delta
;
2446 pressure
= curr_reg_pressure
[cl
] + delta
;
2448 if (pressure
> group
->limits
[pci
].pressure
)
2449 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2456 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2460 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2462 int point
, pci
, cl
, cost
, this_cost
, delta
;
2463 struct reg_pressure_data
*insn_reg_pressure
;
2464 int insn_death
[N_REG_CLASSES
];
2466 calculate_reg_deaths (insn
, insn_death
);
2467 point
= model_index (insn
);
2468 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2472 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2473 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2475 /* Sum up the individual costs for each register class. */
2476 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2478 cl
= ira_pressure_classes
[pci
];
2479 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2480 this_cost
= model_excess_group_cost (&model_before_pressure
,
2484 fprintf (sched_dump
, " %s:[%d base cost %d]",
2485 reg_class_names
[cl
], delta
, this_cost
);
2489 fprintf (sched_dump
, "\n");
2494 /* Dump the next points of maximum pressure for GROUP. */
2497 model_dump_pressure_points (struct model_pressure_group
*group
)
2501 fprintf (sched_dump
, ";;\t\t| pressure points");
2502 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2504 cl
= ira_pressure_classes
[pci
];
2505 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2506 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2507 if (group
->limits
[pci
].point
< model_num_insns
)
2508 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2509 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2511 fprintf (sched_dump
, "end]");
2513 fprintf (sched_dump
, "\n");
2516 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2519 model_set_excess_costs (rtx_insn
**insns
, int count
)
2521 int i
, cost
, priority_base
, priority
;
2524 /* Record the baseECC value for each instruction in the model schedule,
2525 except that negative costs are converted to zero ones now rather than
2526 later. Do not assign a cost to debug instructions, since they must
2527 not change code-generation decisions. Experiments suggest we also
2528 get better results by not assigning a cost to instructions from
2531 Set PRIORITY_BASE to baseP in the block comment above. This is the
2532 maximum priority of the "cheap" instructions, which should always
2533 include the next model instruction. */
2536 for (i
= 0; i
< count
; i
++)
2537 if (INSN_MODEL_INDEX (insns
[i
]))
2539 if (sched_verbose
>= 6 && !print_p
)
2541 fprintf (sched_dump
, MODEL_BAR
);
2542 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2543 model_dump_pressure_points (&model_before_pressure
);
2544 fprintf (sched_dump
, MODEL_BAR
);
2547 cost
= model_excess_cost (insns
[i
], print_p
);
2550 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2551 priority_base
= MAX (priority_base
, priority
);
2554 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2557 fprintf (sched_dump
, MODEL_BAR
);
2559 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2561 for (i
= 0; i
< count
; i
++)
2563 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2564 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2565 if (cost
> 0 && priority
> priority_base
)
2567 cost
+= priority_base
- priority
;
2568 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2574 /* Enum of rank_for_schedule heuristic decisions. */
2576 RFS_DEBUG
, RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2577 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2578 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2579 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2580 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_N
};
2582 /* Corresponding strings for print outs. */
2583 static const char *rfs_str
[RFS_N
] = {
2584 "RFS_DEBUG", "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2585 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2586 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2587 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2588 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION" };
2590 /* Statistical breakdown of rank_for_schedule decisions. */
2591 typedef struct { unsigned stats
[RFS_N
]; } rank_for_schedule_stats_t
;
2592 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2594 /* Return the result of comparing insns TMP and TMP2 and update
2595 Rank_For_Schedule statistics. */
2597 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2599 ++rank_for_schedule_stats
.stats
[decision
];
2601 INSN_LAST_RFS_WIN (tmp
) = decision
;
2602 else if (result
> 0)
2603 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2609 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2610 keeping normal insns in original order. */
2613 rank_for_schedule_debug (const void *x
, const void *y
)
2615 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2616 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2618 /* Schedule debug insns as early as possible. */
2619 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2620 return rfs_result (RFS_DEBUG
, -1, tmp
, tmp2
);
2621 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2622 return rfs_result (RFS_DEBUG
, 1, tmp
, tmp2
);
2623 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2624 return rfs_result (RFS_DEBUG
, INSN_LUID (tmp
) - INSN_LUID (tmp2
),
2627 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2630 /* Returns a positive value if x is preferred; returns a negative value if
2631 y is preferred. Should never return 0, since that will make the sort
2635 rank_for_schedule (const void *x
, const void *y
)
2637 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2638 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2639 int tmp_class
, tmp2_class
;
2640 int val
, priority_val
, info_val
, diff
;
2642 if (live_range_shrinkage_p
)
2644 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2646 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2647 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2648 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2649 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2650 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2651 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2652 /* Sort by INSN_LUID (original insn order), so that we make the
2653 sort stable. This minimizes instruction movement, thus
2654 minimizing sched's effect on debugging and cross-jumping. */
2655 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2656 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2659 /* The insn in a schedule group should be issued the first. */
2660 if (flag_sched_group_heuristic
&&
2661 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2662 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2665 /* Make sure that priority of TMP and TMP2 are initialized. */
2666 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2670 /* The instruction that has the same fusion priority as the last
2671 instruction is the instruction we picked next. If that is not
2672 the case, we sort ready list firstly by fusion priority, then
2673 by priority, and at last by INSN_LUID. */
2674 int a
= INSN_FUSION_PRIORITY (tmp
);
2675 int b
= INSN_FUSION_PRIORITY (tmp2
);
2678 if (last_nondebug_scheduled_insn
2679 && !NOTE_P (last_nondebug_scheduled_insn
)
2680 && BLOCK_FOR_INSN (tmp
)
2681 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2682 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2684 if (a
!= last
&& b
!= last
)
2688 a
= INSN_PRIORITY (tmp
);
2689 b
= INSN_PRIORITY (tmp2
);
2692 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2694 return rfs_result (RFS_FUSION
,
2695 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2699 gcc_assert (last_nondebug_scheduled_insn
2700 && !NOTE_P (last_nondebug_scheduled_insn
));
2701 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2703 a
= abs (INSN_PRIORITY (tmp
) - last
);
2704 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2706 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2708 return rfs_result (RFS_FUSION
,
2709 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2712 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2714 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2717 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2719 /* Prefer insn whose scheduling results in the smallest register
2721 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2723 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2724 - insn_delay (tmp2
))))
2725 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2728 if (sched_pressure
!= SCHED_PRESSURE_NONE
2729 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2730 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2732 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2733 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2736 /* If we are doing backtracking in this schedule, prefer insns that
2737 have forward dependencies with negative cost against an insn that
2738 was already scheduled. */
2739 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2741 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2743 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2746 /* Prefer insn with higher priority. */
2747 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2749 if (flag_sched_critical_path_heuristic
&& priority_val
)
2750 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2752 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2754 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2759 /* Prefer speculative insn with greater dependencies weakness. */
2760 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2766 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2768 dw1
= ds_weak (ds1
);
2772 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2774 dw2
= ds_weak (ds2
);
2779 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2780 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2783 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2784 if (flag_sched_rank_heuristic
&& info_val
)
2785 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2787 /* Compare insns based on their relation to the last scheduled
2789 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2793 rtx last
= last_nondebug_scheduled_insn
;
2795 /* Classify the instructions into three classes:
2796 1) Data dependent on last schedule insn.
2797 2) Anti/Output dependent on last scheduled insn.
2798 3) Independent of last scheduled insn, or has latency of one.
2799 Choose the insn from the highest numbered class if different. */
2800 dep1
= sd_find_dep_between (last
, tmp
, true);
2802 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2804 else if (/* Data dependence. */
2805 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2810 dep2
= sd_find_dep_between (last
, tmp2
, true);
2812 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2814 else if (/* Data dependence. */
2815 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2820 if ((val
= tmp2_class
- tmp_class
))
2821 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2824 /* Prefer instructions that occur earlier in the model schedule. */
2825 if (sched_pressure
== SCHED_PRESSURE_MODEL
2826 && INSN_BB (tmp
) == target_bb
&& INSN_BB (tmp2
) == target_bb
)
2828 diff
= model_index (tmp
) - model_index (tmp2
);
2829 gcc_assert (diff
!= 0);
2830 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2833 /* Prefer the insn which has more later insns that depend on it.
2834 This gives the scheduler more freedom when scheduling later
2835 instructions at the expense of added register pressure. */
2837 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2838 - dep_list_size (tmp
, SD_LIST_FORW
));
2840 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2841 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2843 /* If insns are equally good, sort by INSN_LUID (original insn order),
2844 so that we make the sort stable. This minimizes instruction movement,
2845 thus minimizing sched's effect on debugging and cross-jumping. */
2846 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2849 /* Resort the array A in which only element at index N may be out of order. */
2851 HAIFA_INLINE
static void
2852 swap_sort (rtx_insn
**a
, int n
)
2854 rtx_insn
*insn
= a
[n
- 1];
2857 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2865 /* Add INSN to the insn queue so that it can be executed at least
2866 N_CYCLES after the currently executing insn. Preserve insns
2867 chain for debugging purposes. REASON will be printed in debugging
2870 HAIFA_INLINE
static void
2871 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2873 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2874 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2877 gcc_assert (n_cycles
<= max_insn_queue_index
);
2878 gcc_assert (!DEBUG_INSN_P (insn
));
2880 insn_queue
[next_q
] = link
;
2883 if (sched_verbose
>= 2)
2885 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2886 (*current_sched_info
->print_insn
) (insn
, 0));
2888 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2891 QUEUE_INDEX (insn
) = next_q
;
2893 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2895 new_tick
= clock_var
+ n_cycles
;
2896 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2897 INSN_TICK (insn
) = new_tick
;
2899 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2900 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2902 must_backtrack
= true;
2903 if (sched_verbose
>= 2)
2904 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2909 /* Remove INSN from queue. */
2911 queue_remove (rtx_insn
*insn
)
2913 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2914 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2916 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2919 /* Return a pointer to the bottom of the ready list, i.e. the insn
2920 with the lowest priority. */
2923 ready_lastpos (struct ready_list
*ready
)
2925 gcc_assert (ready
->n_ready
>= 1);
2926 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2929 /* Add an element INSN to the ready list so that it ends up with the
2930 lowest/highest priority depending on FIRST_P. */
2932 HAIFA_INLINE
static void
2933 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2937 if (ready
->first
== ready
->n_ready
)
2939 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2940 ready_lastpos (ready
),
2941 ready
->n_ready
* sizeof (rtx
));
2942 ready
->first
= ready
->veclen
- 1;
2944 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2948 if (ready
->first
== ready
->veclen
- 1)
2951 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2952 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2953 ready_lastpos (ready
),
2954 ready
->n_ready
* sizeof (rtx
));
2955 ready
->first
= ready
->veclen
- 2;
2957 ready
->vec
[++(ready
->first
)] = insn
;
2961 if (DEBUG_INSN_P (insn
))
2964 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2965 QUEUE_INDEX (insn
) = QUEUE_READY
;
2967 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2968 && INSN_EXACT_TICK (insn
) < clock_var
)
2970 must_backtrack
= true;
2974 /* Remove the element with the highest priority from the ready list and
2977 HAIFA_INLINE
static rtx_insn
*
2978 ready_remove_first (struct ready_list
*ready
)
2982 gcc_assert (ready
->n_ready
);
2983 t
= ready
->vec
[ready
->first
--];
2985 if (DEBUG_INSN_P (t
))
2987 /* If the queue becomes empty, reset it. */
2988 if (ready
->n_ready
== 0)
2989 ready
->first
= ready
->veclen
- 1;
2991 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2992 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2997 /* The following code implements multi-pass scheduling for the first
2998 cycle. In other words, we will try to choose ready insn which
2999 permits to start maximum number of insns on the same cycle. */
3001 /* Return a pointer to the element INDEX from the ready. INDEX for
3002 insn with the highest priority is 0, and the lowest priority has
3006 ready_element (struct ready_list
*ready
, int index
)
3008 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3010 return ready
->vec
[ready
->first
- index
];
3013 /* Remove the element INDEX from the ready list and return it. INDEX
3014 for insn with the highest priority is 0, and the lowest priority
3017 HAIFA_INLINE
static rtx_insn
*
3018 ready_remove (struct ready_list
*ready
, int index
)
3024 return ready_remove_first (ready
);
3025 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
3026 t
= ready
->vec
[ready
->first
- index
];
3028 if (DEBUG_INSN_P (t
))
3030 for (i
= index
; i
< ready
->n_ready
; i
++)
3031 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3032 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3036 /* Remove INSN from the ready list. */
3038 ready_remove_insn (rtx insn
)
3042 for (i
= 0; i
< readyp
->n_ready
; i
++)
3043 if (ready_element (readyp
, i
) == insn
)
3045 ready_remove (readyp
, i
);
3051 /* Calculate difference of two statistics set WAS and NOW.
3052 Result returned in WAS. */
3054 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3055 const rank_for_schedule_stats_t
*now
)
3057 for (int i
= 0; i
< RFS_N
; ++i
)
3058 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3061 /* Print rank_for_schedule statistics. */
3063 print_rank_for_schedule_stats (const char *prefix
,
3064 const rank_for_schedule_stats_t
*stats
,
3065 struct ready_list
*ready
)
3067 for (int i
= 0; i
< RFS_N
; ++i
)
3068 if (stats
->stats
[i
])
3070 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3073 /* Print out insns that won due to RFS_<I>. */
3075 rtx_insn
**p
= ready_lastpos (ready
);
3077 fprintf (sched_dump
, ":");
3078 /* Start with 1 since least-priority insn didn't have any wins. */
3079 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3080 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3081 fprintf (sched_dump
, " %s",
3082 (*current_sched_info
->print_insn
) (p
[j
], 0));
3084 fprintf (sched_dump
, "\n");
3088 /* Sort the ready list READY by ascending priority, using the SCHED_SORT
3092 ready_sort (struct ready_list
*ready
)
3095 rtx_insn
**first
= ready_lastpos (ready
);
3096 int n_ready_non_debug
= ready
->n_ready
;
3098 for (i
= 0; i
< ready
->n_ready
; ++i
)
3100 if (DEBUG_INSN_P (first
[i
]))
3101 --n_ready_non_debug
;
3104 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3106 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3107 setup_insn_reg_pressure_info (first
[i
]);
3111 if (sched_pressure
== SCHED_PRESSURE_MODEL
3112 && model_curr_point
< model_num_insns
)
3113 model_set_excess_costs (first
, ready
->n_ready
);
3115 rank_for_schedule_stats_t stats1
;
3116 if (sched_verbose
>= 4)
3117 stats1
= rank_for_schedule_stats
;
3119 if (n_ready_non_debug
< ready
->n_ready
)
3120 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3122 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3125 if (n_ready_non_debug
== 2)
3126 swap_sort (first
, n_ready_non_debug
);
3127 else if (n_ready_non_debug
> 2)
3128 qsort (first
, n_ready_non_debug
, sizeof (rtx
), rank_for_schedule
);
3131 if (sched_verbose
>= 4)
3133 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3134 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3138 /* PREV is an insn that is ready to execute. Adjust its priority if that
3139 will help shorten or lengthen register lifetimes as appropriate. Also
3140 provide a hook for the target to tweak itself. */
3142 HAIFA_INLINE
static void
3143 adjust_priority (rtx_insn
*prev
)
3145 /* ??? There used to be code here to try and estimate how an insn
3146 affected register lifetimes, but it did it by looking at REG_DEAD
3147 notes, which we removed in schedule_region. Nor did it try to
3148 take into account register pressure or anything useful like that.
3150 Revisit when we have a machine model to work with and not before. */
3152 if (targetm
.sched
.adjust_priority
)
3153 INSN_PRIORITY (prev
) =
3154 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3157 /* Advance DFA state STATE on one cycle. */
3159 advance_state (state_t state
)
3161 if (targetm
.sched
.dfa_pre_advance_cycle
)
3162 targetm
.sched
.dfa_pre_advance_cycle ();
3164 if (targetm
.sched
.dfa_pre_cycle_insn
)
3165 state_transition (state
,
3166 targetm
.sched
.dfa_pre_cycle_insn ());
3168 state_transition (state
, NULL
);
3170 if (targetm
.sched
.dfa_post_cycle_insn
)
3171 state_transition (state
,
3172 targetm
.sched
.dfa_post_cycle_insn ());
3174 if (targetm
.sched
.dfa_post_advance_cycle
)
3175 targetm
.sched
.dfa_post_advance_cycle ();
3178 /* Advance time on one cycle. */
3179 HAIFA_INLINE
static void
3180 advance_one_cycle (void)
3182 advance_state (curr_state
);
3183 if (sched_verbose
>= 4)
3184 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3187 /* Update register pressure after scheduling INSN. */
3189 update_register_pressure (rtx_insn
*insn
)
3191 struct reg_use_data
*use
;
3192 struct reg_set_data
*set
;
3194 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3196 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3197 if (dying_use_p (use
))
3198 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3200 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3201 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3205 /* Set up or update (if UPDATE_P) max register pressure (see its
3206 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3207 after insn AFTER. */
3209 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3214 static int max_reg_pressure
[N_REG_CLASSES
];
3216 save_reg_pressure ();
3217 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3218 max_reg_pressure
[ira_pressure_classes
[i
]]
3219 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3220 for (insn
= NEXT_INSN (after
);
3221 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3222 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3223 insn
= NEXT_INSN (insn
))
3224 if (NONDEBUG_INSN_P (insn
))
3227 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3229 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3230 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3233 INSN_MAX_REG_PRESSURE (insn
)[i
]
3234 = max_reg_pressure
[ira_pressure_classes
[i
]];
3237 if (update_p
&& eq_p
)
3239 update_register_pressure (insn
);
3240 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3241 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3242 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3243 max_reg_pressure
[ira_pressure_classes
[i
]]
3244 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3246 restore_reg_pressure ();
3249 /* Update the current register pressure after scheduling INSN. Update
3250 also max register pressure for unscheduled insns of the current
3253 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3256 int before
[N_REG_CLASSES
];
3258 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3259 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3260 update_register_pressure (insn
);
3261 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3262 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3264 if (i
< ira_pressure_classes_num
)
3265 setup_insn_max_reg_pressure (insn
, true);
3268 /* Set up register pressure at the beginning of basic block BB whose
3269 insns starting after insn AFTER. Set up also max register pressure
3270 for all insns of the basic block. */
3272 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3274 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3275 initiate_bb_reg_pressure_info (bb
);
3276 setup_insn_max_reg_pressure (after
, false);
3279 /* If doing predication while scheduling, verify whether INSN, which
3280 has just been scheduled, clobbers the conditions of any
3281 instructions that must be predicated in order to break their
3282 dependencies. If so, remove them from the queues so that they will
3283 only be scheduled once their control dependency is resolved. */
3286 check_clobbered_conditions (rtx insn
)
3291 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3294 find_all_hard_reg_sets (insn
, &t
, true);
3297 for (i
= 0; i
< ready
.n_ready
; i
++)
3299 rtx_insn
*x
= ready_element (&ready
, i
);
3300 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3302 ready_remove_insn (x
);
3306 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3308 rtx_insn_list
*link
;
3309 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3312 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3314 rtx_insn
*x
= link
->insn ();
3315 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3324 /* Return (in order):
3326 - positive if INSN adversely affects the pressure on one
3329 - negative if INSN reduces the pressure on one register class
3331 - 0 if INSN doesn't affect the pressure on any register class. */
3334 model_classify_pressure (struct model_insn_info
*insn
)
3336 struct reg_pressure_data
*reg_pressure
;
3337 int death
[N_REG_CLASSES
];
3340 calculate_reg_deaths (insn
->insn
, death
);
3341 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3343 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3345 cl
= ira_pressure_classes
[pci
];
3346 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3348 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3353 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3356 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3358 unsigned int height1
, height2
;
3359 unsigned int priority1
, priority2
;
3361 /* Prefer instructions with a higher model priority. */
3362 if (insn1
->model_priority
!= insn2
->model_priority
)
3363 return insn1
->model_priority
> insn2
->model_priority
;
3365 /* Combine the length of the longest path of satisfied true dependencies
3366 that leads to each instruction (depth) with the length of the longest
3367 path of any dependencies that leads from the instruction (alap).
3368 Prefer instructions with the greatest combined length. If the combined
3369 lengths are equal, prefer instructions with the greatest depth.
3371 The idea is that, if we have a set S of "equal" instructions that each
3372 have ALAP value X, and we pick one such instruction I, any true-dependent
3373 successors of I that have ALAP value X - 1 should be preferred over S.
3374 This encourages the schedule to be "narrow" rather than "wide".
3375 However, if I is a low-priority instruction that we decided to
3376 schedule because of its model_classify_pressure, and if there
3377 is a set of higher-priority instructions T, the aforementioned
3378 successors of I should not have the edge over T. */
3379 height1
= insn1
->depth
+ insn1
->alap
;
3380 height2
= insn2
->depth
+ insn2
->alap
;
3381 if (height1
!= height2
)
3382 return height1
> height2
;
3383 if (insn1
->depth
!= insn2
->depth
)
3384 return insn1
->depth
> insn2
->depth
;
3386 /* We have no real preference between INSN1 an INSN2 as far as attempts
3387 to reduce pressure go. Prefer instructions with higher priorities. */
3388 priority1
= INSN_PRIORITY (insn1
->insn
);
3389 priority2
= INSN_PRIORITY (insn2
->insn
);
3390 if (priority1
!= priority2
)
3391 return priority1
> priority2
;
3393 /* Use the original rtl sequence as a tie-breaker. */
3394 return insn1
< insn2
;
3397 /* Add INSN to the model worklist immediately after PREV. Add it to the
3398 beginning of the list if PREV is null. */
3401 model_add_to_worklist_at (struct model_insn_info
*insn
,
3402 struct model_insn_info
*prev
)
3404 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3405 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3410 insn
->next
= prev
->next
;
3415 insn
->next
= model_worklist
;
3416 model_worklist
= insn
;
3419 insn
->next
->prev
= insn
;
3422 /* Remove INSN from the model worklist. */
3425 model_remove_from_worklist (struct model_insn_info
*insn
)
3427 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3428 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3431 insn
->prev
->next
= insn
->next
;
3433 model_worklist
= insn
->next
;
3435 insn
->next
->prev
= insn
->prev
;
3438 /* Add INSN to the model worklist. Start looking for a suitable position
3439 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3440 insns either side. A null PREV indicates the beginning of the list and
3441 a null NEXT indicates the end. */
3444 model_add_to_worklist (struct model_insn_info
*insn
,
3445 struct model_insn_info
*prev
,
3446 struct model_insn_info
*next
)
3450 count
= MAX_SCHED_READY_INSNS
;
3451 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3457 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3459 while (count
> 0 && next
&& model_order_p (next
, insn
))
3465 model_add_to_worklist_at (insn
, prev
);
3468 /* INSN may now have a higher priority (in the model_order_p sense)
3469 than before. Move it up the worklist if necessary. */
3472 model_promote_insn (struct model_insn_info
*insn
)
3474 struct model_insn_info
*prev
;
3478 count
= MAX_SCHED_READY_INSNS
;
3479 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3484 if (prev
!= insn
->prev
)
3486 model_remove_from_worklist (insn
);
3487 model_add_to_worklist_at (insn
, prev
);
3491 /* Add INSN to the end of the model schedule. */
3494 model_add_to_schedule (rtx_insn
*insn
)
3498 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3499 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3501 point
= model_schedule
.length ();
3502 model_schedule
.quick_push (insn
);
3503 INSN_MODEL_INDEX (insn
) = point
+ 1;
3506 /* Analyze the instructions that are to be scheduled, setting up
3507 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3508 instructions to model_worklist. */
3511 model_analyze_insns (void)
3513 rtx_insn
*start
, *end
, *iter
;
3514 sd_iterator_def sd_it
;
3516 struct model_insn_info
*insn
, *con
;
3518 model_num_insns
= 0;
3519 start
= PREV_INSN (current_sched_info
->next_tail
);
3520 end
= current_sched_info
->prev_head
;
3521 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3522 if (NONDEBUG_INSN_P (iter
))
3524 insn
= MODEL_INSN_INFO (iter
);
3526 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3528 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3529 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3530 insn
->alap
= con
->alap
+ 1;
3533 insn
->old_queue
= QUEUE_INDEX (iter
);
3534 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3536 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3537 if (insn
->unscheduled_preds
== 0)
3538 model_add_to_worklist (insn
, NULL
, model_worklist
);
3544 /* The global state describes the register pressure at the start of the
3545 model schedule. Initialize GROUP accordingly. */
3548 model_init_pressure_group (struct model_pressure_group
*group
)
3552 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3554 cl
= ira_pressure_classes
[pci
];
3555 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3556 group
->limits
[pci
].point
= 0;
3558 /* Use index model_num_insns to record the state after the last
3559 instruction in the model schedule. */
3560 group
->model
= XNEWVEC (struct model_pressure_data
,
3561 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3564 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3565 Update the maximum pressure for the whole schedule. */
3568 model_record_pressure (struct model_pressure_group
*group
,
3569 int point
, int pci
, int pressure
)
3571 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3572 if (group
->limits
[pci
].pressure
< pressure
)
3574 group
->limits
[pci
].pressure
= pressure
;
3575 group
->limits
[pci
].point
= point
;
3579 /* INSN has just been added to the end of the model schedule. Record its
3580 register-pressure information. */
3583 model_record_pressures (struct model_insn_info
*insn
)
3585 struct reg_pressure_data
*reg_pressure
;
3586 int point
, pci
, cl
, delta
;
3587 int death
[N_REG_CLASSES
];
3589 point
= model_index (insn
->insn
);
3590 if (sched_verbose
>= 2)
3594 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3595 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3597 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3598 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3599 insn
->depth
+ insn
->alap
, insn
->depth
,
3600 INSN_PRIORITY (insn
->insn
),
3601 str_pattern_slim (PATTERN (insn
->insn
)));
3603 calculate_reg_deaths (insn
->insn
, death
);
3604 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3605 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3607 cl
= ira_pressure_classes
[pci
];
3608 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3609 if (sched_verbose
>= 2)
3610 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3611 curr_reg_pressure
[cl
], delta
);
3612 model_record_pressure (&model_before_pressure
, point
, pci
,
3613 curr_reg_pressure
[cl
]);
3615 if (sched_verbose
>= 2)
3616 fprintf (sched_dump
, "\n");
3619 /* All instructions have been added to the model schedule. Record the
3620 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3623 model_record_final_pressures (struct model_pressure_group
*group
)
3625 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3627 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3629 /* Record the final pressure for this class. */
3630 cl
= ira_pressure_classes
[pci
];
3631 point
= model_num_insns
;
3632 ref_pressure
= curr_reg_pressure
[cl
];
3633 model_record_pressure (group
, point
, pci
, ref_pressure
);
3635 /* Record the original maximum pressure. */
3636 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3638 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3639 max_pressure
= ref_pressure
;
3640 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3644 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3645 max_pressure
= MAX (max_pressure
, ref_pressure
);
3646 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3651 /* Update all successors of INSN, given that INSN has just been scheduled. */
3654 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3656 sd_iterator_def sd_it
;
3657 struct model_insn_info
*con
;
3660 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3662 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3663 /* Ignore debug instructions, and instructions from other blocks. */
3666 con
->unscheduled_preds
--;
3668 /* Update the depth field of each true-dependent successor.
3669 Increasing the depth gives them a higher priority than
3671 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3673 con
->depth
= insn
->depth
+ 1;
3674 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3675 model_promote_insn (con
);
3678 /* If this is a true dependency, or if there are no remaining
3679 dependencies for CON (meaning that CON only had non-true
3680 dependencies), make sure that CON is on the worklist.
3681 We don't bother otherwise because it would tend to fill the
3682 worklist with a lot of low-priority instructions that are not
3683 yet ready to issue. */
3684 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3685 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3686 model_add_to_worklist (con
, insn
, insn
->next
);
3691 /* Give INSN a higher priority than any current instruction, then give
3692 unscheduled predecessors of INSN a higher priority still. If any of
3693 those predecessors are not on the model worklist, do the same for its
3694 predecessors, and so on. */
3697 model_promote_predecessors (struct model_insn_info
*insn
)
3699 struct model_insn_info
*pro
, *first
;
3700 sd_iterator_def sd_it
;
3703 if (sched_verbose
>= 7)
3704 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3705 INSN_UID (insn
->insn
), model_next_priority
);
3706 insn
->model_priority
= model_next_priority
++;
3707 model_remove_from_worklist (insn
);
3708 model_add_to_worklist_at (insn
, NULL
);
3713 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3715 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3716 /* The first test is to ignore debug instructions, and instructions
3717 from other blocks. */
3719 && pro
->model_priority
!= model_next_priority
3720 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3722 pro
->model_priority
= model_next_priority
;
3723 if (sched_verbose
>= 7)
3724 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3725 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3727 /* PRO is already in the worklist, but it now has
3728 a higher priority than before. Move it at the
3729 appropriate place. */
3730 model_remove_from_worklist (pro
);
3731 model_add_to_worklist (pro
, NULL
, model_worklist
);
3735 /* PRO isn't in the worklist. Recursively process
3736 its predecessors until we find one that is. */
3747 if (sched_verbose
>= 7)
3748 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3749 model_next_priority
++;
3752 /* Pick one instruction from model_worklist and process it. */
3755 model_choose_insn (void)
3757 struct model_insn_info
*insn
, *fallback
;
3760 if (sched_verbose
>= 7)
3762 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3763 insn
= model_worklist
;
3764 count
= MAX_SCHED_READY_INSNS
;
3765 while (count
> 0 && insn
)
3767 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3768 INSN_UID (insn
->insn
), insn
->model_priority
,
3769 insn
->depth
+ insn
->alap
, insn
->depth
,
3770 INSN_PRIORITY (insn
->insn
));
3776 /* Look for a ready instruction whose model_classify_priority is zero
3777 or negative, picking the highest-priority one. Adding such an
3778 instruction to the schedule now should do no harm, and may actually
3781 Failing that, see whether there is an instruction with the highest
3782 extant model_priority that is not yet ready, but which would reduce
3783 pressure if it became ready. This is designed to catch cases like:
3785 (set (mem (reg R1)) (reg R2))
3787 where the instruction is the last remaining use of R1 and where the
3788 value of R2 is not yet available (or vice versa). The death of R1
3789 means that this instruction already reduces pressure. It is of
3790 course possible that the computation of R2 involves other registers
3791 that are hard to kill, but such cases are rare enough for this
3792 heuristic to be a win in general.
3794 Failing that, just pick the highest-priority instruction in the
3796 count
= MAX_SCHED_READY_INSNS
;
3797 insn
= model_worklist
;
3801 if (count
== 0 || !insn
)
3803 insn
= fallback
? fallback
: model_worklist
;
3806 if (insn
->unscheduled_preds
)
3808 if (model_worklist
->model_priority
== insn
->model_priority
3810 && model_classify_pressure (insn
) < 0)
3815 if (model_classify_pressure (insn
) <= 0)
3822 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3824 if (insn
->unscheduled_preds
)
3825 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3826 INSN_UID (insn
->insn
));
3828 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3829 INSN_UID (insn
->insn
));
3831 if (insn
->unscheduled_preds
)
3832 /* INSN isn't yet ready to issue. Give all its predecessors the
3833 highest priority. */
3834 model_promote_predecessors (insn
);
3837 /* INSN is ready. Add it to the end of model_schedule and
3838 process its successors. */
3839 model_add_successors_to_worklist (insn
);
3840 model_remove_from_worklist (insn
);
3841 model_add_to_schedule (insn
->insn
);
3842 model_record_pressures (insn
);
3843 update_register_pressure (insn
->insn
);
3847 /* Restore all QUEUE_INDEXs to the values that they had before
3848 model_start_schedule was called. */
3851 model_reset_queue_indices (void)
3856 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3857 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3860 /* We have calculated the model schedule and spill costs. Print a summary
3864 model_dump_pressure_summary (void)
3868 fprintf (sched_dump
, ";; Pressure summary:");
3869 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3871 cl
= ira_pressure_classes
[pci
];
3872 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3873 model_before_pressure
.limits
[pci
].pressure
);
3875 fprintf (sched_dump
, "\n\n");
3878 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3879 scheduling region. */
3882 model_start_schedule (basic_block bb
)
3884 model_next_priority
= 1;
3885 model_schedule
.create (sched_max_luid
);
3886 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3888 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3889 initiate_reg_pressure_info (df_get_live_in (bb
));
3891 model_analyze_insns ();
3892 model_init_pressure_group (&model_before_pressure
);
3893 while (model_worklist
)
3894 model_choose_insn ();
3895 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3896 if (sched_verbose
>= 2)
3897 fprintf (sched_dump
, "\n");
3899 model_record_final_pressures (&model_before_pressure
);
3900 model_reset_queue_indices ();
3902 XDELETEVEC (model_insns
);
3904 model_curr_point
= 0;
3905 initiate_reg_pressure_info (df_get_live_in (bb
));
3906 if (sched_verbose
>= 1)
3907 model_dump_pressure_summary ();
3910 /* Free the information associated with GROUP. */
3913 model_finalize_pressure_group (struct model_pressure_group
*group
)
3915 XDELETEVEC (group
->model
);
3918 /* Free the information created by model_start_schedule. */
3921 model_end_schedule (void)
3923 model_finalize_pressure_group (&model_before_pressure
);
3924 model_schedule
.release ();
3927 /* Prepare reg pressure scheduling for basic block BB. */
3929 sched_pressure_start_bb (basic_block bb
)
3931 /* Set the number of available registers for each class taking into account
3932 relative probability of current basic block versus function prologue and
3934 * If the basic block executes much more often than the prologue/epilogue
3935 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3936 nil, so the effective number of available registers is
3937 (ira_class_hard_regs_num[cl] - 0).
3938 * If the basic block executes as often as the prologue/epilogue,
3939 then spill in the block is as costly as in the prologue, so the effective
3940 number of available registers is
3941 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]).
3942 Note that all-else-equal, we prefer to spill in the prologue, since that
3943 allows "extra" registers for other basic blocks of the function.
3944 * If the basic block is on the cold path of the function and executes
3945 rarely, then we should always prefer to spill in the block, rather than
3946 in the prologue/epilogue. The effective number of available register is
3947 (ira_class_hard_regs_num[cl] - call_used_regs_num[cl]). */
3950 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->frequency
;
3951 int bb_freq
= bb
->frequency
;
3955 if (entry_freq
== 0)
3956 entry_freq
= bb_freq
= 1;
3958 if (bb_freq
< entry_freq
)
3959 bb_freq
= entry_freq
;
3961 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3963 enum reg_class cl
= ira_pressure_classes
[i
];
3964 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
];
3965 sched_class_regs_num
[cl
]
3966 -= (call_used_regs_num
[cl
] * entry_freq
) / bb_freq
;
3970 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3971 model_start_schedule (bb
);
3974 /* A structure that holds local state for the loop in schedule_block. */
3975 struct sched_block_state
3977 /* True if no real insns have been scheduled in the current cycle. */
3978 bool first_cycle_insn_p
;
3979 /* True if a shadow insn has been scheduled in the current cycle, which
3980 means that no more normal insns can be issued. */
3981 bool shadows_only_p
;
3982 /* True if we're winding down a modulo schedule, which means that we only
3983 issue insns with INSN_EXACT_TICK set. */
3984 bool modulo_epilogue
;
3985 /* Initialized with the machine's issue rate every cycle, and updated
3986 by calls to the variable_issue hook. */
3990 /* INSN is the "currently executing insn". Launch each insn which was
3991 waiting on INSN. READY is the ready list which contains the insns
3992 that are ready to fire. CLOCK is the current cycle. The function
3993 returns necessary cycle advance after issuing the insn (it is not
3994 zero for insns in a schedule group). */
3997 schedule_insn (rtx_insn
*insn
)
3999 sd_iterator_def sd_it
;
4004 if (sched_verbose
>= 1)
4006 struct reg_pressure_data
*pressure_info
;
4007 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
4008 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
4009 str_pattern_slim (PATTERN (insn
)));
4011 if (recog_memoized (insn
) < 0)
4012 fprintf (sched_dump
, "nothing");
4014 print_reservation (sched_dump
, insn
);
4015 pressure_info
= INSN_REG_PRESSURE (insn
);
4016 if (pressure_info
!= NULL
)
4018 fputc (':', sched_dump
);
4019 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4020 fprintf (sched_dump
, "%s%s%+d(%d)",
4021 scheduled_insns
.length () > 1
4023 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4024 reg_class_names
[ira_pressure_classes
[i
]],
4025 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4027 if (sched_pressure
== SCHED_PRESSURE_MODEL
4028 && model_curr_point
< model_num_insns
4029 && model_index (insn
) == model_curr_point
)
4030 fprintf (sched_dump
, ":model %d", model_curr_point
);
4031 fputc ('\n', sched_dump
);
4034 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4035 update_reg_and_insn_max_reg_pressure (insn
);
4037 /* Scheduling instruction should have all its dependencies resolved and
4038 should have been removed from the ready list. */
4039 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4041 /* Reset debug insns invalidated by moving this insn. */
4042 if (MAY_HAVE_DEBUG_INSNS
&& !DEBUG_INSN_P (insn
))
4043 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4044 sd_iterator_cond (&sd_it
, &dep
);)
4046 rtx_insn
*dbg
= DEP_PRO (dep
);
4047 struct reg_use_data
*use
, *next
;
4049 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4051 sd_iterator_next (&sd_it
);
4055 gcc_assert (DEBUG_INSN_P (dbg
));
4057 if (sched_verbose
>= 6)
4058 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4061 /* ??? Rather than resetting the debug insn, we might be able
4062 to emit a debug temp before the just-scheduled insn, but
4063 this would involve checking that the expression at the
4064 point of the debug insn is equivalent to the expression
4065 before the just-scheduled insn. They might not be: the
4066 expression in the debug insn may depend on other insns not
4067 yet scheduled that set MEMs, REGs or even other debug
4068 insns. It's not clear that attempting to preserve debug
4069 information in these cases is worth the effort, given how
4070 uncommon these resets are and the likelihood that the debug
4071 temps introduced won't survive the schedule change. */
4072 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4073 df_insn_rescan (dbg
);
4075 /* Unknown location doesn't use any registers. */
4076 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4078 struct reg_use_data
*prev
= use
;
4080 /* Remove use from the cyclic next_regno_use chain first. */
4081 while (prev
->next_regno_use
!= use
)
4082 prev
= prev
->next_regno_use
;
4083 prev
->next_regno_use
= use
->next_regno_use
;
4084 next
= use
->next_insn_use
;
4087 INSN_REG_USE_LIST (dbg
) = NULL
;
4089 /* We delete rather than resolve these deps, otherwise we
4090 crash in sched_free_deps(), because forward deps are
4091 expected to be released before backward deps. */
4092 sd_delete_dep (sd_it
);
4095 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4096 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4098 if (sched_pressure
== SCHED_PRESSURE_MODEL
4099 && model_curr_point
< model_num_insns
4100 && NONDEBUG_INSN_P (insn
))
4102 if (model_index (insn
) == model_curr_point
)
4105 while (model_curr_point
< model_num_insns
4106 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4107 == QUEUE_SCHEDULED
));
4109 model_recompute (insn
);
4110 model_update_limit_points ();
4111 update_register_pressure (insn
);
4112 if (sched_verbose
>= 2)
4113 print_curr_reg_pressure ();
4116 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4117 if (INSN_TICK (insn
) > clock_var
)
4118 /* INSN has been prematurely moved from the queue to the ready list.
4119 This is possible only if following flags are set. */
4120 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4122 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4123 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4124 INSN_TICK (insn
) = clock_var
;
4126 check_clobbered_conditions (insn
);
4128 /* Update dependent instructions. First, see if by scheduling this insn
4129 now we broke a dependence in a way that requires us to change another
4131 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4132 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4134 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4135 rtx_insn
*pro
= DEP_PRO (dep
);
4136 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4137 && desc
!= NULL
&& desc
->insn
== pro
)
4138 apply_replacement (dep
, false);
4141 /* Go through and resolve forward dependencies. */
4142 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4143 sd_iterator_cond (&sd_it
, &dep
);)
4145 rtx_insn
*next
= DEP_CON (dep
);
4146 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4148 /* Resolve the dependence between INSN and NEXT.
4149 sd_resolve_dep () moves current dep to another list thus
4150 advancing the iterator. */
4151 sd_resolve_dep (sd_it
);
4155 if (must_restore_pattern_p (next
, dep
))
4156 restore_pattern (dep
, false);
4160 /* Don't bother trying to mark next as ready if insn is a debug
4161 insn. If insn is the last hard dependency, it will have
4162 already been discounted. */
4163 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4166 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4170 effective_cost
= try_ready (next
);
4172 if (effective_cost
>= 0
4173 && SCHED_GROUP_P (next
)
4174 && advance
< effective_cost
)
4175 advance
= effective_cost
;
4178 /* Check always has only one forward dependence (to the first insn in
4179 the recovery block), therefore, this will be executed only once. */
4181 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4182 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4186 /* Annotate the instruction with issue information -- TImode
4187 indicates that the instruction is expected not to be able
4188 to issue on the same cycle as the previous insn. A machine
4189 may use this information to decide how the instruction should
4192 && GET_CODE (PATTERN (insn
)) != USE
4193 && GET_CODE (PATTERN (insn
)) != CLOBBER
4194 && !DEBUG_INSN_P (insn
))
4196 if (reload_completed
)
4197 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4198 last_clock_var
= clock_var
;
4201 if (nonscheduled_insns_begin
!= NULL_RTX
)
4202 /* Indicate to debug counters that INSN is scheduled. */
4203 nonscheduled_insns_begin
= insn
;
4208 /* Functions for handling of notes. */
4210 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4212 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4214 rtx_insn
*from_start
;
4216 /* It's easy when have nothing to concat. */
4217 if (from_end
== NULL
)
4220 /* It's also easy when destination is empty. */
4221 if (*to_endp
== NULL
)
4223 *to_endp
= from_end
;
4227 from_start
= from_end
;
4228 while (PREV_INSN (from_start
) != NULL
)
4229 from_start
= PREV_INSN (from_start
);
4231 SET_PREV_INSN (from_start
) = *to_endp
;
4232 SET_NEXT_INSN (*to_endp
) = from_start
;
4233 *to_endp
= from_end
;
4236 /* Delete notes between HEAD and TAIL and put them in the chain
4237 of notes ended by NOTE_LIST. */
4239 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4241 rtx_insn
*next_tail
, *insn
, *next
;
4244 if (head
== tail
&& !INSN_P (head
))
4247 next_tail
= NEXT_INSN (tail
);
4248 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4250 next
= NEXT_INSN (insn
);
4254 switch (NOTE_KIND (insn
))
4256 case NOTE_INSN_BASIC_BLOCK
:
4259 case NOTE_INSN_EPILOGUE_BEG
:
4263 add_reg_note (next
, REG_SAVE_NOTE
,
4264 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4272 /* Add the note to list that ends at NOTE_LIST. */
4273 SET_PREV_INSN (insn
) = note_list
;
4274 SET_NEXT_INSN (insn
) = NULL_RTX
;
4276 SET_NEXT_INSN (note_list
) = insn
;
4281 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4285 /* A structure to record enough data to allow us to backtrack the scheduler to
4286 a previous state. */
4287 struct haifa_saved_data
4289 /* Next entry on the list. */
4290 struct haifa_saved_data
*next
;
4292 /* Backtracking is associated with scheduling insns that have delay slots.
4293 DELAY_PAIR points to the structure that contains the insns involved, and
4294 the number of cycles between them. */
4295 struct delay_pair
*delay_pair
;
4297 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4298 void *fe_saved_data
;
4299 /* Data used by the backend. */
4300 void *be_saved_data
;
4302 /* Copies of global state. */
4303 int clock_var
, last_clock_var
;
4304 struct ready_list ready
;
4307 rtx_insn
*last_scheduled_insn
;
4308 rtx last_nondebug_scheduled_insn
;
4309 rtx_insn
*nonscheduled_insns_begin
;
4310 int cycle_issued_insns
;
4312 /* Copies of state used in the inner loop of schedule_block. */
4313 struct sched_block_state sched_block
;
4315 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4316 to 0 when restoring. */
4318 rtx_insn_list
**insn_queue
;
4320 /* Describe pattern replacements that occurred since this backtrack point
4322 vec
<dep_t
> replacement_deps
;
4323 vec
<int> replace_apply
;
4325 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4327 vec
<dep_t
> next_cycle_deps
;
4328 vec
<int> next_cycle_apply
;
4331 /* A record, in reverse order, of all scheduled insns which have delay slots
4332 and may require backtracking. */
4333 static struct haifa_saved_data
*backtrack_queue
;
4335 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4338 mark_backtrack_feeds (rtx insn
, int set_p
)
4340 sd_iterator_def sd_it
;
4342 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4344 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4348 /* Save the current scheduler state so that we can backtrack to it
4349 later if necessary. PAIR gives the insns that make it necessary to
4350 save this point. SCHED_BLOCK is the local state of schedule_block
4351 that need to be saved. */
4353 save_backtrack_point (struct delay_pair
*pair
,
4354 struct sched_block_state sched_block
)
4357 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4359 save
->curr_state
= xmalloc (dfa_state_size
);
4360 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4362 save
->ready
.first
= ready
.first
;
4363 save
->ready
.n_ready
= ready
.n_ready
;
4364 save
->ready
.n_debug
= ready
.n_debug
;
4365 save
->ready
.veclen
= ready
.veclen
;
4366 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4367 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4369 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4370 save
->q_size
= q_size
;
4371 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4373 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4374 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4377 save
->clock_var
= clock_var
;
4378 save
->last_clock_var
= last_clock_var
;
4379 save
->cycle_issued_insns
= cycle_issued_insns
;
4380 save
->last_scheduled_insn
= last_scheduled_insn
;
4381 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4382 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4384 save
->sched_block
= sched_block
;
4386 save
->replacement_deps
.create (0);
4387 save
->replace_apply
.create (0);
4388 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4389 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4391 if (current_sched_info
->save_state
)
4392 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4394 if (targetm
.sched
.alloc_sched_context
)
4396 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4397 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4400 save
->be_saved_data
= NULL
;
4402 save
->delay_pair
= pair
;
4404 save
->next
= backtrack_queue
;
4405 backtrack_queue
= save
;
4409 mark_backtrack_feeds (pair
->i2
, 1);
4410 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4411 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4412 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4413 pair
= pair
->next_same_i1
;
4417 /* Walk the ready list and all queues. If any insns have unresolved backwards
4418 dependencies, these must be cancelled deps, broken by predication. Set or
4419 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4422 toggle_cancelled_flags (bool set
)
4425 sd_iterator_def sd_it
;
4428 if (ready
.n_ready
> 0)
4430 rtx_insn
**first
= ready_lastpos (&ready
);
4431 for (i
= 0; i
< ready
.n_ready
; i
++)
4432 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4433 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4436 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4438 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4441 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4443 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4444 rtx_insn_list
*link
;
4445 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4447 rtx_insn
*insn
= link
->insn ();
4448 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4449 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4452 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4454 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4460 /* Undo the replacements that have occurred after backtrack point SAVE
4463 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4465 while (!save
->replacement_deps
.is_empty ())
4467 dep_t dep
= save
->replacement_deps
.pop ();
4468 int apply_p
= save
->replace_apply
.pop ();
4471 restore_pattern (dep
, true);
4473 apply_replacement (dep
, true);
4475 save
->replacement_deps
.release ();
4476 save
->replace_apply
.release ();
4479 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4480 Restore their dependencies to an unresolved state, and mark them as
4484 unschedule_insns_until (rtx insn
)
4486 auto_vec
<rtx_insn
*> recompute_vec
;
4488 /* Make two passes over the insns to be unscheduled. First, we clear out
4489 dependencies and other trivial bookkeeping. */
4493 sd_iterator_def sd_it
;
4496 last
= scheduled_insns
.pop ();
4498 /* This will be changed by restore_backtrack_point if the insn is in
4500 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4502 INSN_TICK (last
) = INVALID_TICK
;
4504 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4505 modulo_insns_scheduled
--;
4507 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4508 sd_iterator_cond (&sd_it
, &dep
);)
4510 rtx_insn
*con
= DEP_CON (dep
);
4511 sd_unresolve_dep (sd_it
);
4512 if (!MUST_RECOMPUTE_SPEC_P (con
))
4514 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4515 recompute_vec
.safe_push (con
);
4523 /* A second pass, to update ready and speculation status for insns
4524 depending on the unscheduled ones. The first pass must have
4525 popped the scheduled_insns vector up to the point where we
4526 restart scheduling, as recompute_todo_spec requires it to be
4528 while (!recompute_vec
.is_empty ())
4532 con
= recompute_vec
.pop ();
4533 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4534 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4536 TODO_SPEC (con
) = HARD_DEP
;
4537 INSN_TICK (con
) = INVALID_TICK
;
4538 if (PREDICATED_PAT (con
) != NULL_RTX
)
4539 haifa_change_pattern (con
, ORIG_PAT (con
));
4541 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4542 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4546 /* Restore scheduler state from the topmost entry on the backtracking queue.
4547 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4548 overwrite with the saved data.
4549 The caller must already have called unschedule_insns_until. */
4552 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4555 struct haifa_saved_data
*save
= backtrack_queue
;
4557 backtrack_queue
= save
->next
;
4559 if (current_sched_info
->restore_state
)
4560 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4562 if (targetm
.sched
.alloc_sched_context
)
4564 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4565 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4568 /* Do this first since it clobbers INSN_TICK of the involved
4570 undo_replacements_for_backtrack (save
);
4572 /* Clear the QUEUE_INDEX of everything in the ready list or one
4574 if (ready
.n_ready
> 0)
4576 rtx_insn
**first
= ready_lastpos (&ready
);
4577 for (i
= 0; i
< ready
.n_ready
; i
++)
4579 rtx_insn
*insn
= first
[i
];
4580 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4581 INSN_TICK (insn
) = INVALID_TICK
;
4584 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4586 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4588 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4590 rtx_insn
*x
= link
->insn ();
4591 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4592 INSN_TICK (x
) = INVALID_TICK
;
4594 free_INSN_LIST_list (&insn_queue
[q
]);
4598 ready
= save
->ready
;
4600 if (ready
.n_ready
> 0)
4602 rtx_insn
**first
= ready_lastpos (&ready
);
4603 for (i
= 0; i
< ready
.n_ready
; i
++)
4605 rtx_insn
*insn
= first
[i
];
4606 QUEUE_INDEX (insn
) = QUEUE_READY
;
4607 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4608 INSN_TICK (insn
) = save
->clock_var
;
4613 q_size
= save
->q_size
;
4614 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4616 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4618 insn_queue
[q
] = save
->insn_queue
[q
];
4620 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4622 rtx_insn
*x
= link
->insn ();
4623 QUEUE_INDEX (x
) = i
;
4624 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4625 INSN_TICK (x
) = save
->clock_var
+ i
;
4628 free (save
->insn_queue
);
4630 toggle_cancelled_flags (true);
4632 clock_var
= save
->clock_var
;
4633 last_clock_var
= save
->last_clock_var
;
4634 cycle_issued_insns
= save
->cycle_issued_insns
;
4635 last_scheduled_insn
= save
->last_scheduled_insn
;
4636 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4637 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4639 *psched_block
= save
->sched_block
;
4641 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4642 free (save
->curr_state
);
4644 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4646 gcc_assert (next_cycle_replace_deps
.is_empty ());
4647 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4648 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4652 for (save
= backtrack_queue
; save
; save
= save
->next
)
4654 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4658 /* Discard all data associated with the topmost entry in the backtrack
4659 queue. If RESET_TICK is false, we just want to free the data. If true,
4660 we are doing this because we discovered a reason to backtrack. In the
4661 latter case, also reset the INSN_TICK for the shadow insn. */
4663 free_topmost_backtrack_point (bool reset_tick
)
4665 struct haifa_saved_data
*save
= backtrack_queue
;
4668 backtrack_queue
= save
->next
;
4672 struct delay_pair
*pair
= save
->delay_pair
;
4675 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4676 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4677 pair
= pair
->next_same_i1
;
4679 undo_replacements_for_backtrack (save
);
4683 save
->replacement_deps
.release ();
4684 save
->replace_apply
.release ();
4687 if (targetm
.sched
.free_sched_context
)
4688 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4689 if (current_sched_info
->restore_state
)
4690 free (save
->fe_saved_data
);
4691 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4692 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4693 free (save
->insn_queue
);
4694 free (save
->curr_state
);
4695 free (save
->ready
.vec
);
4699 /* Free the entire backtrack queue. */
4701 free_backtrack_queue (void)
4703 while (backtrack_queue
)
4704 free_topmost_backtrack_point (false);
4707 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4708 may have to postpone the replacement until the start of the next cycle,
4709 at which point we will be called again with IMMEDIATELY true. This is
4710 only done for machines which have instruction packets with explicit
4711 parallelism however. */
4713 apply_replacement (dep_t dep
, bool immediately
)
4715 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4716 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4718 next_cycle_replace_deps
.safe_push (dep
);
4719 next_cycle_apply
.safe_push (1);
4725 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4728 if (sched_verbose
>= 5)
4729 fprintf (sched_dump
, "applying replacement for insn %d\n",
4730 INSN_UID (desc
->insn
));
4732 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4733 gcc_assert (success
);
4735 update_insn_after_change (desc
->insn
);
4736 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4737 fix_tick_ready (desc
->insn
);
4739 if (backtrack_queue
!= NULL
)
4741 backtrack_queue
->replacement_deps
.safe_push (dep
);
4742 backtrack_queue
->replace_apply
.safe_push (1);
4747 /* We have determined that a pattern involved in DEP must be restored.
4748 If IMMEDIATELY is false, we may have to postpone the replacement
4749 until the start of the next cycle, at which point we will be called
4750 again with IMMEDIATELY true. */
4752 restore_pattern (dep_t dep
, bool immediately
)
4754 rtx_insn
*next
= DEP_CON (dep
);
4755 int tick
= INSN_TICK (next
);
4757 /* If we already scheduled the insn, the modified version is
4759 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4762 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4764 next_cycle_replace_deps
.safe_push (dep
);
4765 next_cycle_apply
.safe_push (0);
4770 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4772 if (sched_verbose
>= 5)
4773 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4775 haifa_change_pattern (next
, ORIG_PAT (next
));
4779 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4782 if (sched_verbose
>= 5)
4783 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4784 INSN_UID (desc
->insn
));
4785 tick
= INSN_TICK (desc
->insn
);
4787 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4788 gcc_assert (success
);
4789 update_insn_after_change (desc
->insn
);
4790 if (backtrack_queue
!= NULL
)
4792 backtrack_queue
->replacement_deps
.safe_push (dep
);
4793 backtrack_queue
->replace_apply
.safe_push (0);
4796 INSN_TICK (next
) = tick
;
4797 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4800 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4801 TODO_SPEC (next
) = 0;
4802 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4803 TODO_SPEC (next
) = HARD_DEP
;
4806 /* Perform pattern replacements that were queued up until the next
4809 perform_replacements_new_cycle (void)
4813 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4815 int apply_p
= next_cycle_apply
[i
];
4817 apply_replacement (dep
, true);
4819 restore_pattern (dep
, true);
4821 next_cycle_replace_deps
.truncate (0);
4822 next_cycle_apply
.truncate (0);
4825 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4826 instructions we've previously encountered, a set bit prevents
4827 recursion. BUDGET is a limit on how far ahead we look, it is
4828 reduced on recursive calls. Return true if we produced a good
4829 estimate, or false if we exceeded the budget. */
4831 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4833 sd_iterator_def sd_it
;
4835 int earliest
= INSN_TICK (insn
);
4837 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4839 rtx_insn
*pro
= DEP_PRO (dep
);
4842 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4845 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4846 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4849 int cost
= dep_cost (dep
);
4852 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4854 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4857 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4858 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4859 if (earliest
== INVALID_TICK
|| t
> earliest
)
4863 bitmap_set_bit (processed
, INSN_LUID (insn
));
4864 INSN_TICK_ESTIMATE (insn
) = earliest
;
4868 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4869 infinite resources) the cycle in which the delayed shadow can be issued.
4870 Return the number of cycles that must pass before the real insn can be
4871 issued in order to meet this constraint. */
4873 estimate_shadow_tick (struct delay_pair
*p
)
4875 bitmap_head processed
;
4878 bitmap_initialize (&processed
, 0);
4880 cutoff
= !estimate_insn_tick (&processed
, p
->i2
,
4881 max_insn_queue_index
+ pair_delay (p
));
4882 bitmap_clear (&processed
);
4884 return max_insn_queue_index
;
4885 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4891 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4892 recursively resolve all its forward dependencies. */
4894 resolve_dependencies (rtx_insn
*insn
)
4896 sd_iterator_def sd_it
;
4899 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4900 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4901 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4904 if (sched_verbose
>= 4)
4905 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4907 if (QUEUE_INDEX (insn
) >= 0)
4908 queue_remove (insn
);
4910 scheduled_insns
.safe_push (insn
);
4912 /* Update dependent instructions. */
4913 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4914 sd_iterator_cond (&sd_it
, &dep
);)
4916 rtx_insn
*next
= DEP_CON (dep
);
4918 if (sched_verbose
>= 4)
4919 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4922 /* Resolve the dependence between INSN and NEXT.
4923 sd_resolve_dep () moves current dep to another list thus
4924 advancing the iterator. */
4925 sd_resolve_dep (sd_it
);
4927 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4929 resolve_dependencies (next
);
4932 /* Check always has only one forward dependence (to the first insn in
4933 the recovery block), therefore, this will be executed only once. */
4935 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4941 /* Return the head and tail pointers of ebb starting at BEG and ending
4944 get_ebb_head_tail (basic_block beg
, basic_block end
,
4945 rtx_insn
**headp
, rtx_insn
**tailp
)
4947 rtx_insn
*beg_head
= BB_HEAD (beg
);
4948 rtx_insn
* beg_tail
= BB_END (beg
);
4949 rtx_insn
* end_head
= BB_HEAD (end
);
4950 rtx_insn
* end_tail
= BB_END (end
);
4952 /* Don't include any notes or labels at the beginning of the BEG
4953 basic block, or notes at the end of the END basic blocks. */
4955 if (LABEL_P (beg_head
))
4956 beg_head
= NEXT_INSN (beg_head
);
4958 while (beg_head
!= beg_tail
)
4959 if (NOTE_P (beg_head
))
4960 beg_head
= NEXT_INSN (beg_head
);
4961 else if (DEBUG_INSN_P (beg_head
))
4963 rtx_insn
* note
, *next
;
4965 for (note
= NEXT_INSN (beg_head
);
4969 next
= NEXT_INSN (note
);
4972 if (sched_verbose
>= 9)
4973 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4975 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4977 if (BLOCK_FOR_INSN (note
) != beg
)
4978 df_insn_change_bb (note
, beg
);
4980 else if (!DEBUG_INSN_P (note
))
4992 end_head
= beg_head
;
4993 else if (LABEL_P (end_head
))
4994 end_head
= NEXT_INSN (end_head
);
4996 while (end_head
!= end_tail
)
4997 if (NOTE_P (end_tail
))
4998 end_tail
= PREV_INSN (end_tail
);
4999 else if (DEBUG_INSN_P (end_tail
))
5001 rtx_insn
* note
, *prev
;
5003 for (note
= PREV_INSN (end_tail
);
5007 prev
= PREV_INSN (note
);
5010 if (sched_verbose
>= 9)
5011 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5013 reorder_insns_nobb (note
, note
, end_tail
);
5015 if (end_tail
== BB_END (end
))
5016 BB_END (end
) = note
;
5018 if (BLOCK_FOR_INSN (note
) != end
)
5019 df_insn_change_bb (note
, end
);
5021 else if (!DEBUG_INSN_P (note
))
5033 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5036 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5038 while (head
!= NEXT_INSN (tail
))
5040 if (!NOTE_P (head
) && !LABEL_P (head
))
5042 head
= NEXT_INSN (head
);
5047 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5048 previously found among the insns. Insert them just before HEAD. */
5050 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5054 rtx_insn
*note_head
= note_list
;
5057 head_bb
= BLOCK_FOR_INSN (head
);
5059 head
= NEXT_INSN (bb_note (head_bb
));
5061 while (PREV_INSN (note_head
))
5063 set_block_for_insn (note_head
, head_bb
);
5064 note_head
= PREV_INSN (note_head
);
5066 /* In the above cycle we've missed this note. */
5067 set_block_for_insn (note_head
, head_bb
);
5069 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5070 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5071 SET_PREV_INSN (head
) = note_list
;
5072 SET_NEXT_INSN (note_list
) = head
;
5074 if (BLOCK_FOR_INSN (head
) != head_bb
)
5075 BB_END (head_bb
) = note_list
;
5083 /* When we know we are going to discard the schedule due to a failed attempt
5084 at modulo scheduling, undo all replacements. */
5086 undo_all_replacements (void)
5091 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5093 sd_iterator_def sd_it
;
5096 /* See if we must undo a replacement. */
5097 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5098 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5100 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5102 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5107 /* Return first non-scheduled insn in the current scheduling block.
5108 This is mostly used for debug-counter purposes. */
5110 first_nonscheduled_insn (void)
5112 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5113 ? nonscheduled_insns_begin
5114 : current_sched_info
->prev_head
);
5118 insn
= next_nonnote_nondebug_insn (insn
);
5120 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5125 /* Move insns that became ready to fire from queue to ready list. */
5128 queue_to_ready (struct ready_list
*ready
)
5131 rtx_insn_list
*link
;
5134 q_ptr
= NEXT_Q (q_ptr
);
5136 if (dbg_cnt (sched_insn
) == false)
5137 /* If debug counter is activated do not requeue the first
5138 nonscheduled insn. */
5139 skip_insn
= first_nonscheduled_insn ();
5141 skip_insn
= NULL_RTX
;
5143 /* Add all pending insns that can be scheduled without stalls to the
5145 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5147 insn
= link
->insn ();
5150 if (sched_verbose
>= 2)
5151 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5152 (*current_sched_info
->print_insn
) (insn
, 0));
5154 /* If the ready list is full, delay the insn for 1 cycle.
5155 See the comment in schedule_block for the rationale. */
5156 if (!reload_completed
5157 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5158 || (sched_pressure
== SCHED_PRESSURE_MODEL
5159 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5160 instructions too. */
5161 && model_index (insn
) > (model_curr_point
5162 + MAX_SCHED_READY_INSNS
)))
5163 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5164 && model_curr_point
< model_num_insns
5165 /* Always allow the next model instruction to issue. */
5166 && model_index (insn
) == model_curr_point
)
5167 && !SCHED_GROUP_P (insn
)
5168 && insn
!= skip_insn
)
5170 if (sched_verbose
>= 2)
5171 fprintf (sched_dump
, "keeping in queue, ready full\n");
5172 queue_insn (insn
, 1, "ready full");
5176 ready_add (ready
, insn
, false);
5177 if (sched_verbose
>= 2)
5178 fprintf (sched_dump
, "moving to ready without stalls\n");
5181 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5183 /* If there are no ready insns, stall until one is ready and add all
5184 of the pending insns at that point to the ready list. */
5185 if (ready
->n_ready
== 0)
5189 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5191 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5193 for (; link
; link
= link
->next ())
5195 insn
= link
->insn ();
5198 if (sched_verbose
>= 2)
5199 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5200 (*current_sched_info
->print_insn
) (insn
, 0));
5202 ready_add (ready
, insn
, false);
5203 if (sched_verbose
>= 2)
5204 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5206 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5208 advance_one_cycle ();
5213 advance_one_cycle ();
5216 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5217 clock_var
+= stalls
;
5218 if (sched_verbose
>= 2)
5219 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5224 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5225 prematurely move INSN from the queue to the ready list. Currently,
5226 if a target defines the hook 'is_costly_dependence', this function
5227 uses the hook to check whether there exist any dependences which are
5228 considered costly by the target, between INSN and other insns that
5229 have already been scheduled. Dependences are checked up to Y cycles
5230 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5231 controlling this value.
5232 (Other considerations could be taken into account instead (or in
5233 addition) depending on user flags and target hooks. */
5236 ok_for_early_queue_removal (rtx insn
)
5238 if (targetm
.sched
.is_costly_dependence
)
5242 int i
= scheduled_insns
.length ();
5243 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5249 prev_insn
= scheduled_insns
[i
];
5251 if (!NOTE_P (prev_insn
))
5255 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5259 cost
= dep_cost (dep
);
5261 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5262 flag_sched_stalled_insns_dep
- n_cycles
))
5267 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5280 /* Remove insns from the queue, before they become "ready" with respect
5281 to FU latency considerations. */
5284 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5287 rtx_insn_list
*link
;
5288 rtx_insn_list
*next_link
;
5289 rtx_insn_list
*prev_link
;
5292 state_t temp_state
= alloca (dfa_state_size
);
5294 int insns_removed
= 0;
5297 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5300 X == 0: There is no limit on how many queued insns can be removed
5301 prematurely. (flag_sched_stalled_insns = -1).
5303 X >= 1: Only X queued insns can be removed prematurely in each
5304 invocation. (flag_sched_stalled_insns = X).
5306 Otherwise: Early queue removal is disabled.
5307 (flag_sched_stalled_insns = 0)
5310 if (! flag_sched_stalled_insns
)
5313 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5315 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5317 if (sched_verbose
> 6)
5318 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5323 next_link
= link
->next ();
5324 insn
= link
->insn ();
5325 if (insn
&& sched_verbose
> 6)
5326 print_rtl_single (sched_dump
, insn
);
5328 memcpy (temp_state
, state
, dfa_state_size
);
5329 if (recog_memoized (insn
) < 0)
5330 /* non-negative to indicate that it's not ready
5331 to avoid infinite Q->R->Q->R... */
5334 cost
= state_transition (temp_state
, insn
);
5336 if (sched_verbose
>= 6)
5337 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5339 move_to_ready
= false;
5342 move_to_ready
= ok_for_early_queue_removal (insn
);
5343 if (move_to_ready
== true)
5345 /* move from Q to R */
5347 ready_add (ready
, insn
, false);
5350 XEXP (prev_link
, 1) = next_link
;
5352 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5354 free_INSN_LIST_node (link
);
5356 if (sched_verbose
>= 2)
5357 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5358 (*current_sched_info
->print_insn
) (insn
, 0));
5361 if (insns_removed
== flag_sched_stalled_insns
)
5362 /* Remove no more than flag_sched_stalled_insns insns
5363 from Q at a time. */
5364 return insns_removed
;
5368 if (move_to_ready
== false)
5375 } /* for stalls.. */
5377 return insns_removed
;
5381 /* Print the ready list for debugging purposes.
5382 If READY_TRY is non-zero then only print insns that max_issue
5385 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5390 if (ready
->n_ready
== 0)
5392 fprintf (sched_dump
, "\n");
5396 p
= ready_lastpos (ready
);
5397 for (i
= 0; i
< ready
->n_ready
; i
++)
5399 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5402 fprintf (sched_dump
, " %s:%d",
5403 (*current_sched_info
->print_insn
) (p
[i
], 0),
5405 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5406 fprintf (sched_dump
, "(cost=%d",
5407 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5408 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5409 if (INSN_TICK (p
[i
]) > clock_var
)
5410 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5411 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5412 fprintf (sched_dump
, ":idx=%d",
5413 model_index (p
[i
]));
5414 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5415 fprintf (sched_dump
, ")");
5417 fprintf (sched_dump
, "\n");
5420 /* Print the ready list. Callable from debugger. */
5422 debug_ready_list (struct ready_list
*ready
)
5424 debug_ready_list_1 (ready
, NULL
);
5427 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5428 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5429 replaces the epilogue note in the correct basic block. */
5431 reemit_notes (rtx_insn
*insn
)
5434 rtx_insn
*last
= insn
;
5436 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5438 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5440 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5442 last
= emit_note_before (note_type
, last
);
5443 remove_note (insn
, note
);
5448 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5450 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5452 if (PREV_INSN (insn
) != last
)
5458 bb
= BLOCK_FOR_INSN (insn
);
5460 /* BB_HEAD is either LABEL or NOTE. */
5461 gcc_assert (BB_HEAD (bb
) != insn
);
5463 if (BB_END (bb
) == insn
)
5464 /* If this is last instruction in BB, move end marker one
5467 /* Jumps are always placed at the end of basic block. */
5468 jump_p
= control_flow_insn_p (insn
);
5471 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5472 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5473 || (common_sched_info
->sched_pass_id
5474 == SCHED_EBB_PASS
));
5476 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5478 BB_END (bb
) = PREV_INSN (insn
);
5481 gcc_assert (BB_END (bb
) != last
);
5484 /* We move the block note along with jump. */
5488 note
= NEXT_INSN (insn
);
5489 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5490 note
= NEXT_INSN (note
);
5494 || BARRIER_P (note
)))
5495 note
= NEXT_INSN (note
);
5497 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5502 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5503 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5505 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5506 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5508 SET_NEXT_INSN (last
) = insn
;
5509 SET_PREV_INSN (insn
) = last
;
5511 bb
= BLOCK_FOR_INSN (last
);
5515 fix_jump_move (insn
);
5517 if (BLOCK_FOR_INSN (insn
) != bb
)
5518 move_block_after_check (insn
);
5520 gcc_assert (BB_END (bb
) == last
);
5523 df_insn_change_bb (insn
, bb
);
5525 /* Update BB_END, if needed. */
5526 if (BB_END (bb
) == last
)
5530 SCHED_GROUP_P (insn
) = 0;
5533 /* Return true if scheduling INSN will finish current clock cycle. */
5535 insn_finishes_cycle_p (rtx_insn
*insn
)
5537 if (SCHED_GROUP_P (insn
))
5538 /* After issuing INSN, rest of the sched_group will be forced to issue
5539 in order. Don't make any plans for the rest of cycle. */
5542 /* Finishing the block will, apparently, finish the cycle. */
5543 if (current_sched_info
->insn_finishes_block_p
5544 && current_sched_info
->insn_finishes_block_p (insn
))
5550 /* Functions to model cache auto-prefetcher.
5552 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5553 memory prefetches if it sees instructions with consequitive memory accesses
5554 in the instruction stream. Details of such hardware units are not published,
5555 so we can only guess what exactly is going on there.
5556 In the scheduler, we model abstract auto-prefetcher. If there are memory
5557 insns in the ready list (or the queue) that have same memory base, but
5558 different offsets, then we delay the insns with larger offsets until insns
5559 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5560 is "1", then we look at the ready list; if it is N>1, then we also look
5561 through N-1 queue entries.
5562 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5563 among its heuristics.
5564 Param value of "-1" disables modelling of the auto-prefetcher. */
5566 /* Initialize autoprefetcher model data for INSN. */
5568 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5570 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5572 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5573 data
->base
= NULL_RTX
;
5575 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5576 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5578 rtx set
= single_set (insn
);
5579 if (set
== NULL_RTX
)
5582 rtx mem
= write
? SET_DEST (set
) : SET_SRC (set
);
5586 struct address_info info
;
5587 decompose_mem_address (&info
, mem
);
5589 /* TODO: Currently only (base+const) addressing is supported. */
5590 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5591 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5594 /* This insn is relevant for auto-prefetcher. */
5595 data
->base
= *info
.base
;
5596 data
->offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5597 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5600 /* Helper function for rank_for_schedule sorting. */
5602 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5604 for (int write
= 0; write
< 2; ++write
)
5606 autopref_multipass_data_t data1
5607 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5608 autopref_multipass_data_t data2
5609 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5611 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5612 autopref_multipass_init (insn1
, write
);
5613 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5616 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5617 autopref_multipass_init (insn2
, write
);
5618 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5621 if (!rtx_equal_p (data1
->base
, data2
->base
))
5624 return data1
->offset
- data2
->offset
;
5630 /* True if header of debug dump was printed. */
5631 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5633 /* Helper for autopref_multipass_dfa_lookahead_guard.
5634 Return "1" if INSN1 should be delayed in favor of INSN2. */
5636 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5637 const rtx_insn
*insn2
, int write
)
5639 autopref_multipass_data_t data1
5640 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5641 autopref_multipass_data_t data2
5642 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5644 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5645 autopref_multipass_init (insn2
, write
);
5646 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5649 if (rtx_equal_p (data1
->base
, data2
->base
)
5650 && data1
->offset
> data2
->offset
)
5652 if (sched_verbose
>= 2)
5654 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5656 fprintf (sched_dump
,
5657 ";;\t\tnot trying in max_issue due to autoprefetch "
5659 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5662 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5673 We could have also hooked autoprefetcher model into
5674 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5675 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5676 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5677 unblocked). We don't bother about this yet because target of interest
5678 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5680 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5681 Return "1" if INSN1 should not be considered in max_issue due to
5682 auto-prefetcher considerations. */
5684 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5688 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5691 if (sched_verbose
>= 2 && ready_index
== 0)
5692 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5694 for (int write
= 0; write
< 2; ++write
)
5696 autopref_multipass_data_t data1
5697 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5699 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5700 autopref_multipass_init (insn1
, write
);
5701 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5704 if (ready_index
== 0
5705 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5706 /* We allow only a single delay on priviledged instructions.
5707 Doing otherwise would cause infinite loop. */
5709 if (sched_verbose
>= 2)
5711 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5713 fprintf (sched_dump
,
5714 ";;\t\tnot trying in max_issue due to autoprefetch "
5716 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5719 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5724 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5726 rtx_insn
*insn2
= get_ready_element (i2
);
5729 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5732 if (ready_index
== 0)
5735 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5741 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5744 /* Everything from the current queue slot should have been moved to
5746 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5748 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5749 if (n_stalls
> max_insn_queue_index
)
5750 n_stalls
= max_insn_queue_index
;
5752 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5754 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5756 link
= link
->next ())
5758 rtx_insn
*insn2
= link
->insn ();
5759 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5763 /* Queue INSN1 until INSN2 can issue. */
5765 if (ready_index
== 0)
5766 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5774 if (sched_verbose
>= 2
5775 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5776 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5777 /* This does not /always/ trigger. We don't output EOL if the last
5778 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5779 called. We can live with this. */
5780 fprintf (sched_dump
, "\n");
5785 /* Define type for target data used in multipass scheduling. */
5786 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5787 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5789 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5791 /* The following structure describe an entry of the stack of choices. */
5794 /* Ordinal number of the issued insn in the ready queue. */
5796 /* The number of the rest insns whose issues we should try. */
5798 /* The number of issued essential insns. */
5800 /* State after issuing the insn. */
5802 /* Target-specific data. */
5803 first_cycle_multipass_data_t target_data
;
5806 /* The following array is used to implement a stack of choices used in
5807 function max_issue. */
5808 static struct choice_entry
*choice_stack
;
5810 /* This holds the value of the target dfa_lookahead hook. */
5813 /* The following variable value is maximal number of tries of issuing
5814 insns for the first cycle multipass insn scheduling. We define
5815 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5816 need this constraint if all real insns (with non-negative codes)
5817 had reservations because in this case the algorithm complexity is
5818 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5819 might be incomplete and such insn might occur. For such
5820 descriptions, the complexity of algorithm (without the constraint)
5821 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5822 static int max_lookahead_tries
;
5824 /* The following function returns maximal (or close to maximal) number
5825 of insns which can be issued on the same cycle and one of which
5826 insns is insns with the best rank (the first insn in READY). To
5827 make this function tries different samples of ready insns. READY
5828 is current queue `ready'. Global array READY_TRY reflects what
5829 insns are already issued in this try. The function stops immediately,
5830 if it reached the such a solution, that all instruction can be issued.
5831 INDEX will contain index of the best insn in READY. The following
5832 function is used only for first cycle multipass scheduling.
5836 This function expects recognized insns only. All USEs,
5837 CLOBBERs, etc must be filtered elsewhere. */
5839 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5840 bool first_cycle_insn_p
, int *index
)
5842 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5844 struct choice_entry
*top
;
5850 n_ready
= ready
->n_ready
;
5851 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5852 && privileged_n
<= n_ready
);
5854 /* Init MAX_LOOKAHEAD_TRIES. */
5855 if (max_lookahead_tries
== 0)
5857 max_lookahead_tries
= 100;
5858 for (i
= 0; i
< issue_rate
; i
++)
5859 max_lookahead_tries
*= dfa_lookahead
;
5862 /* Init max_points. */
5863 more_issue
= issue_rate
- cycle_issued_insns
;
5864 gcc_assert (more_issue
>= 0);
5866 /* The number of the issued insns in the best solution. */
5871 /* Set initial state of the search. */
5872 memcpy (top
->state
, state
, dfa_state_size
);
5873 top
->rest
= dfa_lookahead
;
5875 if (targetm
.sched
.first_cycle_multipass_begin
)
5876 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5878 first_cycle_insn_p
);
5880 /* Count the number of the insns to search among. */
5881 for (all
= i
= 0; i
< n_ready
; i
++)
5885 if (sched_verbose
>= 2)
5887 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5888 debug_ready_list_1 (ready
, ready_try
);
5891 /* I is the index of the insn to try next. */
5896 if (/* If we've reached a dead end or searched enough of what we have
5899 /* or have nothing else to try... */
5901 /* or should not issue more. */
5902 || top
->n
>= more_issue
)
5904 /* ??? (... || i == n_ready). */
5905 gcc_assert (i
<= n_ready
);
5907 /* We should not issue more than issue_rate instructions. */
5908 gcc_assert (top
->n
<= more_issue
);
5910 if (top
== choice_stack
)
5913 if (best
< top
- choice_stack
)
5918 /* Try to find issued privileged insn. */
5919 while (n
&& !ready_try
[--n
])
5923 if (/* If all insns are equally good... */
5925 /* Or a privileged insn will be issued. */
5927 /* Then we have a solution. */
5929 best
= top
- choice_stack
;
5930 /* This is the index of the insn issued first in this
5932 *index
= choice_stack
[1].index
;
5933 if (top
->n
== more_issue
|| best
== all
)
5938 /* Set ready-list index to point to the last insn
5939 ('i++' below will advance it to the next insn). */
5945 if (targetm
.sched
.first_cycle_multipass_backtrack
)
5946 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
5947 ready_try
, n_ready
);
5950 memcpy (state
, top
->state
, dfa_state_size
);
5952 else if (!ready_try
[i
])
5955 if (tries_num
> max_lookahead_tries
)
5957 insn
= ready_element (ready
, i
);
5958 delay
= state_transition (state
, insn
);
5961 if (state_dead_lock_p (state
)
5962 || insn_finishes_cycle_p (insn
))
5963 /* We won't issue any more instructions in the next
5970 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
5973 /* Advance to the next choice_entry. */
5975 /* Initialize it. */
5976 top
->rest
= dfa_lookahead
;
5979 memcpy (top
->state
, state
, dfa_state_size
);
5982 if (targetm
.sched
.first_cycle_multipass_issue
)
5983 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
5993 /* Increase ready-list index. */
5997 if (targetm
.sched
.first_cycle_multipass_end
)
5998 targetm
.sched
.first_cycle_multipass_end (best
!= 0
5999 ? &choice_stack
[1].target_data
6002 /* Restore the original state of the DFA. */
6003 memcpy (state
, choice_stack
->state
, dfa_state_size
);
6008 /* The following function chooses insn from READY and modifies
6009 READY. The following function is used only for first
6010 cycle multipass scheduling.
6012 -1 if cycle should be advanced,
6013 0 if INSN_PTR is set to point to the desirable insn,
6014 1 if choose_ready () should be restarted without advancing the cycle. */
6016 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6017 rtx_insn
**insn_ptr
)
6019 if (dbg_cnt (sched_insn
) == false)
6021 if (nonscheduled_insns_begin
== NULL_RTX
)
6022 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6024 rtx_insn
*insn
= first_nonscheduled_insn ();
6026 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6027 /* INSN is in the ready_list. */
6029 ready_remove_insn (insn
);
6034 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6035 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6039 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6040 || DEBUG_INSN_P (ready_element (ready
, 0)))
6042 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6043 *insn_ptr
= ready_remove_first_dispatch (ready
);
6045 *insn_ptr
= ready_remove_first (ready
);
6051 /* Try to choose the best insn. */
6055 insn
= ready_element (ready
, 0);
6056 if (INSN_CODE (insn
) < 0)
6058 *insn_ptr
= ready_remove_first (ready
);
6062 /* Filter the search space. */
6063 for (i
= 0; i
< ready
->n_ready
; i
++)
6067 insn
= ready_element (ready
, i
);
6069 /* If this insn is recognizable we should have already
6070 recognized it earlier.
6071 ??? Not very clear where this is supposed to be done.
6073 gcc_checking_assert (INSN_CODE (insn
) >= 0
6074 || recog_memoized (insn
) < 0);
6075 if (INSN_CODE (insn
) < 0)
6077 /* Non-recognized insns at position 0 are handled above. */
6083 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6086 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6089 if (ready_try
[i
] < 0)
6090 /* Queue instruction for several cycles.
6091 We need to restart choose_ready as we have changed
6094 change_queue_index (insn
, -ready_try
[i
]);
6098 /* Make sure that we didn't end up with 0'th insn filtered out.
6099 Don't be tempted to make life easier for backends and just
6100 requeue 0'th insn if (ready_try[0] == 0) and restart
6101 choose_ready. Backends should be very considerate about
6102 requeueing instructions -- especially the highest priority
6103 one at position 0. */
6104 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6109 gcc_assert (ready_try
[i
] == 0);
6110 /* INSN made it through the scrutiny of filters! */
6113 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6115 *insn_ptr
= ready_remove_first (ready
);
6116 if (sched_verbose
>= 4)
6117 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6118 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6123 if (sched_verbose
>= 4)
6124 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6125 (*current_sched_info
->print_insn
)
6126 (ready_element (ready
, index
), 0));
6128 *insn_ptr
= ready_remove (ready
, index
);
6134 /* This function is called when we have successfully scheduled a
6135 block. It uses the schedule stored in the scheduled_insns vector
6136 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6137 append the scheduled insns; TAIL is the insn after the scheduled
6138 block. TARGET_BB is the argument passed to schedule_block. */
6141 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6146 last_scheduled_insn
= prev_head
;
6148 scheduled_insns
.iterate (i
, &insn
);
6151 if (control_flow_insn_p (last_scheduled_insn
)
6152 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6154 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6160 x
= next_real_insn (last_scheduled_insn
);
6162 dump_new_block_header (1, *target_bb
, x
, tail
);
6165 last_scheduled_insn
= bb_note (*target_bb
);
6168 if (current_sched_info
->begin_move_insn
)
6169 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6170 move_insn (insn
, last_scheduled_insn
,
6171 current_sched_info
->next_tail
);
6172 if (!DEBUG_INSN_P (insn
))
6173 reemit_notes (insn
);
6174 last_scheduled_insn
= insn
;
6177 scheduled_insns
.truncate (0);
6180 /* Examine all insns on the ready list and queue those which can't be
6181 issued in this cycle. TEMP_STATE is temporary scheduler state we
6182 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6183 have been issued for the current cycle, which means it is valid to
6184 issue an asm statement.
6186 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6187 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6188 we only leave insns which have an INSN_EXACT_TICK. */
6191 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6192 bool shadows_only_p
, bool modulo_epilogue_p
)
6195 bool sched_group_found
= false;
6196 int min_cost_group
= 1;
6201 for (i
= 0; i
< ready
.n_ready
; i
++)
6203 rtx_insn
*insn
= ready_element (&ready
, i
);
6204 if (SCHED_GROUP_P (insn
))
6206 sched_group_found
= true;
6211 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6212 such an insn first and note its cost, then schedule all other insns
6213 for one cycle later. */
6214 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6216 int n
= ready
.n_ready
;
6217 for (i
= 0; i
< n
; i
++)
6219 rtx_insn
*insn
= ready_element (&ready
, i
);
6221 const char *reason
= "resource conflict";
6223 if (DEBUG_INSN_P (insn
))
6226 if (sched_group_found
&& !SCHED_GROUP_P (insn
))
6230 cost
= min_cost_group
;
6231 reason
= "not in sched group";
6233 else if (modulo_epilogue_p
6234 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6236 cost
= max_insn_queue_index
;
6237 reason
= "not an epilogue insn";
6239 else if (shadows_only_p
&& !SHADOW_P (insn
))
6242 reason
= "not a shadow";
6244 else if (recog_memoized (insn
) < 0)
6246 if (!first_cycle_insn_p
6247 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6248 || asm_noperands (PATTERN (insn
)) >= 0))
6252 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6254 if (sched_pressure
== SCHED_PRESSURE_MODEL
6255 && INSN_TICK (insn
) <= clock_var
)
6257 memcpy (temp_state
, curr_state
, dfa_state_size
);
6258 if (state_transition (temp_state
, insn
) >= 0)
6259 INSN_TICK (insn
) = clock_var
+ 1;
6269 struct delay_pair
*delay_entry
;
6271 = delay_htab
->find_with_hash (insn
,
6272 htab_hash_pointer (insn
));
6273 while (delay_entry
&& delay_cost
== 0)
6275 delay_cost
= estimate_shadow_tick (delay_entry
);
6276 if (delay_cost
> max_insn_queue_index
)
6277 delay_cost
= max_insn_queue_index
;
6278 delay_entry
= delay_entry
->next_same_i1
;
6282 memcpy (temp_state
, curr_state
, dfa_state_size
);
6283 cost
= state_transition (temp_state
, insn
);
6288 if (cost
< delay_cost
)
6291 reason
= "shadow tick";
6296 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6297 min_cost_group
= cost
;
6298 ready_remove (&ready
, i
);
6299 /* Normally we'd want to queue INSN for COST cycles. However,
6300 if SCHED_GROUP_P is set, then we must ensure that nothing
6301 else comes between INSN and its predecessor. If there is
6302 some other insn ready to fire on the next cycle, then that
6303 invariant would be broken.
6305 So when SCHED_GROUP_P is set, just queue this insn for a
6307 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6317 /* Called when we detect that the schedule is impossible. We examine the
6318 backtrack queue to find the earliest insn that caused this condition. */
6320 static struct haifa_saved_data
*
6321 verify_shadows (void)
6323 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6324 for (save
= backtrack_queue
; save
; save
= save
->next
)
6327 struct delay_pair
*pair
= save
->delay_pair
;
6328 rtx_insn
*i1
= pair
->i1
;
6330 for (; pair
; pair
= pair
->next_same_i1
)
6332 rtx_insn
*i2
= pair
->i2
;
6334 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6337 t
= INSN_TICK (i1
) + pair_delay (pair
);
6340 if (sched_verbose
>= 2)
6341 fprintf (sched_dump
,
6342 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6344 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6345 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6346 earliest_fail
= save
;
6349 if (QUEUE_INDEX (i2
) >= 0)
6351 int queued_for
= INSN_TICK (i2
);
6355 if (sched_verbose
>= 2)
6356 fprintf (sched_dump
,
6357 ";;\t\tfailed delay requirements for %d/%d"
6358 " (%d->%d), queued too late\n",
6359 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6360 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6361 earliest_fail
= save
;
6368 return earliest_fail
;
6371 /* Print instructions together with useful scheduling information between
6372 HEAD and TAIL (inclusive). */
6374 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6376 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6378 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6379 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6381 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6382 const char *pattern
= (NOTE_P (insn
)
6384 : str_pattern_slim (PATTERN (insn
)));
6386 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6387 INSN_UID (insn
), priority
, pattern
);
6389 if (sched_verbose
>= 4)
6391 if (NOTE_P (insn
) || recog_memoized (insn
) < 0)
6392 fprintf (sched_dump
, "nothing");
6394 print_reservation (sched_dump
, insn
);
6396 fprintf (sched_dump
, "\n");
6400 /* Use forward list scheduling to rearrange insns of block pointed to by
6401 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6405 schedule_block (basic_block
*target_bb
, state_t init_state
)
6408 bool success
= modulo_ii
== 0;
6409 struct sched_block_state ls
;
6410 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6411 int sort_p
, advance
, start_clock_var
;
6413 /* Head/tail info for this block. */
6414 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6415 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6416 rtx_insn
*head
= NEXT_INSN (prev_head
);
6417 rtx_insn
*tail
= PREV_INSN (next_tail
);
6419 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6420 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6421 find_modifiable_mems (head
, tail
);
6423 /* We used to have code to avoid getting parameters moved from hard
6424 argument registers into pseudos.
6426 However, it was removed when it proved to be of marginal benefit
6427 and caused problems because schedule_block and compute_forward_dependences
6428 had different notions of what the "head" insn was. */
6430 gcc_assert (head
!= tail
|| INSN_P (head
));
6432 haifa_recovery_bb_recently_added_p
= false;
6434 backtrack_queue
= NULL
;
6439 dump_new_block_header (0, *target_bb
, head
, tail
);
6441 if (sched_verbose
>= 2)
6443 dump_insn_stream (head
, tail
);
6444 memset (&rank_for_schedule_stats
, 0,
6445 sizeof (rank_for_schedule_stats
));
6449 if (init_state
== NULL
)
6450 state_reset (curr_state
);
6452 memcpy (curr_state
, init_state
, dfa_state_size
);
6454 /* Clear the ready list. */
6455 ready
.first
= ready
.veclen
- 1;
6459 /* It is used for first cycle multipass scheduling. */
6460 temp_state
= alloca (dfa_state_size
);
6462 if (targetm
.sched
.init
)
6463 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6465 /* We start inserting insns after PREV_HEAD. */
6466 last_scheduled_insn
= prev_head
;
6467 last_nondebug_scheduled_insn
= NULL_RTX
;
6468 nonscheduled_insns_begin
= NULL
;
6470 gcc_assert ((NOTE_P (last_scheduled_insn
)
6471 || DEBUG_INSN_P (last_scheduled_insn
))
6472 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6474 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6479 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6480 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6482 /* Start just before the beginning of time. */
6485 /* We need queue and ready lists and clock_var be initialized
6486 in try_ready () (which is called through init_ready_list ()). */
6487 (*current_sched_info
->init_ready_list
) ();
6490 sched_pressure_start_bb (*target_bb
);
6492 /* The algorithm is O(n^2) in the number of ready insns at any given
6493 time in the worst case. Before reload we are more likely to have
6494 big lists so truncate them to a reasonable size. */
6495 if (!reload_completed
6496 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6498 ready_sort (&ready
);
6500 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6501 If there are debug insns, we know they're first. */
6502 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6503 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6506 if (sched_verbose
>= 2)
6508 fprintf (sched_dump
,
6509 ";;\t\tReady list on entry: %d insns\n", ready
.n_ready
);
6510 fprintf (sched_dump
,
6511 ";;\t\t before reload => truncated to %d insns\n", i
);
6514 /* Delay all insns past it for 1 cycle. If debug counter is
6515 activated make an exception for the insn right after
6516 nonscheduled_insns_begin. */
6518 rtx_insn
*skip_insn
;
6520 if (dbg_cnt (sched_insn
) == false)
6521 skip_insn
= first_nonscheduled_insn ();
6525 while (i
< ready
.n_ready
)
6529 insn
= ready_remove (&ready
, i
);
6531 if (insn
!= skip_insn
)
6532 queue_insn (insn
, 1, "list truncated");
6535 ready_add (&ready
, skip_insn
, true);
6539 /* Now we can restore basic block notes and maintain precise cfg. */
6540 restore_bb_notes (*target_bb
);
6542 last_clock_var
= -1;
6546 gcc_assert (scheduled_insns
.length () == 0);
6548 must_backtrack
= false;
6549 modulo_insns_scheduled
= 0;
6551 ls
.modulo_epilogue
= false;
6552 ls
.first_cycle_insn_p
= true;
6554 /* Loop until all the insns in BB are scheduled. */
6555 while ((*current_sched_info
->schedule_more_p
) ())
6557 perform_replacements_new_cycle ();
6560 start_clock_var
= clock_var
;
6564 advance_one_cycle ();
6566 /* Add to the ready list all pending insns that can be issued now.
6567 If there are no ready insns, increment clock until one
6568 is ready and add all pending insns at that point to the ready
6570 queue_to_ready (&ready
);
6572 gcc_assert (ready
.n_ready
);
6574 if (sched_verbose
>= 2)
6576 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6577 debug_ready_list (&ready
);
6579 advance
-= clock_var
- start_clock_var
;
6581 while (advance
> 0);
6583 if (ls
.modulo_epilogue
)
6585 int stage
= clock_var
/ modulo_ii
;
6586 if (stage
> modulo_last_stage
* 2 + 2)
6588 if (sched_verbose
>= 2)
6589 fprintf (sched_dump
,
6590 ";;\t\tmodulo scheduled succeeded at II %d\n",
6596 else if (modulo_ii
> 0)
6598 int stage
= clock_var
/ modulo_ii
;
6599 if (stage
> modulo_max_stages
)
6601 if (sched_verbose
>= 2)
6602 fprintf (sched_dump
,
6603 ";;\t\tfailing schedule due to excessive stages\n");
6606 if (modulo_n_insns
== modulo_insns_scheduled
6607 && stage
> modulo_last_stage
)
6609 if (sched_verbose
>= 2)
6610 fprintf (sched_dump
,
6611 ";;\t\tfound kernel after %d stages, II %d\n",
6613 ls
.modulo_epilogue
= true;
6617 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6618 if (ready
.n_ready
== 0)
6623 ls
.shadows_only_p
= false;
6624 cycle_issued_insns
= 0;
6625 ls
.can_issue_more
= issue_rate
;
6632 if (sort_p
&& ready
.n_ready
> 0)
6634 /* Sort the ready list based on priority. This must be
6635 done every iteration through the loop, as schedule_insn
6636 may have readied additional insns that will not be
6637 sorted correctly. */
6638 ready_sort (&ready
);
6640 if (sched_verbose
>= 2)
6642 fprintf (sched_dump
,
6643 ";;\t\tReady list after ready_sort: ");
6644 debug_ready_list (&ready
);
6648 /* We don't want md sched reorder to even see debug isns, so put
6649 them out right away. */
6650 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6651 && (*current_sched_info
->schedule_more_p
) ())
6653 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6655 rtx_insn
*insn
= ready_remove_first (&ready
);
6656 gcc_assert (DEBUG_INSN_P (insn
));
6657 (*current_sched_info
->begin_schedule_ready
) (insn
);
6658 scheduled_insns
.safe_push (insn
);
6659 last_scheduled_insn
= insn
;
6660 advance
= schedule_insn (insn
);
6661 gcc_assert (advance
== 0);
6662 if (ready
.n_ready
> 0)
6663 ready_sort (&ready
);
6667 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6670 resume_after_backtrack
:
6671 /* Allow the target to reorder the list, typically for
6672 better instruction bundling. */
6674 && (ready
.n_ready
== 0
6675 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6677 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6679 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6680 ready_lastpos (&ready
),
6681 &ready
.n_ready
, clock_var
);
6682 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6684 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6686 ? ready_lastpos (&ready
) : NULL
,
6687 &ready
.n_ready
, clock_var
);
6690 restart_choose_ready
:
6691 if (sched_verbose
>= 2)
6693 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6695 debug_ready_list (&ready
);
6696 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6697 print_curr_reg_pressure ();
6700 if (ready
.n_ready
== 0
6701 && ls
.can_issue_more
6702 && reload_completed
)
6704 /* Allow scheduling insns directly from the queue in case
6705 there's nothing better to do (ready list is empty) but
6706 there are still vacant dispatch slots in the current cycle. */
6707 if (sched_verbose
>= 6)
6708 fprintf (sched_dump
,";;\t\tSecond chance\n");
6709 memcpy (temp_state
, curr_state
, dfa_state_size
);
6710 if (early_queue_to_ready (temp_state
, &ready
))
6711 ready_sort (&ready
);
6714 if (ready
.n_ready
== 0
6715 || !ls
.can_issue_more
6716 || state_dead_lock_p (curr_state
)
6717 || !(*current_sched_info
->schedule_more_p
) ())
6720 /* Select and remove the insn from the ready list. */
6726 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6732 goto restart_choose_ready
;
6734 gcc_assert (insn
!= NULL_RTX
);
6737 insn
= ready_remove_first (&ready
);
6739 if (sched_pressure
!= SCHED_PRESSURE_NONE
6740 && INSN_TICK (insn
) > clock_var
)
6742 ready_add (&ready
, insn
, true);
6747 if (targetm
.sched
.dfa_new_cycle
6748 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6749 insn
, last_clock_var
,
6750 clock_var
, &sort_p
))
6751 /* SORT_P is used by the target to override sorting
6752 of the ready list. This is needed when the target
6753 has modified its internal structures expecting that
6754 the insn will be issued next. As we need the insn
6755 to have the highest priority (so it will be returned by
6756 the ready_remove_first call above), we invoke
6757 ready_add (&ready, insn, true).
6758 But, still, there is one issue: INSN can be later
6759 discarded by scheduler's front end through
6760 current_sched_info->can_schedule_ready_p, hence, won't
6763 ready_add (&ready
, insn
, true);
6769 if (current_sched_info
->can_schedule_ready_p
6770 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6771 /* We normally get here only if we don't want to move
6772 insn from the split block. */
6774 TODO_SPEC (insn
) = DEP_POSTPONED
;
6775 goto restart_choose_ready
;
6780 /* If this insn is the first part of a delay-slot pair, record a
6782 struct delay_pair
*delay_entry
;
6784 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6787 save_backtrack_point (delay_entry
, ls
);
6788 if (sched_verbose
>= 2)
6789 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6793 /* DECISION is made. */
6795 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6797 modulo_insns_scheduled
++;
6798 modulo_last_stage
= clock_var
/ modulo_ii
;
6800 if (TODO_SPEC (insn
) & SPECULATIVE
)
6801 generate_recovery_code (insn
);
6803 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6804 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6806 /* Update counters, etc in the scheduler's front end. */
6807 (*current_sched_info
->begin_schedule_ready
) (insn
);
6808 scheduled_insns
.safe_push (insn
);
6809 gcc_assert (NONDEBUG_INSN_P (insn
));
6810 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6812 if (recog_memoized (insn
) >= 0)
6814 memcpy (temp_state
, curr_state
, dfa_state_size
);
6815 cost
= state_transition (curr_state
, insn
);
6816 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6817 gcc_assert (cost
< 0);
6818 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6819 cycle_issued_insns
++;
6823 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6824 || asm_noperands (PATTERN (insn
)) >= 0);
6826 if (targetm
.sched
.variable_issue
)
6828 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6829 insn
, ls
.can_issue_more
);
6830 /* A naked CLOBBER or USE generates no instruction, so do
6831 not count them against the issue rate. */
6832 else if (GET_CODE (PATTERN (insn
)) != USE
6833 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6834 ls
.can_issue_more
--;
6835 advance
= schedule_insn (insn
);
6837 if (SHADOW_P (insn
))
6838 ls
.shadows_only_p
= true;
6840 /* After issuing an asm insn we should start a new cycle. */
6841 if (advance
== 0 && asm_p
)
6850 ls
.first_cycle_insn_p
= false;
6851 if (ready
.n_ready
> 0)
6852 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6853 ls
.modulo_epilogue
);
6857 if (!must_backtrack
)
6858 for (i
= 0; i
< ready
.n_ready
; i
++)
6860 rtx_insn
*insn
= ready_element (&ready
, i
);
6861 if (INSN_EXACT_TICK (insn
) == clock_var
)
6863 must_backtrack
= true;
6868 if (must_backtrack
&& modulo_ii
> 0)
6870 if (modulo_backtracks_left
== 0)
6872 modulo_backtracks_left
--;
6874 while (must_backtrack
)
6876 struct haifa_saved_data
*failed
;
6877 rtx_insn
*failed_insn
;
6879 must_backtrack
= false;
6880 failed
= verify_shadows ();
6881 gcc_assert (failed
);
6883 failed_insn
= failed
->delay_pair
->i1
;
6884 /* Clear these queues. */
6885 perform_replacements_new_cycle ();
6886 toggle_cancelled_flags (false);
6887 unschedule_insns_until (failed_insn
);
6888 while (failed
!= backtrack_queue
)
6889 free_topmost_backtrack_point (true);
6890 restore_last_backtrack_point (&ls
);
6891 if (sched_verbose
>= 2)
6892 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6893 /* Delay by at least a cycle. This could cause additional
6895 queue_insn (failed_insn
, 1, "backtracked");
6899 if (ready
.n_ready
> 0)
6900 goto resume_after_backtrack
;
6903 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6909 ls
.first_cycle_insn_p
= true;
6911 if (ls
.modulo_epilogue
)
6914 if (!ls
.first_cycle_insn_p
|| advance
)
6915 advance_one_cycle ();
6916 perform_replacements_new_cycle ();
6919 /* Once again, debug insn suckiness: they can be on the ready list
6920 even if they have unresolved dependencies. To make our view
6921 of the world consistent, remove such "ready" insns. */
6922 restart_debug_insn_loop
:
6923 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6927 x
= ready_element (&ready
, i
);
6928 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
6929 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
6931 ready_remove (&ready
, i
);
6932 goto restart_debug_insn_loop
;
6935 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6939 x
= ready_element (&ready
, i
);
6940 resolve_dependencies (x
);
6942 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6944 rtx_insn_list
*link
;
6945 while ((link
= insn_queue
[i
]) != NULL
)
6947 rtx_insn
*x
= link
->insn ();
6948 insn_queue
[i
] = link
->next ();
6949 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6950 free_INSN_LIST_node (link
);
6951 resolve_dependencies (x
);
6957 undo_all_replacements ();
6962 fprintf (sched_dump
, ";;\tReady list (final): ");
6963 debug_ready_list (&ready
);
6966 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
6967 /* Sanity check -- queue must be empty now. Meaningless if region has
6969 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
6970 else if (modulo_ii
== 0)
6972 /* We must maintain QUEUE_INDEX between blocks in region. */
6973 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6977 x
= ready_element (&ready
, i
);
6978 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6979 TODO_SPEC (x
) = HARD_DEP
;
6983 for (i
= 0; i
<= max_insn_queue_index
; i
++)
6985 rtx_insn_list
*link
;
6986 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
6991 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
6992 TODO_SPEC (x
) = HARD_DEP
;
6994 free_INSN_LIST_list (&insn_queue
[i
]);
6998 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
6999 model_end_schedule ();
7003 commit_schedule (prev_head
, tail
, target_bb
);
7005 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7008 last_scheduled_insn
= tail
;
7010 scheduled_insns
.truncate (0);
7012 if (!current_sched_info
->queue_must_finish_empty
7013 || haifa_recovery_bb_recently_added_p
)
7015 /* INSN_TICK (minimum clock tick at which the insn becomes
7016 ready) may be not correct for the insn in the subsequent
7017 blocks of the region. We should use a correct value of
7018 `clock_var' or modify INSN_TICK. It is better to keep
7019 clock_var value equal to 0 at the start of a basic block.
7020 Therefore we modify INSN_TICK here. */
7021 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7024 if (targetm
.sched
.finish
)
7026 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7027 /* Target might have added some instructions to the scheduled block
7028 in its md_finish () hook. These new insns don't have any data
7029 initialized and to identify them we extend h_i_d so that they'll
7031 sched_extend_luids ();
7034 /* Update head/tail boundaries. */
7035 head
= NEXT_INSN (prev_head
);
7036 tail
= last_scheduled_insn
;
7040 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7041 INSN_UID (head
), INSN_UID (tail
));
7043 if (sched_verbose
>= 2)
7045 dump_insn_stream (head
, tail
);
7046 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7050 fprintf (sched_dump
, "\n");
7053 head
= restore_other_notes (head
, NULL
);
7055 current_sched_info
->head
= head
;
7056 current_sched_info
->tail
= tail
;
7058 free_backtrack_queue ();
7063 /* Set_priorities: compute priority of each insn in the block. */
7066 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7070 int sched_max_insns_priority
=
7071 current_sched_info
->sched_max_insns_priority
;
7072 rtx_insn
*prev_head
;
7074 if (head
== tail
&& ! INSN_P (head
))
7079 prev_head
= PREV_INSN (head
);
7080 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7086 (void) priority (insn
);
7088 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7090 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7091 INSN_PRIORITY (insn
));
7094 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7099 /* Set dump and sched_verbose for the desired debugging output. If no
7100 dump-file was specified, but -fsched-verbose=N (any N), print to stderr.
7101 For -fsched-verbose=N, N>=10, print everything to stderr. */
7103 setup_sched_dump (void)
7105 sched_verbose
= sched_verbose_param
;
7106 if (sched_verbose_param
== 0 && dump_file
)
7108 sched_dump
= ((sched_verbose_param
>= 10 || !dump_file
)
7109 ? stderr
: dump_file
);
7112 /* Allocate data for register pressure sensitive scheduling. */
7114 alloc_global_sched_pressure_data (void)
7116 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7118 int i
, max_regno
= max_reg_num ();
7120 if (sched_dump
!= NULL
)
7121 /* We need info about pseudos for rtl dumps about pseudo
7122 classes and costs. */
7123 regstat_init_n_sets_and_refs ();
7124 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7125 sched_regno_pressure_class
7126 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7127 for (i
= 0; i
< max_regno
; i
++)
7128 sched_regno_pressure_class
[i
]
7129 = (i
< FIRST_PSEUDO_REGISTER
7130 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7131 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7132 curr_reg_live
= BITMAP_ALLOC (NULL
);
7133 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7135 saved_reg_live
= BITMAP_ALLOC (NULL
);
7136 region_ref_regs
= BITMAP_ALLOC (NULL
);
7139 /* Calculate number of CALL_USED_REGS in register classes that
7140 we calculate register pressure for. */
7141 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7143 enum reg_class cl
= ira_pressure_classes
[c
];
7145 call_used_regs_num
[cl
] = 0;
7147 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7148 if (call_used_regs
[ira_class_hard_regs
[cl
][i
]])
7149 ++call_used_regs_num
[cl
];
7154 /* Free data for register pressure sensitive scheduling. Also called
7155 from schedule_region when stopping sched-pressure early. */
7157 free_global_sched_pressure_data (void)
7159 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7161 if (regstat_n_sets_and_refs
!= NULL
)
7162 regstat_free_n_sets_and_refs ();
7163 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7165 BITMAP_FREE (region_ref_regs
);
7166 BITMAP_FREE (saved_reg_live
);
7168 BITMAP_FREE (curr_reg_live
);
7169 free (sched_regno_pressure_class
);
7173 /* Initialize some global state for the scheduler. This function works
7174 with the common data shared between all the schedulers. It is called
7175 from the scheduler specific initialization routine. */
7180 /* Disable speculative loads in their presence if cc0 defined. */
7182 flag_schedule_speculative_load
= 0;
7185 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7186 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7188 if (live_range_shrinkage_p
)
7189 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7190 else if (flag_sched_pressure
7191 && !reload_completed
7192 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7193 sched_pressure
= ((enum sched_pressure_algorithm
)
7194 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7196 sched_pressure
= SCHED_PRESSURE_NONE
;
7198 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7199 ira_setup_eliminable_regset ();
7201 /* Initialize SPEC_INFO. */
7202 if (targetm
.sched
.set_sched_flags
)
7204 spec_info
= &spec_info_var
;
7205 targetm
.sched
.set_sched_flags (spec_info
);
7207 if (spec_info
->mask
!= 0)
7209 spec_info
->data_weakness_cutoff
=
7210 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7211 spec_info
->control_weakness_cutoff
=
7212 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7213 * REG_BR_PROB_BASE
) / 100;
7216 /* So we won't read anything accidentally. */
7221 /* So we won't read anything accidentally. */
7224 /* Initialize issue_rate. */
7225 if (targetm
.sched
.issue_rate
)
7226 issue_rate
= targetm
.sched
.issue_rate ();
7230 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7231 /* Don't use max_issue with reg_pressure scheduling. Multipass
7232 scheduling and reg_pressure scheduling undo each other's decisions. */
7233 && sched_pressure
== SCHED_PRESSURE_NONE
)
7234 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7238 /* Set to "0" so that we recalculate. */
7239 max_lookahead_tries
= 0;
7241 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7242 targetm
.sched
.init_dfa_pre_cycle_insn ();
7244 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7245 targetm
.sched
.init_dfa_post_cycle_insn ();
7248 dfa_state_size
= state_size ();
7250 init_alias_analysis ();
7253 df_set_flags (DF_LR_RUN_DCE
);
7254 df_note_add_problem ();
7256 /* More problems needed for interloop dep calculation in SMS. */
7257 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7259 df_rd_add_problem ();
7260 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7265 /* Do not run DCE after reload, as this can kill nops inserted
7267 if (reload_completed
)
7268 df_clear_flags (DF_LR_RUN_DCE
);
7270 regstat_compute_calls_crossed ();
7272 if (targetm
.sched
.init_global
)
7273 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7275 alloc_global_sched_pressure_data ();
7277 curr_state
= xmalloc (dfa_state_size
);
7280 static void haifa_init_only_bb (basic_block
, basic_block
);
7282 /* Initialize data structures specific to the Haifa scheduler. */
7284 haifa_sched_init (void)
7286 setup_sched_dump ();
7289 scheduled_insns
.create (0);
7291 if (spec_info
!= NULL
)
7293 sched_deps_info
->use_deps_list
= 1;
7294 sched_deps_info
->generate_spec_deps
= 1;
7297 /* Initialize luids, dependency caches, target and h_i_d for the
7301 bbs
.create (n_basic_blocks_for_fn (cfun
));
7306 FOR_EACH_BB_FN (bb
, cfun
)
7307 bbs
.quick_push (bb
);
7308 sched_init_luids (bbs
);
7309 sched_deps_init (true);
7310 sched_extend_target ();
7311 haifa_init_h_i_d (bbs
);
7316 sched_init_only_bb
= haifa_init_only_bb
;
7317 sched_split_block
= sched_split_block_1
;
7318 sched_create_empty_bb
= sched_create_empty_bb_1
;
7319 haifa_recovery_bb_ever_added_p
= false;
7321 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7322 before_recovery
= 0;
7328 /* Finish work with the data specific to the Haifa scheduler. */
7330 haifa_sched_finish (void)
7332 sched_create_empty_bb
= NULL
;
7333 sched_split_block
= NULL
;
7334 sched_init_only_bb
= NULL
;
7336 if (spec_info
&& spec_info
->dump
)
7338 char c
= reload_completed
? 'a' : 'b';
7340 fprintf (spec_info
->dump
,
7341 ";; %s:\n", current_function_name ());
7343 fprintf (spec_info
->dump
,
7344 ";; Procedure %cr-begin-data-spec motions == %d\n",
7346 fprintf (spec_info
->dump
,
7347 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7349 fprintf (spec_info
->dump
,
7350 ";; Procedure %cr-begin-control-spec motions == %d\n",
7351 c
, nr_begin_control
);
7352 fprintf (spec_info
->dump
,
7353 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7354 c
, nr_be_in_control
);
7357 scheduled_insns
.release ();
7359 /* Finalize h_i_d, dependency caches, and luids for the whole
7360 function. Target will be finalized in md_global_finish (). */
7361 sched_deps_finish ();
7362 sched_finish_luids ();
7363 current_sched_info
= NULL
;
7367 /* Free global data used during insn scheduling. This function works with
7368 the common data shared between the schedulers. */
7373 haifa_finish_h_i_d ();
7374 free_global_sched_pressure_data ();
7377 if (targetm
.sched
.finish_global
)
7378 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7380 end_alias_analysis ();
7382 regstat_free_calls_crossed ();
7387 /* Free all delay_pair structures that were recorded. */
7389 free_delay_pairs (void)
7393 delay_htab
->empty ();
7394 delay_htab_i2
->empty ();
7398 /* Fix INSN_TICKs of the instructions in the current block as well as
7399 INSN_TICKs of their dependents.
7400 HEAD and TAIL are the begin and the end of the current scheduled block. */
7402 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7404 /* Set of instructions with corrected INSN_TICK. */
7405 bitmap_head processed
;
7406 /* ??? It is doubtful if we should assume that cycle advance happens on
7407 basic block boundaries. Basically insns that are unconditionally ready
7408 on the start of the block are more preferable then those which have
7409 a one cycle dependency over insn from the previous block. */
7410 int next_clock
= clock_var
+ 1;
7412 bitmap_initialize (&processed
, 0);
7414 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7415 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7416 across different blocks. */
7417 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7422 sd_iterator_def sd_it
;
7425 tick
= INSN_TICK (head
);
7426 gcc_assert (tick
>= MIN_TICK
);
7428 /* Fix INSN_TICK of instruction from just scheduled block. */
7429 if (bitmap_set_bit (&processed
, INSN_LUID (head
)))
7433 if (tick
< MIN_TICK
)
7436 INSN_TICK (head
) = tick
;
7439 if (DEBUG_INSN_P (head
))
7442 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7446 next
= DEP_CON (dep
);
7447 tick
= INSN_TICK (next
);
7449 if (tick
!= INVALID_TICK
7450 /* If NEXT has its INSN_TICK calculated, fix it.
7451 If not - it will be properly calculated from
7452 scratch later in fix_tick_ready. */
7453 && bitmap_set_bit (&processed
, INSN_LUID (next
)))
7457 if (tick
< MIN_TICK
)
7460 if (tick
> INTER_TICK (next
))
7461 INTER_TICK (next
) = tick
;
7463 tick
= INTER_TICK (next
);
7465 INSN_TICK (next
) = tick
;
7470 bitmap_clear (&processed
);
7473 /* Check if NEXT is ready to be added to the ready or queue list.
7474 If "yes", add it to the proper list.
7476 -1 - is not ready yet,
7477 0 - added to the ready list,
7478 0 < N - queued for N cycles. */
7480 try_ready (rtx_insn
*next
)
7482 ds_t old_ts
, new_ts
;
7484 old_ts
= TODO_SPEC (next
);
7486 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7487 && (old_ts
== HARD_DEP
7488 || old_ts
== DEP_POSTPONED
7489 || (old_ts
& SPECULATIVE
)
7490 || old_ts
== DEP_CONTROL
));
7492 new_ts
= recompute_todo_spec (next
, false);
7494 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7495 gcc_assert (new_ts
== old_ts
7496 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7497 else if (current_sched_info
->new_ready
)
7498 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7500 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7501 have its original pattern or changed (speculative) one. This is due
7502 to changing ebb in region scheduling.
7503 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7504 has speculative pattern.
7506 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7507 control-speculative NEXT could have been discarded by sched-rgn.c
7508 (the same case as when discarded by can_schedule_ready_p ()). */
7510 if ((new_ts
& SPECULATIVE
)
7511 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7512 need to change anything. */
7513 && new_ts
!= old_ts
)
7518 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7520 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7525 /* It would be nice to change DEP_STATUS of all dependences,
7526 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7527 so we won't reanalyze anything. */
7532 /* We follow the rule, that every speculative insn
7533 has non-null ORIG_PAT. */
7534 if (!ORIG_PAT (next
))
7535 ORIG_PAT (next
) = PATTERN (next
);
7539 if (!ORIG_PAT (next
))
7540 /* If we gonna to overwrite the original pattern of insn,
7542 ORIG_PAT (next
) = PATTERN (next
);
7544 res
= haifa_change_pattern (next
, new_pat
);
7553 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7554 either correct (new_ts & SPECULATIVE),
7555 or we simply don't care (new_ts & HARD_DEP). */
7557 gcc_assert (!ORIG_PAT (next
)
7558 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7560 TODO_SPEC (next
) = new_ts
;
7562 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7564 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7565 control-speculative NEXT could have been discarded by sched-rgn.c
7566 (the same case as when discarded by can_schedule_ready_p ()). */
7567 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7569 change_queue_index (next
, QUEUE_NOWHERE
);
7573 else if (!(new_ts
& BEGIN_SPEC
)
7574 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7575 && !IS_SPECULATION_CHECK_P (next
))
7576 /* We should change pattern of every previously speculative
7577 instruction - and we determine if NEXT was speculative by using
7578 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7579 pat too, so skip them. */
7581 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7582 gcc_assert (success
);
7583 ORIG_PAT (next
) = 0;
7586 if (sched_verbose
>= 2)
7588 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7589 (*current_sched_info
->print_insn
) (next
, 0));
7591 if (spec_info
&& spec_info
->dump
)
7593 if (new_ts
& BEGIN_DATA
)
7594 fprintf (spec_info
->dump
, "; data-spec;");
7595 if (new_ts
& BEGIN_CONTROL
)
7596 fprintf (spec_info
->dump
, "; control-spec;");
7597 if (new_ts
& BE_IN_CONTROL
)
7598 fprintf (spec_info
->dump
, "; in-control-spec;");
7600 if (TODO_SPEC (next
) & DEP_CONTROL
)
7601 fprintf (sched_dump
, " predicated");
7602 fprintf (sched_dump
, "\n");
7605 adjust_priority (next
);
7607 return fix_tick_ready (next
);
7610 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7612 fix_tick_ready (rtx_insn
*next
)
7616 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7619 sd_iterator_def sd_it
;
7622 tick
= INSN_TICK (next
);
7623 /* if tick is not equal to INVALID_TICK, then update
7624 INSN_TICK of NEXT with the most recent resolved dependence
7625 cost. Otherwise, recalculate from scratch. */
7626 full_p
= (tick
== INVALID_TICK
);
7628 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7630 rtx_insn
*pro
= DEP_PRO (dep
);
7633 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7635 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7646 INSN_TICK (next
) = tick
;
7648 delay
= tick
- clock_var
;
7649 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7650 delay
= QUEUE_READY
;
7652 change_queue_index (next
, delay
);
7657 /* Move NEXT to the proper queue list with (DELAY >= 1),
7658 or add it to the ready list (DELAY == QUEUE_READY),
7659 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7661 change_queue_index (rtx_insn
*next
, int delay
)
7663 int i
= QUEUE_INDEX (next
);
7665 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7667 gcc_assert (i
!= QUEUE_SCHEDULED
);
7669 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7670 || (delay
< 0 && delay
== i
))
7671 /* We have nothing to do. */
7674 /* Remove NEXT from wherever it is now. */
7675 if (i
== QUEUE_READY
)
7676 ready_remove_insn (next
);
7678 queue_remove (next
);
7680 /* Add it to the proper place. */
7681 if (delay
== QUEUE_READY
)
7682 ready_add (readyp
, next
, false);
7683 else if (delay
>= 1)
7684 queue_insn (next
, delay
, "change queue index");
7686 if (sched_verbose
>= 2)
7688 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7689 (*current_sched_info
->print_insn
) (next
, 0));
7691 if (delay
== QUEUE_READY
)
7692 fprintf (sched_dump
, " into ready\n");
7693 else if (delay
>= 1)
7694 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7696 fprintf (sched_dump
, " removed from ready or queue lists\n");
7700 static int sched_ready_n_insns
= -1;
7702 /* Initialize per region data structures. */
7704 sched_extend_ready_list (int new_sched_ready_n_insns
)
7708 if (sched_ready_n_insns
== -1)
7709 /* At the first call we need to initialize one more choice_stack
7713 sched_ready_n_insns
= 0;
7714 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7717 i
= sched_ready_n_insns
+ 1;
7719 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7720 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7722 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7724 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7725 sched_ready_n_insns
,
7726 sizeof (*ready_try
));
7728 /* We allocate +1 element to save initial state in the choice_stack[0]
7730 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7731 new_sched_ready_n_insns
+ 1);
7733 for (; i
<= new_sched_ready_n_insns
; i
++)
7735 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7737 if (targetm
.sched
.first_cycle_multipass_init
)
7738 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7742 sched_ready_n_insns
= new_sched_ready_n_insns
;
7745 /* Free per region data structures. */
7747 sched_finish_ready_list (void)
7758 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7760 if (targetm
.sched
.first_cycle_multipass_fini
)
7761 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7764 free (choice_stack
[i
].state
);
7766 free (choice_stack
);
7767 choice_stack
= NULL
;
7769 sched_ready_n_insns
= -1;
7773 haifa_luid_for_non_insn (rtx x
)
7775 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7780 /* Generates recovery code for INSN. */
7782 generate_recovery_code (rtx_insn
*insn
)
7784 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7785 begin_speculative_block (insn
);
7787 /* Here we have insn with no dependencies to
7788 instructions other then CHECK_SPEC ones. */
7790 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7791 add_to_speculative_block (insn
);
7795 Tries to add speculative dependencies of type FS between instructions
7796 in deps_list L and TWIN. */
7798 process_insn_forw_deps_be_in_spec (rtx insn
, rtx_insn
*twin
, ds_t fs
)
7800 sd_iterator_def sd_it
;
7803 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7808 consumer
= DEP_CON (dep
);
7810 ds
= DEP_STATUS (dep
);
7812 if (/* If we want to create speculative dep. */
7814 /* And we can do that because this is a true dep. */
7815 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7817 gcc_assert (!(ds
& BE_IN_SPEC
));
7819 if (/* If this dep can be overcome with 'begin speculation'. */
7821 /* Then we have a choice: keep the dep 'begin speculative'
7822 or transform it into 'be in speculative'. */
7824 if (/* In try_ready we assert that if insn once became ready
7825 it can be removed from the ready (or queue) list only
7826 due to backend decision. Hence we can't let the
7827 probability of the speculative dep to decrease. */
7828 ds_weak (ds
) <= ds_weak (fs
))
7832 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7834 if (/* consumer can 'be in speculative'. */
7835 sched_insn_is_legitimate_for_speculation_p (consumer
,
7837 /* Transform it to be in speculative. */
7842 /* Mark the dep as 'be in speculative'. */
7847 dep_def _new_dep
, *new_dep
= &_new_dep
;
7849 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7850 sd_add_dep (new_dep
, false);
7855 /* Generates recovery code for BEGIN speculative INSN. */
7857 begin_speculative_block (rtx_insn
*insn
)
7859 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7861 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7864 create_check_block_twin (insn
, false);
7866 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7869 static void haifa_init_insn (rtx_insn
*);
7871 /* Generates recovery code for BE_IN speculative INSN. */
7873 add_to_speculative_block (rtx_insn
*insn
)
7876 sd_iterator_def sd_it
;
7878 rtx_insn_list
*twins
= NULL
;
7879 rtx_vec_t priorities_roots
;
7881 ts
= TODO_SPEC (insn
);
7882 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7884 if (ts
& BE_IN_DATA
)
7886 if (ts
& BE_IN_CONTROL
)
7889 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7890 gcc_assert (!TODO_SPEC (insn
));
7892 DONE_SPEC (insn
) |= ts
;
7894 /* First we convert all simple checks to branchy. */
7895 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7896 sd_iterator_cond (&sd_it
, &dep
);)
7898 rtx_insn
*check
= DEP_PRO (dep
);
7900 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7902 create_check_block_twin (check
, true);
7904 /* Restart search. */
7905 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7908 /* Continue search. */
7909 sd_iterator_next (&sd_it
);
7912 priorities_roots
.create (0);
7913 clear_priorities (insn
, &priorities_roots
);
7917 rtx_insn
*check
, *twin
;
7920 /* Get the first backward dependency of INSN. */
7921 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7922 if (!sd_iterator_cond (&sd_it
, &dep
))
7923 /* INSN has no backward dependencies left. */
7926 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
7927 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
7928 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
7930 check
= DEP_PRO (dep
);
7932 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
7933 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
7935 rec
= BLOCK_FOR_INSN (check
);
7937 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
7938 haifa_init_insn (twin
);
7940 sd_copy_back_deps (twin
, insn
, true);
7942 if (sched_verbose
&& spec_info
->dump
)
7943 /* INSN_BB (insn) isn't determined for twin insns yet.
7944 So we can't use current_sched_info->print_insn. */
7945 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
7946 INSN_UID (twin
), rec
->index
);
7948 twins
= alloc_INSN_LIST (twin
, twins
);
7950 /* Add dependences between TWIN and all appropriate
7951 instructions from REC. */
7952 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
7954 rtx_insn
*pro
= DEP_PRO (dep
);
7956 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
7958 /* INSN might have dependencies from the instructions from
7959 several recovery blocks. At this iteration we process those
7960 producers that reside in REC. */
7961 if (BLOCK_FOR_INSN (pro
) == rec
)
7963 dep_def _new_dep
, *new_dep
= &_new_dep
;
7965 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
7966 sd_add_dep (new_dep
, false);
7970 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
7972 /* Remove all dependencies between INSN and insns in REC. */
7973 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7974 sd_iterator_cond (&sd_it
, &dep
);)
7976 rtx_insn
*pro
= DEP_PRO (dep
);
7978 if (BLOCK_FOR_INSN (pro
) == rec
)
7979 sd_delete_dep (sd_it
);
7981 sd_iterator_next (&sd_it
);
7985 /* We couldn't have added the dependencies between INSN and TWINS earlier
7986 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
7990 rtx_insn_list
*next_node
;
7992 twin
= twins
->insn ();
7995 dep_def _new_dep
, *new_dep
= &_new_dep
;
7997 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
7998 sd_add_dep (new_dep
, false);
8001 next_node
= twins
->next ();
8002 free_INSN_LIST_node (twins
);
8006 calc_priorities (priorities_roots
);
8007 priorities_roots
.release ();
8010 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8012 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8014 gcc_assert (new_nmemb
>= old_nmemb
);
8015 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8016 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8021 Find fallthru edge from PRED. */
8023 find_fallthru_edge_from (basic_block pred
)
8028 succ
= pred
->next_bb
;
8029 gcc_assert (succ
->prev_bb
== pred
);
8031 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8033 e
= find_fallthru_edge (pred
->succs
);
8037 gcc_assert (e
->dest
== succ
);
8043 e
= find_fallthru_edge (succ
->preds
);
8047 gcc_assert (e
->src
== pred
);
8055 /* Extend per basic block data structures. */
8057 sched_extend_bb (void)
8059 /* The following is done to keep current_sched_info->next_tail non null. */
8060 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8061 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8062 if (NEXT_INSN (end
) == 0
8065 /* Don't emit a NOTE if it would end up before a BARRIER. */
8066 && !BARRIER_P (NEXT_INSN (end
))))
8068 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8069 /* Make note appear outside BB. */
8070 set_block_for_insn (note
, NULL
);
8071 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8075 /* Init per basic block data structures. */
8077 sched_init_bbs (void)
8082 /* Initialize BEFORE_RECOVERY variable. */
8084 init_before_recovery (basic_block
*before_recovery_ptr
)
8089 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8090 e
= find_fallthru_edge_from (last
);
8094 /* We create two basic blocks:
8095 1. Single instruction block is inserted right after E->SRC
8097 2. Empty block right before EXIT_BLOCK.
8098 Between these two blocks recovery blocks will be emitted. */
8100 basic_block single
, empty
;
8104 /* If the fallthrough edge to exit we've found is from the block we've
8105 created before, don't do anything more. */
8106 if (last
== after_recovery
)
8109 adding_bb_to_current_region_p
= false;
8111 single
= sched_create_empty_bb (last
);
8112 empty
= sched_create_empty_bb (single
);
8114 /* Add new blocks to the root loop. */
8115 if (current_loops
!= NULL
)
8117 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8118 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8121 single
->count
= last
->count
;
8122 empty
->count
= last
->count
;
8123 single
->frequency
= last
->frequency
;
8124 empty
->frequency
= last
->frequency
;
8125 BB_COPY_PARTITION (single
, last
);
8126 BB_COPY_PARTITION (empty
, last
);
8128 redirect_edge_succ (e
, single
);
8129 make_single_succ_edge (single
, empty
, 0);
8130 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8133 label
= block_label (empty
);
8134 x
= emit_jump_insn_after (gen_jump (label
), BB_END (single
));
8135 JUMP_LABEL (x
) = label
;
8136 LABEL_NUSES (label
)++;
8137 haifa_init_insn (x
);
8139 emit_barrier_after (x
);
8141 sched_init_only_bb (empty
, NULL
);
8142 sched_init_only_bb (single
, NULL
);
8145 adding_bb_to_current_region_p
= true;
8146 before_recovery
= single
;
8147 after_recovery
= empty
;
8149 if (before_recovery_ptr
)
8150 *before_recovery_ptr
= before_recovery
;
8152 if (sched_verbose
>= 2 && spec_info
->dump
)
8153 fprintf (spec_info
->dump
,
8154 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8155 last
->index
, single
->index
, empty
->index
);
8158 before_recovery
= last
;
8161 /* Returns new recovery block. */
8163 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8169 haifa_recovery_bb_recently_added_p
= true;
8170 haifa_recovery_bb_ever_added_p
= true;
8172 init_before_recovery (before_recovery_ptr
);
8174 barrier
= get_last_bb_insn (before_recovery
);
8175 gcc_assert (BARRIER_P (barrier
));
8177 label
= emit_label_after (gen_label_rtx (), barrier
);
8179 rec
= create_basic_block (label
, label
, before_recovery
);
8181 /* A recovery block always ends with an unconditional jump. */
8182 emit_barrier_after (BB_END (rec
));
8184 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8185 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8187 if (sched_verbose
&& spec_info
->dump
)
8188 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8194 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8195 and emit necessary jumps. */
8197 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8198 basic_block second_bb
)
8204 /* This is fixing of incoming edge. */
8205 /* ??? Which other flags should be specified? */
8206 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8207 /* Partition type is the same, if it is "unpartitioned". */
8208 edge_flags
= EDGE_CROSSING
;
8212 make_edge (first_bb
, rec
, edge_flags
);
8213 label
= block_label (second_bb
);
8214 jump
= emit_jump_insn_after (gen_jump (label
), BB_END (rec
));
8215 JUMP_LABEL (jump
) = label
;
8216 LABEL_NUSES (label
)++;
8218 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8219 /* Partition type is the same, if it is "unpartitioned". */
8221 /* Rewritten from cfgrtl.c. */
8222 if (flag_reorder_blocks_and_partition
8223 && targetm_common
.have_named_sections
)
8225 /* We don't need the same note for the check because
8226 any_condjump_p (check) == true. */
8227 CROSSING_JUMP_P (jump
) = 1;
8229 edge_flags
= EDGE_CROSSING
;
8234 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8235 if (dom_info_available_p (CDI_DOMINATORS
))
8236 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8239 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8240 INSN is a simple check, that should be converted to branchy one. */
8242 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8245 rtx_insn
*label
, *check
, *twin
;
8248 sd_iterator_def sd_it
;
8250 dep_def _new_dep
, *new_dep
= &_new_dep
;
8253 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8256 todo_spec
= TODO_SPEC (insn
);
8259 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8260 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8262 todo_spec
= CHECK_SPEC (insn
);
8265 todo_spec
&= SPECULATIVE
;
8267 /* Create recovery block. */
8268 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8270 rec
= sched_create_recovery_block (NULL
);
8271 label
= BB_HEAD (rec
);
8275 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8280 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8282 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8284 /* To have mem_reg alive at the beginning of second_bb,
8285 we emit check BEFORE insn, so insn after splitting
8286 insn will be at the beginning of second_bb, which will
8287 provide us with the correct life information. */
8288 check
= emit_jump_insn_before (check_pat
, insn
);
8289 JUMP_LABEL (check
) = label
;
8290 LABEL_NUSES (label
)++;
8293 check
= emit_insn_before (check_pat
, insn
);
8295 /* Extend data structures. */
8296 haifa_init_insn (check
);
8298 /* CHECK is being added to current region. Extend ready list. */
8299 gcc_assert (sched_ready_n_insns
!= -1);
8300 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8302 if (current_sched_info
->add_remove_insn
)
8303 current_sched_info
->add_remove_insn (insn
, 0);
8305 RECOVERY_BLOCK (check
) = rec
;
8307 if (sched_verbose
&& spec_info
->dump
)
8308 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8309 (*current_sched_info
->print_insn
) (check
, 0));
8311 gcc_assert (ORIG_PAT (insn
));
8313 /* Initialize TWIN (twin is a duplicate of original instruction
8314 in the recovery block). */
8315 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8317 sd_iterator_def sd_it
;
8320 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8321 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8323 struct _dep _dep2
, *dep2
= &_dep2
;
8325 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8327 sd_add_dep (dep2
, true);
8330 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8331 haifa_init_insn (twin
);
8333 if (sched_verbose
&& spec_info
->dump
)
8334 /* INSN_BB (insn) isn't determined for twin insns yet.
8335 So we can't use current_sched_info->print_insn. */
8336 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8337 INSN_UID (twin
), rec
->index
);
8341 ORIG_PAT (check
) = ORIG_PAT (insn
);
8342 HAS_INTERNAL_DEP (check
) = 1;
8344 /* ??? We probably should change all OUTPUT dependencies to
8348 /* Copy all resolved back dependencies of INSN to TWIN. This will
8349 provide correct value for INSN_TICK (TWIN). */
8350 sd_copy_back_deps (twin
, insn
, true);
8352 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8353 /* In case of branchy check, fix CFG. */
8355 basic_block first_bb
, second_bb
;
8358 first_bb
= BLOCK_FOR_INSN (check
);
8359 second_bb
= sched_split_block (first_bb
, check
);
8361 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8363 sched_init_only_bb (second_bb
, first_bb
);
8364 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8366 jump
= BB_END (rec
);
8367 haifa_init_insn (jump
);
8370 /* Move backward dependences from INSN to CHECK and
8371 move forward dependences from INSN to TWIN. */
8373 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8374 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8376 rtx_insn
*pro
= DEP_PRO (dep
);
8379 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8380 check --TRUE--> producer ??? or ANTI ???
8381 twin --TRUE--> producer
8382 twin --ANTI--> check
8384 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8385 check --ANTI--> producer
8386 twin --ANTI--> producer
8387 twin --ANTI--> check
8389 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8390 check ~~TRUE~~> producer
8391 twin ~~TRUE~~> producer
8392 twin --ANTI--> check */
8394 ds
= DEP_STATUS (dep
);
8396 if (ds
& BEGIN_SPEC
)
8398 gcc_assert (!mutate_p
);
8402 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8403 sd_add_dep (new_dep
, false);
8405 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8407 DEP_CON (new_dep
) = twin
;
8408 sd_add_dep (new_dep
, false);
8412 /* Second, remove backward dependencies of INSN. */
8413 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8414 sd_iterator_cond (&sd_it
, &dep
);)
8416 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8418 /* We can delete this dep because we overcome it with
8419 BEGIN_SPECULATION. */
8420 sd_delete_dep (sd_it
);
8422 sd_iterator_next (&sd_it
);
8425 /* Future Speculations. Determine what BE_IN speculations will be like. */
8428 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8431 gcc_assert (!DONE_SPEC (insn
));
8435 ds_t ts
= TODO_SPEC (insn
);
8437 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8438 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8440 /* Luckiness of future speculations solely depends upon initial
8441 BEGIN speculation. */
8442 if (ts
& BEGIN_DATA
)
8443 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8444 if (ts
& BEGIN_CONTROL
)
8445 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8446 get_dep_weak (ts
, BEGIN_CONTROL
));
8449 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8451 /* Future speculations: call the helper. */
8452 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8454 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8456 /* Which types of dependencies should we use here is,
8457 generally, machine-dependent question... But, for now,
8462 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8463 sd_add_dep (new_dep
, false);
8465 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8466 sd_add_dep (new_dep
, false);
8470 if (spec_info
->dump
)
8471 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8472 (*current_sched_info
->print_insn
) (insn
, 0));
8474 /* Remove all dependencies of the INSN. */
8476 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8478 | SD_LIST_RES_BACK
));
8479 while (sd_iterator_cond (&sd_it
, &dep
))
8480 sd_delete_dep (sd_it
);
8483 /* If former check (INSN) already was moved to the ready (or queue)
8484 list, add new check (CHECK) there too. */
8485 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8488 /* Remove old check from instruction stream and free its
8490 sched_remove_insn (insn
);
8493 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8494 sd_add_dep (new_dep
, false);
8498 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8499 sd_add_dep (new_dep
, false);
8503 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8504 because it'll be done later in add_to_speculative_block. */
8506 rtx_vec_t priorities_roots
= rtx_vec_t ();
8508 clear_priorities (twin
, &priorities_roots
);
8509 calc_priorities (priorities_roots
);
8510 priorities_roots
.release ();
8514 /* Removes dependency between instructions in the recovery block REC
8515 and usual region instructions. It keeps inner dependences so it
8516 won't be necessary to recompute them. */
8518 fix_recovery_deps (basic_block rec
)
8520 rtx_insn
*note
, *insn
, *jump
;
8521 rtx_insn_list
*ready_list
= 0;
8522 bitmap_head in_ready
;
8523 rtx_insn_list
*link
;
8525 bitmap_initialize (&in_ready
, 0);
8527 /* NOTE - a basic block note. */
8528 note
= NEXT_INSN (BB_HEAD (rec
));
8529 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8530 insn
= BB_END (rec
);
8531 gcc_assert (JUMP_P (insn
));
8532 insn
= PREV_INSN (insn
);
8536 sd_iterator_def sd_it
;
8539 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8540 sd_iterator_cond (&sd_it
, &dep
);)
8542 rtx_insn
*consumer
= DEP_CON (dep
);
8544 if (BLOCK_FOR_INSN (consumer
) != rec
)
8546 sd_delete_dep (sd_it
);
8548 if (bitmap_set_bit (&in_ready
, INSN_LUID (consumer
)))
8549 ready_list
= alloc_INSN_LIST (consumer
, ready_list
);
8553 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8555 sd_iterator_next (&sd_it
);
8559 insn
= PREV_INSN (insn
);
8561 while (insn
!= note
);
8563 bitmap_clear (&in_ready
);
8565 /* Try to add instructions to the ready or queue list. */
8566 for (link
= ready_list
; link
; link
= link
->next ())
8567 try_ready (link
->insn ());
8568 free_INSN_LIST_list (&ready_list
);
8570 /* Fixing jump's dependences. */
8571 insn
= BB_HEAD (rec
);
8572 jump
= BB_END (rec
);
8574 gcc_assert (LABEL_P (insn
));
8575 insn
= NEXT_INSN (insn
);
8577 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8578 add_jump_dependencies (insn
, jump
);
8581 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8582 instruction data. */
8584 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8588 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8592 update_insn_after_change (insn
);
8596 /* -1 - can't speculate,
8597 0 - for speculation with REQUEST mode it is OK to use
8598 current instruction pattern,
8599 1 - need to change pattern for *NEW_PAT to be speculative. */
8601 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8603 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8604 && (request
& SPECULATIVE
)
8605 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8607 if ((request
& spec_info
->mask
) != request
)
8610 if (request
& BE_IN_SPEC
8611 && !(request
& BEGIN_SPEC
))
8614 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8618 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8620 gcc_assert (sched_deps_info
->generate_spec_deps
8621 && !IS_SPECULATION_CHECK_P (insn
));
8623 if (HAS_INTERNAL_DEP (insn
)
8624 || SCHED_GROUP_P (insn
))
8627 return sched_speculate_insn (insn
, request
, new_pat
);
8630 /* Print some information about block BB, which starts with HEAD and
8631 ends with TAIL, before scheduling it.
8632 I is zero, if scheduler is about to start with the fresh ebb. */
8634 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8637 fprintf (sched_dump
,
8638 ";; ======================================================\n");
8640 fprintf (sched_dump
,
8641 ";; =====================ADVANCING TO=====================\n");
8642 fprintf (sched_dump
,
8643 ";; -- basic block %d from %d to %d -- %s reload\n",
8644 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8645 (reload_completed
? "after" : "before"));
8646 fprintf (sched_dump
,
8647 ";; ======================================================\n");
8648 fprintf (sched_dump
, "\n");
8651 /* Unlink basic block notes and labels and saves them, so they
8652 can be easily restored. We unlink basic block notes in EBB to
8653 provide back-compatibility with the previous code, as target backends
8654 assume, that there'll be only instructions between
8655 current_sched_info->{head and tail}. We restore these notes as soon
8657 FIRST (LAST) is the first (last) basic block in the ebb.
8658 NB: In usual case (FIRST == LAST) nothing is really done. */
8660 unlink_bb_notes (basic_block first
, basic_block last
)
8662 /* We DON'T unlink basic block notes of the first block in the ebb. */
8666 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8668 /* Make a sentinel. */
8669 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8670 bb_header
[last
->next_bb
->index
] = 0;
8672 first
= first
->next_bb
;
8675 rtx_insn
*prev
, *label
, *note
, *next
;
8677 label
= BB_HEAD (last
);
8678 if (LABEL_P (label
))
8679 note
= NEXT_INSN (label
);
8682 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8684 prev
= PREV_INSN (label
);
8685 next
= NEXT_INSN (note
);
8686 gcc_assert (prev
&& next
);
8688 SET_NEXT_INSN (prev
) = next
;
8689 SET_PREV_INSN (next
) = prev
;
8691 bb_header
[last
->index
] = label
;
8696 last
= last
->prev_bb
;
8701 /* Restore basic block notes.
8702 FIRST is the first basic block in the ebb. */
8704 restore_bb_notes (basic_block first
)
8709 /* We DON'T unlink basic block notes of the first block in the ebb. */
8710 first
= first
->next_bb
;
8711 /* Remember: FIRST is actually a second basic block in the ebb. */
8713 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8714 && bb_header
[first
->index
])
8716 rtx_insn
*prev
, *label
, *note
, *next
;
8718 label
= bb_header
[first
->index
];
8719 prev
= PREV_INSN (label
);
8720 next
= NEXT_INSN (prev
);
8722 if (LABEL_P (label
))
8723 note
= NEXT_INSN (label
);
8726 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8728 bb_header
[first
->index
] = 0;
8730 SET_NEXT_INSN (prev
) = label
;
8731 SET_NEXT_INSN (note
) = next
;
8732 SET_PREV_INSN (next
) = note
;
8734 first
= first
->next_bb
;
8742 Fix CFG after both in- and inter-block movement of
8743 control_flow_insn_p JUMP. */
8745 fix_jump_move (rtx_insn
*jump
)
8747 basic_block bb
, jump_bb
, jump_bb_next
;
8749 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8750 jump_bb
= BLOCK_FOR_INSN (jump
);
8751 jump_bb_next
= jump_bb
->next_bb
;
8753 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8754 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8756 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8757 /* if jump_bb_next is not empty. */
8758 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8760 if (BB_END (bb
) != PREV_INSN (jump
))
8761 /* Then there are instruction after jump that should be placed
8763 BB_END (jump_bb_next
) = BB_END (bb
);
8765 /* Otherwise jump_bb_next is empty. */
8766 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8768 /* To make assertion in move_insn happy. */
8769 BB_END (bb
) = PREV_INSN (jump
);
8771 update_bb_for_insn (jump_bb_next
);
8774 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8776 move_block_after_check (rtx_insn
*jump
)
8778 basic_block bb
, jump_bb
, jump_bb_next
;
8779 vec
<edge
, va_gc
> *t
;
8781 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8782 jump_bb
= BLOCK_FOR_INSN (jump
);
8783 jump_bb_next
= jump_bb
->next_bb
;
8785 update_bb_for_insn (jump_bb
);
8787 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8788 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8790 unlink_block (jump_bb_next
);
8791 link_block (jump_bb_next
, bb
);
8795 move_succs (&(jump_bb
->succs
), bb
);
8796 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8797 move_succs (&t
, jump_bb_next
);
8799 df_mark_solutions_dirty ();
8801 common_sched_info
->fix_recovery_cfg
8802 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8805 /* Helper function for move_block_after_check.
8806 This functions attaches edge vector pointed to by SUCCSP to
8809 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8814 gcc_assert (to
->succs
== 0);
8816 to
->succs
= *succsp
;
8818 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8824 /* Remove INSN from the instruction stream.
8825 INSN should have any dependencies. */
8827 sched_remove_insn (rtx_insn
*insn
)
8829 sd_finish_insn (insn
);
8831 change_queue_index (insn
, QUEUE_NOWHERE
);
8832 current_sched_info
->add_remove_insn (insn
, 1);
8836 /* Clear priorities of all instructions, that are forward dependent on INSN.
8837 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8838 be invoked to initialize all cleared priorities. */
8840 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8842 sd_iterator_def sd_it
;
8844 bool insn_is_root_p
= true;
8846 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8848 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8850 rtx_insn
*pro
= DEP_PRO (dep
);
8852 if (INSN_PRIORITY_STATUS (pro
) >= 0
8853 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8855 /* If DEP doesn't contribute to priority then INSN itself should
8856 be added to priority roots. */
8857 if (contributes_to_priority_p (dep
))
8858 insn_is_root_p
= false;
8860 INSN_PRIORITY_STATUS (pro
) = -1;
8861 clear_priorities (pro
, roots_ptr
);
8866 roots_ptr
->safe_push (insn
);
8869 /* Recompute priorities of instructions, whose priorities might have been
8870 changed. ROOTS is a vector of instructions whose priority computation will
8871 trigger initialization of all cleared priorities. */
8873 calc_priorities (rtx_vec_t roots
)
8878 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8883 /* Add dependences between JUMP and other instructions in the recovery
8884 block. INSN is the first insn the recovery block. */
8886 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8890 insn
= NEXT_INSN (insn
);
8894 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8896 dep_def _new_dep
, *new_dep
= &_new_dep
;
8898 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8899 sd_add_dep (new_dep
, false);
8904 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8907 /* Extend data structures for logical insn UID. */
8909 sched_extend_luids (void)
8911 int new_luids_max_uid
= get_max_uid () + 1;
8913 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
8916 /* Initialize LUID for INSN. */
8918 sched_init_insn_luid (rtx_insn
*insn
)
8920 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8925 luid
= sched_max_luid
;
8926 sched_max_luid
+= i
;
8931 SET_INSN_LUID (insn
, luid
);
8934 /* Initialize luids for BBS.
8935 The hook common_sched_info->luid_for_non_insn () is used to determine
8936 if notes, labels, etc. need luids. */
8938 sched_init_luids (bb_vec_t bbs
)
8943 sched_extend_luids ();
8944 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
8948 FOR_BB_INSNS (bb
, insn
)
8949 sched_init_insn_luid (insn
);
8955 sched_finish_luids (void)
8957 sched_luids
.release ();
8961 /* Return logical uid of INSN. Helpful while debugging. */
8963 insn_luid (rtx_insn
*insn
)
8965 return INSN_LUID (insn
);
8968 /* Extend per insn data in the target. */
8970 sched_extend_target (void)
8972 if (targetm
.sched
.h_i_d_extended
)
8973 targetm
.sched
.h_i_d_extended ();
8976 /* Extend global scheduler structures (those, that live across calls to
8977 schedule_block) to include information about just emitted INSN. */
8981 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
8983 && ! h_i_d
.space (reserve
))
8985 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
8986 sched_extend_target ();
8990 /* Initialize h_i_d entry of the INSN with default values.
8991 Values, that are not explicitly initialized here, hold zero. */
8993 init_h_i_d (rtx_insn
*insn
)
8995 if (INSN_LUID (insn
) > 0)
8997 INSN_COST (insn
) = -1;
8998 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
8999 INSN_TICK (insn
) = INVALID_TICK
;
9000 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
9001 INTER_TICK (insn
) = INVALID_TICK
;
9002 TODO_SPEC (insn
) = HARD_DEP
;
9003 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
9004 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9005 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
9006 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9010 /* Initialize haifa_insn_data for BBS. */
9012 haifa_init_h_i_d (bb_vec_t bbs
)
9018 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9022 FOR_BB_INSNS (bb
, insn
)
9027 /* Finalize haifa_insn_data. */
9029 haifa_finish_h_i_d (void)
9032 haifa_insn_data_t data
;
9033 struct reg_use_data
*use
, *next
;
9035 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9037 free (data
->max_reg_pressure
);
9038 free (data
->reg_pressure
);
9039 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next
)
9041 next
= use
->next_insn_use
;
9048 /* Init data for the new insn INSN. */
9050 haifa_init_insn (rtx_insn
*insn
)
9052 gcc_assert (insn
!= NULL
);
9054 sched_extend_luids ();
9055 sched_init_insn_luid (insn
);
9056 sched_extend_target ();
9057 sched_deps_init (false);
9061 if (adding_bb_to_current_region_p
)
9063 sd_init_insn (insn
);
9065 /* Extend dependency caches by one element. */
9066 extend_dependency_caches (1, false);
9068 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9069 init_insn_reg_pressure_info (insn
);
9072 /* Init data for the new basic block BB which comes after AFTER. */
9074 haifa_init_only_bb (basic_block bb
, basic_block after
)
9076 gcc_assert (bb
!= NULL
);
9080 if (common_sched_info
->add_block
)
9081 /* This changes only data structures of the front-end. */
9082 common_sched_info
->add_block (bb
, after
);
9085 /* A generic version of sched_split_block (). */
9087 sched_split_block_1 (basic_block first_bb
, rtx after
)
9091 e
= split_block (first_bb
, after
);
9092 gcc_assert (e
->src
== first_bb
);
9094 /* sched_split_block emits note if *check == BB_END. Probably it
9095 is better to rip that note off. */
9100 /* A generic version of sched_create_empty_bb (). */
9102 sched_create_empty_bb_1 (basic_block after
)
9104 return create_empty_bb (after
);
9107 /* Insert PAT as an INSN into the schedule and update the necessary data
9108 structures to account for it. */
9110 sched_emit_insn (rtx pat
)
9112 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9113 haifa_init_insn (insn
);
9115 if (current_sched_info
->add_remove_insn
)
9116 current_sched_info
->add_remove_insn (insn
, 0);
9118 (*current_sched_info
->begin_schedule_ready
) (insn
);
9119 scheduled_insns
.safe_push (insn
);
9121 last_scheduled_insn
= insn
;
9125 /* This function returns a candidate satisfying dispatch constraints from
9129 ready_remove_first_dispatch (struct ready_list
*ready
)
9132 rtx_insn
*insn
= ready_element (ready
, 0);
9134 if (ready
->n_ready
== 1
9136 || INSN_CODE (insn
) < 0
9137 || !active_insn_p (insn
)
9138 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9139 return ready_remove_first (ready
);
9141 for (i
= 1; i
< ready
->n_ready
; i
++)
9143 insn
= ready_element (ready
, i
);
9146 || INSN_CODE (insn
) < 0
9147 || !active_insn_p (insn
))
9150 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9152 /* Return ith element of ready. */
9153 insn
= ready_remove (ready
, i
);
9158 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9159 return ready_remove_first (ready
);
9161 for (i
= 1; i
< ready
->n_ready
; i
++)
9163 insn
= ready_element (ready
, i
);
9166 || INSN_CODE (insn
) < 0
9167 || !active_insn_p (insn
))
9170 /* Return i-th element of ready. */
9171 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9172 return ready_remove (ready
, i
);
9175 return ready_remove_first (ready
);
9178 /* Get number of ready insn in the ready list. */
9181 number_in_ready (void)
9183 return ready
.n_ready
;
9186 /* Get number of ready's in the ready list. */
9189 get_ready_element (int i
)
9191 return ready_element (&ready
, i
);
9194 #endif /* INSN_SCHEDULING */