1 /* Instruction scheduling pass.
2 Copyright (C) 1992-2019 Free Software Foundation, Inc.
3 Contributed by Michael Tiemann (tiemann@cygnus.com) Enhanced by,
4 and currently maintained by, Jim Wilson (wilson@cygnus.com)
6 This file is part of GCC.
8 GCC is free software; you can redistribute it and/or modify it under
9 the terms of the GNU General Public License as published by the Free
10 Software Foundation; either version 3, or (at your option) any later
13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
18 You should have received a copy of the GNU General Public License
19 along with GCC; see the file COPYING3. If not see
20 <http://www.gnu.org/licenses/>. */
22 /* Instruction scheduling pass. This file, along with sched-deps.c,
23 contains the generic parts. The actual entry point for
24 the normal instruction scheduling pass is found in sched-rgn.c.
26 We compute insn priorities based on data dependencies. Flow
27 analysis only creates a fraction of the data-dependencies we must
28 observe: namely, only those dependencies which the combiner can be
29 expected to use. For this pass, we must therefore create the
30 remaining dependencies we need to observe: register dependencies,
31 memory dependencies, dependencies to keep function calls in order,
32 and the dependence between a conditional branch and the setting of
33 condition codes are all dealt with here.
35 The scheduler first traverses the data flow graph, starting with
36 the last instruction, and proceeding to the first, assigning values
37 to insn_priority as it goes. This sorts the instructions
38 topologically by data dependence.
40 Once priorities have been established, we order the insns using
41 list scheduling. This works as follows: starting with a list of
42 all the ready insns, and sorted according to priority number, we
43 schedule the insn from the end of the list by placing its
44 predecessors in the list according to their priority order. We
45 consider this insn scheduled by setting the pointer to the "end" of
46 the list to point to the previous insn. When an insn has no
47 predecessors, we either queue it until sufficient time has elapsed
48 or add it to the ready list. As the instructions are scheduled or
49 when stalls are introduced, the queue advances and dumps insns into
50 the ready list. When all insns down to the lowest priority have
51 been scheduled, the critical path of the basic block has been made
52 as short as possible. The remaining insns are then scheduled in
55 The following list shows the order in which we want to break ties
56 among insns in the ready list:
58 1. choose insn with the longest path to end of bb, ties
60 2. choose insn with least contribution to register pressure,
62 3. prefer in-block upon interblock motion, ties broken by
63 4. prefer useful upon speculative motion, ties broken by
64 5. choose insn with largest control flow probability, ties
66 6. choose insn with the least dependences upon the previously
67 scheduled insn, or finally
68 7 choose the insn which has the most insns dependent on it.
69 8. choose insn with lowest UID.
71 Memory references complicate matters. Only if we can be certain
72 that memory references are not part of the data dependency graph
73 (via true, anti, or output dependence), can we move operations past
74 memory references. To first approximation, reads can be done
75 independently, while writes introduce dependencies. Better
76 approximations will yield fewer dependencies.
78 Before reload, an extended analysis of interblock data dependences
79 is required for interblock scheduling. This is performed in
80 compute_block_dependences ().
82 Dependencies set up by memory references are treated in exactly the
83 same way as other dependencies, by using insn backward dependences
84 INSN_BACK_DEPS. INSN_BACK_DEPS are translated into forward dependences
85 INSN_FORW_DEPS for the purpose of forward list scheduling.
87 Having optimized the critical path, we may have also unduly
88 extended the lifetimes of some registers. If an operation requires
89 that constants be loaded into registers, it is certainly desirable
90 to load those constants as early as necessary, but no earlier.
91 I.e., it will not do to load up a bunch of registers at the
92 beginning of a basic block only to use them at the end, if they
93 could be loaded later, since this may result in excessive register
96 Note that since branches are never in basic blocks, but only end
97 basic blocks, this pass will not move branches. But that is ok,
98 since we can use GNU's delayed branch scheduling pass to take care
101 Also note that no further optimizations based on algebraic
102 identities are performed, so this pass would be a good one to
103 perform instruction splitting, such as breaking up a multiply
104 instruction into shifts and adds where that is profitable.
106 Given the memory aliasing analysis that this pass should perform,
107 it should be possible to remove redundant stores to memory, and to
108 load values from registers instead of hitting memory.
110 Before reload, speculative insns are moved only if a 'proof' exists
111 that no exception will be caused by this, and if no live registers
112 exist that inhibit the motion (live registers constraints are not
113 represented by data dependence edges).
115 This pass must update information that subsequent passes expect to
116 be correct. Namely: reg_n_refs, reg_n_sets, reg_n_deaths,
117 reg_n_calls_crossed, and reg_live_length. Also, BB_HEAD, BB_END.
119 The information in the line number notes is carefully retained by
120 this pass. Notes that refer to the starting and ending of
121 exception regions are also carefully retained by this pass. All
122 other NOTE insns are grouped in their same relative order at the
123 beginning of basic blocks and regions that have been scheduled. */
127 #include "coretypes.h"
131 #include "cfghooks.h"
133 #include "memmodel.h"
135 #include "insn-config.h"
139 #include "insn-attr.h"
141 #include "cfgbuild.h"
142 #include "sched-int.h"
143 #include "common/common-target.h"
147 #include "dumpfile.h"
148 #include "print-rtl.h"
149 #include "function-abi.h"
151 #ifdef INSN_SCHEDULING
153 /* True if we do register pressure relief through live-range
155 static bool live_range_shrinkage_p
;
157 /* Switch on live range shrinkage. */
159 initialize_live_range_shrinkage (void)
161 live_range_shrinkage_p
= true;
164 /* Switch off live range shrinkage. */
166 finish_live_range_shrinkage (void)
168 live_range_shrinkage_p
= false;
171 /* issue_rate is the number of insns that can be scheduled in the same
172 machine cycle. It can be defined in the config/mach/mach.h file,
173 otherwise we set it to 1. */
177 /* This can be set to true by a backend if the scheduler should not
178 enable a DCE pass. */
181 /* The current initiation interval used when modulo scheduling. */
182 static int modulo_ii
;
184 /* The maximum number of stages we are prepared to handle. */
185 static int modulo_max_stages
;
187 /* The number of insns that exist in each iteration of the loop. We use this
188 to detect when we've scheduled all insns from the first iteration. */
189 static int modulo_n_insns
;
191 /* The current count of insns in the first iteration of the loop that have
192 already been scheduled. */
193 static int modulo_insns_scheduled
;
195 /* The maximum uid of insns from the first iteration of the loop. */
196 static int modulo_iter0_max_uid
;
198 /* The number of times we should attempt to backtrack when modulo scheduling.
199 Decreased each time we have to backtrack. */
200 static int modulo_backtracks_left
;
202 /* The stage in which the last insn from the original loop was
204 static int modulo_last_stage
;
206 /* sched-verbose controls the amount of debugging output the
207 scheduler prints. It is controlled by -fsched-verbose=N:
208 N=0: no debugging output.
210 N=2: bb's probabilities, detailed ready list info, unit/insn info.
211 N=3: rtl at abort point, control-flow, regions info.
212 N=5: dependences info. */
213 int sched_verbose
= 0;
215 /* Debugging file. All printouts are sent to dump. */
216 FILE *sched_dump
= 0;
218 /* This is a placeholder for the scheduler parameters common
219 to all schedulers. */
220 struct common_sched_info_def
*common_sched_info
;
222 #define INSN_TICK(INSN) (HID (INSN)->tick)
223 #define INSN_EXACT_TICK(INSN) (HID (INSN)->exact_tick)
224 #define INSN_TICK_ESTIMATE(INSN) (HID (INSN)->tick_estimate)
225 #define INTER_TICK(INSN) (HID (INSN)->inter_tick)
226 #define FEEDS_BACKTRACK_INSN(INSN) (HID (INSN)->feeds_backtrack_insn)
227 #define SHADOW_P(INSN) (HID (INSN)->shadow_p)
228 #define MUST_RECOMPUTE_SPEC_P(INSN) (HID (INSN)->must_recompute_spec)
229 /* Cached cost of the instruction. Use insn_sched_cost to get cost of the
230 insn. -1 here means that the field is not initialized. */
231 #define INSN_COST(INSN) (HID (INSN)->cost)
233 /* If INSN_TICK of an instruction is equal to INVALID_TICK,
234 then it should be recalculated from scratch. */
235 #define INVALID_TICK (-(max_insn_queue_index + 1))
236 /* The minimal value of the INSN_TICK of an instruction. */
237 #define MIN_TICK (-max_insn_queue_index)
239 /* Original order of insns in the ready list.
240 Used to keep order of normal insns while separating DEBUG_INSNs. */
241 #define INSN_RFS_DEBUG_ORIG_ORDER(INSN) (HID (INSN)->rfs_debug_orig_order)
243 /* The deciding reason for INSN's place in the ready list. */
244 #define INSN_LAST_RFS_WIN(INSN) (HID (INSN)->last_rfs_win)
246 /* List of important notes we must keep around. This is a pointer to the
247 last element in the list. */
250 static struct spec_info_def spec_info_var
;
251 /* Description of the speculative part of the scheduling.
252 If NULL - no speculation. */
253 spec_info_t spec_info
= NULL
;
255 /* True, if recovery block was added during scheduling of current block.
256 Used to determine, if we need to fix INSN_TICKs. */
257 static bool haifa_recovery_bb_recently_added_p
;
259 /* True, if recovery block was added during this scheduling pass.
260 Used to determine if we should have empty memory pools of dependencies
261 after finishing current region. */
262 bool haifa_recovery_bb_ever_added_p
;
264 /* Counters of different types of speculative instructions. */
265 static int nr_begin_data
, nr_be_in_data
, nr_begin_control
, nr_be_in_control
;
267 /* Array used in {unlink, restore}_bb_notes. */
268 static rtx_insn
**bb_header
= 0;
270 /* Basic block after which recovery blocks will be created. */
271 static basic_block before_recovery
;
273 /* Basic block just before the EXIT_BLOCK and after recovery, if we have
275 basic_block after_recovery
;
277 /* FALSE if we add bb to another region, so we don't need to initialize it. */
278 bool adding_bb_to_current_region_p
= true;
282 /* An instruction is ready to be scheduled when all insns preceding it
283 have already been scheduled. It is important to ensure that all
284 insns which use its result will not be executed until its result
285 has been computed. An insn is maintained in one of four structures:
287 (P) the "Pending" set of insns which cannot be scheduled until
288 their dependencies have been satisfied.
289 (Q) the "Queued" set of insns that can be scheduled when sufficient
291 (R) the "Ready" list of unscheduled, uncommitted insns.
292 (S) the "Scheduled" list of insns.
294 Initially, all insns are either "Pending" or "Ready" depending on
295 whether their dependencies are satisfied.
297 Insns move from the "Ready" list to the "Scheduled" list as they
298 are committed to the schedule. As this occurs, the insns in the
299 "Pending" list have their dependencies satisfied and move to either
300 the "Ready" list or the "Queued" set depending on whether
301 sufficient time has passed to make them ready. As time passes,
302 insns move from the "Queued" set to the "Ready" list.
304 The "Pending" list (P) are the insns in the INSN_FORW_DEPS of the
305 unscheduled insns, i.e., those that are ready, queued, and pending.
306 The "Queued" set (Q) is implemented by the variable `insn_queue'.
307 The "Ready" list (R) is implemented by the variables `ready' and
309 The "Scheduled" list (S) is the new insn chain built by this pass.
311 The transition (R->S) is implemented in the scheduling loop in
312 `schedule_block' when the best insn to schedule is chosen.
313 The transitions (P->R and P->Q) are implemented in `schedule_insn' as
314 insns move from the ready list to the scheduled list.
315 The transition (Q->R) is implemented in 'queue_to_insn' as time
316 passes or stalls are introduced. */
318 /* Implement a circular buffer to delay instructions until sufficient
319 time has passed. For the new pipeline description interface,
320 MAX_INSN_QUEUE_INDEX is a power of two minus one which is not less
321 than maximal time of instruction execution computed by genattr.c on
322 the base maximal time of functional unit reservations and getting a
323 result. This is the longest time an insn may be queued. */
325 static rtx_insn_list
**insn_queue
;
326 static int q_ptr
= 0;
327 static int q_size
= 0;
328 #define NEXT_Q(X) (((X)+1) & max_insn_queue_index)
329 #define NEXT_Q_AFTER(X, C) (((X)+C) & max_insn_queue_index)
331 #define QUEUE_SCHEDULED (-3)
332 #define QUEUE_NOWHERE (-2)
333 #define QUEUE_READY (-1)
334 /* QUEUE_SCHEDULED - INSN is scheduled.
335 QUEUE_NOWHERE - INSN isn't scheduled yet and is neither in
337 QUEUE_READY - INSN is in ready list.
338 N >= 0 - INSN queued for X [where NEXT_Q_AFTER (q_ptr, X) == N] cycles. */
340 #define QUEUE_INDEX(INSN) (HID (INSN)->queue_index)
342 /* The following variable value refers for all current and future
343 reservations of the processor units. */
346 /* The following variable value is size of memory representing all
347 current and future reservations of the processor units. */
348 size_t dfa_state_size
;
350 /* The following array is used to find the best insn from ready when
351 the automaton pipeline interface is used. */
352 signed char *ready_try
= NULL
;
354 /* The ready list. */
355 struct ready_list ready
= {NULL
, 0, 0, 0, 0};
357 /* The pointer to the ready list (to be removed). */
358 static struct ready_list
*readyp
= &ready
;
360 /* Scheduling clock. */
361 static int clock_var
;
363 /* Clock at which the previous instruction was issued. */
364 static int last_clock_var
;
366 /* Set to true if, when queuing a shadow insn, we discover that it would be
367 scheduled too late. */
368 static bool must_backtrack
;
370 /* The following variable value is number of essential insns issued on
371 the current cycle. An insn is essential one if it changes the
373 int cycle_issued_insns
;
375 /* This records the actual schedule. It is built up during the main phase
376 of schedule_block, and afterwards used to reorder the insns in the RTL. */
377 static vec
<rtx_insn
*> scheduled_insns
;
379 static int may_trap_exp (const_rtx
, int);
381 /* Nonzero iff the address is comprised from at most 1 register. */
382 #define CONST_BASED_ADDRESS_P(x) \
384 || ((GET_CODE (x) == PLUS || GET_CODE (x) == MINUS \
385 || (GET_CODE (x) == LO_SUM)) \
386 && (CONSTANT_P (XEXP (x, 0)) \
387 || CONSTANT_P (XEXP (x, 1)))))
389 /* Returns a class that insn with GET_DEST(insn)=x may belong to,
390 as found by analyzing insn's expression. */
393 static int haifa_luid_for_non_insn (rtx x
);
395 /* Haifa version of sched_info hooks common to all headers. */
396 const struct common_sched_info_def haifa_common_sched_info
=
398 NULL
, /* fix_recovery_cfg */
399 NULL
, /* add_block */
400 NULL
, /* estimate_number_of_insns */
401 haifa_luid_for_non_insn
, /* luid_for_non_insn */
402 SCHED_PASS_UNKNOWN
/* sched_pass_id */
405 /* Mapping from instruction UID to its Logical UID. */
406 vec
<int> sched_luids
;
408 /* Next LUID to assign to an instruction. */
409 int sched_max_luid
= 1;
411 /* Haifa Instruction Data. */
412 vec
<haifa_insn_data_def
> h_i_d
;
414 void (* sched_init_only_bb
) (basic_block
, basic_block
);
416 /* Split block function. Different schedulers might use different functions
417 to handle their internal data consistent. */
418 basic_block (* sched_split_block
) (basic_block
, rtx
);
420 /* Create empty basic block after the specified block. */
421 basic_block (* sched_create_empty_bb
) (basic_block
);
423 /* Return the number of cycles until INSN is expected to be ready.
424 Return zero if it already is. */
426 insn_delay (rtx_insn
*insn
)
428 return MAX (INSN_TICK (insn
) - clock_var
, 0);
432 may_trap_exp (const_rtx x
, int is_store
)
441 if (code
== MEM
&& may_trap_p (x
))
448 /* The insn uses memory: a volatile load. */
449 if (MEM_VOLATILE_P (x
))
451 /* An exception-free load. */
454 /* A load with 1 base register, to be further checked. */
455 if (CONST_BASED_ADDRESS_P (XEXP (x
, 0)))
456 return PFREE_CANDIDATE
;
457 /* No info on the load, to be further checked. */
458 return PRISKY_CANDIDATE
;
463 int i
, insn_class
= TRAP_FREE
;
465 /* Neither store nor load, check if it may cause a trap. */
468 /* Recursive step: walk the insn... */
469 fmt
= GET_RTX_FORMAT (code
);
470 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
474 int tmp_class
= may_trap_exp (XEXP (x
, i
), is_store
);
475 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
477 else if (fmt
[i
] == 'E')
480 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
482 int tmp_class
= may_trap_exp (XVECEXP (x
, i
, j
), is_store
);
483 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
484 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
488 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
495 /* Classifies rtx X of an insn for the purpose of verifying that X can be
496 executed speculatively (and consequently the insn can be moved
497 speculatively), by examining X, returning:
498 TRAP_RISKY: store, or risky non-load insn (e.g. division by variable).
499 TRAP_FREE: non-load insn.
500 IFREE: load from a globally safe location.
501 IRISKY: volatile load.
502 PFREE_CANDIDATE, PRISKY_CANDIDATE: load that need to be checked for
503 being either PFREE or PRISKY. */
506 haifa_classify_rtx (const_rtx x
)
508 int tmp_class
= TRAP_FREE
;
509 int insn_class
= TRAP_FREE
;
512 if (GET_CODE (x
) == PARALLEL
)
514 int i
, len
= XVECLEN (x
, 0);
516 for (i
= len
- 1; i
>= 0; i
--)
518 tmp_class
= haifa_classify_rtx (XVECEXP (x
, 0, i
));
519 insn_class
= WORST_CLASS (insn_class
, tmp_class
);
520 if (insn_class
== TRAP_RISKY
|| insn_class
== IRISKY
)
530 /* Test if it is a 'store'. */
531 tmp_class
= may_trap_exp (XEXP (x
, 0), 1);
534 gcc_assert (REG_P (XEXP (x
, 0)));
537 /* Test if it is a store. */
538 tmp_class
= may_trap_exp (SET_DEST (x
), 1);
539 if (tmp_class
== TRAP_RISKY
)
541 /* Test if it is a load. */
543 WORST_CLASS (tmp_class
,
544 may_trap_exp (SET_SRC (x
), 0));
547 tmp_class
= haifa_classify_rtx (COND_EXEC_CODE (x
));
548 if (tmp_class
== TRAP_RISKY
)
550 tmp_class
= WORST_CLASS (tmp_class
,
551 may_trap_exp (COND_EXEC_TEST (x
), 0));
554 tmp_class
= TRAP_RISKY
;
558 insn_class
= tmp_class
;
565 haifa_classify_insn (const_rtx insn
)
567 return haifa_classify_rtx (PATTERN (insn
));
570 /* After the scheduler initialization function has been called, this function
571 can be called to enable modulo scheduling. II is the initiation interval
572 we should use, it affects the delays for delay_pairs that were recorded as
573 separated by a given number of stages.
575 MAX_STAGES provides us with a limit
576 after which we give up scheduling; the caller must have unrolled at least
577 as many copies of the loop body and recorded delay_pairs for them.
579 INSNS is the number of real (non-debug) insns in one iteration of
580 the loop. MAX_UID can be used to test whether an insn belongs to
581 the first iteration of the loop; all of them have a uid lower than
584 set_modulo_params (int ii
, int max_stages
, int insns
, int max_uid
)
587 modulo_max_stages
= max_stages
;
588 modulo_n_insns
= insns
;
589 modulo_iter0_max_uid
= max_uid
;
590 modulo_backtracks_left
= PARAM_VALUE (PARAM_MAX_MODULO_BACKTRACK_ATTEMPTS
);
593 /* A structure to record a pair of insns where the first one is a real
594 insn that has delay slots, and the second is its delayed shadow.
595 I1 is scheduled normally and will emit an assembly instruction,
596 while I2 describes the side effect that takes place at the
597 transition between cycles CYCLES and (CYCLES + 1) after I1. */
600 struct delay_pair
*next_same_i1
;
603 /* When doing modulo scheduling, we a delay_pair can also be used to
604 show that I1 and I2 are the same insn in a different stage. If that
605 is the case, STAGES will be nonzero. */
609 /* Helpers for delay hashing. */
611 struct delay_i1_hasher
: nofree_ptr_hash
<delay_pair
>
613 typedef void *compare_type
;
614 static inline hashval_t
hash (const delay_pair
*);
615 static inline bool equal (const delay_pair
*, const void *);
618 /* Returns a hash value for X, based on hashing just I1. */
621 delay_i1_hasher::hash (const delay_pair
*x
)
623 return htab_hash_pointer (x
->i1
);
626 /* Return true if I1 of pair X is the same as that of pair Y. */
629 delay_i1_hasher::equal (const delay_pair
*x
, const void *y
)
634 struct delay_i2_hasher
: free_ptr_hash
<delay_pair
>
636 typedef void *compare_type
;
637 static inline hashval_t
hash (const delay_pair
*);
638 static inline bool equal (const delay_pair
*, const void *);
641 /* Returns a hash value for X, based on hashing just I2. */
644 delay_i2_hasher::hash (const delay_pair
*x
)
646 return htab_hash_pointer (x
->i2
);
649 /* Return true if I2 of pair X is the same as that of pair Y. */
652 delay_i2_hasher::equal (const delay_pair
*x
, const void *y
)
657 /* Two hash tables to record delay_pairs, one indexed by I1 and the other
659 static hash_table
<delay_i1_hasher
> *delay_htab
;
660 static hash_table
<delay_i2_hasher
> *delay_htab_i2
;
662 /* Called through htab_traverse. Walk the hashtable using I2 as
663 index, and delete all elements involving an UID higher than
664 that pointed to by *DATA. */
666 haifa_htab_i2_traverse (delay_pair
**slot
, int *data
)
669 struct delay_pair
*p
= *slot
;
670 if (INSN_UID (p
->i2
) >= maxuid
|| INSN_UID (p
->i1
) >= maxuid
)
672 delay_htab_i2
->clear_slot (slot
);
677 /* Called through htab_traverse. Walk the hashtable using I2 as
678 index, and delete all elements involving an UID higher than
679 that pointed to by *DATA. */
681 haifa_htab_i1_traverse (delay_pair
**pslot
, int *data
)
684 struct delay_pair
*p
, *first
, **pprev
;
686 if (INSN_UID ((*pslot
)->i1
) >= maxuid
)
688 delay_htab
->clear_slot (pslot
);
692 for (p
= *pslot
; p
; p
= p
->next_same_i1
)
694 if (INSN_UID (p
->i2
) < maxuid
)
697 pprev
= &p
->next_same_i1
;
702 delay_htab
->clear_slot (pslot
);
708 /* Discard all delay pairs which involve an insn with an UID higher
711 discard_delay_pairs_above (int max_uid
)
713 delay_htab
->traverse
<int *, haifa_htab_i1_traverse
> (&max_uid
);
714 delay_htab_i2
->traverse
<int *, haifa_htab_i2_traverse
> (&max_uid
);
717 /* This function can be called by a port just before it starts the final
718 scheduling pass. It records the fact that an instruction with delay
719 slots has been split into two insns, I1 and I2. The first one will be
720 scheduled normally and initiates the operation. The second one is a
721 shadow which must follow a specific number of cycles after I1; its only
722 purpose is to show the side effect that occurs at that cycle in the RTL.
723 If a JUMP_INSN or a CALL_INSN has been split, I1 should be a normal INSN,
724 while I2 retains the original insn type.
726 There are two ways in which the number of cycles can be specified,
727 involving the CYCLES and STAGES arguments to this function. If STAGES
728 is zero, we just use the value of CYCLES. Otherwise, STAGES is a factor
729 which is multiplied by MODULO_II to give the number of cycles. This is
730 only useful if the caller also calls set_modulo_params to enable modulo
734 record_delay_slot_pair (rtx_insn
*i1
, rtx_insn
*i2
, int cycles
, int stages
)
736 struct delay_pair
*p
= XNEW (struct delay_pair
);
737 struct delay_pair
**slot
;
746 delay_htab
= new hash_table
<delay_i1_hasher
> (10);
747 delay_htab_i2
= new hash_table
<delay_i2_hasher
> (10);
749 slot
= delay_htab
->find_slot_with_hash (i1
, htab_hash_pointer (i1
), INSERT
);
750 p
->next_same_i1
= *slot
;
752 slot
= delay_htab_i2
->find_slot (p
, INSERT
);
756 /* Examine the delay pair hashtable to see if INSN is a shadow for another,
757 and return the other insn if so. Return NULL otherwise. */
759 real_insn_for_shadow (rtx_insn
*insn
)
761 struct delay_pair
*pair
;
766 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
767 if (!pair
|| pair
->stages
> 0)
772 /* For a pair P of insns, return the fixed distance in cycles from the first
773 insn after which the second must be scheduled. */
775 pair_delay (struct delay_pair
*p
)
780 return p
->stages
* modulo_ii
;
783 /* Given an insn INSN, add a dependence on its delayed shadow if it
784 has one. Also try to find situations where shadows depend on each other
785 and add dependencies to the real insns to limit the amount of backtracking
788 add_delay_dependencies (rtx_insn
*insn
)
790 struct delay_pair
*pair
;
791 sd_iterator_def sd_it
;
797 pair
= delay_htab_i2
->find_with_hash (insn
, htab_hash_pointer (insn
));
800 add_dependence (insn
, pair
->i1
, REG_DEP_ANTI
);
804 FOR_EACH_DEP (pair
->i2
, SD_LIST_BACK
, sd_it
, dep
)
806 rtx_insn
*pro
= DEP_PRO (dep
);
807 struct delay_pair
*other_pair
808 = delay_htab_i2
->find_with_hash (pro
, htab_hash_pointer (pro
));
809 if (!other_pair
|| other_pair
->stages
)
811 if (pair_delay (other_pair
) >= pair_delay (pair
))
813 if (sched_verbose
>= 4)
815 fprintf (sched_dump
, ";;\tadding dependence %d <- %d\n",
816 INSN_UID (other_pair
->i1
),
817 INSN_UID (pair
->i1
));
818 fprintf (sched_dump
, ";;\tpair1 %d <- %d, cost %d\n",
822 fprintf (sched_dump
, ";;\tpair2 %d <- %d, cost %d\n",
823 INSN_UID (other_pair
->i1
),
824 INSN_UID (other_pair
->i2
),
825 pair_delay (other_pair
));
827 add_dependence (pair
->i1
, other_pair
->i1
, REG_DEP_ANTI
);
832 /* Forward declarations. */
834 static int priority (rtx_insn
*, bool force_recompute
= false);
835 static int autopref_rank_for_schedule (const rtx_insn
*, const rtx_insn
*);
836 static int rank_for_schedule (const void *, const void *);
837 static void swap_sort (rtx_insn
**, int);
838 static void queue_insn (rtx_insn
*, int, const char *);
839 static int schedule_insn (rtx_insn
*);
840 static void adjust_priority (rtx_insn
*);
841 static void advance_one_cycle (void);
842 static void extend_h_i_d (void);
845 /* Notes handling mechanism:
846 =========================
847 Generally, NOTES are saved before scheduling and restored after scheduling.
848 The scheduler distinguishes between two types of notes:
850 (1) LOOP_BEGIN, LOOP_END, SETJMP, EHREGION_BEG, EHREGION_END notes:
851 Before scheduling a region, a pointer to the note is added to the insn
852 that follows or precedes it. (This happens as part of the data dependence
853 computation). After scheduling an insn, the pointer contained in it is
854 used for regenerating the corresponding note (in reemit_notes).
856 (2) All other notes (e.g. INSN_DELETED): Before scheduling a block,
857 these notes are put in a list (in rm_other_notes() and
858 unlink_other_notes ()). After scheduling the block, these notes are
859 inserted at the beginning of the block (in schedule_block()). */
861 static void ready_add (struct ready_list
*, rtx_insn
*, bool);
862 static rtx_insn
*ready_remove_first (struct ready_list
*);
863 static rtx_insn
*ready_remove_first_dispatch (struct ready_list
*ready
);
865 static void queue_to_ready (struct ready_list
*);
866 static int early_queue_to_ready (state_t
, struct ready_list
*);
868 /* The following functions are used to implement multi-pass scheduling
869 on the first cycle. */
870 static rtx_insn
*ready_remove (struct ready_list
*, int);
871 static void ready_remove_insn (rtx_insn
*);
873 static void fix_inter_tick (rtx_insn
*, rtx_insn
*);
874 static int fix_tick_ready (rtx_insn
*);
875 static void change_queue_index (rtx_insn
*, int);
877 /* The following functions are used to implement scheduling of data/control
878 speculative instructions. */
880 static void extend_h_i_d (void);
881 static void init_h_i_d (rtx_insn
*);
882 static int haifa_speculate_insn (rtx_insn
*, ds_t
, rtx
*);
883 static void generate_recovery_code (rtx_insn
*);
884 static void process_insn_forw_deps_be_in_spec (rtx_insn
*, rtx_insn
*, ds_t
);
885 static void begin_speculative_block (rtx_insn
*);
886 static void add_to_speculative_block (rtx_insn
*);
887 static void init_before_recovery (basic_block
*);
888 static void create_check_block_twin (rtx_insn
*, bool);
889 static void fix_recovery_deps (basic_block
);
890 static bool haifa_change_pattern (rtx_insn
*, rtx
);
891 static void dump_new_block_header (int, basic_block
, rtx_insn
*, rtx_insn
*);
892 static void restore_bb_notes (basic_block
);
893 static void fix_jump_move (rtx_insn
*);
894 static void move_block_after_check (rtx_insn
*);
895 static void move_succs (vec
<edge
, va_gc
> **, basic_block
);
896 static void sched_remove_insn (rtx_insn
*);
897 static void clear_priorities (rtx_insn
*, rtx_vec_t
*);
898 static void calc_priorities (rtx_vec_t
);
899 static void add_jump_dependencies (rtx_insn
*, rtx_insn
*);
901 #endif /* INSN_SCHEDULING */
903 /* Point to state used for the current scheduling pass. */
904 struct haifa_sched_info
*current_sched_info
;
906 #ifndef INSN_SCHEDULING
908 schedule_insns (void)
913 /* Do register pressure sensitive insn scheduling if the flag is set
915 enum sched_pressure_algorithm sched_pressure
;
917 /* Map regno -> its pressure class. The map defined only when
918 SCHED_PRESSURE != SCHED_PRESSURE_NONE. */
919 enum reg_class
*sched_regno_pressure_class
;
921 /* The current register pressure. Only elements corresponding pressure
922 classes are defined. */
923 static int curr_reg_pressure
[N_REG_CLASSES
];
925 /* Saved value of the previous array. */
926 static int saved_reg_pressure
[N_REG_CLASSES
];
928 /* Register living at given scheduling point. */
929 static bitmap curr_reg_live
;
931 /* Saved value of the previous array. */
932 static bitmap saved_reg_live
;
934 /* Registers mentioned in the current region. */
935 static bitmap region_ref_regs
;
937 /* Temporary bitmap used for SCHED_PRESSURE_MODEL. */
938 static bitmap tmp_bitmap
;
940 /* Effective number of available registers of a given class (see comment
941 in sched_pressure_start_bb). */
942 static int sched_class_regs_num
[N_REG_CLASSES
];
943 /* The number of registers that the function would need to save before it
944 uses them, and the number of fixed_regs. Helpers for calculating of
945 sched_class_regs_num. */
946 static int call_saved_regs_num
[N_REG_CLASSES
];
947 static int fixed_regs_num
[N_REG_CLASSES
];
949 /* Initiate register pressure relative info for scheduling the current
950 region. Currently it is only clearing register mentioned in the
953 sched_init_region_reg_pressure_info (void)
955 bitmap_clear (region_ref_regs
);
958 /* PRESSURE[CL] describes the pressure on register class CL. Update it
959 for the birth (if BIRTH_P) or death (if !BIRTH_P) of register REGNO.
960 LIVE tracks the set of live registers; if it is null, assume that
961 every birth or death is genuine. */
963 mark_regno_birth_or_death (bitmap live
, int *pressure
, int regno
, bool birth_p
)
965 enum reg_class pressure_class
;
967 pressure_class
= sched_regno_pressure_class
[regno
];
968 if (regno
>= FIRST_PSEUDO_REGISTER
)
970 if (pressure_class
!= NO_REGS
)
974 if (!live
|| bitmap_set_bit (live
, regno
))
975 pressure
[pressure_class
]
976 += (ira_reg_class_max_nregs
977 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
981 if (!live
|| bitmap_clear_bit (live
, regno
))
982 pressure
[pressure_class
]
983 -= (ira_reg_class_max_nregs
984 [pressure_class
][PSEUDO_REGNO_MODE (regno
)]);
988 else if (pressure_class
!= NO_REGS
989 && ! TEST_HARD_REG_BIT (ira_no_alloc_regs
, regno
))
993 if (!live
|| bitmap_set_bit (live
, regno
))
994 pressure
[pressure_class
]++;
998 if (!live
|| bitmap_clear_bit (live
, regno
))
999 pressure
[pressure_class
]--;
1004 /* Initiate current register pressure related info from living
1005 registers given by LIVE. */
1007 initiate_reg_pressure_info (bitmap live
)
1013 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1014 curr_reg_pressure
[ira_pressure_classes
[i
]] = 0;
1015 bitmap_clear (curr_reg_live
);
1016 EXECUTE_IF_SET_IN_BITMAP (live
, 0, j
, bi
)
1017 if (sched_pressure
== SCHED_PRESSURE_MODEL
1018 || current_nr_blocks
== 1
1019 || bitmap_bit_p (region_ref_regs
, j
))
1020 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
, j
, true);
1023 /* Mark registers in X as mentioned in the current region. */
1025 setup_ref_regs (rtx x
)
1028 const RTX_CODE code
= GET_CODE (x
);
1033 bitmap_set_range (region_ref_regs
, REGNO (x
), REG_NREGS (x
));
1036 fmt
= GET_RTX_FORMAT (code
);
1037 for (i
= GET_RTX_LENGTH (code
) - 1; i
>= 0; i
--)
1039 setup_ref_regs (XEXP (x
, i
));
1040 else if (fmt
[i
] == 'E')
1042 for (j
= 0; j
< XVECLEN (x
, i
); j
++)
1043 setup_ref_regs (XVECEXP (x
, i
, j
));
1047 /* Initiate current register pressure related info at the start of
1050 initiate_bb_reg_pressure_info (basic_block bb
)
1052 unsigned int i ATTRIBUTE_UNUSED
;
1055 if (current_nr_blocks
> 1)
1056 FOR_BB_INSNS (bb
, insn
)
1057 if (NONDEBUG_INSN_P (insn
))
1058 setup_ref_regs (PATTERN (insn
));
1059 initiate_reg_pressure_info (df_get_live_in (bb
));
1060 if (bb_has_eh_pred (bb
))
1063 unsigned int regno
= EH_RETURN_DATA_REGNO (i
);
1065 if (regno
== INVALID_REGNUM
)
1067 if (! bitmap_bit_p (df_get_live_in (bb
), regno
))
1068 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
1073 /* Save current register pressure related info. */
1075 save_reg_pressure (void)
1079 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1080 saved_reg_pressure
[ira_pressure_classes
[i
]]
1081 = curr_reg_pressure
[ira_pressure_classes
[i
]];
1082 bitmap_copy (saved_reg_live
, curr_reg_live
);
1085 /* Restore saved register pressure related info. */
1087 restore_reg_pressure (void)
1091 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1092 curr_reg_pressure
[ira_pressure_classes
[i
]]
1093 = saved_reg_pressure
[ira_pressure_classes
[i
]];
1094 bitmap_copy (curr_reg_live
, saved_reg_live
);
1097 /* Return TRUE if the register is dying after its USE. */
1099 dying_use_p (struct reg_use_data
*use
)
1101 struct reg_use_data
*next
;
1103 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
1104 if (NONDEBUG_INSN_P (next
->insn
)
1105 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
1110 /* Print info about the current register pressure and its excess for
1111 each pressure class. */
1113 print_curr_reg_pressure (void)
1118 fprintf (sched_dump
, ";;\t");
1119 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1121 cl
= ira_pressure_classes
[i
];
1122 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1123 fprintf (sched_dump
, " %s:%d(%d)", reg_class_names
[cl
],
1124 curr_reg_pressure
[cl
],
1125 curr_reg_pressure
[cl
] - sched_class_regs_num
[cl
]);
1127 fprintf (sched_dump
, "\n");
1130 /* Determine if INSN has a condition that is clobbered if a register
1131 in SET_REGS is modified. */
1133 cond_clobbered_p (rtx_insn
*insn
, HARD_REG_SET set_regs
)
1135 rtx pat
= PATTERN (insn
);
1136 gcc_assert (GET_CODE (pat
) == COND_EXEC
);
1137 if (TEST_HARD_REG_BIT (set_regs
, REGNO (XEXP (COND_EXEC_TEST (pat
), 0))))
1139 sd_iterator_def sd_it
;
1141 haifa_change_pattern (insn
, ORIG_PAT (insn
));
1142 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
1143 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1144 TODO_SPEC (insn
) = HARD_DEP
;
1145 if (sched_verbose
>= 2)
1146 fprintf (sched_dump
,
1147 ";;\t\tdequeue insn %s because of clobbered condition\n",
1148 (*current_sched_info
->print_insn
) (insn
, 0));
1155 /* This function should be called after modifying the pattern of INSN,
1156 to update scheduler data structures as needed. */
1158 update_insn_after_change (rtx_insn
*insn
)
1160 sd_iterator_def sd_it
;
1163 dfa_clear_single_insn_cache (insn
);
1165 sd_it
= sd_iterator_start (insn
,
1166 SD_LIST_FORW
| SD_LIST_BACK
| SD_LIST_RES_BACK
);
1167 while (sd_iterator_cond (&sd_it
, &dep
))
1169 DEP_COST (dep
) = UNKNOWN_DEP_COST
;
1170 sd_iterator_next (&sd_it
);
1173 /* Invalidate INSN_COST, so it'll be recalculated. */
1174 INSN_COST (insn
) = -1;
1175 /* Invalidate INSN_TICK, so it'll be recalculated. */
1176 INSN_TICK (insn
) = INVALID_TICK
;
1178 /* Invalidate autoprefetch data entry. */
1179 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
1180 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1181 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
1182 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
1186 /* Two VECs, one to hold dependencies for which pattern replacements
1187 need to be applied or restored at the start of the next cycle, and
1188 another to hold an integer that is either one, to apply the
1189 corresponding replacement, or zero to restore it. */
1190 static vec
<dep_t
> next_cycle_replace_deps
;
1191 static vec
<int> next_cycle_apply
;
1193 static void apply_replacement (dep_t
, bool);
1194 static void restore_pattern (dep_t
, bool);
1196 /* Look at the remaining dependencies for insn NEXT, and compute and return
1197 the TODO_SPEC value we should use for it. This is called after one of
1198 NEXT's dependencies has been resolved.
1199 We also perform pattern replacements for predication, and for broken
1200 replacement dependencies. The latter is only done if FOR_BACKTRACK is
1204 recompute_todo_spec (rtx_insn
*next
, bool for_backtrack
)
1207 sd_iterator_def sd_it
;
1208 dep_t dep
, modify_dep
= NULL
;
1212 bool first_p
= true;
1214 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
1215 /* NEXT has all its dependencies resolved. */
1218 if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
1221 /* If NEXT is intended to sit adjacent to this instruction, we don't
1222 want to try to break any dependencies. Treat it as a HARD_DEP. */
1223 if (SCHED_GROUP_P (next
))
1226 /* Now we've got NEXT with speculative deps only.
1227 1. Look at the deps to see what we have to do.
1228 2. Check if we can do 'todo'. */
1231 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1233 rtx_insn
*pro
= DEP_PRO (dep
);
1234 ds_t ds
= DEP_STATUS (dep
) & SPECULATIVE
;
1236 if (DEBUG_INSN_P (pro
) && !DEBUG_INSN_P (next
))
1249 new_ds
= ds_merge (new_ds
, ds
);
1251 else if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
1253 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1258 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1260 else if (DEP_REPLACE (dep
) != NULL
)
1262 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
)
1267 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
1271 if (n_replace
> 0 && n_control
== 0 && n_spec
== 0)
1273 if (!dbg_cnt (sched_breakdep
))
1275 FOR_EACH_DEP (next
, SD_LIST_BACK
, sd_it
, dep
)
1277 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
1280 if (desc
->insn
== next
&& !for_backtrack
)
1282 gcc_assert (n_replace
== 1);
1283 apply_replacement (dep
, true);
1285 DEP_STATUS (dep
) |= DEP_CANCELLED
;
1291 else if (n_control
== 1 && n_replace
== 0 && n_spec
== 0)
1293 rtx_insn
*pro
, *other
;
1295 rtx cond
= NULL_RTX
;
1297 rtx_insn
*prev
= NULL
;
1301 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0
1302 || (ORIG_PAT (next
) != NULL_RTX
1303 && PREDICATED_PAT (next
) == NULL_RTX
))
1306 pro
= DEP_PRO (modify_dep
);
1307 other
= real_insn_for_shadow (pro
);
1308 if (other
!= NULL_RTX
)
1311 cond
= sched_get_reverse_condition_uncached (pro
);
1312 regno
= REGNO (XEXP (cond
, 0));
1314 /* Find the last scheduled insn that modifies the condition register.
1315 We can stop looking once we find the insn we depend on through the
1316 REG_DEP_CONTROL; if the condition register isn't modified after it,
1317 we know that it still has the right value. */
1318 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
1319 FOR_EACH_VEC_ELT_REVERSE (scheduled_insns
, i
, prev
)
1323 find_all_hard_reg_sets (prev
, &t
, true);
1324 if (TEST_HARD_REG_BIT (t
, regno
))
1329 if (ORIG_PAT (next
) == NULL_RTX
)
1331 ORIG_PAT (next
) = PATTERN (next
);
1333 new_pat
= gen_rtx_COND_EXEC (VOIDmode
, cond
, PATTERN (next
));
1334 success
= haifa_change_pattern (next
, new_pat
);
1337 PREDICATED_PAT (next
) = new_pat
;
1339 else if (PATTERN (next
) != PREDICATED_PAT (next
))
1341 bool success
= haifa_change_pattern (next
,
1342 PREDICATED_PAT (next
));
1343 gcc_assert (success
);
1345 DEP_STATUS (modify_dep
) |= DEP_CANCELLED
;
1349 if (PREDICATED_PAT (next
) != NULL_RTX
)
1351 int tick
= INSN_TICK (next
);
1352 bool success
= haifa_change_pattern (next
,
1354 INSN_TICK (next
) = tick
;
1355 gcc_assert (success
);
1358 /* We can't handle the case where there are both speculative and control
1359 dependencies, so we return HARD_DEP in such a case. Also fail if
1360 we have speculative dependencies with not enough points, or more than
1361 one control dependency. */
1362 if ((n_spec
> 0 && (n_control
> 0 || n_replace
> 0))
1364 /* Too few points? */
1365 && ds_weak (new_ds
) < spec_info
->data_weakness_cutoff
)
1373 /* Pointer to the last instruction scheduled. */
1374 static rtx_insn
*last_scheduled_insn
;
1376 /* Pointer to the last nondebug instruction scheduled within the
1377 block, or the prev_head of the scheduling block. Used by
1378 rank_for_schedule, so that insns independent of the last scheduled
1379 insn will be preferred over dependent instructions. */
1380 static rtx_insn
*last_nondebug_scheduled_insn
;
1382 /* Pointer that iterates through the list of unscheduled insns if we
1383 have a dbg_cnt enabled. It always points at an insn prior to the
1384 first unscheduled one. */
1385 static rtx_insn
*nonscheduled_insns_begin
;
1387 /* Compute cost of executing INSN.
1388 This is the number of cycles between instruction issue and
1389 instruction results. */
1391 insn_sched_cost (rtx_insn
*insn
)
1400 if (recog_memoized (insn
) < 0)
1403 cost
= insn_default_latency (insn
);
1410 cost
= INSN_COST (insn
);
1414 /* A USE insn, or something else we don't need to
1415 understand. We can't pass these directly to
1416 result_ready_cost or insn_default_latency because it will
1417 trigger a fatal error for unrecognizable insns. */
1418 if (recog_memoized (insn
) < 0)
1420 INSN_COST (insn
) = 0;
1425 cost
= insn_default_latency (insn
);
1429 INSN_COST (insn
) = cost
;
1436 /* Compute cost of dependence LINK.
1437 This is the number of cycles between instruction issue and
1438 instruction results.
1439 ??? We also use this function to call recog_memoized on all insns. */
1441 dep_cost_1 (dep_t link
, dw_t dw
)
1443 rtx_insn
*insn
= DEP_PRO (link
);
1444 rtx_insn
*used
= DEP_CON (link
);
1447 if (DEP_COST (link
) != UNKNOWN_DEP_COST
)
1448 return DEP_COST (link
);
1452 struct delay_pair
*delay_entry
;
1454 = delay_htab_i2
->find_with_hash (used
, htab_hash_pointer (used
));
1457 if (delay_entry
->i1
== insn
)
1459 DEP_COST (link
) = pair_delay (delay_entry
);
1460 return DEP_COST (link
);
1465 /* A USE insn should never require the value used to be computed.
1466 This allows the computation of a function's result and parameter
1467 values to overlap the return and call. We don't care about the
1468 dependence cost when only decreasing register pressure. */
1469 if (recog_memoized (used
) < 0)
1472 recog_memoized (insn
);
1476 enum reg_note dep_type
= DEP_TYPE (link
);
1478 cost
= insn_sched_cost (insn
);
1480 if (INSN_CODE (insn
) >= 0)
1482 if (dep_type
== REG_DEP_ANTI
)
1484 else if (dep_type
== REG_DEP_OUTPUT
)
1486 cost
= (insn_default_latency (insn
)
1487 - insn_default_latency (used
));
1491 else if (bypass_p (insn
))
1492 cost
= insn_latency (insn
, used
);
1496 if (targetm
.sched
.adjust_cost
)
1497 cost
= targetm
.sched
.adjust_cost (used
, (int) dep_type
, insn
, cost
,
1504 DEP_COST (link
) = cost
;
1508 /* Compute cost of dependence LINK.
1509 This is the number of cycles between instruction issue and
1510 instruction results. */
1512 dep_cost (dep_t link
)
1514 return dep_cost_1 (link
, 0);
1517 /* Use this sel-sched.c friendly function in reorder2 instead of increasing
1518 INSN_PRIORITY explicitly. */
1520 increase_insn_priority (rtx_insn
*insn
, int amount
)
1522 if (!sel_sched_p ())
1524 /* We're dealing with haifa-sched.c INSN_PRIORITY. */
1525 if (INSN_PRIORITY_KNOWN (insn
))
1526 INSN_PRIORITY (insn
) += amount
;
1530 /* In sel-sched.c INSN_PRIORITY is not kept up to date.
1531 Use EXPR_PRIORITY instead. */
1532 sel_add_to_insn_priority (insn
, amount
);
1536 /* Return 'true' if DEP should be included in priority calculations. */
1538 contributes_to_priority_p (dep_t dep
)
1540 if (DEBUG_INSN_P (DEP_CON (dep
))
1541 || DEBUG_INSN_P (DEP_PRO (dep
)))
1544 /* Critical path is meaningful in block boundaries only. */
1545 if (!current_sched_info
->contributes_to_priority (DEP_CON (dep
),
1549 if (DEP_REPLACE (dep
) != NULL
)
1552 /* If flag COUNT_SPEC_IN_CRITICAL_PATH is set,
1553 then speculative instructions will less likely be
1554 scheduled. That is because the priority of
1555 their producers will increase, and, thus, the
1556 producers will more likely be scheduled, thus,
1557 resolving the dependence. */
1558 if (sched_deps_info
->generate_spec_deps
1559 && !(spec_info
->flags
& COUNT_SPEC_IN_CRITICAL_PATH
)
1560 && (DEP_STATUS (dep
) & SPECULATIVE
))
1566 /* Compute the number of nondebug deps in list LIST for INSN. */
1569 dep_list_size (rtx_insn
*insn
, sd_list_types_def list
)
1571 sd_iterator_def sd_it
;
1573 int dbgcount
= 0, nodbgcount
= 0;
1575 if (!MAY_HAVE_DEBUG_INSNS
)
1576 return sd_lists_size (insn
, list
);
1578 FOR_EACH_DEP (insn
, list
, sd_it
, dep
)
1580 if (DEBUG_INSN_P (DEP_CON (dep
)))
1582 else if (!DEBUG_INSN_P (DEP_PRO (dep
)))
1586 gcc_assert (dbgcount
+ nodbgcount
== sd_lists_size (insn
, list
));
1593 /* Compute the priority number for INSN. */
1595 priority (rtx_insn
*insn
, bool force_recompute
)
1597 if (! INSN_P (insn
))
1600 /* We should not be interested in priority of an already scheduled insn. */
1601 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
1603 if (force_recompute
|| !INSN_PRIORITY_KNOWN (insn
))
1605 int this_priority
= -1;
1609 int this_fusion_priority
;
1611 targetm
.sched
.fusion_priority (insn
, FUSION_MAX_PRIORITY
,
1612 &this_fusion_priority
, &this_priority
);
1613 INSN_FUSION_PRIORITY (insn
) = this_fusion_priority
;
1615 else if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
1616 /* ??? We should set INSN_PRIORITY to insn_sched_cost when and insn
1617 has some forward deps but all of them are ignored by
1618 contributes_to_priority hook. At the moment we set priority of
1620 this_priority
= insn_sched_cost (insn
);
1623 rtx_insn
*prev_first
, *twin
;
1626 /* For recovery check instructions we calculate priority slightly
1627 different than that of normal instructions. Instead of walking
1628 through INSN_FORW_DEPS (check) list, we walk through
1629 INSN_FORW_DEPS list of each instruction in the corresponding
1632 /* Selective scheduling does not define RECOVERY_BLOCK macro. */
1633 rec
= sel_sched_p () ? NULL
: RECOVERY_BLOCK (insn
);
1634 if (!rec
|| rec
== EXIT_BLOCK_PTR_FOR_FN (cfun
))
1636 prev_first
= PREV_INSN (insn
);
1641 prev_first
= NEXT_INSN (BB_HEAD (rec
));
1642 twin
= PREV_INSN (BB_END (rec
));
1647 sd_iterator_def sd_it
;
1650 FOR_EACH_DEP (twin
, SD_LIST_FORW
, sd_it
, dep
)
1655 next
= DEP_CON (dep
);
1657 if (BLOCK_FOR_INSN (next
) != rec
)
1661 if (!contributes_to_priority_p (dep
))
1665 cost
= dep_cost (dep
);
1668 struct _dep _dep1
, *dep1
= &_dep1
;
1670 init_dep (dep1
, insn
, next
, REG_DEP_ANTI
);
1672 cost
= dep_cost (dep1
);
1675 next_priority
= cost
+ priority (next
);
1677 if (next_priority
> this_priority
)
1678 this_priority
= next_priority
;
1682 twin
= PREV_INSN (twin
);
1684 while (twin
!= prev_first
);
1687 if (this_priority
< 0)
1689 gcc_assert (this_priority
== -1);
1691 this_priority
= insn_sched_cost (insn
);
1694 INSN_PRIORITY (insn
) = this_priority
;
1695 INSN_PRIORITY_STATUS (insn
) = 1;
1698 return INSN_PRIORITY (insn
);
1701 /* Macros and functions for keeping the priority queue sorted, and
1702 dealing with queuing and dequeuing of instructions. */
1704 /* For each pressure class CL, set DEATH[CL] to the number of registers
1705 in that class that die in INSN. */
1708 calculate_reg_deaths (rtx_insn
*insn
, int *death
)
1711 struct reg_use_data
*use
;
1713 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1714 death
[ira_pressure_classes
[i
]] = 0;
1715 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
1716 if (dying_use_p (use
))
1717 mark_regno_birth_or_death (0, death
, use
->regno
, true);
1720 /* Setup info about the current register pressure impact of scheduling
1721 INSN at the current scheduling point. */
1723 setup_insn_reg_pressure_info (rtx_insn
*insn
)
1725 int i
, change
, before
, after
, hard_regno
;
1726 int excess_cost_change
;
1729 struct reg_pressure_data
*pressure_info
;
1730 int *max_reg_pressure
;
1731 static int death
[N_REG_CLASSES
];
1733 gcc_checking_assert (!DEBUG_INSN_P (insn
));
1735 excess_cost_change
= 0;
1736 calculate_reg_deaths (insn
, death
);
1737 pressure_info
= INSN_REG_PRESSURE (insn
);
1738 max_reg_pressure
= INSN_MAX_REG_PRESSURE (insn
);
1739 gcc_assert (pressure_info
!= NULL
&& max_reg_pressure
!= NULL
);
1740 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
1742 cl
= ira_pressure_classes
[i
];
1743 gcc_assert (curr_reg_pressure
[cl
] >= 0);
1744 change
= (int) pressure_info
[i
].set_increase
- death
[cl
];
1745 before
= MAX (0, max_reg_pressure
[i
] - sched_class_regs_num
[cl
]);
1746 after
= MAX (0, max_reg_pressure
[i
] + change
1747 - sched_class_regs_num
[cl
]);
1748 hard_regno
= ira_class_hard_regs
[cl
][0];
1749 gcc_assert (hard_regno
>= 0);
1750 mode
= reg_raw_mode
[hard_regno
];
1751 excess_cost_change
+= ((after
- before
)
1752 * (ira_memory_move_cost
[mode
][cl
][0]
1753 + ira_memory_move_cost
[mode
][cl
][1]));
1755 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insn
) = excess_cost_change
;
1758 /* This is the first page of code related to SCHED_PRESSURE_MODEL.
1759 It tries to make the scheduler take register pressure into account
1760 without introducing too many unnecessary stalls. It hooks into the
1761 main scheduling algorithm at several points:
1763 - Before scheduling starts, model_start_schedule constructs a
1764 "model schedule" for the current block. This model schedule is
1765 chosen solely to keep register pressure down. It does not take the
1766 target's pipeline or the original instruction order into account,
1767 except as a tie-breaker. It also doesn't work to a particular
1770 This model schedule gives us an idea of what pressure can be
1771 achieved for the block and gives us an example of a schedule that
1772 keeps to that pressure. It also makes the final schedule less
1773 dependent on the original instruction order. This is important
1774 because the original order can either be "wide" (many values live
1775 at once, such as in user-scheduled code) or "narrow" (few values
1776 live at once, such as after loop unrolling, where several
1777 iterations are executed sequentially).
1779 We do not apply this model schedule to the rtx stream. We simply
1780 record it in model_schedule. We also compute the maximum pressure,
1781 MP, that was seen during this schedule.
1783 - Instructions are added to the ready queue even if they require
1784 a stall. The length of the stall is instead computed as:
1786 MAX (INSN_TICK (INSN) - clock_var, 0)
1788 (= insn_delay). This allows rank_for_schedule to choose between
1789 introducing a deliberate stall or increasing pressure.
1791 - Before sorting the ready queue, model_set_excess_costs assigns
1792 a pressure-based cost to each ready instruction in the queue.
1793 This is the instruction's INSN_REG_PRESSURE_EXCESS_COST_CHANGE
1794 (ECC for short) and is effectively measured in cycles.
1796 - rank_for_schedule ranks instructions based on:
1798 ECC (insn) + insn_delay (insn)
1804 So, for example, an instruction X1 with an ECC of 1 that can issue
1805 now will win over an instruction X0 with an ECC of zero that would
1806 introduce a stall of one cycle. However, an instruction X2 with an
1807 ECC of 2 that can issue now will lose to both X0 and X1.
1809 - When an instruction is scheduled, model_recompute updates the model
1810 schedule with the new pressures (some of which might now exceed the
1811 original maximum pressure MP). model_update_limit_points then searches
1812 for the new point of maximum pressure, if not already known. */
1814 /* Used to separate high-verbosity debug information for SCHED_PRESSURE_MODEL
1815 from surrounding debug information. */
1817 ";;\t\t+------------------------------------------------------\n"
1819 /* Information about the pressure on a particular register class at a
1820 particular point of the model schedule. */
1821 struct model_pressure_data
{
1822 /* The pressure at this point of the model schedule, or -1 if the
1823 point is associated with an instruction that has already been
1827 /* The maximum pressure during or after this point of the model schedule. */
1831 /* Per-instruction information that is used while building the model
1832 schedule. Here, "schedule" refers to the model schedule rather
1833 than the main schedule. */
1834 struct model_insn_info
{
1835 /* The instruction itself. */
1838 /* If this instruction is in model_worklist, these fields link to the
1839 previous (higher-priority) and next (lower-priority) instructions
1841 struct model_insn_info
*prev
;
1842 struct model_insn_info
*next
;
1844 /* While constructing the schedule, QUEUE_INDEX describes whether an
1845 instruction has already been added to the schedule (QUEUE_SCHEDULED),
1846 is in model_worklist (QUEUE_READY), or neither (QUEUE_NOWHERE).
1847 old_queue records the value that QUEUE_INDEX had before scheduling
1848 started, so that we can restore it once the schedule is complete. */
1851 /* The relative importance of an unscheduled instruction. Higher
1852 values indicate greater importance. */
1853 unsigned int model_priority
;
1855 /* The length of the longest path of satisfied true dependencies
1856 that leads to this instruction. */
1859 /* The length of the longest path of dependencies of any kind
1860 that leads from this instruction. */
1863 /* The number of predecessor nodes that must still be scheduled. */
1864 int unscheduled_preds
;
1867 /* Information about the pressure limit for a particular register class.
1868 This structure is used when applying a model schedule to the main
1870 struct model_pressure_limit
{
1871 /* The maximum register pressure seen in the original model schedule. */
1874 /* The maximum register pressure seen in the current model schedule
1875 (which excludes instructions that have already been scheduled). */
1878 /* The point of the current model schedule at which PRESSURE is first
1879 reached. It is set to -1 if the value needs to be recomputed. */
1883 /* Describes a particular way of measuring register pressure. */
1884 struct model_pressure_group
{
1885 /* Index PCI describes the maximum pressure on ira_pressure_classes[PCI]. */
1886 struct model_pressure_limit limits
[N_REG_CLASSES
];
1888 /* Index (POINT * ira_num_pressure_classes + PCI) describes the pressure
1889 on register class ira_pressure_classes[PCI] at point POINT of the
1890 current model schedule. A POINT of model_num_insns describes the
1891 pressure at the end of the schedule. */
1892 struct model_pressure_data
*model
;
1895 /* Index POINT gives the instruction at point POINT of the model schedule.
1896 This array doesn't change during main scheduling. */
1897 static vec
<rtx_insn
*> model_schedule
;
1899 /* The list of instructions in the model worklist, sorted in order of
1900 decreasing priority. */
1901 static struct model_insn_info
*model_worklist
;
1903 /* Index I describes the instruction with INSN_LUID I. */
1904 static struct model_insn_info
*model_insns
;
1906 /* The number of instructions in the model schedule. */
1907 static int model_num_insns
;
1909 /* The index of the first instruction in model_schedule that hasn't yet been
1910 added to the main schedule, or model_num_insns if all of them have. */
1911 static int model_curr_point
;
1913 /* Describes the pressure before each instruction in the model schedule. */
1914 static struct model_pressure_group model_before_pressure
;
1916 /* The first unused model_priority value (as used in model_insn_info). */
1917 static unsigned int model_next_priority
;
1920 /* The model_pressure_data for ira_pressure_classes[PCI] in GROUP
1921 at point POINT of the model schedule. */
1922 #define MODEL_PRESSURE_DATA(GROUP, POINT, PCI) \
1923 (&(GROUP)->model[(POINT) * ira_pressure_classes_num + (PCI)])
1925 /* The maximum pressure on ira_pressure_classes[PCI] in GROUP at or
1926 after point POINT of the model schedule. */
1927 #define MODEL_MAX_PRESSURE(GROUP, POINT, PCI) \
1928 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->max_pressure)
1930 /* The pressure on ira_pressure_classes[PCI] in GROUP at point POINT
1931 of the model schedule. */
1932 #define MODEL_REF_PRESSURE(GROUP, POINT, PCI) \
1933 (MODEL_PRESSURE_DATA (GROUP, POINT, PCI)->ref_pressure)
1935 /* Information about INSN that is used when creating the model schedule. */
1936 #define MODEL_INSN_INFO(INSN) \
1937 (&model_insns[INSN_LUID (INSN)])
1939 /* The instruction at point POINT of the model schedule. */
1940 #define MODEL_INSN(POINT) \
1941 (model_schedule[POINT])
1944 /* Return INSN's index in the model schedule, or model_num_insns if it
1945 doesn't belong to that schedule. */
1948 model_index (rtx_insn
*insn
)
1950 if (INSN_MODEL_INDEX (insn
) == 0)
1951 return model_num_insns
;
1952 return INSN_MODEL_INDEX (insn
) - 1;
1955 /* Make sure that GROUP->limits is up-to-date for the current point
1956 of the model schedule. */
1959 model_update_limit_points_in_group (struct model_pressure_group
*group
)
1961 int pci
, max_pressure
, point
;
1963 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
1965 /* We may have passed the final point at which the pressure in
1966 group->limits[pci].pressure was reached. Update the limit if so. */
1967 max_pressure
= MODEL_MAX_PRESSURE (group
, model_curr_point
, pci
);
1968 group
->limits
[pci
].pressure
= max_pressure
;
1970 /* Find the point at which MAX_PRESSURE is first reached. We need
1971 to search in three cases:
1973 - We've already moved past the previous pressure point.
1974 In this case we search forward from model_curr_point.
1976 - We scheduled the previous point of maximum pressure ahead of
1977 its position in the model schedule, but doing so didn't bring
1978 the pressure point earlier. In this case we search forward
1979 from that previous pressure point.
1981 - Scheduling an instruction early caused the maximum pressure
1982 to decrease. In this case we will have set the pressure
1983 point to -1, and we search forward from model_curr_point. */
1984 point
= MAX (group
->limits
[pci
].point
, model_curr_point
);
1985 while (point
< model_num_insns
1986 && MODEL_REF_PRESSURE (group
, point
, pci
) < max_pressure
)
1988 group
->limits
[pci
].point
= point
;
1990 gcc_assert (MODEL_REF_PRESSURE (group
, point
, pci
) == max_pressure
);
1991 gcc_assert (MODEL_MAX_PRESSURE (group
, point
, pci
) == max_pressure
);
1995 /* Make sure that all register-pressure limits are up-to-date for the
1996 current position in the model schedule. */
1999 model_update_limit_points (void)
2001 model_update_limit_points_in_group (&model_before_pressure
);
2004 /* Return the model_index of the last unscheduled use in chain USE
2005 outside of USE's instruction. Return -1 if there are no other uses,
2006 or model_num_insns if the register is live at the end of the block. */
2009 model_last_use_except (struct reg_use_data
*use
)
2011 struct reg_use_data
*next
;
2015 for (next
= use
->next_regno_use
; next
!= use
; next
= next
->next_regno_use
)
2016 if (NONDEBUG_INSN_P (next
->insn
)
2017 && QUEUE_INDEX (next
->insn
) != QUEUE_SCHEDULED
)
2019 index
= model_index (next
->insn
);
2020 if (index
== model_num_insns
)
2021 return model_num_insns
;
2028 /* An instruction with model_index POINT has just been scheduled, and it
2029 adds DELTA to the pressure on ira_pressure_classes[PCI] after POINT - 1.
2030 Update MODEL_REF_PRESSURE (GROUP, POINT, PCI) and
2031 MODEL_MAX_PRESSURE (GROUP, POINT, PCI) accordingly. */
2034 model_start_update_pressure (struct model_pressure_group
*group
,
2035 int point
, int pci
, int delta
)
2037 int next_max_pressure
;
2039 if (point
== model_num_insns
)
2041 /* The instruction wasn't part of the model schedule; it was moved
2042 from a different block. Update the pressure for the end of
2043 the model schedule. */
2044 MODEL_REF_PRESSURE (group
, point
, pci
) += delta
;
2045 MODEL_MAX_PRESSURE (group
, point
, pci
) += delta
;
2049 /* Record that this instruction has been scheduled. Nothing now
2050 changes between POINT and POINT + 1, so get the maximum pressure
2051 from the latter. If the maximum pressure decreases, the new
2052 pressure point may be before POINT. */
2053 MODEL_REF_PRESSURE (group
, point
, pci
) = -1;
2054 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2055 if (MODEL_MAX_PRESSURE (group
, point
, pci
) > next_max_pressure
)
2057 MODEL_MAX_PRESSURE (group
, point
, pci
) = next_max_pressure
;
2058 if (group
->limits
[pci
].point
== point
)
2059 group
->limits
[pci
].point
= -1;
2064 /* Record that scheduling a later instruction has changed the pressure
2065 at point POINT of the model schedule by DELTA (which might be 0).
2066 Update GROUP accordingly. Return nonzero if these changes might
2067 trigger changes to previous points as well. */
2070 model_update_pressure (struct model_pressure_group
*group
,
2071 int point
, int pci
, int delta
)
2073 int ref_pressure
, max_pressure
, next_max_pressure
;
2075 /* If POINT hasn't yet been scheduled, update its pressure. */
2076 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
2077 if (ref_pressure
>= 0 && delta
!= 0)
2079 ref_pressure
+= delta
;
2080 MODEL_REF_PRESSURE (group
, point
, pci
) = ref_pressure
;
2082 /* Check whether the maximum pressure in the overall schedule
2083 has increased. (This means that the MODEL_MAX_PRESSURE of
2084 every point <= POINT will need to increase too; see below.) */
2085 if (group
->limits
[pci
].pressure
< ref_pressure
)
2086 group
->limits
[pci
].pressure
= ref_pressure
;
2088 /* If we are at maximum pressure, and the maximum pressure
2089 point was previously unknown or later than POINT,
2090 bring it forward. */
2091 if (group
->limits
[pci
].pressure
== ref_pressure
2092 && !IN_RANGE (group
->limits
[pci
].point
, 0, point
))
2093 group
->limits
[pci
].point
= point
;
2095 /* If POINT used to be the point of maximum pressure, but isn't
2096 any longer, we need to recalculate it using a forward walk. */
2097 if (group
->limits
[pci
].pressure
> ref_pressure
2098 && group
->limits
[pci
].point
== point
)
2099 group
->limits
[pci
].point
= -1;
2102 /* Update the maximum pressure at POINT. Changes here might also
2103 affect the maximum pressure at POINT - 1. */
2104 next_max_pressure
= MODEL_MAX_PRESSURE (group
, point
+ 1, pci
);
2105 max_pressure
= MAX (ref_pressure
, next_max_pressure
);
2106 if (MODEL_MAX_PRESSURE (group
, point
, pci
) != max_pressure
)
2108 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
2114 /* INSN has just been scheduled. Update the model schedule accordingly. */
2117 model_recompute (rtx_insn
*insn
)
2122 } uses
[FIRST_PSEUDO_REGISTER
+ MAX_RECOG_OPERANDS
];
2123 struct reg_use_data
*use
;
2124 struct reg_pressure_data
*reg_pressure
;
2125 int delta
[N_REG_CLASSES
];
2126 int pci
, point
, mix
, new_last
, cl
, ref_pressure
, queue
;
2127 unsigned int i
, num_uses
, num_pending_births
;
2130 /* The destinations of INSN were previously live from POINT onwards, but are
2131 now live from model_curr_point onwards. Set up DELTA accordingly. */
2132 point
= model_index (insn
);
2133 reg_pressure
= INSN_REG_PRESSURE (insn
);
2134 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2136 cl
= ira_pressure_classes
[pci
];
2137 delta
[cl
] = reg_pressure
[pci
].set_increase
;
2140 /* Record which registers previously died at POINT, but which now die
2141 before POINT. Adjust DELTA so that it represents the effect of
2142 this change after POINT - 1. Set NUM_PENDING_BIRTHS to the number of
2143 registers that will be born in the range [model_curr_point, POINT). */
2145 num_pending_births
= 0;
2146 bitmap_clear (tmp_bitmap
);
2147 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
2149 new_last
= model_last_use_except (use
);
2150 if (new_last
< point
&& bitmap_set_bit (tmp_bitmap
, use
->regno
))
2152 gcc_assert (num_uses
< ARRAY_SIZE (uses
));
2153 uses
[num_uses
].last_use
= new_last
;
2154 uses
[num_uses
].regno
= use
->regno
;
2155 /* This register is no longer live after POINT - 1. */
2156 mark_regno_birth_or_death (NULL
, delta
, use
->regno
, false);
2159 num_pending_births
++;
2163 /* Update the MODEL_REF_PRESSURE and MODEL_MAX_PRESSURE for POINT.
2164 Also set each group pressure limit for POINT. */
2165 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2167 cl
= ira_pressure_classes
[pci
];
2168 model_start_update_pressure (&model_before_pressure
,
2169 point
, pci
, delta
[cl
]);
2172 /* Walk the model schedule backwards, starting immediately before POINT. */
2174 if (point
!= model_curr_point
)
2178 insn
= MODEL_INSN (point
);
2179 queue
= QUEUE_INDEX (insn
);
2181 if (queue
!= QUEUE_SCHEDULED
)
2183 /* DELTA describes the effect of the move on the register pressure
2184 after POINT. Make it describe the effect on the pressure
2187 while (i
< num_uses
)
2189 if (uses
[i
].last_use
== point
)
2191 /* This register is now live again. */
2192 mark_regno_birth_or_death (NULL
, delta
,
2193 uses
[i
].regno
, true);
2195 /* Remove this use from the array. */
2196 uses
[i
] = uses
[num_uses
- 1];
2198 num_pending_births
--;
2204 if (sched_verbose
>= 5)
2208 fprintf (sched_dump
, MODEL_BAR
);
2209 fprintf (sched_dump
, ";;\t\t| New pressure for model"
2211 fprintf (sched_dump
, MODEL_BAR
);
2215 fprintf (sched_dump
, ";;\t\t| %3d %4d %-30s ",
2216 point
, INSN_UID (insn
),
2217 str_pattern_slim (PATTERN (insn
)));
2218 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2220 cl
= ira_pressure_classes
[pci
];
2221 ref_pressure
= MODEL_REF_PRESSURE (&model_before_pressure
,
2223 fprintf (sched_dump
, " %s:[%d->%d]",
2224 reg_class_names
[ira_pressure_classes
[pci
]],
2225 ref_pressure
, ref_pressure
+ delta
[cl
]);
2227 fprintf (sched_dump
, "\n");
2231 /* Adjust the pressure at POINT. Set MIX to nonzero if POINT - 1
2232 might have changed as well. */
2233 mix
= num_pending_births
;
2234 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2236 cl
= ira_pressure_classes
[pci
];
2238 mix
|= model_update_pressure (&model_before_pressure
,
2239 point
, pci
, delta
[cl
]);
2242 while (mix
&& point
> model_curr_point
);
2245 fprintf (sched_dump
, MODEL_BAR
);
2248 /* After DEP, which was cancelled, has been resolved for insn NEXT,
2249 check whether the insn's pattern needs restoring. */
2251 must_restore_pattern_p (rtx_insn
*next
, dep_t dep
)
2253 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
2256 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
2258 gcc_assert (ORIG_PAT (next
) != NULL_RTX
);
2259 gcc_assert (next
== DEP_CON (dep
));
2263 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
2264 if (desc
->insn
!= next
)
2266 gcc_assert (*desc
->loc
== desc
->orig
);
2273 /* model_spill_cost (CL, P, P') returns the cost of increasing the
2274 pressure on CL from P to P'. We use this to calculate a "base ECC",
2275 baseECC (CL, X), for each pressure class CL and each instruction X.
2276 Supposing X changes the pressure on CL from P to P', and that the
2277 maximum pressure on CL in the current model schedule is MP', then:
2279 * if X occurs before or at the next point of maximum pressure in
2280 the model schedule and P' > MP', then:
2282 baseECC (CL, X) = model_spill_cost (CL, MP, P')
2284 The idea is that the pressure after scheduling a fixed set of
2285 instructions -- in this case, the set up to and including the
2286 next maximum pressure point -- is going to be the same regardless
2287 of the order; we simply want to keep the intermediate pressure
2288 under control. Thus X has a cost of zero unless scheduling it
2289 now would exceed MP'.
2291 If all increases in the set are by the same amount, no zero-cost
2292 instruction will ever cause the pressure to exceed MP'. However,
2293 if X is instead moved past an instruction X' with pressure in the
2294 range (MP' - (P' - P), MP'), the pressure at X' will increase
2295 beyond MP'. Since baseECC is very much a heuristic anyway,
2296 it doesn't seem worth the overhead of tracking cases like these.
2298 The cost of exceeding MP' is always based on the original maximum
2299 pressure MP. This is so that going 2 registers over the original
2300 limit has the same cost regardless of whether it comes from two
2301 separate +1 deltas or from a single +2 delta.
2303 * if X occurs after the next point of maximum pressure in the model
2304 schedule and P' > P, then:
2306 baseECC (CL, X) = model_spill_cost (CL, MP, MP' + (P' - P))
2308 That is, if we move X forward across a point of maximum pressure,
2309 and if X increases the pressure by P' - P, then we conservatively
2310 assume that scheduling X next would increase the maximum pressure
2311 by P' - P. Again, the cost of doing this is based on the original
2312 maximum pressure MP, for the same reason as above.
2314 * if P' < P, P > MP, and X occurs at or after the next point of
2315 maximum pressure, then:
2317 baseECC (CL, X) = -model_spill_cost (CL, MAX (MP, P'), P)
2319 That is, if we have already exceeded the original maximum pressure MP,
2320 and if X might reduce the maximum pressure again -- or at least push
2321 it further back, and thus allow more scheduling freedom -- it is given
2322 a negative cost to reflect the improvement.
2328 In this case, X is not expected to affect the maximum pressure MP',
2329 so it has zero cost.
2331 We then create a combined value baseECC (X) that is the sum of
2332 baseECC (CL, X) for each pressure class CL.
2334 baseECC (X) could itself be used as the ECC value described above.
2335 However, this is often too conservative, in the sense that it
2336 tends to make high-priority instructions that increase pressure
2337 wait too long in cases where introducing a spill would be better.
2338 For this reason the final ECC is a priority-adjusted form of
2339 baseECC (X). Specifically, we calculate:
2341 P (X) = INSN_PRIORITY (X) - insn_delay (X) - baseECC (X)
2342 baseP = MAX { P (X) | baseECC (X) <= 0 }
2346 ECC (X) = MAX (MIN (baseP - P (X), baseECC (X)), 0)
2348 Thus an instruction's effect on pressure is ignored if it has a high
2349 enough priority relative to the ones that don't increase pressure.
2350 Negative values of baseECC (X) do not increase the priority of X
2351 itself, but they do make it harder for other instructions to
2352 increase the pressure further.
2354 This pressure cost is deliberately timid. The intention has been
2355 to choose a heuristic that rarely interferes with the normal list
2356 scheduler in cases where that scheduler would produce good code.
2357 We simply want to curb some of its worst excesses. */
2359 /* Return the cost of increasing the pressure in class CL from FROM to TO.
2361 Here we use the very simplistic cost model that every register above
2362 sched_class_regs_num[CL] has a spill cost of 1. We could use other
2363 measures instead, such as one based on MEMORY_MOVE_COST. However:
2365 (1) In order for an instruction to be scheduled, the higher cost
2366 would need to be justified in a single saving of that many stalls.
2367 This is overly pessimistic, because the benefit of spilling is
2368 often to avoid a sequence of several short stalls rather than
2371 (2) The cost is still arbitrary. Because we are not allocating
2372 registers during scheduling, we have no way of knowing for
2373 sure how many memory accesses will be required by each spill,
2374 where the spills will be placed within the block, or even
2375 which block(s) will contain the spills.
2377 So a higher cost than 1 is often too conservative in practice,
2378 forcing blocks to contain unnecessary stalls instead of spill code.
2379 The simple cost below seems to be the best compromise. It reduces
2380 the interference with the normal list scheduler, which helps make
2381 it more suitable for a default-on option. */
2384 model_spill_cost (int cl
, int from
, int to
)
2386 from
= MAX (from
, sched_class_regs_num
[cl
]);
2387 return MAX (to
, from
) - from
;
2390 /* Return baseECC (ira_pressure_classes[PCI], POINT), given that
2391 P = curr_reg_pressure[ira_pressure_classes[PCI]] and that
2395 model_excess_group_cost (struct model_pressure_group
*group
,
2396 int point
, int pci
, int delta
)
2400 cl
= ira_pressure_classes
[pci
];
2401 if (delta
< 0 && point
>= group
->limits
[pci
].point
)
2403 pressure
= MAX (group
->limits
[pci
].orig_pressure
,
2404 curr_reg_pressure
[cl
] + delta
);
2405 return -model_spill_cost (cl
, pressure
, curr_reg_pressure
[cl
]);
2410 if (point
> group
->limits
[pci
].point
)
2411 pressure
= group
->limits
[pci
].pressure
+ delta
;
2413 pressure
= curr_reg_pressure
[cl
] + delta
;
2415 if (pressure
> group
->limits
[pci
].pressure
)
2416 return model_spill_cost (cl
, group
->limits
[pci
].orig_pressure
,
2423 /* Return baseECC (MODEL_INSN (INSN)). Dump the costs to sched_dump
2427 model_excess_cost (rtx_insn
*insn
, bool print_p
)
2429 int point
, pci
, cl
, cost
, this_cost
, delta
;
2430 struct reg_pressure_data
*insn_reg_pressure
;
2431 int insn_death
[N_REG_CLASSES
];
2433 calculate_reg_deaths (insn
, insn_death
);
2434 point
= model_index (insn
);
2435 insn_reg_pressure
= INSN_REG_PRESSURE (insn
);
2439 fprintf (sched_dump
, ";;\t\t| %3d %4d | %4d %+3d |", point
,
2440 INSN_UID (insn
), INSN_PRIORITY (insn
), insn_delay (insn
));
2442 /* Sum up the individual costs for each register class. */
2443 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2445 cl
= ira_pressure_classes
[pci
];
2446 delta
= insn_reg_pressure
[pci
].set_increase
- insn_death
[cl
];
2447 this_cost
= model_excess_group_cost (&model_before_pressure
,
2451 fprintf (sched_dump
, " %s:[%d base cost %d]",
2452 reg_class_names
[cl
], delta
, this_cost
);
2456 fprintf (sched_dump
, "\n");
2461 /* Dump the next points of maximum pressure for GROUP. */
2464 model_dump_pressure_points (struct model_pressure_group
*group
)
2468 fprintf (sched_dump
, ";;\t\t| pressure points");
2469 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
2471 cl
= ira_pressure_classes
[pci
];
2472 fprintf (sched_dump
, " %s:[%d->%d at ", reg_class_names
[cl
],
2473 curr_reg_pressure
[cl
], group
->limits
[pci
].pressure
);
2474 if (group
->limits
[pci
].point
< model_num_insns
)
2475 fprintf (sched_dump
, "%d:%d]", group
->limits
[pci
].point
,
2476 INSN_UID (MODEL_INSN (group
->limits
[pci
].point
)));
2478 fprintf (sched_dump
, "end]");
2480 fprintf (sched_dump
, "\n");
2483 /* Set INSN_REG_PRESSURE_EXCESS_COST_CHANGE for INSNS[0...COUNT-1]. */
2486 model_set_excess_costs (rtx_insn
**insns
, int count
)
2488 int i
, cost
, priority_base
, priority
;
2491 /* Record the baseECC value for each instruction in the model schedule,
2492 except that negative costs are converted to zero ones now rather than
2493 later. Do not assign a cost to debug instructions, since they must
2494 not change code-generation decisions. Experiments suggest we also
2495 get better results by not assigning a cost to instructions from
2498 Set PRIORITY_BASE to baseP in the block comment above. This is the
2499 maximum priority of the "cheap" instructions, which should always
2500 include the next model instruction. */
2503 for (i
= 0; i
< count
; i
++)
2504 if (INSN_MODEL_INDEX (insns
[i
]))
2506 if (sched_verbose
>= 6 && !print_p
)
2508 fprintf (sched_dump
, MODEL_BAR
);
2509 fprintf (sched_dump
, ";;\t\t| Pressure costs for ready queue\n");
2510 model_dump_pressure_points (&model_before_pressure
);
2511 fprintf (sched_dump
, MODEL_BAR
);
2514 cost
= model_excess_cost (insns
[i
], print_p
);
2517 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]) - cost
;
2518 priority_base
= MAX (priority_base
, priority
);
2521 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = cost
;
2524 fprintf (sched_dump
, MODEL_BAR
);
2526 /* Use MAX (baseECC, 0) and baseP to calculcate ECC for each
2528 for (i
= 0; i
< count
; i
++)
2530 cost
= INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]);
2531 priority
= INSN_PRIORITY (insns
[i
]) - insn_delay (insns
[i
]);
2532 if (cost
> 0 && priority
> priority_base
)
2534 cost
+= priority_base
- priority
;
2535 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (insns
[i
]) = MAX (cost
, 0);
2541 /* Enum of rank_for_schedule heuristic decisions. */
2543 RFS_LIVE_RANGE_SHRINK1
, RFS_LIVE_RANGE_SHRINK2
,
2544 RFS_SCHED_GROUP
, RFS_PRESSURE_DELAY
, RFS_PRESSURE_TICK
,
2545 RFS_FEEDS_BACKTRACK_INSN
, RFS_PRIORITY
, RFS_SPECULATION
,
2546 RFS_SCHED_RANK
, RFS_LAST_INSN
, RFS_PRESSURE_INDEX
,
2547 RFS_DEP_COUNT
, RFS_TIE
, RFS_FUSION
, RFS_COST
, RFS_N
};
2549 /* Corresponding strings for print outs. */
2550 static const char *rfs_str
[RFS_N
] = {
2551 "RFS_LIVE_RANGE_SHRINK1", "RFS_LIVE_RANGE_SHRINK2",
2552 "RFS_SCHED_GROUP", "RFS_PRESSURE_DELAY", "RFS_PRESSURE_TICK",
2553 "RFS_FEEDS_BACKTRACK_INSN", "RFS_PRIORITY", "RFS_SPECULATION",
2554 "RFS_SCHED_RANK", "RFS_LAST_INSN", "RFS_PRESSURE_INDEX",
2555 "RFS_DEP_COUNT", "RFS_TIE", "RFS_FUSION", "RFS_COST" };
2557 /* Statistical breakdown of rank_for_schedule decisions. */
2558 struct rank_for_schedule_stats_t
{ unsigned stats
[RFS_N
]; };
2559 static rank_for_schedule_stats_t rank_for_schedule_stats
;
2561 /* Return the result of comparing insns TMP and TMP2 and update
2562 Rank_For_Schedule statistics. */
2564 rfs_result (enum rfs_decision decision
, int result
, rtx tmp
, rtx tmp2
)
2566 ++rank_for_schedule_stats
.stats
[decision
];
2568 INSN_LAST_RFS_WIN (tmp
) = decision
;
2569 else if (result
> 0)
2570 INSN_LAST_RFS_WIN (tmp2
) = decision
;
2576 /* Sorting predicate to move DEBUG_INSNs to the top of ready list, while
2577 keeping normal insns in original order. */
2580 rank_for_schedule_debug (const void *x
, const void *y
)
2582 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2583 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2585 /* Schedule debug insns as early as possible. */
2586 if (DEBUG_INSN_P (tmp
) && !DEBUG_INSN_P (tmp2
))
2588 else if (!DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2590 else if (DEBUG_INSN_P (tmp
) && DEBUG_INSN_P (tmp2
))
2591 return INSN_LUID (tmp
) - INSN_LUID (tmp2
);
2593 return INSN_RFS_DEBUG_ORIG_ORDER (tmp2
) - INSN_RFS_DEBUG_ORIG_ORDER (tmp
);
2596 /* Returns a positive value if x is preferred; returns a negative value if
2597 y is preferred. Should never return 0, since that will make the sort
2601 rank_for_schedule (const void *x
, const void *y
)
2603 rtx_insn
*tmp
= *(rtx_insn
* const *) y
;
2604 rtx_insn
*tmp2
= *(rtx_insn
* const *) x
;
2605 int tmp_class
, tmp2_class
;
2606 int val
, priority_val
, info_val
, diff
;
2608 if (live_range_shrinkage_p
)
2610 /* Don't use SCHED_PRESSURE_MODEL -- it results in much worse
2612 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
2613 if ((INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
) < 0
2614 || INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
) < 0)
2615 && (diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2616 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
))) != 0)
2617 return rfs_result (RFS_LIVE_RANGE_SHRINK1
, diff
, tmp
, tmp2
);
2618 /* Sort by INSN_LUID (original insn order), so that we make the
2619 sort stable. This minimizes instruction movement, thus
2620 minimizing sched's effect on debugging and cross-jumping. */
2621 return rfs_result (RFS_LIVE_RANGE_SHRINK2
,
2622 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2625 /* The insn in a schedule group should be issued the first. */
2626 if (flag_sched_group_heuristic
&&
2627 SCHED_GROUP_P (tmp
) != SCHED_GROUP_P (tmp2
))
2628 return rfs_result (RFS_SCHED_GROUP
, SCHED_GROUP_P (tmp2
) ? 1 : -1,
2631 /* Make sure that priority of TMP and TMP2 are initialized. */
2632 gcc_assert (INSN_PRIORITY_KNOWN (tmp
) && INSN_PRIORITY_KNOWN (tmp2
));
2636 /* The instruction that has the same fusion priority as the last
2637 instruction is the instruction we picked next. If that is not
2638 the case, we sort ready list firstly by fusion priority, then
2639 by priority, and at last by INSN_LUID. */
2640 int a
= INSN_FUSION_PRIORITY (tmp
);
2641 int b
= INSN_FUSION_PRIORITY (tmp2
);
2644 if (last_nondebug_scheduled_insn
2645 && !NOTE_P (last_nondebug_scheduled_insn
)
2646 && BLOCK_FOR_INSN (tmp
)
2647 == BLOCK_FOR_INSN (last_nondebug_scheduled_insn
))
2648 last
= INSN_FUSION_PRIORITY (last_nondebug_scheduled_insn
);
2650 if (a
!= last
&& b
!= last
)
2654 a
= INSN_PRIORITY (tmp
);
2655 b
= INSN_PRIORITY (tmp2
);
2658 return rfs_result (RFS_FUSION
, b
- a
, tmp
, tmp2
);
2660 return rfs_result (RFS_FUSION
,
2661 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2665 gcc_assert (last_nondebug_scheduled_insn
2666 && !NOTE_P (last_nondebug_scheduled_insn
));
2667 last
= INSN_PRIORITY (last_nondebug_scheduled_insn
);
2669 a
= abs (INSN_PRIORITY (tmp
) - last
);
2670 b
= abs (INSN_PRIORITY (tmp2
) - last
);
2672 return rfs_result (RFS_FUSION
, a
- b
, tmp
, tmp2
);
2674 return rfs_result (RFS_FUSION
,
2675 INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2678 return rfs_result (RFS_FUSION
, -1, tmp
, tmp2
);
2680 return rfs_result (RFS_FUSION
, 1, tmp
, tmp2
);
2683 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
2685 /* Prefer insn whose scheduling results in the smallest register
2687 if ((diff
= (INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp
)
2689 - INSN_REG_PRESSURE_EXCESS_COST_CHANGE (tmp2
)
2690 - insn_delay (tmp2
))))
2691 return rfs_result (RFS_PRESSURE_DELAY
, diff
, tmp
, tmp2
);
2694 if (sched_pressure
!= SCHED_PRESSURE_NONE
2695 && (INSN_TICK (tmp2
) > clock_var
|| INSN_TICK (tmp
) > clock_var
)
2696 && INSN_TICK (tmp2
) != INSN_TICK (tmp
))
2698 diff
= INSN_TICK (tmp
) - INSN_TICK (tmp2
);
2699 return rfs_result (RFS_PRESSURE_TICK
, diff
, tmp
, tmp2
);
2702 /* If we are doing backtracking in this schedule, prefer insns that
2703 have forward dependencies with negative cost against an insn that
2704 was already scheduled. */
2705 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2707 priority_val
= FEEDS_BACKTRACK_INSN (tmp2
) - FEEDS_BACKTRACK_INSN (tmp
);
2709 return rfs_result (RFS_FEEDS_BACKTRACK_INSN
, priority_val
, tmp
, tmp2
);
2712 /* Prefer insn with higher priority. */
2713 priority_val
= INSN_PRIORITY (tmp2
) - INSN_PRIORITY (tmp
);
2715 if (flag_sched_critical_path_heuristic
&& priority_val
)
2716 return rfs_result (RFS_PRIORITY
, priority_val
, tmp
, tmp2
);
2718 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) >= 0)
2720 int autopref
= autopref_rank_for_schedule (tmp
, tmp2
);
2725 /* Prefer speculative insn with greater dependencies weakness. */
2726 if (flag_sched_spec_insn_heuristic
&& spec_info
)
2732 ds1
= TODO_SPEC (tmp
) & SPECULATIVE
;
2734 dw1
= ds_weak (ds1
);
2738 ds2
= TODO_SPEC (tmp2
) & SPECULATIVE
;
2740 dw2
= ds_weak (ds2
);
2745 if (dw
> (NO_DEP_WEAK
/ 8) || dw
< -(NO_DEP_WEAK
/ 8))
2746 return rfs_result (RFS_SPECULATION
, dw
, tmp
, tmp2
);
2749 info_val
= (*current_sched_info
->rank
) (tmp
, tmp2
);
2750 if (flag_sched_rank_heuristic
&& info_val
)
2751 return rfs_result (RFS_SCHED_RANK
, info_val
, tmp
, tmp2
);
2753 /* Compare insns based on their relation to the last scheduled
2755 if (flag_sched_last_insn_heuristic
&& last_nondebug_scheduled_insn
)
2759 rtx_insn
*last
= last_nondebug_scheduled_insn
;
2761 /* Classify the instructions into three classes:
2762 1) Data dependent on last schedule insn.
2763 2) Anti/Output dependent on last scheduled insn.
2764 3) Independent of last scheduled insn, or has latency of one.
2765 Choose the insn from the highest numbered class if different. */
2766 dep1
= sd_find_dep_between (last
, tmp
, true);
2768 if (dep1
== NULL
|| dep_cost (dep1
) == 1)
2770 else if (/* Data dependence. */
2771 DEP_TYPE (dep1
) == REG_DEP_TRUE
)
2776 dep2
= sd_find_dep_between (last
, tmp2
, true);
2778 if (dep2
== NULL
|| dep_cost (dep2
) == 1)
2780 else if (/* Data dependence. */
2781 DEP_TYPE (dep2
) == REG_DEP_TRUE
)
2786 if ((val
= tmp2_class
- tmp_class
))
2787 return rfs_result (RFS_LAST_INSN
, val
, tmp
, tmp2
);
2790 /* Prefer instructions that occur earlier in the model schedule. */
2791 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
2793 diff
= model_index (tmp
) - model_index (tmp2
);
2795 return rfs_result (RFS_PRESSURE_INDEX
, diff
, tmp
, tmp2
);
2798 /* Prefer the insn which has more later insns that depend on it.
2799 This gives the scheduler more freedom when scheduling later
2800 instructions at the expense of added register pressure. */
2802 val
= (dep_list_size (tmp2
, SD_LIST_FORW
)
2803 - dep_list_size (tmp
, SD_LIST_FORW
));
2805 if (flag_sched_dep_count_heuristic
&& val
!= 0)
2806 return rfs_result (RFS_DEP_COUNT
, val
, tmp
, tmp2
);
2808 /* Sort by INSN_COST rather than INSN_LUID. This means that instructions
2809 which take longer to execute are prioritised and it leads to more
2810 dual-issue opportunities on in-order cores which have this feature. */
2812 if (INSN_COST (tmp
) != INSN_COST (tmp2
))
2813 return rfs_result (RFS_COST
, INSN_COST (tmp2
) - INSN_COST (tmp
),
2816 /* If insns are equally good, sort by INSN_LUID (original insn order),
2817 so that we make the sort stable. This minimizes instruction movement,
2818 thus minimizing sched's effect on debugging and cross-jumping. */
2819 return rfs_result (RFS_TIE
, INSN_LUID (tmp
) - INSN_LUID (tmp2
), tmp
, tmp2
);
2822 /* Resort the array A in which only element at index N may be out of order. */
2824 HAIFA_INLINE
static void
2825 swap_sort (rtx_insn
**a
, int n
)
2827 rtx_insn
*insn
= a
[n
- 1];
2830 while (i
>= 0 && rank_for_schedule (a
+ i
, &insn
) >= 0)
2838 /* Add INSN to the insn queue so that it can be executed at least
2839 N_CYCLES after the currently executing insn. Preserve insns
2840 chain for debugging purposes. REASON will be printed in debugging
2843 HAIFA_INLINE
static void
2844 queue_insn (rtx_insn
*insn
, int n_cycles
, const char *reason
)
2846 int next_q
= NEXT_Q_AFTER (q_ptr
, n_cycles
);
2847 rtx_insn_list
*link
= alloc_INSN_LIST (insn
, insn_queue
[next_q
]);
2850 gcc_assert (n_cycles
<= max_insn_queue_index
);
2851 gcc_assert (!DEBUG_INSN_P (insn
));
2853 insn_queue
[next_q
] = link
;
2856 if (sched_verbose
>= 2)
2858 fprintf (sched_dump
, ";;\t\tReady-->Q: insn %s: ",
2859 (*current_sched_info
->print_insn
) (insn
, 0));
2861 fprintf (sched_dump
, "queued for %d cycles (%s).\n", n_cycles
, reason
);
2864 QUEUE_INDEX (insn
) = next_q
;
2866 if (current_sched_info
->flags
& DO_BACKTRACKING
)
2868 new_tick
= clock_var
+ n_cycles
;
2869 if (INSN_TICK (insn
) == INVALID_TICK
|| INSN_TICK (insn
) < new_tick
)
2870 INSN_TICK (insn
) = new_tick
;
2872 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2873 && INSN_EXACT_TICK (insn
) < clock_var
+ n_cycles
)
2875 must_backtrack
= true;
2876 if (sched_verbose
>= 2)
2877 fprintf (sched_dump
, ";;\t\tcausing a backtrack.\n");
2882 /* Remove INSN from queue. */
2884 queue_remove (rtx_insn
*insn
)
2886 gcc_assert (QUEUE_INDEX (insn
) >= 0);
2887 remove_free_INSN_LIST_elem (insn
, &insn_queue
[QUEUE_INDEX (insn
)]);
2889 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
2892 /* Return a pointer to the bottom of the ready list, i.e. the insn
2893 with the lowest priority. */
2896 ready_lastpos (struct ready_list
*ready
)
2898 gcc_assert (ready
->n_ready
>= 1);
2899 return ready
->vec
+ ready
->first
- ready
->n_ready
+ 1;
2902 /* Add an element INSN to the ready list so that it ends up with the
2903 lowest/highest priority depending on FIRST_P. */
2905 HAIFA_INLINE
static void
2906 ready_add (struct ready_list
*ready
, rtx_insn
*insn
, bool first_p
)
2910 if (ready
->first
== ready
->n_ready
)
2912 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
,
2913 ready_lastpos (ready
),
2914 ready
->n_ready
* sizeof (rtx
));
2915 ready
->first
= ready
->veclen
- 1;
2917 ready
->vec
[ready
->first
- ready
->n_ready
] = insn
;
2921 if (ready
->first
== ready
->veclen
- 1)
2924 /* ready_lastpos() fails when called with (ready->n_ready == 0). */
2925 memmove (ready
->vec
+ ready
->veclen
- ready
->n_ready
- 1,
2926 ready_lastpos (ready
),
2927 ready
->n_ready
* sizeof (rtx
));
2928 ready
->first
= ready
->veclen
- 2;
2930 ready
->vec
[++(ready
->first
)] = insn
;
2934 if (DEBUG_INSN_P (insn
))
2937 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_READY
);
2938 QUEUE_INDEX (insn
) = QUEUE_READY
;
2940 if (INSN_EXACT_TICK (insn
) != INVALID_TICK
2941 && INSN_EXACT_TICK (insn
) < clock_var
)
2943 must_backtrack
= true;
2947 /* Remove the element with the highest priority from the ready list and
2950 HAIFA_INLINE
static rtx_insn
*
2951 ready_remove_first (struct ready_list
*ready
)
2955 gcc_assert (ready
->n_ready
);
2956 t
= ready
->vec
[ready
->first
--];
2958 if (DEBUG_INSN_P (t
))
2960 /* If the queue becomes empty, reset it. */
2961 if (ready
->n_ready
== 0)
2962 ready
->first
= ready
->veclen
- 1;
2964 gcc_assert (QUEUE_INDEX (t
) == QUEUE_READY
);
2965 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
2970 /* The following code implements multi-pass scheduling for the first
2971 cycle. In other words, we will try to choose ready insn which
2972 permits to start maximum number of insns on the same cycle. */
2974 /* Return a pointer to the element INDEX from the ready. INDEX for
2975 insn with the highest priority is 0, and the lowest priority has
2979 ready_element (struct ready_list
*ready
, int index
)
2981 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2983 return ready
->vec
[ready
->first
- index
];
2986 /* Remove the element INDEX from the ready list and return it. INDEX
2987 for insn with the highest priority is 0, and the lowest priority
2990 HAIFA_INLINE
static rtx_insn
*
2991 ready_remove (struct ready_list
*ready
, int index
)
2997 return ready_remove_first (ready
);
2998 gcc_assert (ready
->n_ready
&& index
< ready
->n_ready
);
2999 t
= ready
->vec
[ready
->first
- index
];
3001 if (DEBUG_INSN_P (t
))
3003 for (i
= index
; i
< ready
->n_ready
; i
++)
3004 ready
->vec
[ready
->first
- i
] = ready
->vec
[ready
->first
- i
- 1];
3005 QUEUE_INDEX (t
) = QUEUE_NOWHERE
;
3009 /* Remove INSN from the ready list. */
3011 ready_remove_insn (rtx_insn
*insn
)
3015 for (i
= 0; i
< readyp
->n_ready
; i
++)
3016 if (ready_element (readyp
, i
) == insn
)
3018 ready_remove (readyp
, i
);
3024 /* Calculate difference of two statistics set WAS and NOW.
3025 Result returned in WAS. */
3027 rank_for_schedule_stats_diff (rank_for_schedule_stats_t
*was
,
3028 const rank_for_schedule_stats_t
*now
)
3030 for (int i
= 0; i
< RFS_N
; ++i
)
3031 was
->stats
[i
] = now
->stats
[i
] - was
->stats
[i
];
3034 /* Print rank_for_schedule statistics. */
3036 print_rank_for_schedule_stats (const char *prefix
,
3037 const rank_for_schedule_stats_t
*stats
,
3038 struct ready_list
*ready
)
3040 for (int i
= 0; i
< RFS_N
; ++i
)
3041 if (stats
->stats
[i
])
3043 fprintf (sched_dump
, "%s%20s: %u", prefix
, rfs_str
[i
], stats
->stats
[i
]);
3046 /* Print out insns that won due to RFS_<I>. */
3048 rtx_insn
**p
= ready_lastpos (ready
);
3050 fprintf (sched_dump
, ":");
3051 /* Start with 1 since least-priority insn didn't have any wins. */
3052 for (int j
= 1; j
< ready
->n_ready
; ++j
)
3053 if (INSN_LAST_RFS_WIN (p
[j
]) == i
)
3054 fprintf (sched_dump
, " %s",
3055 (*current_sched_info
->print_insn
) (p
[j
], 0));
3057 fprintf (sched_dump
, "\n");
3061 /* Separate DEBUG_INSNS from normal insns. DEBUG_INSNs go to the end
3064 ready_sort_debug (struct ready_list
*ready
)
3067 rtx_insn
**first
= ready_lastpos (ready
);
3069 for (i
= 0; i
< ready
->n_ready
; ++i
)
3070 if (!DEBUG_INSN_P (first
[i
]))
3071 INSN_RFS_DEBUG_ORIG_ORDER (first
[i
]) = i
;
3073 qsort (first
, ready
->n_ready
, sizeof (rtx
), rank_for_schedule_debug
);
3076 /* Sort non-debug insns in the ready list READY by ascending priority.
3077 Assumes that all debug insns are separated from the real insns. */
3079 ready_sort_real (struct ready_list
*ready
)
3082 rtx_insn
**first
= ready_lastpos (ready
);
3083 int n_ready_real
= ready
->n_ready
- ready
->n_debug
;
3085 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
3086 for (i
= 0; i
< n_ready_real
; ++i
)
3087 setup_insn_reg_pressure_info (first
[i
]);
3088 else if (sched_pressure
== SCHED_PRESSURE_MODEL
3089 && model_curr_point
< model_num_insns
)
3090 model_set_excess_costs (first
, n_ready_real
);
3092 rank_for_schedule_stats_t stats1
;
3093 if (sched_verbose
>= 4)
3094 stats1
= rank_for_schedule_stats
;
3096 if (n_ready_real
== 2)
3097 swap_sort (first
, n_ready_real
);
3098 else if (n_ready_real
> 2)
3099 qsort (first
, n_ready_real
, sizeof (rtx
), rank_for_schedule
);
3101 if (sched_verbose
>= 4)
3103 rank_for_schedule_stats_diff (&stats1
, &rank_for_schedule_stats
);
3104 print_rank_for_schedule_stats (";;\t\t", &stats1
, ready
);
3108 /* Sort the ready list READY by ascending priority. */
3110 ready_sort (struct ready_list
*ready
)
3112 if (ready
->n_debug
> 0)
3113 ready_sort_debug (ready
);
3115 ready_sort_real (ready
);
3118 /* PREV is an insn that is ready to execute. Adjust its priority if that
3119 will help shorten or lengthen register lifetimes as appropriate. Also
3120 provide a hook for the target to tweak itself. */
3122 HAIFA_INLINE
static void
3123 adjust_priority (rtx_insn
*prev
)
3125 /* ??? There used to be code here to try and estimate how an insn
3126 affected register lifetimes, but it did it by looking at REG_DEAD
3127 notes, which we removed in schedule_region. Nor did it try to
3128 take into account register pressure or anything useful like that.
3130 Revisit when we have a machine model to work with and not before. */
3132 if (targetm
.sched
.adjust_priority
)
3133 INSN_PRIORITY (prev
) =
3134 targetm
.sched
.adjust_priority (prev
, INSN_PRIORITY (prev
));
3137 /* Advance DFA state STATE on one cycle. */
3139 advance_state (state_t state
)
3141 if (targetm
.sched
.dfa_pre_advance_cycle
)
3142 targetm
.sched
.dfa_pre_advance_cycle ();
3144 if (targetm
.sched
.dfa_pre_cycle_insn
)
3145 state_transition (state
,
3146 targetm
.sched
.dfa_pre_cycle_insn ());
3148 state_transition (state
, NULL
);
3150 if (targetm
.sched
.dfa_post_cycle_insn
)
3151 state_transition (state
,
3152 targetm
.sched
.dfa_post_cycle_insn ());
3154 if (targetm
.sched
.dfa_post_advance_cycle
)
3155 targetm
.sched
.dfa_post_advance_cycle ();
3158 /* Advance time on one cycle. */
3159 HAIFA_INLINE
static void
3160 advance_one_cycle (void)
3162 advance_state (curr_state
);
3163 if (sched_verbose
>= 4)
3164 fprintf (sched_dump
, ";;\tAdvance the current state.\n");
3167 /* Update register pressure after scheduling INSN. */
3169 update_register_pressure (rtx_insn
*insn
)
3171 struct reg_use_data
*use
;
3172 struct reg_set_data
*set
;
3174 gcc_checking_assert (!DEBUG_INSN_P (insn
));
3176 for (use
= INSN_REG_USE_LIST (insn
); use
!= NULL
; use
= use
->next_insn_use
)
3177 if (dying_use_p (use
))
3178 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3180 for (set
= INSN_REG_SET_LIST (insn
); set
!= NULL
; set
= set
->next_insn_set
)
3181 mark_regno_birth_or_death (curr_reg_live
, curr_reg_pressure
,
3185 /* Set up or update (if UPDATE_P) max register pressure (see its
3186 meaning in sched-int.h::_haifa_insn_data) for all current BB insns
3187 after insn AFTER. */
3189 setup_insn_max_reg_pressure (rtx_insn
*after
, bool update_p
)
3194 static int max_reg_pressure
[N_REG_CLASSES
];
3196 save_reg_pressure ();
3197 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3198 max_reg_pressure
[ira_pressure_classes
[i
]]
3199 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3200 for (insn
= NEXT_INSN (after
);
3201 insn
!= NULL_RTX
&& ! BARRIER_P (insn
)
3202 && BLOCK_FOR_INSN (insn
) == BLOCK_FOR_INSN (after
);
3203 insn
= NEXT_INSN (insn
))
3204 if (NONDEBUG_INSN_P (insn
))
3207 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3209 p
= max_reg_pressure
[ira_pressure_classes
[i
]];
3210 if (INSN_MAX_REG_PRESSURE (insn
)[i
] != p
)
3213 INSN_MAX_REG_PRESSURE (insn
)[i
]
3214 = max_reg_pressure
[ira_pressure_classes
[i
]];
3217 if (update_p
&& eq_p
)
3219 update_register_pressure (insn
);
3220 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3221 if (max_reg_pressure
[ira_pressure_classes
[i
]]
3222 < curr_reg_pressure
[ira_pressure_classes
[i
]])
3223 max_reg_pressure
[ira_pressure_classes
[i
]]
3224 = curr_reg_pressure
[ira_pressure_classes
[i
]];
3226 restore_reg_pressure ();
3229 /* Update the current register pressure after scheduling INSN. Update
3230 also max register pressure for unscheduled insns of the current
3233 update_reg_and_insn_max_reg_pressure (rtx_insn
*insn
)
3236 int before
[N_REG_CLASSES
];
3238 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3239 before
[i
] = curr_reg_pressure
[ira_pressure_classes
[i
]];
3240 update_register_pressure (insn
);
3241 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
3242 if (curr_reg_pressure
[ira_pressure_classes
[i
]] != before
[i
])
3244 if (i
< ira_pressure_classes_num
)
3245 setup_insn_max_reg_pressure (insn
, true);
3248 /* Set up register pressure at the beginning of basic block BB whose
3249 insns starting after insn AFTER. Set up also max register pressure
3250 for all insns of the basic block. */
3252 sched_setup_bb_reg_pressure_info (basic_block bb
, rtx_insn
*after
)
3254 gcc_assert (sched_pressure
== SCHED_PRESSURE_WEIGHTED
);
3255 initiate_bb_reg_pressure_info (bb
);
3256 setup_insn_max_reg_pressure (after
, false);
3259 /* If doing predication while scheduling, verify whether INSN, which
3260 has just been scheduled, clobbers the conditions of any
3261 instructions that must be predicated in order to break their
3262 dependencies. If so, remove them from the queues so that they will
3263 only be scheduled once their control dependency is resolved. */
3266 check_clobbered_conditions (rtx_insn
*insn
)
3271 if ((current_sched_info
->flags
& DO_PREDICATION
) == 0)
3274 find_all_hard_reg_sets (insn
, &t
, true);
3277 for (i
= 0; i
< ready
.n_ready
; i
++)
3279 rtx_insn
*x
= ready_element (&ready
, i
);
3280 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3282 ready_remove_insn (x
);
3286 for (i
= 0; i
<= max_insn_queue_index
; i
++)
3288 rtx_insn_list
*link
;
3289 int q
= NEXT_Q_AFTER (q_ptr
, i
);
3292 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
3294 rtx_insn
*x
= link
->insn ();
3295 if (TODO_SPEC (x
) == DEP_CONTROL
&& cond_clobbered_p (x
, t
))
3304 /* Return (in order):
3306 - positive if INSN adversely affects the pressure on one
3309 - negative if INSN reduces the pressure on one register class
3311 - 0 if INSN doesn't affect the pressure on any register class. */
3314 model_classify_pressure (struct model_insn_info
*insn
)
3316 struct reg_pressure_data
*reg_pressure
;
3317 int death
[N_REG_CLASSES
];
3320 calculate_reg_deaths (insn
->insn
, death
);
3321 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3323 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3325 cl
= ira_pressure_classes
[pci
];
3326 if (death
[cl
] < reg_pressure
[pci
].set_increase
)
3328 sum
+= reg_pressure
[pci
].set_increase
- death
[cl
];
3333 /* Return true if INSN1 should come before INSN2 in the model schedule. */
3336 model_order_p (struct model_insn_info
*insn1
, struct model_insn_info
*insn2
)
3338 unsigned int height1
, height2
;
3339 unsigned int priority1
, priority2
;
3341 /* Prefer instructions with a higher model priority. */
3342 if (insn1
->model_priority
!= insn2
->model_priority
)
3343 return insn1
->model_priority
> insn2
->model_priority
;
3345 /* Combine the length of the longest path of satisfied true dependencies
3346 that leads to each instruction (depth) with the length of the longest
3347 path of any dependencies that leads from the instruction (alap).
3348 Prefer instructions with the greatest combined length. If the combined
3349 lengths are equal, prefer instructions with the greatest depth.
3351 The idea is that, if we have a set S of "equal" instructions that each
3352 have ALAP value X, and we pick one such instruction I, any true-dependent
3353 successors of I that have ALAP value X - 1 should be preferred over S.
3354 This encourages the schedule to be "narrow" rather than "wide".
3355 However, if I is a low-priority instruction that we decided to
3356 schedule because of its model_classify_pressure, and if there
3357 is a set of higher-priority instructions T, the aforementioned
3358 successors of I should not have the edge over T. */
3359 height1
= insn1
->depth
+ insn1
->alap
;
3360 height2
= insn2
->depth
+ insn2
->alap
;
3361 if (height1
!= height2
)
3362 return height1
> height2
;
3363 if (insn1
->depth
!= insn2
->depth
)
3364 return insn1
->depth
> insn2
->depth
;
3366 /* We have no real preference between INSN1 an INSN2 as far as attempts
3367 to reduce pressure go. Prefer instructions with higher priorities. */
3368 priority1
= INSN_PRIORITY (insn1
->insn
);
3369 priority2
= INSN_PRIORITY (insn2
->insn
);
3370 if (priority1
!= priority2
)
3371 return priority1
> priority2
;
3373 /* Use the original rtl sequence as a tie-breaker. */
3374 return insn1
< insn2
;
3377 /* Add INSN to the model worklist immediately after PREV. Add it to the
3378 beginning of the list if PREV is null. */
3381 model_add_to_worklist_at (struct model_insn_info
*insn
,
3382 struct model_insn_info
*prev
)
3384 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_NOWHERE
);
3385 QUEUE_INDEX (insn
->insn
) = QUEUE_READY
;
3390 insn
->next
= prev
->next
;
3395 insn
->next
= model_worklist
;
3396 model_worklist
= insn
;
3399 insn
->next
->prev
= insn
;
3402 /* Remove INSN from the model worklist. */
3405 model_remove_from_worklist (struct model_insn_info
*insn
)
3407 gcc_assert (QUEUE_INDEX (insn
->insn
) == QUEUE_READY
);
3408 QUEUE_INDEX (insn
->insn
) = QUEUE_NOWHERE
;
3411 insn
->prev
->next
= insn
->next
;
3413 model_worklist
= insn
->next
;
3415 insn
->next
->prev
= insn
->prev
;
3418 /* Add INSN to the model worklist. Start looking for a suitable position
3419 between neighbors PREV and NEXT, testing at most MAX_SCHED_READY_INSNS
3420 insns either side. A null PREV indicates the beginning of the list and
3421 a null NEXT indicates the end. */
3424 model_add_to_worklist (struct model_insn_info
*insn
,
3425 struct model_insn_info
*prev
,
3426 struct model_insn_info
*next
)
3430 count
= MAX_SCHED_READY_INSNS
;
3431 if (count
> 0 && prev
&& model_order_p (insn
, prev
))
3437 while (count
> 0 && prev
&& model_order_p (insn
, prev
));
3439 while (count
> 0 && next
&& model_order_p (next
, insn
))
3445 model_add_to_worklist_at (insn
, prev
);
3448 /* INSN may now have a higher priority (in the model_order_p sense)
3449 than before. Move it up the worklist if necessary. */
3452 model_promote_insn (struct model_insn_info
*insn
)
3454 struct model_insn_info
*prev
;
3458 count
= MAX_SCHED_READY_INSNS
;
3459 while (count
> 0 && prev
&& model_order_p (insn
, prev
))
3464 if (prev
!= insn
->prev
)
3466 model_remove_from_worklist (insn
);
3467 model_add_to_worklist_at (insn
, prev
);
3471 /* Add INSN to the end of the model schedule. */
3474 model_add_to_schedule (rtx_insn
*insn
)
3478 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
3479 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
3481 point
= model_schedule
.length ();
3482 model_schedule
.quick_push (insn
);
3483 INSN_MODEL_INDEX (insn
) = point
+ 1;
3486 /* Analyze the instructions that are to be scheduled, setting up
3487 MODEL_INSN_INFO (...) and model_num_insns accordingly. Add ready
3488 instructions to model_worklist. */
3491 model_analyze_insns (void)
3493 rtx_insn
*start
, *end
, *iter
;
3494 sd_iterator_def sd_it
;
3496 struct model_insn_info
*insn
, *con
;
3498 model_num_insns
= 0;
3499 start
= PREV_INSN (current_sched_info
->next_tail
);
3500 end
= current_sched_info
->prev_head
;
3501 for (iter
= start
; iter
!= end
; iter
= PREV_INSN (iter
))
3502 if (NONDEBUG_INSN_P (iter
))
3504 insn
= MODEL_INSN_INFO (iter
);
3506 FOR_EACH_DEP (iter
, SD_LIST_FORW
, sd_it
, dep
)
3508 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3509 if (con
->insn
&& insn
->alap
< con
->alap
+ 1)
3510 insn
->alap
= con
->alap
+ 1;
3513 insn
->old_queue
= QUEUE_INDEX (iter
);
3514 QUEUE_INDEX (iter
) = QUEUE_NOWHERE
;
3516 insn
->unscheduled_preds
= dep_list_size (iter
, SD_LIST_HARD_BACK
);
3517 if (insn
->unscheduled_preds
== 0)
3518 model_add_to_worklist (insn
, NULL
, model_worklist
);
3524 /* The global state describes the register pressure at the start of the
3525 model schedule. Initialize GROUP accordingly. */
3528 model_init_pressure_group (struct model_pressure_group
*group
)
3532 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3534 cl
= ira_pressure_classes
[pci
];
3535 group
->limits
[pci
].pressure
= curr_reg_pressure
[cl
];
3536 group
->limits
[pci
].point
= 0;
3538 /* Use index model_num_insns to record the state after the last
3539 instruction in the model schedule. */
3540 group
->model
= XNEWVEC (struct model_pressure_data
,
3541 (model_num_insns
+ 1) * ira_pressure_classes_num
);
3544 /* Record that MODEL_REF_PRESSURE (GROUP, POINT, PCI) is PRESSURE.
3545 Update the maximum pressure for the whole schedule. */
3548 model_record_pressure (struct model_pressure_group
*group
,
3549 int point
, int pci
, int pressure
)
3551 MODEL_REF_PRESSURE (group
, point
, pci
) = pressure
;
3552 if (group
->limits
[pci
].pressure
< pressure
)
3554 group
->limits
[pci
].pressure
= pressure
;
3555 group
->limits
[pci
].point
= point
;
3559 /* INSN has just been added to the end of the model schedule. Record its
3560 register-pressure information. */
3563 model_record_pressures (struct model_insn_info
*insn
)
3565 struct reg_pressure_data
*reg_pressure
;
3566 int point
, pci
, cl
, delta
;
3567 int death
[N_REG_CLASSES
];
3569 point
= model_index (insn
->insn
);
3570 if (sched_verbose
>= 2)
3574 fprintf (sched_dump
, "\n;;\tModel schedule:\n;;\n");
3575 fprintf (sched_dump
, ";;\t| idx insn | mpri hght dpth prio |\n");
3577 fprintf (sched_dump
, ";;\t| %3d %4d | %4d %4d %4d %4d | %-30s ",
3578 point
, INSN_UID (insn
->insn
), insn
->model_priority
,
3579 insn
->depth
+ insn
->alap
, insn
->depth
,
3580 INSN_PRIORITY (insn
->insn
),
3581 str_pattern_slim (PATTERN (insn
->insn
)));
3583 calculate_reg_deaths (insn
->insn
, death
);
3584 reg_pressure
= INSN_REG_PRESSURE (insn
->insn
);
3585 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3587 cl
= ira_pressure_classes
[pci
];
3588 delta
= reg_pressure
[pci
].set_increase
- death
[cl
];
3589 if (sched_verbose
>= 2)
3590 fprintf (sched_dump
, " %s:[%d,%+d]", reg_class_names
[cl
],
3591 curr_reg_pressure
[cl
], delta
);
3592 model_record_pressure (&model_before_pressure
, point
, pci
,
3593 curr_reg_pressure
[cl
]);
3595 if (sched_verbose
>= 2)
3596 fprintf (sched_dump
, "\n");
3599 /* All instructions have been added to the model schedule. Record the
3600 final register pressure in GROUP and set up all MODEL_MAX_PRESSUREs. */
3603 model_record_final_pressures (struct model_pressure_group
*group
)
3605 int point
, pci
, max_pressure
, ref_pressure
, cl
;
3607 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3609 /* Record the final pressure for this class. */
3610 cl
= ira_pressure_classes
[pci
];
3611 point
= model_num_insns
;
3612 ref_pressure
= curr_reg_pressure
[cl
];
3613 model_record_pressure (group
, point
, pci
, ref_pressure
);
3615 /* Record the original maximum pressure. */
3616 group
->limits
[pci
].orig_pressure
= group
->limits
[pci
].pressure
;
3618 /* Update the MODEL_MAX_PRESSURE for every point of the schedule. */
3619 max_pressure
= ref_pressure
;
3620 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3624 ref_pressure
= MODEL_REF_PRESSURE (group
, point
, pci
);
3625 max_pressure
= MAX (max_pressure
, ref_pressure
);
3626 MODEL_MAX_PRESSURE (group
, point
, pci
) = max_pressure
;
3631 /* Update all successors of INSN, given that INSN has just been scheduled. */
3634 model_add_successors_to_worklist (struct model_insn_info
*insn
)
3636 sd_iterator_def sd_it
;
3637 struct model_insn_info
*con
;
3640 FOR_EACH_DEP (insn
->insn
, SD_LIST_FORW
, sd_it
, dep
)
3642 con
= MODEL_INSN_INFO (DEP_CON (dep
));
3643 /* Ignore debug instructions, and instructions from other blocks. */
3646 con
->unscheduled_preds
--;
3648 /* Update the depth field of each true-dependent successor.
3649 Increasing the depth gives them a higher priority than
3651 if (DEP_TYPE (dep
) == REG_DEP_TRUE
&& con
->depth
< insn
->depth
+ 1)
3653 con
->depth
= insn
->depth
+ 1;
3654 if (QUEUE_INDEX (con
->insn
) == QUEUE_READY
)
3655 model_promote_insn (con
);
3658 /* If this is a true dependency, or if there are no remaining
3659 dependencies for CON (meaning that CON only had non-true
3660 dependencies), make sure that CON is on the worklist.
3661 We don't bother otherwise because it would tend to fill the
3662 worklist with a lot of low-priority instructions that are not
3663 yet ready to issue. */
3664 if ((con
->depth
> 0 || con
->unscheduled_preds
== 0)
3665 && QUEUE_INDEX (con
->insn
) == QUEUE_NOWHERE
)
3666 model_add_to_worklist (con
, insn
, insn
->next
);
3671 /* Give INSN a higher priority than any current instruction, then give
3672 unscheduled predecessors of INSN a higher priority still. If any of
3673 those predecessors are not on the model worklist, do the same for its
3674 predecessors, and so on. */
3677 model_promote_predecessors (struct model_insn_info
*insn
)
3679 struct model_insn_info
*pro
, *first
;
3680 sd_iterator_def sd_it
;
3683 if (sched_verbose
>= 7)
3684 fprintf (sched_dump
, ";;\t+--- priority of %d = %d, priority of",
3685 INSN_UID (insn
->insn
), model_next_priority
);
3686 insn
->model_priority
= model_next_priority
++;
3687 model_remove_from_worklist (insn
);
3688 model_add_to_worklist_at (insn
, NULL
);
3693 FOR_EACH_DEP (insn
->insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
3695 pro
= MODEL_INSN_INFO (DEP_PRO (dep
));
3696 /* The first test is to ignore debug instructions, and instructions
3697 from other blocks. */
3699 && pro
->model_priority
!= model_next_priority
3700 && QUEUE_INDEX (pro
->insn
) != QUEUE_SCHEDULED
)
3702 pro
->model_priority
= model_next_priority
;
3703 if (sched_verbose
>= 7)
3704 fprintf (sched_dump
, " %d", INSN_UID (pro
->insn
));
3705 if (QUEUE_INDEX (pro
->insn
) == QUEUE_READY
)
3707 /* PRO is already in the worklist, but it now has
3708 a higher priority than before. Move it at the
3709 appropriate place. */
3710 model_remove_from_worklist (pro
);
3711 model_add_to_worklist (pro
, NULL
, model_worklist
);
3715 /* PRO isn't in the worklist. Recursively process
3716 its predecessors until we find one that is. */
3727 if (sched_verbose
>= 7)
3728 fprintf (sched_dump
, " = %d\n", model_next_priority
);
3729 model_next_priority
++;
3732 /* Pick one instruction from model_worklist and process it. */
3735 model_choose_insn (void)
3737 struct model_insn_info
*insn
, *fallback
;
3740 if (sched_verbose
>= 7)
3742 fprintf (sched_dump
, ";;\t+--- worklist:\n");
3743 insn
= model_worklist
;
3744 count
= MAX_SCHED_READY_INSNS
;
3745 while (count
> 0 && insn
)
3747 fprintf (sched_dump
, ";;\t+--- %d [%d, %d, %d, %d]\n",
3748 INSN_UID (insn
->insn
), insn
->model_priority
,
3749 insn
->depth
+ insn
->alap
, insn
->depth
,
3750 INSN_PRIORITY (insn
->insn
));
3756 /* Look for a ready instruction whose model_classify_priority is zero
3757 or negative, picking the highest-priority one. Adding such an
3758 instruction to the schedule now should do no harm, and may actually
3761 Failing that, see whether there is an instruction with the highest
3762 extant model_priority that is not yet ready, but which would reduce
3763 pressure if it became ready. This is designed to catch cases like:
3765 (set (mem (reg R1)) (reg R2))
3767 where the instruction is the last remaining use of R1 and where the
3768 value of R2 is not yet available (or vice versa). The death of R1
3769 means that this instruction already reduces pressure. It is of
3770 course possible that the computation of R2 involves other registers
3771 that are hard to kill, but such cases are rare enough for this
3772 heuristic to be a win in general.
3774 Failing that, just pick the highest-priority instruction in the
3776 count
= MAX_SCHED_READY_INSNS
;
3777 insn
= model_worklist
;
3781 if (count
== 0 || !insn
)
3783 insn
= fallback
? fallback
: model_worklist
;
3786 if (insn
->unscheduled_preds
)
3788 if (model_worklist
->model_priority
== insn
->model_priority
3790 && model_classify_pressure (insn
) < 0)
3795 if (model_classify_pressure (insn
) <= 0)
3802 if (sched_verbose
>= 7 && insn
!= model_worklist
)
3804 if (insn
->unscheduled_preds
)
3805 fprintf (sched_dump
, ";;\t+--- promoting insn %d, with dependencies\n",
3806 INSN_UID (insn
->insn
));
3808 fprintf (sched_dump
, ";;\t+--- promoting insn %d, which is ready\n",
3809 INSN_UID (insn
->insn
));
3811 if (insn
->unscheduled_preds
)
3812 /* INSN isn't yet ready to issue. Give all its predecessors the
3813 highest priority. */
3814 model_promote_predecessors (insn
);
3817 /* INSN is ready. Add it to the end of model_schedule and
3818 process its successors. */
3819 model_add_successors_to_worklist (insn
);
3820 model_remove_from_worklist (insn
);
3821 model_add_to_schedule (insn
->insn
);
3822 model_record_pressures (insn
);
3823 update_register_pressure (insn
->insn
);
3827 /* Restore all QUEUE_INDEXs to the values that they had before
3828 model_start_schedule was called. */
3831 model_reset_queue_indices (void)
3836 FOR_EACH_VEC_ELT (model_schedule
, i
, insn
)
3837 QUEUE_INDEX (insn
) = MODEL_INSN_INFO (insn
)->old_queue
;
3840 /* We have calculated the model schedule and spill costs. Print a summary
3844 model_dump_pressure_summary (void)
3848 fprintf (sched_dump
, ";; Pressure summary:");
3849 for (pci
= 0; pci
< ira_pressure_classes_num
; pci
++)
3851 cl
= ira_pressure_classes
[pci
];
3852 fprintf (sched_dump
, " %s:%d", reg_class_names
[cl
],
3853 model_before_pressure
.limits
[pci
].pressure
);
3855 fprintf (sched_dump
, "\n\n");
3858 /* Initialize the SCHED_PRESSURE_MODEL information for the current
3859 scheduling region. */
3862 model_start_schedule (basic_block bb
)
3864 model_next_priority
= 1;
3865 model_schedule
.create (sched_max_luid
);
3866 model_insns
= XCNEWVEC (struct model_insn_info
, sched_max_luid
);
3868 gcc_assert (bb
== BLOCK_FOR_INSN (NEXT_INSN (current_sched_info
->prev_head
)));
3869 initiate_reg_pressure_info (df_get_live_in (bb
));
3871 model_analyze_insns ();
3872 model_init_pressure_group (&model_before_pressure
);
3873 while (model_worklist
)
3874 model_choose_insn ();
3875 gcc_assert (model_num_insns
== (int) model_schedule
.length ());
3876 if (sched_verbose
>= 2)
3877 fprintf (sched_dump
, "\n");
3879 model_record_final_pressures (&model_before_pressure
);
3880 model_reset_queue_indices ();
3882 XDELETEVEC (model_insns
);
3884 model_curr_point
= 0;
3885 initiate_reg_pressure_info (df_get_live_in (bb
));
3886 if (sched_verbose
>= 1)
3887 model_dump_pressure_summary ();
3890 /* Free the information associated with GROUP. */
3893 model_finalize_pressure_group (struct model_pressure_group
*group
)
3895 XDELETEVEC (group
->model
);
3898 /* Free the information created by model_start_schedule. */
3901 model_end_schedule (void)
3903 model_finalize_pressure_group (&model_before_pressure
);
3904 model_schedule
.release ();
3907 /* Prepare reg pressure scheduling for basic block BB. */
3909 sched_pressure_start_bb (basic_block bb
)
3911 /* Set the number of available registers for each class taking into account
3912 relative probability of current basic block versus function prologue and
3914 * If the basic block executes much more often than the prologue/epilogue
3915 (e.g., inside a hot loop), then cost of spill in the prologue is close to
3916 nil, so the effective number of available registers is
3917 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl] - 0).
3918 * If the basic block executes as often as the prologue/epilogue,
3919 then spill in the block is as costly as in the prologue, so the effective
3920 number of available registers is
3921 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3922 - call_saved_regs_num[cl]).
3923 Note that all-else-equal, we prefer to spill in the prologue, since that
3924 allows "extra" registers for other basic blocks of the function.
3925 * If the basic block is on the cold path of the function and executes
3926 rarely, then we should always prefer to spill in the block, rather than
3927 in the prologue/epilogue. The effective number of available register is
3928 (ira_class_hard_regs_num[cl] - fixed_regs_num[cl]
3929 - call_saved_regs_num[cl]). */
3932 int entry_freq
= ENTRY_BLOCK_PTR_FOR_FN (cfun
)->count
.to_frequency (cfun
);
3933 int bb_freq
= bb
->count
.to_frequency (cfun
);
3937 if (entry_freq
== 0)
3938 entry_freq
= bb_freq
= 1;
3940 if (bb_freq
< entry_freq
)
3941 bb_freq
= entry_freq
;
3943 for (i
= 0; i
< ira_pressure_classes_num
; ++i
)
3945 enum reg_class cl
= ira_pressure_classes
[i
];
3946 sched_class_regs_num
[cl
] = ira_class_hard_regs_num
[cl
]
3947 - fixed_regs_num
[cl
];
3948 sched_class_regs_num
[cl
]
3949 -= (call_saved_regs_num
[cl
] * entry_freq
) / bb_freq
;
3953 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
3954 model_start_schedule (bb
);
3957 /* A structure that holds local state for the loop in schedule_block. */
3958 struct sched_block_state
3960 /* True if no real insns have been scheduled in the current cycle. */
3961 bool first_cycle_insn_p
;
3962 /* True if a shadow insn has been scheduled in the current cycle, which
3963 means that no more normal insns can be issued. */
3964 bool shadows_only_p
;
3965 /* True if we're winding down a modulo schedule, which means that we only
3966 issue insns with INSN_EXACT_TICK set. */
3967 bool modulo_epilogue
;
3968 /* Initialized with the machine's issue rate every cycle, and updated
3969 by calls to the variable_issue hook. */
3973 /* INSN is the "currently executing insn". Launch each insn which was
3974 waiting on INSN. READY is the ready list which contains the insns
3975 that are ready to fire. CLOCK is the current cycle. The function
3976 returns necessary cycle advance after issuing the insn (it is not
3977 zero for insns in a schedule group). */
3980 schedule_insn (rtx_insn
*insn
)
3982 sd_iterator_def sd_it
;
3987 if (sched_verbose
>= 1)
3989 struct reg_pressure_data
*pressure_info
;
3990 fprintf (sched_dump
, ";;\t%3i--> %s %-40s:",
3991 clock_var
, (*current_sched_info
->print_insn
) (insn
, 1),
3992 str_pattern_slim (PATTERN (insn
)));
3994 if (recog_memoized (insn
) < 0)
3995 fprintf (sched_dump
, "nothing");
3997 print_reservation (sched_dump
, insn
);
3998 pressure_info
= INSN_REG_PRESSURE (insn
);
3999 if (pressure_info
!= NULL
)
4001 fputc (':', sched_dump
);
4002 for (i
= 0; i
< ira_pressure_classes_num
; i
++)
4003 fprintf (sched_dump
, "%s%s%+d(%d)",
4004 scheduled_insns
.length () > 1
4006 < INSN_LUID (scheduled_insns
[scheduled_insns
.length () - 2]) ? "@" : "",
4007 reg_class_names
[ira_pressure_classes
[i
]],
4008 pressure_info
[i
].set_increase
, pressure_info
[i
].change
);
4010 if (sched_pressure
== SCHED_PRESSURE_MODEL
4011 && model_curr_point
< model_num_insns
4012 && model_index (insn
) == model_curr_point
)
4013 fprintf (sched_dump
, ":model %d", model_curr_point
);
4014 fputc ('\n', sched_dump
);
4017 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
&& !DEBUG_INSN_P (insn
))
4018 update_reg_and_insn_max_reg_pressure (insn
);
4020 /* Scheduling instruction should have all its dependencies resolved and
4021 should have been removed from the ready list. */
4022 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_HARD_BACK
));
4024 /* Reset debug insns invalidated by moving this insn. */
4025 if (MAY_HAVE_DEBUG_BIND_INSNS
&& !DEBUG_INSN_P (insn
))
4026 for (sd_it
= sd_iterator_start (insn
, SD_LIST_BACK
);
4027 sd_iterator_cond (&sd_it
, &dep
);)
4029 rtx_insn
*dbg
= DEP_PRO (dep
);
4030 struct reg_use_data
*use
, *next
;
4032 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4034 sd_iterator_next (&sd_it
);
4038 gcc_assert (DEBUG_BIND_INSN_P (dbg
));
4040 if (sched_verbose
>= 6)
4041 fprintf (sched_dump
, ";;\t\tresetting: debug insn %d\n",
4044 /* ??? Rather than resetting the debug insn, we might be able
4045 to emit a debug temp before the just-scheduled insn, but
4046 this would involve checking that the expression at the
4047 point of the debug insn is equivalent to the expression
4048 before the just-scheduled insn. They might not be: the
4049 expression in the debug insn may depend on other insns not
4050 yet scheduled that set MEMs, REGs or even other debug
4051 insns. It's not clear that attempting to preserve debug
4052 information in these cases is worth the effort, given how
4053 uncommon these resets are and the likelihood that the debug
4054 temps introduced won't survive the schedule change. */
4055 INSN_VAR_LOCATION_LOC (dbg
) = gen_rtx_UNKNOWN_VAR_LOC ();
4056 df_insn_rescan (dbg
);
4058 /* Unknown location doesn't use any registers. */
4059 for (use
= INSN_REG_USE_LIST (dbg
); use
!= NULL
; use
= next
)
4061 struct reg_use_data
*prev
= use
;
4063 /* Remove use from the cyclic next_regno_use chain first. */
4064 while (prev
->next_regno_use
!= use
)
4065 prev
= prev
->next_regno_use
;
4066 prev
->next_regno_use
= use
->next_regno_use
;
4067 next
= use
->next_insn_use
;
4070 INSN_REG_USE_LIST (dbg
) = NULL
;
4072 /* We delete rather than resolve these deps, otherwise we
4073 crash in sched_free_deps(), because forward deps are
4074 expected to be released before backward deps. */
4075 sd_delete_dep (sd_it
);
4078 gcc_assert (QUEUE_INDEX (insn
) == QUEUE_NOWHERE
);
4079 QUEUE_INDEX (insn
) = QUEUE_SCHEDULED
;
4081 if (sched_pressure
== SCHED_PRESSURE_MODEL
4082 && model_curr_point
< model_num_insns
4083 && NONDEBUG_INSN_P (insn
))
4085 if (model_index (insn
) == model_curr_point
)
4088 while (model_curr_point
< model_num_insns
4089 && (QUEUE_INDEX (MODEL_INSN (model_curr_point
))
4090 == QUEUE_SCHEDULED
));
4092 model_recompute (insn
);
4093 model_update_limit_points ();
4094 update_register_pressure (insn
);
4095 if (sched_verbose
>= 2)
4096 print_curr_reg_pressure ();
4099 gcc_assert (INSN_TICK (insn
) >= MIN_TICK
);
4100 if (INSN_TICK (insn
) > clock_var
)
4101 /* INSN has been prematurely moved from the queue to the ready list.
4102 This is possible only if following flags are set. */
4103 gcc_assert (flag_sched_stalled_insns
|| sched_fusion
);
4105 /* ??? Probably, if INSN is scheduled prematurely, we should leave
4106 INSN_TICK untouched. This is a machine-dependent issue, actually. */
4107 INSN_TICK (insn
) = clock_var
;
4109 check_clobbered_conditions (insn
);
4111 /* Update dependent instructions. First, see if by scheduling this insn
4112 now we broke a dependence in a way that requires us to change another
4114 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
4115 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
4117 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4118 rtx_insn
*pro
= DEP_PRO (dep
);
4119 if (QUEUE_INDEX (pro
) != QUEUE_SCHEDULED
4120 && desc
!= NULL
&& desc
->insn
== pro
)
4121 apply_replacement (dep
, false);
4124 /* Go through and resolve forward dependencies. */
4125 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4126 sd_iterator_cond (&sd_it
, &dep
);)
4128 rtx_insn
*next
= DEP_CON (dep
);
4129 bool cancelled
= (DEP_STATUS (dep
) & DEP_CANCELLED
) != 0;
4131 /* Resolve the dependence between INSN and NEXT.
4132 sd_resolve_dep () moves current dep to another list thus
4133 advancing the iterator. */
4134 sd_resolve_dep (sd_it
);
4138 if (must_restore_pattern_p (next
, dep
))
4139 restore_pattern (dep
, false);
4143 /* Don't bother trying to mark next as ready if insn is a debug
4144 insn. If insn is the last hard dependency, it will have
4145 already been discounted. */
4146 if (DEBUG_INSN_P (insn
) && !DEBUG_INSN_P (next
))
4149 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4153 effective_cost
= try_ready (next
);
4155 if (effective_cost
>= 0
4156 && SCHED_GROUP_P (next
)
4157 && advance
< effective_cost
)
4158 advance
= effective_cost
;
4161 /* Check always has only one forward dependence (to the first insn in
4162 the recovery block), therefore, this will be executed only once. */
4164 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4165 fix_recovery_deps (RECOVERY_BLOCK (insn
));
4169 /* Annotate the instruction with issue information -- TImode
4170 indicates that the instruction is expected not to be able
4171 to issue on the same cycle as the previous insn. A machine
4172 may use this information to decide how the instruction should
4175 && GET_CODE (PATTERN (insn
)) != USE
4176 && GET_CODE (PATTERN (insn
)) != CLOBBER
4177 && !DEBUG_INSN_P (insn
))
4179 if (reload_completed
)
4180 PUT_MODE (insn
, clock_var
> last_clock_var
? TImode
: VOIDmode
);
4181 last_clock_var
= clock_var
;
4184 if (nonscheduled_insns_begin
!= NULL_RTX
)
4185 /* Indicate to debug counters that INSN is scheduled. */
4186 nonscheduled_insns_begin
= insn
;
4191 /* Functions for handling of notes. */
4193 /* Add note list that ends on FROM_END to the end of TO_ENDP. */
4195 concat_note_lists (rtx_insn
*from_end
, rtx_insn
**to_endp
)
4197 rtx_insn
*from_start
;
4199 /* It's easy when have nothing to concat. */
4200 if (from_end
== NULL
)
4203 /* It's also easy when destination is empty. */
4204 if (*to_endp
== NULL
)
4206 *to_endp
= from_end
;
4210 from_start
= from_end
;
4211 while (PREV_INSN (from_start
) != NULL
)
4212 from_start
= PREV_INSN (from_start
);
4214 SET_PREV_INSN (from_start
) = *to_endp
;
4215 SET_NEXT_INSN (*to_endp
) = from_start
;
4216 *to_endp
= from_end
;
4219 /* Delete notes between HEAD and TAIL and put them in the chain
4220 of notes ended by NOTE_LIST. */
4222 remove_notes (rtx_insn
*head
, rtx_insn
*tail
)
4224 rtx_insn
*next_tail
, *insn
, *next
;
4227 if (head
== tail
&& !INSN_P (head
))
4230 next_tail
= NEXT_INSN (tail
);
4231 for (insn
= head
; insn
!= next_tail
; insn
= next
)
4233 next
= NEXT_INSN (insn
);
4237 switch (NOTE_KIND (insn
))
4239 case NOTE_INSN_BASIC_BLOCK
:
4242 case NOTE_INSN_EPILOGUE_BEG
:
4246 add_reg_note (next
, REG_SAVE_NOTE
,
4247 GEN_INT (NOTE_INSN_EPILOGUE_BEG
));
4255 /* Add the note to list that ends at NOTE_LIST. */
4256 SET_PREV_INSN (insn
) = note_list
;
4257 SET_NEXT_INSN (insn
) = NULL_RTX
;
4259 SET_NEXT_INSN (note_list
) = insn
;
4264 gcc_assert ((sel_sched_p () || insn
!= tail
) && insn
!= head
);
4268 /* A structure to record enough data to allow us to backtrack the scheduler to
4269 a previous state. */
4270 struct haifa_saved_data
4272 /* Next entry on the list. */
4273 struct haifa_saved_data
*next
;
4275 /* Backtracking is associated with scheduling insns that have delay slots.
4276 DELAY_PAIR points to the structure that contains the insns involved, and
4277 the number of cycles between them. */
4278 struct delay_pair
*delay_pair
;
4280 /* Data used by the frontend (e.g. sched-ebb or sched-rgn). */
4281 void *fe_saved_data
;
4282 /* Data used by the backend. */
4283 void *be_saved_data
;
4285 /* Copies of global state. */
4286 int clock_var
, last_clock_var
;
4287 struct ready_list ready
;
4290 rtx_insn
*last_scheduled_insn
;
4291 rtx_insn
*last_nondebug_scheduled_insn
;
4292 rtx_insn
*nonscheduled_insns_begin
;
4293 int cycle_issued_insns
;
4295 /* Copies of state used in the inner loop of schedule_block. */
4296 struct sched_block_state sched_block
;
4298 /* We don't need to save q_ptr, as its value is arbitrary and we can set it
4299 to 0 when restoring. */
4301 rtx_insn_list
**insn_queue
;
4303 /* Describe pattern replacements that occurred since this backtrack point
4305 vec
<dep_t
> replacement_deps
;
4306 vec
<int> replace_apply
;
4308 /* A copy of the next-cycle replacement vectors at the time of the backtrack
4310 vec
<dep_t
> next_cycle_deps
;
4311 vec
<int> next_cycle_apply
;
4314 /* A record, in reverse order, of all scheduled insns which have delay slots
4315 and may require backtracking. */
4316 static struct haifa_saved_data
*backtrack_queue
;
4318 /* For every dependency of INSN, set the FEEDS_BACKTRACK_INSN bit according
4321 mark_backtrack_feeds (rtx_insn
*insn
, int set_p
)
4323 sd_iterator_def sd_it
;
4325 FOR_EACH_DEP (insn
, SD_LIST_HARD_BACK
, sd_it
, dep
)
4327 FEEDS_BACKTRACK_INSN (DEP_PRO (dep
)) = set_p
;
4331 /* Save the current scheduler state so that we can backtrack to it
4332 later if necessary. PAIR gives the insns that make it necessary to
4333 save this point. SCHED_BLOCK is the local state of schedule_block
4334 that need to be saved. */
4336 save_backtrack_point (struct delay_pair
*pair
,
4337 struct sched_block_state sched_block
)
4340 struct haifa_saved_data
*save
= XNEW (struct haifa_saved_data
);
4342 save
->curr_state
= xmalloc (dfa_state_size
);
4343 memcpy (save
->curr_state
, curr_state
, dfa_state_size
);
4345 save
->ready
.first
= ready
.first
;
4346 save
->ready
.n_ready
= ready
.n_ready
;
4347 save
->ready
.n_debug
= ready
.n_debug
;
4348 save
->ready
.veclen
= ready
.veclen
;
4349 save
->ready
.vec
= XNEWVEC (rtx_insn
*, ready
.veclen
);
4350 memcpy (save
->ready
.vec
, ready
.vec
, ready
.veclen
* sizeof (rtx
));
4352 save
->insn_queue
= XNEWVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
4353 save
->q_size
= q_size
;
4354 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4356 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4357 save
->insn_queue
[i
] = copy_INSN_LIST (insn_queue
[q
]);
4360 save
->clock_var
= clock_var
;
4361 save
->last_clock_var
= last_clock_var
;
4362 save
->cycle_issued_insns
= cycle_issued_insns
;
4363 save
->last_scheduled_insn
= last_scheduled_insn
;
4364 save
->last_nondebug_scheduled_insn
= last_nondebug_scheduled_insn
;
4365 save
->nonscheduled_insns_begin
= nonscheduled_insns_begin
;
4367 save
->sched_block
= sched_block
;
4369 save
->replacement_deps
.create (0);
4370 save
->replace_apply
.create (0);
4371 save
->next_cycle_deps
= next_cycle_replace_deps
.copy ();
4372 save
->next_cycle_apply
= next_cycle_apply
.copy ();
4374 if (current_sched_info
->save_state
)
4375 save
->fe_saved_data
= (*current_sched_info
->save_state
) ();
4377 if (targetm
.sched
.alloc_sched_context
)
4379 save
->be_saved_data
= targetm
.sched
.alloc_sched_context ();
4380 targetm
.sched
.init_sched_context (save
->be_saved_data
, false);
4383 save
->be_saved_data
= NULL
;
4385 save
->delay_pair
= pair
;
4387 save
->next
= backtrack_queue
;
4388 backtrack_queue
= save
;
4392 mark_backtrack_feeds (pair
->i2
, 1);
4393 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4394 INSN_EXACT_TICK (pair
->i2
) = clock_var
+ pair_delay (pair
);
4395 SHADOW_P (pair
->i2
) = pair
->stages
== 0;
4396 pair
= pair
->next_same_i1
;
4400 /* Walk the ready list and all queues. If any insns have unresolved backwards
4401 dependencies, these must be cancelled deps, broken by predication. Set or
4402 clear (depending on SET) the DEP_CANCELLED bit in DEP_STATUS. */
4405 toggle_cancelled_flags (bool set
)
4408 sd_iterator_def sd_it
;
4411 if (ready
.n_ready
> 0)
4413 rtx_insn
**first
= ready_lastpos (&ready
);
4414 for (i
= 0; i
< ready
.n_ready
; i
++)
4415 FOR_EACH_DEP (first
[i
], SD_LIST_BACK
, sd_it
, dep
)
4416 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4419 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4421 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4424 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4426 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4427 rtx_insn_list
*link
;
4428 for (link
= insn_queue
[q
]; link
; link
= link
->next ())
4430 rtx_insn
*insn
= link
->insn ();
4431 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4432 if (!DEBUG_INSN_P (DEP_PRO (dep
)))
4435 DEP_STATUS (dep
) |= DEP_CANCELLED
;
4437 DEP_STATUS (dep
) &= ~DEP_CANCELLED
;
4443 /* Undo the replacements that have occurred after backtrack point SAVE
4446 undo_replacements_for_backtrack (struct haifa_saved_data
*save
)
4448 while (!save
->replacement_deps
.is_empty ())
4450 dep_t dep
= save
->replacement_deps
.pop ();
4451 int apply_p
= save
->replace_apply
.pop ();
4454 restore_pattern (dep
, true);
4456 apply_replacement (dep
, true);
4458 save
->replacement_deps
.release ();
4459 save
->replace_apply
.release ();
4462 /* Pop entries from the SCHEDULED_INSNS vector up to and including INSN.
4463 Restore their dependencies to an unresolved state, and mark them as
4467 unschedule_insns_until (rtx_insn
*insn
)
4469 auto_vec
<rtx_insn
*> recompute_vec
;
4471 /* Make two passes over the insns to be unscheduled. First, we clear out
4472 dependencies and other trivial bookkeeping. */
4476 sd_iterator_def sd_it
;
4479 last
= scheduled_insns
.pop ();
4481 /* This will be changed by restore_backtrack_point if the insn is in
4483 QUEUE_INDEX (last
) = QUEUE_NOWHERE
;
4485 INSN_TICK (last
) = INVALID_TICK
;
4487 if (modulo_ii
> 0 && INSN_UID (last
) < modulo_iter0_max_uid
)
4488 modulo_insns_scheduled
--;
4490 for (sd_it
= sd_iterator_start (last
, SD_LIST_RES_FORW
);
4491 sd_iterator_cond (&sd_it
, &dep
);)
4493 rtx_insn
*con
= DEP_CON (dep
);
4494 sd_unresolve_dep (sd_it
);
4495 if (!MUST_RECOMPUTE_SPEC_P (con
))
4497 MUST_RECOMPUTE_SPEC_P (con
) = 1;
4498 recompute_vec
.safe_push (con
);
4506 /* A second pass, to update ready and speculation status for insns
4507 depending on the unscheduled ones. The first pass must have
4508 popped the scheduled_insns vector up to the point where we
4509 restart scheduling, as recompute_todo_spec requires it to be
4511 while (!recompute_vec
.is_empty ())
4515 con
= recompute_vec
.pop ();
4516 MUST_RECOMPUTE_SPEC_P (con
) = 0;
4517 if (!sd_lists_empty_p (con
, SD_LIST_HARD_BACK
))
4519 TODO_SPEC (con
) = HARD_DEP
;
4520 INSN_TICK (con
) = INVALID_TICK
;
4521 if (PREDICATED_PAT (con
) != NULL_RTX
)
4522 haifa_change_pattern (con
, ORIG_PAT (con
));
4524 else if (QUEUE_INDEX (con
) != QUEUE_SCHEDULED
)
4525 TODO_SPEC (con
) = recompute_todo_spec (con
, true);
4529 /* Restore scheduler state from the topmost entry on the backtracking queue.
4530 PSCHED_BLOCK_P points to the local data of schedule_block that we must
4531 overwrite with the saved data.
4532 The caller must already have called unschedule_insns_until. */
4535 restore_last_backtrack_point (struct sched_block_state
*psched_block
)
4538 struct haifa_saved_data
*save
= backtrack_queue
;
4540 backtrack_queue
= save
->next
;
4542 if (current_sched_info
->restore_state
)
4543 (*current_sched_info
->restore_state
) (save
->fe_saved_data
);
4545 if (targetm
.sched
.alloc_sched_context
)
4547 targetm
.sched
.set_sched_context (save
->be_saved_data
);
4548 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4551 /* Do this first since it clobbers INSN_TICK of the involved
4553 undo_replacements_for_backtrack (save
);
4555 /* Clear the QUEUE_INDEX of everything in the ready list or one
4557 if (ready
.n_ready
> 0)
4559 rtx_insn
**first
= ready_lastpos (&ready
);
4560 for (i
= 0; i
< ready
.n_ready
; i
++)
4562 rtx_insn
*insn
= first
[i
];
4563 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
4564 INSN_TICK (insn
) = INVALID_TICK
;
4567 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4569 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4571 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4573 rtx_insn
*x
= link
->insn ();
4574 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
4575 INSN_TICK (x
) = INVALID_TICK
;
4577 free_INSN_LIST_list (&insn_queue
[q
]);
4581 ready
= save
->ready
;
4583 if (ready
.n_ready
> 0)
4585 rtx_insn
**first
= ready_lastpos (&ready
);
4586 for (i
= 0; i
< ready
.n_ready
; i
++)
4588 rtx_insn
*insn
= first
[i
];
4589 QUEUE_INDEX (insn
) = QUEUE_READY
;
4590 TODO_SPEC (insn
) = recompute_todo_spec (insn
, true);
4591 INSN_TICK (insn
) = save
->clock_var
;
4596 q_size
= save
->q_size
;
4597 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4599 int q
= NEXT_Q_AFTER (q_ptr
, i
);
4601 insn_queue
[q
] = save
->insn_queue
[q
];
4603 for (rtx_insn_list
*link
= insn_queue
[q
]; link
; link
= link
->next ())
4605 rtx_insn
*x
= link
->insn ();
4606 QUEUE_INDEX (x
) = i
;
4607 TODO_SPEC (x
) = recompute_todo_spec (x
, true);
4608 INSN_TICK (x
) = save
->clock_var
+ i
;
4611 free (save
->insn_queue
);
4613 toggle_cancelled_flags (true);
4615 clock_var
= save
->clock_var
;
4616 last_clock_var
= save
->last_clock_var
;
4617 cycle_issued_insns
= save
->cycle_issued_insns
;
4618 last_scheduled_insn
= save
->last_scheduled_insn
;
4619 last_nondebug_scheduled_insn
= save
->last_nondebug_scheduled_insn
;
4620 nonscheduled_insns_begin
= save
->nonscheduled_insns_begin
;
4622 *psched_block
= save
->sched_block
;
4624 memcpy (curr_state
, save
->curr_state
, dfa_state_size
);
4625 free (save
->curr_state
);
4627 mark_backtrack_feeds (save
->delay_pair
->i2
, 0);
4629 gcc_assert (next_cycle_replace_deps
.is_empty ());
4630 next_cycle_replace_deps
= save
->next_cycle_deps
.copy ();
4631 next_cycle_apply
= save
->next_cycle_apply
.copy ();
4635 for (save
= backtrack_queue
; save
; save
= save
->next
)
4637 mark_backtrack_feeds (save
->delay_pair
->i2
, 1);
4641 /* Discard all data associated with the topmost entry in the backtrack
4642 queue. If RESET_TICK is false, we just want to free the data. If true,
4643 we are doing this because we discovered a reason to backtrack. In the
4644 latter case, also reset the INSN_TICK for the shadow insn. */
4646 free_topmost_backtrack_point (bool reset_tick
)
4648 struct haifa_saved_data
*save
= backtrack_queue
;
4651 backtrack_queue
= save
->next
;
4655 struct delay_pair
*pair
= save
->delay_pair
;
4658 INSN_TICK (pair
->i2
) = INVALID_TICK
;
4659 INSN_EXACT_TICK (pair
->i2
) = INVALID_TICK
;
4660 pair
= pair
->next_same_i1
;
4662 undo_replacements_for_backtrack (save
);
4666 save
->replacement_deps
.release ();
4667 save
->replace_apply
.release ();
4670 if (targetm
.sched
.free_sched_context
)
4671 targetm
.sched
.free_sched_context (save
->be_saved_data
);
4672 if (current_sched_info
->restore_state
)
4673 free (save
->fe_saved_data
);
4674 for (i
= 0; i
<= max_insn_queue_index
; i
++)
4675 free_INSN_LIST_list (&save
->insn_queue
[i
]);
4676 free (save
->insn_queue
);
4677 free (save
->curr_state
);
4678 free (save
->ready
.vec
);
4682 /* Free the entire backtrack queue. */
4684 free_backtrack_queue (void)
4686 while (backtrack_queue
)
4687 free_topmost_backtrack_point (false);
4690 /* Apply a replacement described by DESC. If IMMEDIATELY is false, we
4691 may have to postpone the replacement until the start of the next cycle,
4692 at which point we will be called again with IMMEDIATELY true. This is
4693 only done for machines which have instruction packets with explicit
4694 parallelism however. */
4696 apply_replacement (dep_t dep
, bool immediately
)
4698 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4699 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4701 next_cycle_replace_deps
.safe_push (dep
);
4702 next_cycle_apply
.safe_push (1);
4708 if (QUEUE_INDEX (desc
->insn
) == QUEUE_SCHEDULED
)
4711 if (sched_verbose
>= 5)
4712 fprintf (sched_dump
, "applying replacement for insn %d\n",
4713 INSN_UID (desc
->insn
));
4715 success
= validate_change (desc
->insn
, desc
->loc
, desc
->newval
, 0);
4716 gcc_assert (success
);
4718 rtx_insn
*insn
= DEP_PRO (dep
);
4720 /* Recompute priority since dependent priorities may have changed. */
4721 priority (insn
, true);
4722 update_insn_after_change (desc
->insn
);
4724 if ((TODO_SPEC (desc
->insn
) & (HARD_DEP
| DEP_POSTPONED
)) == 0)
4725 fix_tick_ready (desc
->insn
);
4727 if (backtrack_queue
!= NULL
)
4729 backtrack_queue
->replacement_deps
.safe_push (dep
);
4730 backtrack_queue
->replace_apply
.safe_push (1);
4735 /* We have determined that a pattern involved in DEP must be restored.
4736 If IMMEDIATELY is false, we may have to postpone the replacement
4737 until the start of the next cycle, at which point we will be called
4738 again with IMMEDIATELY true. */
4740 restore_pattern (dep_t dep
, bool immediately
)
4742 rtx_insn
*next
= DEP_CON (dep
);
4743 int tick
= INSN_TICK (next
);
4745 /* If we already scheduled the insn, the modified version is
4747 if (QUEUE_INDEX (next
) == QUEUE_SCHEDULED
)
4750 if (!immediately
&& targetm
.sched
.exposed_pipeline
&& reload_completed
)
4752 next_cycle_replace_deps
.safe_push (dep
);
4753 next_cycle_apply
.safe_push (0);
4758 if (DEP_TYPE (dep
) == REG_DEP_CONTROL
)
4760 if (sched_verbose
>= 5)
4761 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4763 haifa_change_pattern (next
, ORIG_PAT (next
));
4767 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
4770 if (sched_verbose
>= 5)
4771 fprintf (sched_dump
, "restoring pattern for insn %d\n",
4772 INSN_UID (desc
->insn
));
4773 tick
= INSN_TICK (desc
->insn
);
4775 success
= validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
4776 gcc_assert (success
);
4778 rtx_insn
*insn
= DEP_PRO (dep
);
4780 if (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
4782 /* Recompute priority since dependent priorities may have changed. */
4783 priority (insn
, true);
4786 update_insn_after_change (desc
->insn
);
4788 if (backtrack_queue
!= NULL
)
4790 backtrack_queue
->replacement_deps
.safe_push (dep
);
4791 backtrack_queue
->replace_apply
.safe_push (0);
4794 INSN_TICK (next
) = tick
;
4795 if (TODO_SPEC (next
) == DEP_POSTPONED
)
4798 if (sd_lists_empty_p (next
, SD_LIST_BACK
))
4799 TODO_SPEC (next
) = 0;
4800 else if (!sd_lists_empty_p (next
, SD_LIST_HARD_BACK
))
4801 TODO_SPEC (next
) = HARD_DEP
;
4804 /* Perform pattern replacements that were queued up until the next
4807 perform_replacements_new_cycle (void)
4811 FOR_EACH_VEC_ELT (next_cycle_replace_deps
, i
, dep
)
4813 int apply_p
= next_cycle_apply
[i
];
4815 apply_replacement (dep
, true);
4817 restore_pattern (dep
, true);
4819 next_cycle_replace_deps
.truncate (0);
4820 next_cycle_apply
.truncate (0);
4823 /* Compute INSN_TICK_ESTIMATE for INSN. PROCESSED is a bitmap of
4824 instructions we've previously encountered, a set bit prevents
4825 recursion. BUDGET is a limit on how far ahead we look, it is
4826 reduced on recursive calls. Return true if we produced a good
4827 estimate, or false if we exceeded the budget. */
4829 estimate_insn_tick (bitmap processed
, rtx_insn
*insn
, int budget
)
4831 sd_iterator_def sd_it
;
4833 int earliest
= INSN_TICK (insn
);
4835 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
4837 rtx_insn
*pro
= DEP_PRO (dep
);
4840 if (DEP_STATUS (dep
) & DEP_CANCELLED
)
4843 if (QUEUE_INDEX (pro
) == QUEUE_SCHEDULED
)
4844 gcc_assert (INSN_TICK (pro
) + dep_cost (dep
) <= INSN_TICK (insn
));
4847 int cost
= dep_cost (dep
);
4850 if (!bitmap_bit_p (processed
, INSN_LUID (pro
)))
4852 if (!estimate_insn_tick (processed
, pro
, budget
- cost
))
4855 gcc_assert (INSN_TICK_ESTIMATE (pro
) != INVALID_TICK
);
4856 t
= INSN_TICK_ESTIMATE (pro
) + cost
;
4857 if (earliest
== INVALID_TICK
|| t
> earliest
)
4861 bitmap_set_bit (processed
, INSN_LUID (insn
));
4862 INSN_TICK_ESTIMATE (insn
) = earliest
;
4866 /* Examine the pair of insns in P, and estimate (optimistically, assuming
4867 infinite resources) the cycle in which the delayed shadow can be issued.
4868 Return the number of cycles that must pass before the real insn can be
4869 issued in order to meet this constraint. */
4871 estimate_shadow_tick (struct delay_pair
*p
)
4873 auto_bitmap processed
;
4877 cutoff
= !estimate_insn_tick (processed
, p
->i2
,
4878 max_insn_queue_index
+ pair_delay (p
));
4880 return max_insn_queue_index
;
4881 t
= INSN_TICK_ESTIMATE (p
->i2
) - (clock_var
+ pair_delay (p
) + 1);
4887 /* If INSN has no unresolved backwards dependencies, add it to the schedule and
4888 recursively resolve all its forward dependencies. */
4890 resolve_dependencies (rtx_insn
*insn
)
4892 sd_iterator_def sd_it
;
4895 /* Don't use sd_lists_empty_p; it ignores debug insns. */
4896 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (insn
)) != NULL
4897 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (insn
)) != NULL
)
4900 if (sched_verbose
>= 4)
4901 fprintf (sched_dump
, ";;\tquickly resolving %d\n", INSN_UID (insn
));
4903 if (QUEUE_INDEX (insn
) >= 0)
4904 queue_remove (insn
);
4906 scheduled_insns
.safe_push (insn
);
4908 /* Update dependent instructions. */
4909 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
4910 sd_iterator_cond (&sd_it
, &dep
);)
4912 rtx_insn
*next
= DEP_CON (dep
);
4914 if (sched_verbose
>= 4)
4915 fprintf (sched_dump
, ";;\t\tdep %d against %d\n", INSN_UID (insn
),
4918 /* Resolve the dependence between INSN and NEXT.
4919 sd_resolve_dep () moves current dep to another list thus
4920 advancing the iterator. */
4921 sd_resolve_dep (sd_it
);
4923 if (!IS_SPECULATION_BRANCHY_CHECK_P (insn
))
4925 resolve_dependencies (next
);
4928 /* Check always has only one forward dependence (to the first insn in
4929 the recovery block), therefore, this will be executed only once. */
4931 gcc_assert (sd_lists_empty_p (insn
, SD_LIST_FORW
));
4937 /* Return the head and tail pointers of ebb starting at BEG and ending
4940 get_ebb_head_tail (basic_block beg
, basic_block end
,
4941 rtx_insn
**headp
, rtx_insn
**tailp
)
4943 rtx_insn
*beg_head
= BB_HEAD (beg
);
4944 rtx_insn
* beg_tail
= BB_END (beg
);
4945 rtx_insn
* end_head
= BB_HEAD (end
);
4946 rtx_insn
* end_tail
= BB_END (end
);
4948 /* Don't include any notes or labels at the beginning of the BEG
4949 basic block, or notes at the end of the END basic blocks. */
4951 if (LABEL_P (beg_head
))
4952 beg_head
= NEXT_INSN (beg_head
);
4954 while (beg_head
!= beg_tail
)
4955 if (NOTE_P (beg_head
))
4956 beg_head
= NEXT_INSN (beg_head
);
4957 else if (DEBUG_INSN_P (beg_head
))
4959 rtx_insn
* note
, *next
;
4961 for (note
= NEXT_INSN (beg_head
);
4965 next
= NEXT_INSN (note
);
4968 if (sched_verbose
>= 9)
4969 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
4971 reorder_insns_nobb (note
, note
, PREV_INSN (beg_head
));
4973 if (BLOCK_FOR_INSN (note
) != beg
)
4974 df_insn_change_bb (note
, beg
);
4976 else if (!DEBUG_INSN_P (note
))
4988 end_head
= beg_head
;
4989 else if (LABEL_P (end_head
))
4990 end_head
= NEXT_INSN (end_head
);
4992 while (end_head
!= end_tail
)
4993 if (NOTE_P (end_tail
))
4994 end_tail
= PREV_INSN (end_tail
);
4995 else if (DEBUG_INSN_P (end_tail
))
4997 rtx_insn
* note
, *prev
;
4999 for (note
= PREV_INSN (end_tail
);
5003 prev
= PREV_INSN (note
);
5006 if (sched_verbose
>= 9)
5007 fprintf (sched_dump
, "reorder %i\n", INSN_UID (note
));
5009 reorder_insns_nobb (note
, note
, end_tail
);
5011 if (end_tail
== BB_END (end
))
5012 BB_END (end
) = note
;
5014 if (BLOCK_FOR_INSN (note
) != end
)
5015 df_insn_change_bb (note
, end
);
5017 else if (!DEBUG_INSN_P (note
))
5029 /* Return nonzero if there are no real insns in the range [ HEAD, TAIL ]. */
5032 no_real_insns_p (const rtx_insn
*head
, const rtx_insn
*tail
)
5034 while (head
!= NEXT_INSN (tail
))
5036 if (!NOTE_P (head
) && !LABEL_P (head
))
5038 head
= NEXT_INSN (head
);
5043 /* Restore-other-notes: NOTE_LIST is the end of a chain of notes
5044 previously found among the insns. Insert them just before HEAD. */
5046 restore_other_notes (rtx_insn
*head
, basic_block head_bb
)
5050 rtx_insn
*note_head
= note_list
;
5053 head_bb
= BLOCK_FOR_INSN (head
);
5055 head
= NEXT_INSN (bb_note (head_bb
));
5057 while (PREV_INSN (note_head
))
5059 set_block_for_insn (note_head
, head_bb
);
5060 note_head
= PREV_INSN (note_head
);
5062 /* In the above cycle we've missed this note. */
5063 set_block_for_insn (note_head
, head_bb
);
5065 SET_PREV_INSN (note_head
) = PREV_INSN (head
);
5066 SET_NEXT_INSN (PREV_INSN (head
)) = note_head
;
5067 SET_PREV_INSN (head
) = note_list
;
5068 SET_NEXT_INSN (note_list
) = head
;
5070 if (BLOCK_FOR_INSN (head
) != head_bb
)
5071 BB_END (head_bb
) = note_list
;
5079 /* When we know we are going to discard the schedule due to a failed attempt
5080 at modulo scheduling, undo all replacements. */
5082 undo_all_replacements (void)
5087 FOR_EACH_VEC_ELT (scheduled_insns
, i
, insn
)
5089 sd_iterator_def sd_it
;
5092 /* See if we must undo a replacement. */
5093 for (sd_it
= sd_iterator_start (insn
, SD_LIST_RES_FORW
);
5094 sd_iterator_cond (&sd_it
, &dep
); sd_iterator_next (&sd_it
))
5096 struct dep_replacement
*desc
= DEP_REPLACE (dep
);
5098 validate_change (desc
->insn
, desc
->loc
, desc
->orig
, 0);
5103 /* Return first non-scheduled insn in the current scheduling block.
5104 This is mostly used for debug-counter purposes. */
5106 first_nonscheduled_insn (void)
5108 rtx_insn
*insn
= (nonscheduled_insns_begin
!= NULL_RTX
5109 ? nonscheduled_insns_begin
5110 : current_sched_info
->prev_head
);
5114 insn
= next_nonnote_nondebug_insn (insn
);
5116 while (QUEUE_INDEX (insn
) == QUEUE_SCHEDULED
);
5121 /* Move insns that became ready to fire from queue to ready list. */
5124 queue_to_ready (struct ready_list
*ready
)
5127 rtx_insn_list
*link
;
5128 rtx_insn
*skip_insn
;
5130 q_ptr
= NEXT_Q (q_ptr
);
5132 if (dbg_cnt (sched_insn
) == false)
5133 /* If debug counter is activated do not requeue the first
5134 nonscheduled insn. */
5135 skip_insn
= first_nonscheduled_insn ();
5139 /* Add all pending insns that can be scheduled without stalls to the
5141 for (link
= insn_queue
[q_ptr
]; link
; link
= link
->next ())
5143 insn
= link
->insn ();
5146 if (sched_verbose
>= 2)
5147 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5148 (*current_sched_info
->print_insn
) (insn
, 0));
5150 /* If the ready list is full, delay the insn for 1 cycle.
5151 See the comment in schedule_block for the rationale. */
5152 if (!reload_completed
5153 && (ready
->n_ready
- ready
->n_debug
> MAX_SCHED_READY_INSNS
5154 || (sched_pressure
== SCHED_PRESSURE_MODEL
5155 /* Limit pressure recalculations to MAX_SCHED_READY_INSNS
5156 instructions too. */
5157 && model_index (insn
) > (model_curr_point
5158 + MAX_SCHED_READY_INSNS
)))
5159 && !(sched_pressure
== SCHED_PRESSURE_MODEL
5160 && model_curr_point
< model_num_insns
5161 /* Always allow the next model instruction to issue. */
5162 && model_index (insn
) == model_curr_point
)
5163 && !SCHED_GROUP_P (insn
)
5164 && insn
!= skip_insn
)
5166 if (sched_verbose
>= 2)
5167 fprintf (sched_dump
, "keeping in queue, ready full\n");
5168 queue_insn (insn
, 1, "ready full");
5172 ready_add (ready
, insn
, false);
5173 if (sched_verbose
>= 2)
5174 fprintf (sched_dump
, "moving to ready without stalls\n");
5177 free_INSN_LIST_list (&insn_queue
[q_ptr
]);
5179 /* If there are no ready insns, stall until one is ready and add all
5180 of the pending insns at that point to the ready list. */
5181 if (ready
->n_ready
== 0)
5185 for (stalls
= 1; stalls
<= max_insn_queue_index
; stalls
++)
5187 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5189 for (; link
; link
= link
->next ())
5191 insn
= link
->insn ();
5194 if (sched_verbose
>= 2)
5195 fprintf (sched_dump
, ";;\t\tQ-->Ready: insn %s: ",
5196 (*current_sched_info
->print_insn
) (insn
, 0));
5198 ready_add (ready
, insn
, false);
5199 if (sched_verbose
>= 2)
5200 fprintf (sched_dump
, "moving to ready with %d stalls\n", stalls
);
5202 free_INSN_LIST_list (&insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]);
5204 advance_one_cycle ();
5209 advance_one_cycle ();
5212 q_ptr
= NEXT_Q_AFTER (q_ptr
, stalls
);
5213 clock_var
+= stalls
;
5214 if (sched_verbose
>= 2)
5215 fprintf (sched_dump
, ";;\tAdvancing clock by %d cycle[s] to %d\n",
5220 /* Used by early_queue_to_ready. Determines whether it is "ok" to
5221 prematurely move INSN from the queue to the ready list. Currently,
5222 if a target defines the hook 'is_costly_dependence', this function
5223 uses the hook to check whether there exist any dependences which are
5224 considered costly by the target, between INSN and other insns that
5225 have already been scheduled. Dependences are checked up to Y cycles
5226 back, with default Y=1; The flag -fsched-stalled-insns-dep=Y allows
5227 controlling this value.
5228 (Other considerations could be taken into account instead (or in
5229 addition) depending on user flags and target hooks. */
5232 ok_for_early_queue_removal (rtx_insn
*insn
)
5234 if (targetm
.sched
.is_costly_dependence
)
5237 int i
= scheduled_insns
.length ();
5238 for (n_cycles
= flag_sched_stalled_insns_dep
; n_cycles
; n_cycles
--)
5244 rtx_insn
*prev_insn
= scheduled_insns
[i
];
5246 if (!NOTE_P (prev_insn
))
5250 dep
= sd_find_dep_between (prev_insn
, insn
, true);
5254 cost
= dep_cost (dep
);
5256 if (targetm
.sched
.is_costly_dependence (dep
, cost
,
5257 flag_sched_stalled_insns_dep
- n_cycles
))
5262 if (GET_MODE (prev_insn
) == TImode
) /* end of dispatch group */
5275 /* Remove insns from the queue, before they become "ready" with respect
5276 to FU latency considerations. */
5279 early_queue_to_ready (state_t state
, struct ready_list
*ready
)
5282 rtx_insn_list
*link
;
5283 rtx_insn_list
*next_link
;
5284 rtx_insn_list
*prev_link
;
5287 state_t temp_state
= alloca (dfa_state_size
);
5289 int insns_removed
= 0;
5292 Flag '-fsched-stalled-insns=X' determines the aggressiveness of this
5295 X == 0: There is no limit on how many queued insns can be removed
5296 prematurely. (flag_sched_stalled_insns = -1).
5298 X >= 1: Only X queued insns can be removed prematurely in each
5299 invocation. (flag_sched_stalled_insns = X).
5301 Otherwise: Early queue removal is disabled.
5302 (flag_sched_stalled_insns = 0)
5305 if (! flag_sched_stalled_insns
)
5308 for (stalls
= 0; stalls
<= max_insn_queue_index
; stalls
++)
5310 if ((link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)]))
5312 if (sched_verbose
> 6)
5313 fprintf (sched_dump
, ";; look at index %d + %d\n", q_ptr
, stalls
);
5318 next_link
= link
->next ();
5319 insn
= link
->insn ();
5320 if (insn
&& sched_verbose
> 6)
5321 print_rtl_single (sched_dump
, insn
);
5323 memcpy (temp_state
, state
, dfa_state_size
);
5324 if (recog_memoized (insn
) < 0)
5325 /* non-negative to indicate that it's not ready
5326 to avoid infinite Q->R->Q->R... */
5329 cost
= state_transition (temp_state
, insn
);
5331 if (sched_verbose
>= 6)
5332 fprintf (sched_dump
, "transition cost = %d\n", cost
);
5334 move_to_ready
= false;
5337 move_to_ready
= ok_for_early_queue_removal (insn
);
5338 if (move_to_ready
== true)
5340 /* move from Q to R */
5342 ready_add (ready
, insn
, false);
5345 XEXP (prev_link
, 1) = next_link
;
5347 insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)] = next_link
;
5349 free_INSN_LIST_node (link
);
5351 if (sched_verbose
>= 2)
5352 fprintf (sched_dump
, ";;\t\tEarly Q-->Ready: insn %s\n",
5353 (*current_sched_info
->print_insn
) (insn
, 0));
5356 if (insns_removed
== flag_sched_stalled_insns
)
5357 /* Remove no more than flag_sched_stalled_insns insns
5358 from Q at a time. */
5359 return insns_removed
;
5363 if (move_to_ready
== false)
5370 } /* for stalls.. */
5372 return insns_removed
;
5376 /* Print the ready list for debugging purposes.
5377 If READY_TRY is non-zero then only print insns that max_issue
5380 debug_ready_list_1 (struct ready_list
*ready
, signed char *ready_try
)
5385 if (ready
->n_ready
== 0)
5387 fprintf (sched_dump
, "\n");
5391 p
= ready_lastpos (ready
);
5392 for (i
= 0; i
< ready
->n_ready
; i
++)
5394 if (ready_try
!= NULL
&& ready_try
[ready
->n_ready
- i
- 1])
5397 fprintf (sched_dump
, " %s:%d",
5398 (*current_sched_info
->print_insn
) (p
[i
], 0),
5400 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5401 fprintf (sched_dump
, "(cost=%d",
5402 INSN_REG_PRESSURE_EXCESS_COST_CHANGE (p
[i
]));
5403 fprintf (sched_dump
, ":prio=%d", INSN_PRIORITY (p
[i
]));
5404 if (INSN_TICK (p
[i
]) > clock_var
)
5405 fprintf (sched_dump
, ":delay=%d", INSN_TICK (p
[i
]) - clock_var
);
5406 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
5407 fprintf (sched_dump
, ":idx=%d",
5408 model_index (p
[i
]));
5409 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
5410 fprintf (sched_dump
, ")");
5412 fprintf (sched_dump
, "\n");
5415 /* Print the ready list. Callable from debugger. */
5417 debug_ready_list (struct ready_list
*ready
)
5419 debug_ready_list_1 (ready
, NULL
);
5422 /* Search INSN for REG_SAVE_NOTE notes and convert them back into insn
5423 NOTEs. This is used for NOTE_INSN_EPILOGUE_BEG, so that sched-ebb
5424 replaces the epilogue note in the correct basic block. */
5426 reemit_notes (rtx_insn
*insn
)
5429 rtx_insn
*last
= insn
;
5431 for (note
= REG_NOTES (insn
); note
; note
= XEXP (note
, 1))
5433 if (REG_NOTE_KIND (note
) == REG_SAVE_NOTE
)
5435 enum insn_note note_type
= (enum insn_note
) INTVAL (XEXP (note
, 0));
5437 last
= emit_note_before (note_type
, last
);
5438 remove_note (insn
, note
);
5443 /* Move INSN. Reemit notes if needed. Update CFG, if needed. */
5445 move_insn (rtx_insn
*insn
, rtx_insn
*last
, rtx nt
)
5447 if (PREV_INSN (insn
) != last
)
5453 bb
= BLOCK_FOR_INSN (insn
);
5455 /* BB_HEAD is either LABEL or NOTE. */
5456 gcc_assert (BB_HEAD (bb
) != insn
);
5458 if (BB_END (bb
) == insn
)
5459 /* If this is last instruction in BB, move end marker one
5462 /* Jumps are always placed at the end of basic block. */
5463 jump_p
= control_flow_insn_p (insn
);
5466 || ((common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
5467 && IS_SPECULATION_BRANCHY_CHECK_P (insn
))
5468 || (common_sched_info
->sched_pass_id
5469 == SCHED_EBB_PASS
));
5471 gcc_assert (BLOCK_FOR_INSN (PREV_INSN (insn
)) == bb
);
5473 BB_END (bb
) = PREV_INSN (insn
);
5476 gcc_assert (BB_END (bb
) != last
);
5479 /* We move the block note along with jump. */
5483 note
= NEXT_INSN (insn
);
5484 while (NOTE_NOT_BB_P (note
) && note
!= nt
)
5485 note
= NEXT_INSN (note
);
5489 || BARRIER_P (note
)))
5490 note
= NEXT_INSN (note
);
5492 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
5497 SET_NEXT_INSN (PREV_INSN (insn
)) = NEXT_INSN (note
);
5498 SET_PREV_INSN (NEXT_INSN (note
)) = PREV_INSN (insn
);
5500 SET_NEXT_INSN (note
) = NEXT_INSN (last
);
5501 SET_PREV_INSN (NEXT_INSN (last
)) = note
;
5503 SET_NEXT_INSN (last
) = insn
;
5504 SET_PREV_INSN (insn
) = last
;
5506 bb
= BLOCK_FOR_INSN (last
);
5510 fix_jump_move (insn
);
5512 if (BLOCK_FOR_INSN (insn
) != bb
)
5513 move_block_after_check (insn
);
5515 gcc_assert (BB_END (bb
) == last
);
5518 df_insn_change_bb (insn
, bb
);
5520 /* Update BB_END, if needed. */
5521 if (BB_END (bb
) == last
)
5525 SCHED_GROUP_P (insn
) = 0;
5528 /* Return true if scheduling INSN will finish current clock cycle. */
5530 insn_finishes_cycle_p (rtx_insn
*insn
)
5532 if (SCHED_GROUP_P (insn
))
5533 /* After issuing INSN, rest of the sched_group will be forced to issue
5534 in order. Don't make any plans for the rest of cycle. */
5537 /* Finishing the block will, apparently, finish the cycle. */
5538 if (current_sched_info
->insn_finishes_block_p
5539 && current_sched_info
->insn_finishes_block_p (insn
))
5545 /* Helper for autopref_multipass_init. Given a SET in PAT and whether
5546 we're expecting a memory WRITE or not, check that the insn is relevant to
5547 the autoprefetcher modelling code. Return true iff that is the case.
5548 If it is relevant, record the base register of the memory op in BASE and
5549 the offset in OFFSET. */
5552 analyze_set_insn_for_autopref (rtx pat
, bool write
, rtx
*base
, int *offset
)
5554 if (GET_CODE (pat
) != SET
)
5557 rtx mem
= write
? SET_DEST (pat
) : SET_SRC (pat
);
5561 struct address_info info
;
5562 decompose_mem_address (&info
, mem
);
5564 /* TODO: Currently only (base+const) addressing is supported. */
5565 if (info
.base
== NULL
|| !REG_P (*info
.base
)
5566 || (info
.disp
!= NULL
&& !CONST_INT_P (*info
.disp
)))
5570 *offset
= info
.disp
? INTVAL (*info
.disp
) : 0;
5574 /* Functions to model cache auto-prefetcher.
5576 Some of the CPUs have cache auto-prefetcher, which /seems/ to initiate
5577 memory prefetches if it sees instructions with consequitive memory accesses
5578 in the instruction stream. Details of such hardware units are not published,
5579 so we can only guess what exactly is going on there.
5580 In the scheduler, we model abstract auto-prefetcher. If there are memory
5581 insns in the ready list (or the queue) that have same memory base, but
5582 different offsets, then we delay the insns with larger offsets until insns
5583 with smaller offsets get scheduled. If PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
5584 is "1", then we look at the ready list; if it is N>1, then we also look
5585 through N-1 queue entries.
5586 If the param is N>=0, then rank_for_schedule will consider auto-prefetching
5587 among its heuristics.
5588 Param value of "-1" disables modelling of the auto-prefetcher. */
5590 /* Initialize autoprefetcher model data for INSN. */
5592 autopref_multipass_init (const rtx_insn
*insn
, int write
)
5594 autopref_multipass_data_t data
= &INSN_AUTOPREF_MULTIPASS_DATA (insn
)[write
];
5596 gcc_assert (data
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
);
5597 data
->base
= NULL_RTX
;
5599 /* Set insn entry initialized, but not relevant for auto-prefetcher. */
5600 data
->status
= AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5602 rtx pat
= PATTERN (insn
);
5604 /* We have a multi-set insn like a load-multiple or store-multiple.
5605 We care about these as long as all the memory ops inside the PARALLEL
5606 have the same base register. We care about the minimum and maximum
5607 offsets from that base but don't check for the order of those offsets
5608 within the PARALLEL insn itself. */
5609 if (GET_CODE (pat
) == PARALLEL
)
5611 int n_elems
= XVECLEN (pat
, 0);
5614 rtx base
, prev_base
= NULL_RTX
;
5615 int min_offset
= INT_MAX
;
5617 for (i
= 0; i
< n_elems
; i
++)
5619 rtx set
= XVECEXP (pat
, 0, i
);
5620 if (GET_CODE (set
) != SET
)
5623 if (!analyze_set_insn_for_autopref (set
, write
, &base
, &offset
))
5626 /* Ensure that all memory operations in the PARALLEL use the same
5628 if (i
> 0 && REGNO (base
) != REGNO (prev_base
))
5631 min_offset
= MIN (min_offset
, offset
);
5634 /* If we reached here then we have a valid PARALLEL of multiple memory ops
5635 with prev_base as the base and min_offset containing the offset. */
5636 gcc_assert (prev_base
);
5637 data
->base
= prev_base
;
5638 data
->offset
= min_offset
;
5639 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5643 /* Otherwise this is a single set memory operation. */
5644 rtx set
= single_set (insn
);
5645 if (set
== NULL_RTX
)
5648 if (!analyze_set_insn_for_autopref (set
, write
, &data
->base
,
5652 /* This insn is relevant for the auto-prefetcher.
5653 The base and offset fields will have been filled in the
5654 analyze_set_insn_for_autopref call above. */
5655 data
->status
= AUTOPREF_MULTIPASS_DATA_NORMAL
;
5658 /* Helper function for rank_for_schedule sorting. */
5660 autopref_rank_for_schedule (const rtx_insn
*insn1
, const rtx_insn
*insn2
)
5663 for (int write
= 0; write
< 2 && !r
; ++write
)
5665 autopref_multipass_data_t data1
5666 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5667 autopref_multipass_data_t data2
5668 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5670 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5671 autopref_multipass_init (insn1
, write
);
5673 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5674 autopref_multipass_init (insn2
, write
);
5676 int irrel1
= data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5677 int irrel2
= data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
;
5679 if (!irrel1
&& !irrel2
)
5680 r
= data1
->offset
- data2
->offset
;
5682 r
= irrel2
- irrel1
;
5688 /* True if header of debug dump was printed. */
5689 static bool autopref_multipass_dfa_lookahead_guard_started_dump_p
;
5691 /* Helper for autopref_multipass_dfa_lookahead_guard.
5692 Return "1" if INSN1 should be delayed in favor of INSN2. */
5694 autopref_multipass_dfa_lookahead_guard_1 (const rtx_insn
*insn1
,
5695 const rtx_insn
*insn2
, int write
)
5697 autopref_multipass_data_t data1
5698 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5699 autopref_multipass_data_t data2
5700 = &INSN_AUTOPREF_MULTIPASS_DATA (insn2
)[write
];
5702 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5703 autopref_multipass_init (insn2
, write
);
5704 if (data2
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5707 if (rtx_equal_p (data1
->base
, data2
->base
)
5708 && data1
->offset
> data2
->offset
)
5710 if (sched_verbose
>= 2)
5712 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5714 fprintf (sched_dump
,
5715 ";;\t\tnot trying in max_issue due to autoprefetch "
5717 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5720 fprintf (sched_dump
, " %d(%d)", INSN_UID (insn1
), INSN_UID (insn2
));
5731 We could have also hooked autoprefetcher model into
5732 first_cycle_multipass_backtrack / first_cycle_multipass_issue hooks
5733 to enable intelligent selection of "[r1+0]=r2; [r1+4]=r3" on the same cycle
5734 (e.g., once "[r1+0]=r2" is issued in max_issue(), "[r1+4]=r3" gets
5735 unblocked). We don't bother about this yet because target of interest
5736 (ARM Cortex-A15) can issue only 1 memory operation per cycle. */
5738 /* Implementation of first_cycle_multipass_dfa_lookahead_guard hook.
5739 Return "1" if INSN1 should not be considered in max_issue due to
5740 auto-prefetcher considerations. */
5742 autopref_multipass_dfa_lookahead_guard (rtx_insn
*insn1
, int ready_index
)
5746 /* Exit early if the param forbids this or if we're not entering here through
5747 normal haifa scheduling. This can happen if selective scheduling is
5748 explicitly enabled. */
5749 if (!insn_queue
|| PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) <= 0)
5752 if (sched_verbose
>= 2 && ready_index
== 0)
5753 autopref_multipass_dfa_lookahead_guard_started_dump_p
= false;
5755 for (int write
= 0; write
< 2; ++write
)
5757 autopref_multipass_data_t data1
5758 = &INSN_AUTOPREF_MULTIPASS_DATA (insn1
)[write
];
5760 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
)
5761 autopref_multipass_init (insn1
, write
);
5762 if (data1
->status
== AUTOPREF_MULTIPASS_DATA_IRRELEVANT
)
5765 if (ready_index
== 0
5766 && data1
->status
== AUTOPREF_MULTIPASS_DATA_DONT_DELAY
)
5767 /* We allow only a single delay on priviledged instructions.
5768 Doing otherwise would cause infinite loop. */
5770 if (sched_verbose
>= 2)
5772 if (!autopref_multipass_dfa_lookahead_guard_started_dump_p
)
5774 fprintf (sched_dump
,
5775 ";;\t\tnot trying in max_issue due to autoprefetch "
5777 autopref_multipass_dfa_lookahead_guard_started_dump_p
= true;
5780 fprintf (sched_dump
, " *%d*", INSN_UID (insn1
));
5785 for (int i2
= 0; i2
< ready
.n_ready
; ++i2
)
5787 rtx_insn
*insn2
= get_ready_element (i2
);
5790 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
, write
);
5793 if (ready_index
== 0)
5796 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5802 if (PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) == 1)
5805 /* Everything from the current queue slot should have been moved to
5807 gcc_assert (insn_queue
[NEXT_Q_AFTER (q_ptr
, 0)] == NULL_RTX
);
5809 int n_stalls
= PARAM_VALUE (PARAM_SCHED_AUTOPREF_QUEUE_DEPTH
) - 1;
5810 if (n_stalls
> max_insn_queue_index
)
5811 n_stalls
= max_insn_queue_index
;
5813 for (int stalls
= 1; stalls
<= n_stalls
; ++stalls
)
5815 for (rtx_insn_list
*link
= insn_queue
[NEXT_Q_AFTER (q_ptr
, stalls
)];
5817 link
= link
->next ())
5819 rtx_insn
*insn2
= link
->insn ();
5820 r
= autopref_multipass_dfa_lookahead_guard_1 (insn1
, insn2
,
5824 /* Queue INSN1 until INSN2 can issue. */
5826 if (ready_index
== 0)
5827 data1
->status
= AUTOPREF_MULTIPASS_DATA_DONT_DELAY
;
5835 if (sched_verbose
>= 2
5836 && autopref_multipass_dfa_lookahead_guard_started_dump_p
5837 && (ready_index
== ready
.n_ready
- 1 || r
< 0))
5838 /* This does not /always/ trigger. We don't output EOL if the last
5839 insn is not recognized (INSN_CODE < 0) and lookahead_guard is not
5840 called. We can live with this. */
5841 fprintf (sched_dump
, "\n");
5846 /* Define type for target data used in multipass scheduling. */
5847 #ifndef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T
5848 # define TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T int
5850 typedef TARGET_SCHED_FIRST_CYCLE_MULTIPASS_DATA_T first_cycle_multipass_data_t
;
5852 /* The following structure describe an entry of the stack of choices. */
5855 /* Ordinal number of the issued insn in the ready queue. */
5857 /* The number of the rest insns whose issues we should try. */
5859 /* The number of issued essential insns. */
5861 /* State after issuing the insn. */
5863 /* Target-specific data. */
5864 first_cycle_multipass_data_t target_data
;
5867 /* The following array is used to implement a stack of choices used in
5868 function max_issue. */
5869 static struct choice_entry
*choice_stack
;
5871 /* This holds the value of the target dfa_lookahead hook. */
5874 /* The following variable value is maximal number of tries of issuing
5875 insns for the first cycle multipass insn scheduling. We define
5876 this value as constant*(DFA_LOOKAHEAD**ISSUE_RATE). We would not
5877 need this constraint if all real insns (with non-negative codes)
5878 had reservations because in this case the algorithm complexity is
5879 O(DFA_LOOKAHEAD**ISSUE_RATE). Unfortunately, the dfa descriptions
5880 might be incomplete and such insn might occur. For such
5881 descriptions, the complexity of algorithm (without the constraint)
5882 could achieve DFA_LOOKAHEAD ** N , where N is the queue length. */
5883 static int max_lookahead_tries
;
5885 /* The following function returns maximal (or close to maximal) number
5886 of insns which can be issued on the same cycle and one of which
5887 insns is insns with the best rank (the first insn in READY). To
5888 make this function tries different samples of ready insns. READY
5889 is current queue `ready'. Global array READY_TRY reflects what
5890 insns are already issued in this try. The function stops immediately,
5891 if it reached the such a solution, that all instruction can be issued.
5892 INDEX will contain index of the best insn in READY. The following
5893 function is used only for first cycle multipass scheduling.
5897 This function expects recognized insns only. All USEs,
5898 CLOBBERs, etc must be filtered elsewhere. */
5900 max_issue (struct ready_list
*ready
, int privileged_n
, state_t state
,
5901 bool first_cycle_insn_p
, int *index
)
5903 int n
, i
, all
, n_ready
, best
, delay
, tries_num
;
5905 struct choice_entry
*top
;
5911 n_ready
= ready
->n_ready
;
5912 gcc_assert (dfa_lookahead
>= 1 && privileged_n
>= 0
5913 && privileged_n
<= n_ready
);
5915 /* Init MAX_LOOKAHEAD_TRIES. */
5916 if (max_lookahead_tries
== 0)
5918 max_lookahead_tries
= 100;
5919 for (i
= 0; i
< issue_rate
; i
++)
5920 max_lookahead_tries
*= dfa_lookahead
;
5923 /* Init max_points. */
5924 more_issue
= issue_rate
- cycle_issued_insns
;
5925 gcc_assert (more_issue
>= 0);
5927 /* The number of the issued insns in the best solution. */
5932 /* Set initial state of the search. */
5933 memcpy (top
->state
, state
, dfa_state_size
);
5934 top
->rest
= dfa_lookahead
;
5936 if (targetm
.sched
.first_cycle_multipass_begin
)
5937 targetm
.sched
.first_cycle_multipass_begin (&top
->target_data
,
5939 first_cycle_insn_p
);
5941 /* Count the number of the insns to search among. */
5942 for (all
= i
= 0; i
< n_ready
; i
++)
5946 if (sched_verbose
>= 2)
5948 fprintf (sched_dump
, ";;\t\tmax_issue among %d insns:", all
);
5949 debug_ready_list_1 (ready
, ready_try
);
5952 /* I is the index of the insn to try next. */
5957 if (/* If we've reached a dead end or searched enough of what we have
5960 /* or have nothing else to try... */
5962 /* or should not issue more. */
5963 || top
->n
>= more_issue
)
5965 /* ??? (... || i == n_ready). */
5966 gcc_assert (i
<= n_ready
);
5968 /* We should not issue more than issue_rate instructions. */
5969 gcc_assert (top
->n
<= more_issue
);
5971 if (top
== choice_stack
)
5974 if (best
< top
- choice_stack
)
5979 /* Try to find issued privileged insn. */
5980 while (n
&& !ready_try
[--n
])
5984 if (/* If all insns are equally good... */
5986 /* Or a privileged insn will be issued. */
5988 /* Then we have a solution. */
5990 best
= top
- choice_stack
;
5991 /* This is the index of the insn issued first in this
5993 *index
= choice_stack
[1].index
;
5994 if (top
->n
== more_issue
|| best
== all
)
5999 /* Set ready-list index to point to the last insn
6000 ('i++' below will advance it to the next insn). */
6006 if (targetm
.sched
.first_cycle_multipass_backtrack
)
6007 targetm
.sched
.first_cycle_multipass_backtrack (&top
->target_data
,
6008 ready_try
, n_ready
);
6011 memcpy (state
, top
->state
, dfa_state_size
);
6013 else if (!ready_try
[i
])
6016 if (tries_num
> max_lookahead_tries
)
6018 insn
= ready_element (ready
, i
);
6019 delay
= state_transition (state
, insn
);
6022 if (state_dead_lock_p (state
)
6023 || insn_finishes_cycle_p (insn
))
6024 /* We won't issue any more instructions in the next
6031 if (memcmp (top
->state
, state
, dfa_state_size
) != 0)
6034 /* Advance to the next choice_entry. */
6036 /* Initialize it. */
6037 top
->rest
= dfa_lookahead
;
6040 memcpy (top
->state
, state
, dfa_state_size
);
6043 if (targetm
.sched
.first_cycle_multipass_issue
)
6044 targetm
.sched
.first_cycle_multipass_issue (&top
->target_data
,
6054 /* Increase ready-list index. */
6058 if (targetm
.sched
.first_cycle_multipass_end
)
6059 targetm
.sched
.first_cycle_multipass_end (best
!= 0
6060 ? &choice_stack
[1].target_data
6063 /* Restore the original state of the DFA. */
6064 memcpy (state
, choice_stack
->state
, dfa_state_size
);
6069 /* The following function chooses insn from READY and modifies
6070 READY. The following function is used only for first
6071 cycle multipass scheduling.
6073 -1 if cycle should be advanced,
6074 0 if INSN_PTR is set to point to the desirable insn,
6075 1 if choose_ready () should be restarted without advancing the cycle. */
6077 choose_ready (struct ready_list
*ready
, bool first_cycle_insn_p
,
6078 rtx_insn
**insn_ptr
)
6080 if (dbg_cnt (sched_insn
) == false)
6082 if (nonscheduled_insns_begin
== NULL_RTX
)
6083 nonscheduled_insns_begin
= current_sched_info
->prev_head
;
6085 rtx_insn
*insn
= first_nonscheduled_insn ();
6087 if (QUEUE_INDEX (insn
) == QUEUE_READY
)
6088 /* INSN is in the ready_list. */
6090 ready_remove_insn (insn
);
6095 /* INSN is in the queue. Advance cycle to move it to the ready list. */
6096 gcc_assert (QUEUE_INDEX (insn
) >= 0);
6100 if (dfa_lookahead
<= 0 || SCHED_GROUP_P (ready_element (ready
, 0))
6101 || DEBUG_INSN_P (ready_element (ready
, 0)))
6103 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6104 *insn_ptr
= ready_remove_first_dispatch (ready
);
6106 *insn_ptr
= ready_remove_first (ready
);
6112 /* Try to choose the best insn. */
6116 insn
= ready_element (ready
, 0);
6117 if (INSN_CODE (insn
) < 0)
6119 *insn_ptr
= ready_remove_first (ready
);
6123 /* Filter the search space. */
6124 for (i
= 0; i
< ready
->n_ready
; i
++)
6128 insn
= ready_element (ready
, i
);
6130 /* If this insn is recognizable we should have already
6131 recognized it earlier.
6132 ??? Not very clear where this is supposed to be done.
6134 gcc_checking_assert (INSN_CODE (insn
) >= 0
6135 || recog_memoized (insn
) < 0);
6136 if (INSN_CODE (insn
) < 0)
6138 /* Non-recognized insns at position 0 are handled above. */
6144 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
)
6147 = (targetm
.sched
.first_cycle_multipass_dfa_lookahead_guard
6150 if (ready_try
[i
] < 0)
6151 /* Queue instruction for several cycles.
6152 We need to restart choose_ready as we have changed
6155 change_queue_index (insn
, -ready_try
[i
]);
6159 /* Make sure that we didn't end up with 0'th insn filtered out.
6160 Don't be tempted to make life easier for backends and just
6161 requeue 0'th insn if (ready_try[0] == 0) and restart
6162 choose_ready. Backends should be very considerate about
6163 requeueing instructions -- especially the highest priority
6164 one at position 0. */
6165 gcc_assert (ready_try
[i
] == 0 || i
> 0);
6170 gcc_assert (ready_try
[i
] == 0);
6171 /* INSN made it through the scrutiny of filters! */
6174 if (max_issue (ready
, 1, curr_state
, first_cycle_insn_p
, &index
) == 0)
6176 *insn_ptr
= ready_remove_first (ready
);
6177 if (sched_verbose
>= 4)
6178 fprintf (sched_dump
, ";;\t\tChosen insn (but can't issue) : %s \n",
6179 (*current_sched_info
->print_insn
) (*insn_ptr
, 0));
6184 if (sched_verbose
>= 4)
6185 fprintf (sched_dump
, ";;\t\tChosen insn : %s\n",
6186 (*current_sched_info
->print_insn
)
6187 (ready_element (ready
, index
), 0));
6189 *insn_ptr
= ready_remove (ready
, index
);
6195 /* This function is called when we have successfully scheduled a
6196 block. It uses the schedule stored in the scheduled_insns vector
6197 to rearrange the RTL. PREV_HEAD is used as the anchor to which we
6198 append the scheduled insns; TAIL is the insn after the scheduled
6199 block. TARGET_BB is the argument passed to schedule_block. */
6202 commit_schedule (rtx_insn
*prev_head
, rtx_insn
*tail
, basic_block
*target_bb
)
6207 last_scheduled_insn
= prev_head
;
6209 scheduled_insns
.iterate (i
, &insn
);
6212 if (control_flow_insn_p (last_scheduled_insn
)
6213 || current_sched_info
->advance_target_bb (*target_bb
, insn
))
6215 *target_bb
= current_sched_info
->advance_target_bb (*target_bb
, 0);
6221 x
= next_real_insn (last_scheduled_insn
);
6223 dump_new_block_header (1, *target_bb
, x
, tail
);
6226 last_scheduled_insn
= bb_note (*target_bb
);
6229 if (current_sched_info
->begin_move_insn
)
6230 (*current_sched_info
->begin_move_insn
) (insn
, last_scheduled_insn
);
6231 move_insn (insn
, last_scheduled_insn
,
6232 current_sched_info
->next_tail
);
6233 if (!DEBUG_INSN_P (insn
))
6234 reemit_notes (insn
);
6235 last_scheduled_insn
= insn
;
6238 scheduled_insns
.truncate (0);
6241 /* Examine all insns on the ready list and queue those which can't be
6242 issued in this cycle. TEMP_STATE is temporary scheduler state we
6243 can use as scratch space. If FIRST_CYCLE_INSN_P is true, no insns
6244 have been issued for the current cycle, which means it is valid to
6245 issue an asm statement.
6247 If SHADOWS_ONLY_P is true, we eliminate all real insns and only
6248 leave those for which SHADOW_P is true. If MODULO_EPILOGUE is true,
6249 we only leave insns which have an INSN_EXACT_TICK. */
6252 prune_ready_list (state_t temp_state
, bool first_cycle_insn_p
,
6253 bool shadows_only_p
, bool modulo_epilogue_p
)
6256 bool sched_group_found
= false;
6257 int min_cost_group
= 0;
6262 for (i
= 0; i
< ready
.n_ready
; i
++)
6264 rtx_insn
*insn
= ready_element (&ready
, i
);
6265 if (SCHED_GROUP_P (insn
))
6267 sched_group_found
= true;
6272 /* Make two passes if there's a SCHED_GROUP_P insn; make sure to handle
6273 such an insn first and note its cost. If at least one SCHED_GROUP_P insn
6274 gets queued, then all other insns get queued for one cycle later. */
6275 for (pass
= sched_group_found
? 0 : 1; pass
< 2; )
6277 int n
= ready
.n_ready
;
6278 for (i
= 0; i
< n
; i
++)
6280 rtx_insn
*insn
= ready_element (&ready
, i
);
6282 const char *reason
= "resource conflict";
6284 if (DEBUG_INSN_P (insn
))
6287 if (sched_group_found
&& !SCHED_GROUP_P (insn
)
6288 && ((pass
== 0) || (min_cost_group
>= 1)))
6292 cost
= min_cost_group
;
6293 reason
= "not in sched group";
6295 else if (modulo_epilogue_p
6296 && INSN_EXACT_TICK (insn
) == INVALID_TICK
)
6298 cost
= max_insn_queue_index
;
6299 reason
= "not an epilogue insn";
6301 else if (shadows_only_p
&& !SHADOW_P (insn
))
6304 reason
= "not a shadow";
6306 else if (recog_memoized (insn
) < 0)
6308 if (!first_cycle_insn_p
6309 && (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6310 || asm_noperands (PATTERN (insn
)) >= 0))
6314 else if (sched_pressure
!= SCHED_PRESSURE_NONE
)
6316 if (sched_pressure
== SCHED_PRESSURE_MODEL
6317 && INSN_TICK (insn
) <= clock_var
)
6319 memcpy (temp_state
, curr_state
, dfa_state_size
);
6320 if (state_transition (temp_state
, insn
) >= 0)
6321 INSN_TICK (insn
) = clock_var
+ 1;
6331 struct delay_pair
*delay_entry
;
6333 = delay_htab
->find_with_hash (insn
,
6334 htab_hash_pointer (insn
));
6335 while (delay_entry
&& delay_cost
== 0)
6337 delay_cost
= estimate_shadow_tick (delay_entry
);
6338 if (delay_cost
> max_insn_queue_index
)
6339 delay_cost
= max_insn_queue_index
;
6340 delay_entry
= delay_entry
->next_same_i1
;
6344 memcpy (temp_state
, curr_state
, dfa_state_size
);
6345 cost
= state_transition (temp_state
, insn
);
6350 if (cost
< delay_cost
)
6353 reason
= "shadow tick";
6358 if (SCHED_GROUP_P (insn
) && cost
> min_cost_group
)
6359 min_cost_group
= cost
;
6360 ready_remove (&ready
, i
);
6361 /* Normally we'd want to queue INSN for COST cycles. However,
6362 if SCHED_GROUP_P is set, then we must ensure that nothing
6363 else comes between INSN and its predecessor. If there is
6364 some other insn ready to fire on the next cycle, then that
6365 invariant would be broken.
6367 So when SCHED_GROUP_P is set, just queue this insn for a
6369 queue_insn (insn
, SCHED_GROUP_P (insn
) ? 1 : cost
, reason
);
6379 /* Called when we detect that the schedule is impossible. We examine the
6380 backtrack queue to find the earliest insn that caused this condition. */
6382 static struct haifa_saved_data
*
6383 verify_shadows (void)
6385 struct haifa_saved_data
*save
, *earliest_fail
= NULL
;
6386 for (save
= backtrack_queue
; save
; save
= save
->next
)
6389 struct delay_pair
*pair
= save
->delay_pair
;
6390 rtx_insn
*i1
= pair
->i1
;
6392 for (; pair
; pair
= pair
->next_same_i1
)
6394 rtx_insn
*i2
= pair
->i2
;
6396 if (QUEUE_INDEX (i2
) == QUEUE_SCHEDULED
)
6399 t
= INSN_TICK (i1
) + pair_delay (pair
);
6402 if (sched_verbose
>= 2)
6403 fprintf (sched_dump
,
6404 ";;\t\tfailed delay requirements for %d/%d (%d->%d)"
6406 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6407 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6408 earliest_fail
= save
;
6411 if (QUEUE_INDEX (i2
) >= 0)
6413 int queued_for
= INSN_TICK (i2
);
6417 if (sched_verbose
>= 2)
6418 fprintf (sched_dump
,
6419 ";;\t\tfailed delay requirements for %d/%d"
6420 " (%d->%d), queued too late\n",
6421 INSN_UID (pair
->i1
), INSN_UID (pair
->i2
),
6422 INSN_TICK (pair
->i1
), INSN_EXACT_TICK (pair
->i2
));
6423 earliest_fail
= save
;
6430 return earliest_fail
;
6433 /* Print instructions together with useful scheduling information between
6434 HEAD and TAIL (inclusive). */
6436 dump_insn_stream (rtx_insn
*head
, rtx_insn
*tail
)
6438 fprintf (sched_dump
, ";;\t| insn | prio |\n");
6440 rtx_insn
*next_tail
= NEXT_INSN (tail
);
6441 for (rtx_insn
*insn
= head
; insn
!= next_tail
; insn
= NEXT_INSN (insn
))
6443 int priority
= NOTE_P (insn
) ? 0 : INSN_PRIORITY (insn
);
6444 const char *pattern
= (NOTE_P (insn
)
6446 : str_pattern_slim (PATTERN (insn
)));
6448 fprintf (sched_dump
, ";;\t| %4d | %4d | %-30s ",
6449 INSN_UID (insn
), priority
, pattern
);
6451 if (sched_verbose
>= 4)
6453 if (NOTE_P (insn
) || LABEL_P (insn
) || recog_memoized (insn
) < 0)
6454 fprintf (sched_dump
, "nothing");
6456 print_reservation (sched_dump
, insn
);
6458 fprintf (sched_dump
, "\n");
6462 /* Use forward list scheduling to rearrange insns of block pointed to by
6463 TARGET_BB, possibly bringing insns from subsequent blocks in the same
6467 schedule_block (basic_block
*target_bb
, state_t init_state
)
6470 bool success
= modulo_ii
== 0;
6471 struct sched_block_state ls
;
6472 state_t temp_state
= NULL
; /* It is used for multipass scheduling. */
6473 int sort_p
, advance
, start_clock_var
;
6475 /* Head/tail info for this block. */
6476 rtx_insn
*prev_head
= current_sched_info
->prev_head
;
6477 rtx_insn
*next_tail
= current_sched_info
->next_tail
;
6478 rtx_insn
*head
= NEXT_INSN (prev_head
);
6479 rtx_insn
*tail
= PREV_INSN (next_tail
);
6481 if ((current_sched_info
->flags
& DONT_BREAK_DEPENDENCIES
) == 0
6482 && sched_pressure
!= SCHED_PRESSURE_MODEL
&& !sched_fusion
)
6483 find_modifiable_mems (head
, tail
);
6485 /* We used to have code to avoid getting parameters moved from hard
6486 argument registers into pseudos.
6488 However, it was removed when it proved to be of marginal benefit
6489 and caused problems because schedule_block and compute_forward_dependences
6490 had different notions of what the "head" insn was. */
6492 gcc_assert (head
!= tail
|| INSN_P (head
));
6494 haifa_recovery_bb_recently_added_p
= false;
6496 backtrack_queue
= NULL
;
6501 dump_new_block_header (0, *target_bb
, head
, tail
);
6503 if (sched_verbose
>= 2)
6505 dump_insn_stream (head
, tail
);
6506 memset (&rank_for_schedule_stats
, 0,
6507 sizeof (rank_for_schedule_stats
));
6511 if (init_state
== NULL
)
6512 state_reset (curr_state
);
6514 memcpy (curr_state
, init_state
, dfa_state_size
);
6516 /* Clear the ready list. */
6517 ready
.first
= ready
.veclen
- 1;
6521 /* It is used for first cycle multipass scheduling. */
6522 temp_state
= alloca (dfa_state_size
);
6524 if (targetm
.sched
.init
)
6525 targetm
.sched
.init (sched_dump
, sched_verbose
, ready
.veclen
);
6527 /* We start inserting insns after PREV_HEAD. */
6528 last_scheduled_insn
= prev_head
;
6529 last_nondebug_scheduled_insn
= NULL
;
6530 nonscheduled_insns_begin
= NULL
;
6532 gcc_assert ((NOTE_P (last_scheduled_insn
)
6533 || DEBUG_INSN_P (last_scheduled_insn
))
6534 && BLOCK_FOR_INSN (last_scheduled_insn
) == *target_bb
);
6536 /* Initialize INSN_QUEUE. Q_SIZE is the total number of insns in the
6541 insn_queue
= XALLOCAVEC (rtx_insn_list
*, max_insn_queue_index
+ 1);
6542 memset (insn_queue
, 0, (max_insn_queue_index
+ 1) * sizeof (rtx
));
6544 /* Start just before the beginning of time. */
6547 /* We need queue and ready lists and clock_var be initialized
6548 in try_ready () (which is called through init_ready_list ()). */
6549 (*current_sched_info
->init_ready_list
) ();
6552 sched_pressure_start_bb (*target_bb
);
6554 /* The algorithm is O(n^2) in the number of ready insns at any given
6555 time in the worst case. Before reload we are more likely to have
6556 big lists so truncate them to a reasonable size. */
6557 if (!reload_completed
6558 && ready
.n_ready
- ready
.n_debug
> MAX_SCHED_READY_INSNS
)
6560 ready_sort_debug (&ready
);
6561 ready_sort_real (&ready
);
6563 /* Find first free-standing insn past MAX_SCHED_READY_INSNS.
6564 If there are debug insns, we know they're first. */
6565 for (i
= MAX_SCHED_READY_INSNS
+ ready
.n_debug
; i
< ready
.n_ready
; i
++)
6566 if (!SCHED_GROUP_P (ready_element (&ready
, i
)))
6569 if (sched_verbose
>= 2)
6571 fprintf (sched_dump
,
6572 ";;\t\tReady list on entry: %d insns: ", ready
.n_ready
);
6573 debug_ready_list (&ready
);
6574 fprintf (sched_dump
,
6575 ";;\t\t before reload => truncated to %d insns\n", i
);
6578 /* Delay all insns past it for 1 cycle. If debug counter is
6579 activated make an exception for the insn right after
6580 nonscheduled_insns_begin. */
6582 rtx_insn
*skip_insn
;
6584 if (dbg_cnt (sched_insn
) == false)
6585 skip_insn
= first_nonscheduled_insn ();
6589 while (i
< ready
.n_ready
)
6593 insn
= ready_remove (&ready
, i
);
6595 if (insn
!= skip_insn
)
6596 queue_insn (insn
, 1, "list truncated");
6599 ready_add (&ready
, skip_insn
, true);
6603 /* Now we can restore basic block notes and maintain precise cfg. */
6604 restore_bb_notes (*target_bb
);
6606 last_clock_var
= -1;
6610 gcc_assert (scheduled_insns
.length () == 0);
6612 must_backtrack
= false;
6613 modulo_insns_scheduled
= 0;
6615 ls
.modulo_epilogue
= false;
6616 ls
.first_cycle_insn_p
= true;
6618 /* Loop until all the insns in BB are scheduled. */
6619 while ((*current_sched_info
->schedule_more_p
) ())
6621 perform_replacements_new_cycle ();
6624 start_clock_var
= clock_var
;
6628 advance_one_cycle ();
6630 /* Add to the ready list all pending insns that can be issued now.
6631 If there are no ready insns, increment clock until one
6632 is ready and add all pending insns at that point to the ready
6634 queue_to_ready (&ready
);
6636 gcc_assert (ready
.n_ready
);
6638 if (sched_verbose
>= 2)
6640 fprintf (sched_dump
, ";;\t\tReady list after queue_to_ready:");
6641 debug_ready_list (&ready
);
6643 advance
-= clock_var
- start_clock_var
;
6645 while (advance
> 0);
6647 if (ls
.modulo_epilogue
)
6649 int stage
= clock_var
/ modulo_ii
;
6650 if (stage
> modulo_last_stage
* 2 + 2)
6652 if (sched_verbose
>= 2)
6653 fprintf (sched_dump
,
6654 ";;\t\tmodulo scheduled succeeded at II %d\n",
6660 else if (modulo_ii
> 0)
6662 int stage
= clock_var
/ modulo_ii
;
6663 if (stage
> modulo_max_stages
)
6665 if (sched_verbose
>= 2)
6666 fprintf (sched_dump
,
6667 ";;\t\tfailing schedule due to excessive stages\n");
6670 if (modulo_n_insns
== modulo_insns_scheduled
6671 && stage
> modulo_last_stage
)
6673 if (sched_verbose
>= 2)
6674 fprintf (sched_dump
,
6675 ";;\t\tfound kernel after %d stages, II %d\n",
6677 ls
.modulo_epilogue
= true;
6681 prune_ready_list (temp_state
, true, false, ls
.modulo_epilogue
);
6682 if (ready
.n_ready
== 0)
6687 ls
.shadows_only_p
= false;
6688 cycle_issued_insns
= 0;
6689 ls
.can_issue_more
= issue_rate
;
6696 if (sort_p
&& ready
.n_ready
> 0)
6698 /* Sort the ready list based on priority. This must be
6699 done every iteration through the loop, as schedule_insn
6700 may have readied additional insns that will not be
6701 sorted correctly. */
6702 ready_sort (&ready
);
6704 if (sched_verbose
>= 2)
6706 fprintf (sched_dump
,
6707 ";;\t\tReady list after ready_sort: ");
6708 debug_ready_list (&ready
);
6712 /* We don't want md sched reorder to even see debug isns, so put
6713 them out right away. */
6714 if (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0))
6715 && (*current_sched_info
->schedule_more_p
) ())
6717 while (ready
.n_ready
&& DEBUG_INSN_P (ready_element (&ready
, 0)))
6719 rtx_insn
*insn
= ready_remove_first (&ready
);
6720 gcc_assert (DEBUG_INSN_P (insn
));
6721 (*current_sched_info
->begin_schedule_ready
) (insn
);
6722 scheduled_insns
.safe_push (insn
);
6723 last_scheduled_insn
= insn
;
6724 advance
= schedule_insn (insn
);
6725 gcc_assert (advance
== 0);
6726 if (ready
.n_ready
> 0)
6727 ready_sort (&ready
);
6731 if (ls
.first_cycle_insn_p
&& !ready
.n_ready
)
6734 resume_after_backtrack
:
6735 /* Allow the target to reorder the list, typically for
6736 better instruction bundling. */
6738 && (ready
.n_ready
== 0
6739 || !SCHED_GROUP_P (ready_element (&ready
, 0))))
6741 if (ls
.first_cycle_insn_p
&& targetm
.sched
.reorder
)
6743 = targetm
.sched
.reorder (sched_dump
, sched_verbose
,
6744 ready_lastpos (&ready
),
6745 &ready
.n_ready
, clock_var
);
6746 else if (!ls
.first_cycle_insn_p
&& targetm
.sched
.reorder2
)
6748 = targetm
.sched
.reorder2 (sched_dump
, sched_verbose
,
6750 ? ready_lastpos (&ready
) : NULL
,
6751 &ready
.n_ready
, clock_var
);
6754 restart_choose_ready
:
6755 if (sched_verbose
>= 2)
6757 fprintf (sched_dump
, ";;\tReady list (t = %3d): ",
6759 debug_ready_list (&ready
);
6760 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
6761 print_curr_reg_pressure ();
6764 if (ready
.n_ready
== 0
6765 && ls
.can_issue_more
6766 && reload_completed
)
6768 /* Allow scheduling insns directly from the queue in case
6769 there's nothing better to do (ready list is empty) but
6770 there are still vacant dispatch slots in the current cycle. */
6771 if (sched_verbose
>= 6)
6772 fprintf (sched_dump
,";;\t\tSecond chance\n");
6773 memcpy (temp_state
, curr_state
, dfa_state_size
);
6774 if (early_queue_to_ready (temp_state
, &ready
))
6775 ready_sort (&ready
);
6778 if (ready
.n_ready
== 0
6779 || !ls
.can_issue_more
6780 || state_dead_lock_p (curr_state
)
6781 || !(*current_sched_info
->schedule_more_p
) ())
6784 /* Select and remove the insn from the ready list. */
6790 res
= choose_ready (&ready
, ls
.first_cycle_insn_p
, &insn
);
6796 goto restart_choose_ready
;
6798 gcc_assert (insn
!= NULL_RTX
);
6801 insn
= ready_remove_first (&ready
);
6803 if (sched_pressure
!= SCHED_PRESSURE_NONE
6804 && INSN_TICK (insn
) > clock_var
)
6806 ready_add (&ready
, insn
, true);
6811 if (targetm
.sched
.dfa_new_cycle
6812 && targetm
.sched
.dfa_new_cycle (sched_dump
, sched_verbose
,
6813 insn
, last_clock_var
,
6814 clock_var
, &sort_p
))
6815 /* SORT_P is used by the target to override sorting
6816 of the ready list. This is needed when the target
6817 has modified its internal structures expecting that
6818 the insn will be issued next. As we need the insn
6819 to have the highest priority (so it will be returned by
6820 the ready_remove_first call above), we invoke
6821 ready_add (&ready, insn, true).
6822 But, still, there is one issue: INSN can be later
6823 discarded by scheduler's front end through
6824 current_sched_info->can_schedule_ready_p, hence, won't
6827 ready_add (&ready
, insn
, true);
6833 if (current_sched_info
->can_schedule_ready_p
6834 && ! (*current_sched_info
->can_schedule_ready_p
) (insn
))
6835 /* We normally get here only if we don't want to move
6836 insn from the split block. */
6838 TODO_SPEC (insn
) = DEP_POSTPONED
;
6839 goto restart_choose_ready
;
6844 /* If this insn is the first part of a delay-slot pair, record a
6846 struct delay_pair
*delay_entry
;
6848 = delay_htab
->find_with_hash (insn
, htab_hash_pointer (insn
));
6851 save_backtrack_point (delay_entry
, ls
);
6852 if (sched_verbose
>= 2)
6853 fprintf (sched_dump
, ";;\t\tsaving backtrack point\n");
6857 /* DECISION is made. */
6859 if (modulo_ii
> 0 && INSN_UID (insn
) < modulo_iter0_max_uid
)
6861 modulo_insns_scheduled
++;
6862 modulo_last_stage
= clock_var
/ modulo_ii
;
6864 if (TODO_SPEC (insn
) & SPECULATIVE
)
6865 generate_recovery_code (insn
);
6867 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
6868 targetm
.sched
.dispatch_do (insn
, ADD_TO_DISPATCH_WINDOW
);
6870 /* Update counters, etc in the scheduler's front end. */
6871 (*current_sched_info
->begin_schedule_ready
) (insn
);
6872 scheduled_insns
.safe_push (insn
);
6873 gcc_assert (NONDEBUG_INSN_P (insn
));
6874 last_nondebug_scheduled_insn
= last_scheduled_insn
= insn
;
6876 if (recog_memoized (insn
) >= 0)
6878 memcpy (temp_state
, curr_state
, dfa_state_size
);
6879 cost
= state_transition (curr_state
, insn
);
6880 if (sched_pressure
!= SCHED_PRESSURE_WEIGHTED
&& !sched_fusion
)
6881 gcc_assert (cost
< 0);
6882 if (memcmp (temp_state
, curr_state
, dfa_state_size
) != 0)
6883 cycle_issued_insns
++;
6887 asm_p
= (GET_CODE (PATTERN (insn
)) == ASM_INPUT
6888 || asm_noperands (PATTERN (insn
)) >= 0);
6890 if (targetm
.sched
.variable_issue
)
6892 targetm
.sched
.variable_issue (sched_dump
, sched_verbose
,
6893 insn
, ls
.can_issue_more
);
6894 /* A naked CLOBBER or USE generates no instruction, so do
6895 not count them against the issue rate. */
6896 else if (GET_CODE (PATTERN (insn
)) != USE
6897 && GET_CODE (PATTERN (insn
)) != CLOBBER
)
6898 ls
.can_issue_more
--;
6899 advance
= schedule_insn (insn
);
6901 if (SHADOW_P (insn
))
6902 ls
.shadows_only_p
= true;
6904 /* After issuing an asm insn we should start a new cycle. */
6905 if (advance
== 0 && asm_p
)
6914 ls
.first_cycle_insn_p
= false;
6915 if (ready
.n_ready
> 0)
6916 prune_ready_list (temp_state
, false, ls
.shadows_only_p
,
6917 ls
.modulo_epilogue
);
6921 if (!must_backtrack
)
6922 for (i
= 0; i
< ready
.n_ready
; i
++)
6924 rtx_insn
*insn
= ready_element (&ready
, i
);
6925 if (INSN_EXACT_TICK (insn
) == clock_var
)
6927 must_backtrack
= true;
6932 if (must_backtrack
&& modulo_ii
> 0)
6934 if (modulo_backtracks_left
== 0)
6936 modulo_backtracks_left
--;
6938 while (must_backtrack
)
6940 struct haifa_saved_data
*failed
;
6941 rtx_insn
*failed_insn
;
6943 must_backtrack
= false;
6944 failed
= verify_shadows ();
6945 gcc_assert (failed
);
6947 failed_insn
= failed
->delay_pair
->i1
;
6948 /* Clear these queues. */
6949 perform_replacements_new_cycle ();
6950 toggle_cancelled_flags (false);
6951 unschedule_insns_until (failed_insn
);
6952 while (failed
!= backtrack_queue
)
6953 free_topmost_backtrack_point (true);
6954 restore_last_backtrack_point (&ls
);
6955 if (sched_verbose
>= 2)
6956 fprintf (sched_dump
, ";;\t\trewind to cycle %d\n", clock_var
);
6957 /* Delay by at least a cycle. This could cause additional
6959 queue_insn (failed_insn
, 1, "backtracked");
6963 if (ready
.n_ready
> 0)
6964 goto resume_after_backtrack
;
6967 if (clock_var
== 0 && ls
.first_cycle_insn_p
)
6973 ls
.first_cycle_insn_p
= true;
6975 if (ls
.modulo_epilogue
)
6978 if (!ls
.first_cycle_insn_p
|| advance
)
6979 advance_one_cycle ();
6980 perform_replacements_new_cycle ();
6983 /* Once again, debug insn suckiness: they can be on the ready list
6984 even if they have unresolved dependencies. To make our view
6985 of the world consistent, remove such "ready" insns. */
6986 restart_debug_insn_loop
:
6987 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
6991 x
= ready_element (&ready
, i
);
6992 if (DEPS_LIST_FIRST (INSN_HARD_BACK_DEPS (x
)) != NULL
6993 || DEPS_LIST_FIRST (INSN_SPEC_BACK_DEPS (x
)) != NULL
)
6995 ready_remove (&ready
, i
);
6996 goto restart_debug_insn_loop
;
6999 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7003 x
= ready_element (&ready
, i
);
7004 resolve_dependencies (x
);
7006 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7008 rtx_insn_list
*link
;
7009 while ((link
= insn_queue
[i
]) != NULL
)
7011 rtx_insn
*x
= link
->insn ();
7012 insn_queue
[i
] = link
->next ();
7013 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7014 free_INSN_LIST_node (link
);
7015 resolve_dependencies (x
);
7021 undo_all_replacements ();
7026 fprintf (sched_dump
, ";;\tReady list (final): ");
7027 debug_ready_list (&ready
);
7030 if (modulo_ii
== 0 && current_sched_info
->queue_must_finish_empty
)
7031 /* Sanity check -- queue must be empty now. Meaningless if region has
7033 gcc_assert (!q_size
&& !ready
.n_ready
&& !ready
.n_debug
);
7034 else if (modulo_ii
== 0)
7036 /* We must maintain QUEUE_INDEX between blocks in region. */
7037 for (i
= ready
.n_ready
- 1; i
>= 0; i
--)
7041 x
= ready_element (&ready
, i
);
7042 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7043 TODO_SPEC (x
) = HARD_DEP
;
7047 for (i
= 0; i
<= max_insn_queue_index
; i
++)
7049 rtx_insn_list
*link
;
7050 for (link
= insn_queue
[i
]; link
; link
= link
->next ())
7055 QUEUE_INDEX (x
) = QUEUE_NOWHERE
;
7056 TODO_SPEC (x
) = HARD_DEP
;
7058 free_INSN_LIST_list (&insn_queue
[i
]);
7062 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7063 model_end_schedule ();
7067 commit_schedule (prev_head
, tail
, target_bb
);
7069 fprintf (sched_dump
, ";; total time = %d\n", clock_var
);
7072 last_scheduled_insn
= tail
;
7074 scheduled_insns
.truncate (0);
7076 if (!current_sched_info
->queue_must_finish_empty
7077 || haifa_recovery_bb_recently_added_p
)
7079 /* INSN_TICK (minimum clock tick at which the insn becomes
7080 ready) may be not correct for the insn in the subsequent
7081 blocks of the region. We should use a correct value of
7082 `clock_var' or modify INSN_TICK. It is better to keep
7083 clock_var value equal to 0 at the start of a basic block.
7084 Therefore we modify INSN_TICK here. */
7085 fix_inter_tick (NEXT_INSN (prev_head
), last_scheduled_insn
);
7088 if (targetm
.sched
.finish
)
7090 targetm
.sched
.finish (sched_dump
, sched_verbose
);
7091 /* Target might have added some instructions to the scheduled block
7092 in its md_finish () hook. These new insns don't have any data
7093 initialized and to identify them we extend h_i_d so that they'll
7095 sched_extend_luids ();
7098 /* Update head/tail boundaries. */
7099 head
= NEXT_INSN (prev_head
);
7100 tail
= last_scheduled_insn
;
7104 fprintf (sched_dump
, ";; new head = %d\n;; new tail = %d\n",
7105 INSN_UID (head
), INSN_UID (tail
));
7107 if (sched_verbose
>= 2)
7109 dump_insn_stream (head
, tail
);
7110 print_rank_for_schedule_stats (";; TOTAL ", &rank_for_schedule_stats
,
7114 fprintf (sched_dump
, "\n");
7117 head
= restore_other_notes (head
, NULL
);
7119 current_sched_info
->head
= head
;
7120 current_sched_info
->tail
= tail
;
7122 free_backtrack_queue ();
7127 /* Set_priorities: compute priority of each insn in the block. */
7130 set_priorities (rtx_insn
*head
, rtx_insn
*tail
)
7134 int sched_max_insns_priority
=
7135 current_sched_info
->sched_max_insns_priority
;
7136 rtx_insn
*prev_head
;
7138 if (head
== tail
&& ! INSN_P (head
))
7143 prev_head
= PREV_INSN (head
);
7144 for (insn
= tail
; insn
!= prev_head
; insn
= PREV_INSN (insn
))
7150 (void) priority (insn
);
7152 gcc_assert (INSN_PRIORITY_KNOWN (insn
));
7154 sched_max_insns_priority
= MAX (sched_max_insns_priority
,
7155 INSN_PRIORITY (insn
));
7158 current_sched_info
->sched_max_insns_priority
= sched_max_insns_priority
;
7163 /* Set sched_dump and sched_verbose for the desired debugging output. */
7165 setup_sched_dump (void)
7167 sched_verbose
= sched_verbose_param
;
7168 sched_dump
= dump_file
;
7173 /* Allocate data for register pressure sensitive scheduling. */
7175 alloc_global_sched_pressure_data (void)
7177 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7179 int i
, max_regno
= max_reg_num ();
7181 if (sched_dump
!= NULL
)
7182 /* We need info about pseudos for rtl dumps about pseudo
7183 classes and costs. */
7184 regstat_init_n_sets_and_refs ();
7185 ira_set_pseudo_classes (true, sched_verbose
? sched_dump
: NULL
);
7186 sched_regno_pressure_class
7187 = (enum reg_class
*) xmalloc (max_regno
* sizeof (enum reg_class
));
7188 for (i
= 0; i
< max_regno
; i
++)
7189 sched_regno_pressure_class
[i
]
7190 = (i
< FIRST_PSEUDO_REGISTER
7191 ? ira_pressure_class_translate
[REGNO_REG_CLASS (i
)]
7192 : ira_pressure_class_translate
[reg_allocno_class (i
)]);
7193 curr_reg_live
= BITMAP_ALLOC (NULL
);
7194 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7196 saved_reg_live
= BITMAP_ALLOC (NULL
);
7197 region_ref_regs
= BITMAP_ALLOC (NULL
);
7199 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7200 tmp_bitmap
= BITMAP_ALLOC (NULL
);
7202 /* Calculate number of CALL_SAVED_REGS and FIXED_REGS in register classes
7203 that we calculate register pressure for. */
7204 for (int c
= 0; c
< ira_pressure_classes_num
; ++c
)
7206 enum reg_class cl
= ira_pressure_classes
[c
];
7208 call_saved_regs_num
[cl
] = 0;
7209 fixed_regs_num
[cl
] = 0;
7211 for (int i
= 0; i
< ira_class_hard_regs_num
[cl
]; ++i
)
7213 unsigned int regno
= ira_class_hard_regs
[cl
][i
];
7214 if (fixed_regs
[regno
])
7215 ++fixed_regs_num
[cl
];
7216 else if (!crtl
->abi
->clobbers_full_reg_p (regno
))
7217 ++call_saved_regs_num
[cl
];
7223 /* Free data for register pressure sensitive scheduling. Also called
7224 from schedule_region when stopping sched-pressure early. */
7226 free_global_sched_pressure_data (void)
7228 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7230 if (regstat_n_sets_and_refs
!= NULL
)
7231 regstat_free_n_sets_and_refs ();
7232 if (sched_pressure
== SCHED_PRESSURE_WEIGHTED
)
7234 BITMAP_FREE (region_ref_regs
);
7235 BITMAP_FREE (saved_reg_live
);
7237 if (sched_pressure
== SCHED_PRESSURE_MODEL
)
7238 BITMAP_FREE (tmp_bitmap
);
7239 BITMAP_FREE (curr_reg_live
);
7240 free (sched_regno_pressure_class
);
7244 /* Initialize some global state for the scheduler. This function works
7245 with the common data shared between all the schedulers. It is called
7246 from the scheduler specific initialization routine. */
7251 /* Disable speculative loads in their presence if cc0 defined. */
7253 flag_schedule_speculative_load
= 0;
7255 if (targetm
.sched
.dispatch (NULL
, IS_DISPATCH_ON
))
7256 targetm
.sched
.dispatch_do (NULL
, DISPATCH_INIT
);
7258 if (live_range_shrinkage_p
)
7259 sched_pressure
= SCHED_PRESSURE_WEIGHTED
;
7260 else if (flag_sched_pressure
7261 && !reload_completed
7262 && common_sched_info
->sched_pass_id
== SCHED_RGN_PASS
)
7263 sched_pressure
= ((enum sched_pressure_algorithm
)
7264 PARAM_VALUE (PARAM_SCHED_PRESSURE_ALGORITHM
));
7266 sched_pressure
= SCHED_PRESSURE_NONE
;
7268 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
7269 ira_setup_eliminable_regset ();
7271 /* Initialize SPEC_INFO. */
7272 if (targetm
.sched
.set_sched_flags
)
7274 spec_info
= &spec_info_var
;
7275 targetm
.sched
.set_sched_flags (spec_info
);
7277 if (spec_info
->mask
!= 0)
7279 spec_info
->data_weakness_cutoff
=
7280 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
) * MAX_DEP_WEAK
) / 100;
7281 spec_info
->control_weakness_cutoff
=
7282 (PARAM_VALUE (PARAM_SCHED_SPEC_PROB_CUTOFF
)
7283 * REG_BR_PROB_BASE
) / 100;
7286 /* So we won't read anything accidentally. */
7291 /* So we won't read anything accidentally. */
7294 /* Initialize issue_rate. */
7295 if (targetm
.sched
.issue_rate
)
7296 issue_rate
= targetm
.sched
.issue_rate ();
7300 if (targetm
.sched
.first_cycle_multipass_dfa_lookahead
7301 /* Don't use max_issue with reg_pressure scheduling. Multipass
7302 scheduling and reg_pressure scheduling undo each other's decisions. */
7303 && sched_pressure
== SCHED_PRESSURE_NONE
)
7304 dfa_lookahead
= targetm
.sched
.first_cycle_multipass_dfa_lookahead ();
7308 /* Set to "0" so that we recalculate. */
7309 max_lookahead_tries
= 0;
7311 if (targetm
.sched
.init_dfa_pre_cycle_insn
)
7312 targetm
.sched
.init_dfa_pre_cycle_insn ();
7314 if (targetm
.sched
.init_dfa_post_cycle_insn
)
7315 targetm
.sched
.init_dfa_post_cycle_insn ();
7318 dfa_state_size
= state_size ();
7320 init_alias_analysis ();
7323 df_set_flags (DF_LR_RUN_DCE
);
7324 df_note_add_problem ();
7326 /* More problems needed for interloop dep calculation in SMS. */
7327 if (common_sched_info
->sched_pass_id
== SCHED_SMS_PASS
)
7329 df_rd_add_problem ();
7330 df_chain_add_problem (DF_DU_CHAIN
+ DF_UD_CHAIN
);
7335 /* Do not run DCE after reload, as this can kill nops inserted
7337 if (reload_completed
)
7338 df_clear_flags (DF_LR_RUN_DCE
);
7340 regstat_compute_calls_crossed ();
7342 if (targetm
.sched
.init_global
)
7343 targetm
.sched
.init_global (sched_dump
, sched_verbose
, get_max_uid () + 1);
7345 alloc_global_sched_pressure_data ();
7347 curr_state
= xmalloc (dfa_state_size
);
7350 static void haifa_init_only_bb (basic_block
, basic_block
);
7352 /* Initialize data structures specific to the Haifa scheduler. */
7354 haifa_sched_init (void)
7356 setup_sched_dump ();
7359 scheduled_insns
.create (0);
7361 if (spec_info
!= NULL
)
7363 sched_deps_info
->use_deps_list
= 1;
7364 sched_deps_info
->generate_spec_deps
= 1;
7367 /* Initialize luids, dependency caches, target and h_i_d for the
7372 auto_vec
<basic_block
> bbs (n_basic_blocks_for_fn (cfun
));
7374 FOR_EACH_BB_FN (bb
, cfun
)
7375 bbs
.quick_push (bb
);
7376 sched_init_luids (bbs
);
7377 sched_deps_init (true);
7378 sched_extend_target ();
7379 haifa_init_h_i_d (bbs
);
7382 sched_init_only_bb
= haifa_init_only_bb
;
7383 sched_split_block
= sched_split_block_1
;
7384 sched_create_empty_bb
= sched_create_empty_bb_1
;
7385 haifa_recovery_bb_ever_added_p
= false;
7387 nr_begin_data
= nr_begin_control
= nr_be_in_data
= nr_be_in_control
= 0;
7388 before_recovery
= 0;
7394 /* Finish work with the data specific to the Haifa scheduler. */
7396 haifa_sched_finish (void)
7398 sched_create_empty_bb
= NULL
;
7399 sched_split_block
= NULL
;
7400 sched_init_only_bb
= NULL
;
7402 if (spec_info
&& spec_info
->dump
)
7404 char c
= reload_completed
? 'a' : 'b';
7406 fprintf (spec_info
->dump
,
7407 ";; %s:\n", current_function_name ());
7409 fprintf (spec_info
->dump
,
7410 ";; Procedure %cr-begin-data-spec motions == %d\n",
7412 fprintf (spec_info
->dump
,
7413 ";; Procedure %cr-be-in-data-spec motions == %d\n",
7415 fprintf (spec_info
->dump
,
7416 ";; Procedure %cr-begin-control-spec motions == %d\n",
7417 c
, nr_begin_control
);
7418 fprintf (spec_info
->dump
,
7419 ";; Procedure %cr-be-in-control-spec motions == %d\n",
7420 c
, nr_be_in_control
);
7423 scheduled_insns
.release ();
7425 /* Finalize h_i_d, dependency caches, and luids for the whole
7426 function. Target will be finalized in md_global_finish (). */
7427 sched_deps_finish ();
7428 sched_finish_luids ();
7429 current_sched_info
= NULL
;
7434 /* Free global data used during insn scheduling. This function works with
7435 the common data shared between the schedulers. */
7440 haifa_finish_h_i_d ();
7441 free_global_sched_pressure_data ();
7444 if (targetm
.sched
.finish_global
)
7445 targetm
.sched
.finish_global (sched_dump
, sched_verbose
);
7447 end_alias_analysis ();
7449 regstat_free_calls_crossed ();
7454 /* Free all delay_pair structures that were recorded. */
7456 free_delay_pairs (void)
7460 delay_htab
->empty ();
7461 delay_htab_i2
->empty ();
7465 /* Fix INSN_TICKs of the instructions in the current block as well as
7466 INSN_TICKs of their dependents.
7467 HEAD and TAIL are the begin and the end of the current scheduled block. */
7469 fix_inter_tick (rtx_insn
*head
, rtx_insn
*tail
)
7471 /* Set of instructions with corrected INSN_TICK. */
7472 auto_bitmap processed
;
7473 /* ??? It is doubtful if we should assume that cycle advance happens on
7474 basic block boundaries. Basically insns that are unconditionally ready
7475 on the start of the block are more preferable then those which have
7476 a one cycle dependency over insn from the previous block. */
7477 int next_clock
= clock_var
+ 1;
7479 /* Iterates over scheduled instructions and fix their INSN_TICKs and
7480 INSN_TICKs of dependent instructions, so that INSN_TICKs are consistent
7481 across different blocks. */
7482 for (tail
= NEXT_INSN (tail
); head
!= tail
; head
= NEXT_INSN (head
))
7487 sd_iterator_def sd_it
;
7490 tick
= INSN_TICK (head
);
7491 gcc_assert (tick
>= MIN_TICK
);
7493 /* Fix INSN_TICK of instruction from just scheduled block. */
7494 if (bitmap_set_bit (processed
, INSN_LUID (head
)))
7498 if (tick
< MIN_TICK
)
7501 INSN_TICK (head
) = tick
;
7504 if (DEBUG_INSN_P (head
))
7507 FOR_EACH_DEP (head
, SD_LIST_RES_FORW
, sd_it
, dep
)
7511 next
= DEP_CON (dep
);
7512 tick
= INSN_TICK (next
);
7514 if (tick
!= INVALID_TICK
7515 /* If NEXT has its INSN_TICK calculated, fix it.
7516 If not - it will be properly calculated from
7517 scratch later in fix_tick_ready. */
7518 && bitmap_set_bit (processed
, INSN_LUID (next
)))
7522 if (tick
< MIN_TICK
)
7525 if (tick
> INTER_TICK (next
))
7526 INTER_TICK (next
) = tick
;
7528 tick
= INTER_TICK (next
);
7530 INSN_TICK (next
) = tick
;
7537 /* Check if NEXT is ready to be added to the ready or queue list.
7538 If "yes", add it to the proper list.
7540 -1 - is not ready yet,
7541 0 - added to the ready list,
7542 0 < N - queued for N cycles. */
7544 try_ready (rtx_insn
*next
)
7546 ds_t old_ts
, new_ts
;
7548 old_ts
= TODO_SPEC (next
);
7550 gcc_assert (!(old_ts
& ~(SPECULATIVE
| HARD_DEP
| DEP_CONTROL
| DEP_POSTPONED
))
7551 && (old_ts
== HARD_DEP
7552 || old_ts
== DEP_POSTPONED
7553 || (old_ts
& SPECULATIVE
)
7554 || old_ts
== DEP_CONTROL
));
7556 new_ts
= recompute_todo_spec (next
, false);
7558 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7559 gcc_assert (new_ts
== old_ts
7560 && QUEUE_INDEX (next
) == QUEUE_NOWHERE
);
7561 else if (current_sched_info
->new_ready
)
7562 new_ts
= current_sched_info
->new_ready (next
, new_ts
);
7564 /* * if !(old_ts & SPECULATIVE) (e.g. HARD_DEP or 0), then insn might
7565 have its original pattern or changed (speculative) one. This is due
7566 to changing ebb in region scheduling.
7567 * But if (old_ts & SPECULATIVE), then we are pretty sure that insn
7568 has speculative pattern.
7570 We can't assert (!(new_ts & HARD_DEP) || new_ts == old_ts) here because
7571 control-speculative NEXT could have been discarded by sched-rgn.c
7572 (the same case as when discarded by can_schedule_ready_p ()). */
7574 if ((new_ts
& SPECULATIVE
)
7575 /* If (old_ts == new_ts), then (old_ts & SPECULATIVE) and we don't
7576 need to change anything. */
7577 && new_ts
!= old_ts
)
7582 gcc_assert ((new_ts
& SPECULATIVE
) && !(new_ts
& ~SPECULATIVE
));
7584 res
= haifa_speculate_insn (next
, new_ts
, &new_pat
);
7589 /* It would be nice to change DEP_STATUS of all dependences,
7590 which have ((DEP_STATUS & SPECULATIVE) == new_ts) to HARD_DEP,
7591 so we won't reanalyze anything. */
7596 /* We follow the rule, that every speculative insn
7597 has non-null ORIG_PAT. */
7598 if (!ORIG_PAT (next
))
7599 ORIG_PAT (next
) = PATTERN (next
);
7603 if (!ORIG_PAT (next
))
7604 /* If we gonna to overwrite the original pattern of insn,
7606 ORIG_PAT (next
) = PATTERN (next
);
7608 res
= haifa_change_pattern (next
, new_pat
);
7617 /* We need to restore pattern only if (new_ts == 0), because otherwise it is
7618 either correct (new_ts & SPECULATIVE),
7619 or we simply don't care (new_ts & HARD_DEP). */
7621 gcc_assert (!ORIG_PAT (next
)
7622 || !IS_SPECULATION_BRANCHY_CHECK_P (next
));
7624 TODO_SPEC (next
) = new_ts
;
7626 if (new_ts
& (HARD_DEP
| DEP_POSTPONED
))
7628 /* We can't assert (QUEUE_INDEX (next) == QUEUE_NOWHERE) here because
7629 control-speculative NEXT could have been discarded by sched-rgn.c
7630 (the same case as when discarded by can_schedule_ready_p ()). */
7631 /*gcc_assert (QUEUE_INDEX (next) == QUEUE_NOWHERE);*/
7633 change_queue_index (next
, QUEUE_NOWHERE
);
7637 else if (!(new_ts
& BEGIN_SPEC
)
7638 && ORIG_PAT (next
) && PREDICATED_PAT (next
) == NULL_RTX
7639 && !IS_SPECULATION_CHECK_P (next
))
7640 /* We should change pattern of every previously speculative
7641 instruction - and we determine if NEXT was speculative by using
7642 ORIG_PAT field. Except one case - speculation checks have ORIG_PAT
7643 pat too, so skip them. */
7645 bool success
= haifa_change_pattern (next
, ORIG_PAT (next
));
7646 gcc_assert (success
);
7647 ORIG_PAT (next
) = 0;
7650 if (sched_verbose
>= 2)
7652 fprintf (sched_dump
, ";;\t\tdependencies resolved: insn %s",
7653 (*current_sched_info
->print_insn
) (next
, 0));
7655 if (spec_info
&& spec_info
->dump
)
7657 if (new_ts
& BEGIN_DATA
)
7658 fprintf (spec_info
->dump
, "; data-spec;");
7659 if (new_ts
& BEGIN_CONTROL
)
7660 fprintf (spec_info
->dump
, "; control-spec;");
7661 if (new_ts
& BE_IN_CONTROL
)
7662 fprintf (spec_info
->dump
, "; in-control-spec;");
7664 if (TODO_SPEC (next
) & DEP_CONTROL
)
7665 fprintf (sched_dump
, " predicated");
7666 fprintf (sched_dump
, "\n");
7669 adjust_priority (next
);
7671 return fix_tick_ready (next
);
7674 /* Calculate INSN_TICK of NEXT and add it to either ready or queue list. */
7676 fix_tick_ready (rtx_insn
*next
)
7680 if (!DEBUG_INSN_P (next
) && !sd_lists_empty_p (next
, SD_LIST_RES_BACK
))
7683 sd_iterator_def sd_it
;
7686 tick
= INSN_TICK (next
);
7687 /* if tick is not equal to INVALID_TICK, then update
7688 INSN_TICK of NEXT with the most recent resolved dependence
7689 cost. Otherwise, recalculate from scratch. */
7690 full_p
= (tick
== INVALID_TICK
);
7692 FOR_EACH_DEP (next
, SD_LIST_RES_BACK
, sd_it
, dep
)
7694 rtx_insn
*pro
= DEP_PRO (dep
);
7697 gcc_assert (INSN_TICK (pro
) >= MIN_TICK
);
7699 tick1
= INSN_TICK (pro
) + dep_cost (dep
);
7710 INSN_TICK (next
) = tick
;
7712 delay
= tick
- clock_var
;
7713 if (delay
<= 0 || sched_pressure
!= SCHED_PRESSURE_NONE
|| sched_fusion
)
7714 delay
= QUEUE_READY
;
7716 change_queue_index (next
, delay
);
7721 /* Move NEXT to the proper queue list with (DELAY >= 1),
7722 or add it to the ready list (DELAY == QUEUE_READY),
7723 or remove it from ready and queue lists at all (DELAY == QUEUE_NOWHERE). */
7725 change_queue_index (rtx_insn
*next
, int delay
)
7727 int i
= QUEUE_INDEX (next
);
7729 gcc_assert (QUEUE_NOWHERE
<= delay
&& delay
<= max_insn_queue_index
7731 gcc_assert (i
!= QUEUE_SCHEDULED
);
7733 if ((delay
> 0 && NEXT_Q_AFTER (q_ptr
, delay
) == i
)
7734 || (delay
< 0 && delay
== i
))
7735 /* We have nothing to do. */
7738 /* Remove NEXT from wherever it is now. */
7739 if (i
== QUEUE_READY
)
7740 ready_remove_insn (next
);
7742 queue_remove (next
);
7744 /* Add it to the proper place. */
7745 if (delay
== QUEUE_READY
)
7746 ready_add (readyp
, next
, false);
7747 else if (delay
>= 1)
7748 queue_insn (next
, delay
, "change queue index");
7750 if (sched_verbose
>= 2)
7752 fprintf (sched_dump
, ";;\t\ttick updated: insn %s",
7753 (*current_sched_info
->print_insn
) (next
, 0));
7755 if (delay
== QUEUE_READY
)
7756 fprintf (sched_dump
, " into ready\n");
7757 else if (delay
>= 1)
7758 fprintf (sched_dump
, " into queue with cost=%d\n", delay
);
7760 fprintf (sched_dump
, " removed from ready or queue lists\n");
7764 static int sched_ready_n_insns
= -1;
7766 /* Initialize per region data structures. */
7768 sched_extend_ready_list (int new_sched_ready_n_insns
)
7772 if (sched_ready_n_insns
== -1)
7773 /* At the first call we need to initialize one more choice_stack
7777 sched_ready_n_insns
= 0;
7778 scheduled_insns
.reserve (new_sched_ready_n_insns
);
7781 i
= sched_ready_n_insns
+ 1;
7783 ready
.veclen
= new_sched_ready_n_insns
+ issue_rate
;
7784 ready
.vec
= XRESIZEVEC (rtx_insn
*, ready
.vec
, ready
.veclen
);
7786 gcc_assert (new_sched_ready_n_insns
>= sched_ready_n_insns
);
7788 ready_try
= (signed char *) xrecalloc (ready_try
, new_sched_ready_n_insns
,
7789 sched_ready_n_insns
,
7790 sizeof (*ready_try
));
7792 /* We allocate +1 element to save initial state in the choice_stack[0]
7794 choice_stack
= XRESIZEVEC (struct choice_entry
, choice_stack
,
7795 new_sched_ready_n_insns
+ 1);
7797 for (; i
<= new_sched_ready_n_insns
; i
++)
7799 choice_stack
[i
].state
= xmalloc (dfa_state_size
);
7801 if (targetm
.sched
.first_cycle_multipass_init
)
7802 targetm
.sched
.first_cycle_multipass_init (&(choice_stack
[i
]
7806 sched_ready_n_insns
= new_sched_ready_n_insns
;
7809 /* Free per region data structures. */
7811 sched_finish_ready_list (void)
7822 for (i
= 0; i
<= sched_ready_n_insns
; i
++)
7824 if (targetm
.sched
.first_cycle_multipass_fini
)
7825 targetm
.sched
.first_cycle_multipass_fini (&(choice_stack
[i
]
7828 free (choice_stack
[i
].state
);
7830 free (choice_stack
);
7831 choice_stack
= NULL
;
7833 sched_ready_n_insns
= -1;
7837 haifa_luid_for_non_insn (rtx x
)
7839 gcc_assert (NOTE_P (x
) || LABEL_P (x
));
7844 /* Generates recovery code for INSN. */
7846 generate_recovery_code (rtx_insn
*insn
)
7848 if (TODO_SPEC (insn
) & BEGIN_SPEC
)
7849 begin_speculative_block (insn
);
7851 /* Here we have insn with no dependencies to
7852 instructions other then CHECK_SPEC ones. */
7854 if (TODO_SPEC (insn
) & BE_IN_SPEC
)
7855 add_to_speculative_block (insn
);
7859 Tries to add speculative dependencies of type FS between instructions
7860 in deps_list L and TWIN. */
7862 process_insn_forw_deps_be_in_spec (rtx_insn
*insn
, rtx_insn
*twin
, ds_t fs
)
7864 sd_iterator_def sd_it
;
7867 FOR_EACH_DEP (insn
, SD_LIST_FORW
, sd_it
, dep
)
7872 consumer
= DEP_CON (dep
);
7874 ds
= DEP_STATUS (dep
);
7876 if (/* If we want to create speculative dep. */
7878 /* And we can do that because this is a true dep. */
7879 && (ds
& DEP_TYPES
) == DEP_TRUE
)
7881 gcc_assert (!(ds
& BE_IN_SPEC
));
7883 if (/* If this dep can be overcome with 'begin speculation'. */
7885 /* Then we have a choice: keep the dep 'begin speculative'
7886 or transform it into 'be in speculative'. */
7888 if (/* In try_ready we assert that if insn once became ready
7889 it can be removed from the ready (or queue) list only
7890 due to backend decision. Hence we can't let the
7891 probability of the speculative dep to decrease. */
7892 ds_weak (ds
) <= ds_weak (fs
))
7896 new_ds
= (ds
& ~BEGIN_SPEC
) | fs
;
7898 if (/* consumer can 'be in speculative'. */
7899 sched_insn_is_legitimate_for_speculation_p (consumer
,
7901 /* Transform it to be in speculative. */
7906 /* Mark the dep as 'be in speculative'. */
7911 dep_def _new_dep
, *new_dep
= &_new_dep
;
7913 init_dep_1 (new_dep
, twin
, consumer
, DEP_TYPE (dep
), ds
);
7914 sd_add_dep (new_dep
, false);
7919 /* Generates recovery code for BEGIN speculative INSN. */
7921 begin_speculative_block (rtx_insn
*insn
)
7923 if (TODO_SPEC (insn
) & BEGIN_DATA
)
7925 if (TODO_SPEC (insn
) & BEGIN_CONTROL
)
7928 create_check_block_twin (insn
, false);
7930 TODO_SPEC (insn
) &= ~BEGIN_SPEC
;
7933 static void haifa_init_insn (rtx_insn
*);
7935 /* Generates recovery code for BE_IN speculative INSN. */
7937 add_to_speculative_block (rtx_insn
*insn
)
7940 sd_iterator_def sd_it
;
7942 auto_vec
<rtx_insn
*, 10> twins
;
7944 ts
= TODO_SPEC (insn
);
7945 gcc_assert (!(ts
& ~BE_IN_SPEC
));
7947 if (ts
& BE_IN_DATA
)
7949 if (ts
& BE_IN_CONTROL
)
7952 TODO_SPEC (insn
) &= ~BE_IN_SPEC
;
7953 gcc_assert (!TODO_SPEC (insn
));
7955 DONE_SPEC (insn
) |= ts
;
7957 /* First we convert all simple checks to branchy. */
7958 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7959 sd_iterator_cond (&sd_it
, &dep
);)
7961 rtx_insn
*check
= DEP_PRO (dep
);
7963 if (IS_SPECULATION_SIMPLE_CHECK_P (check
))
7965 create_check_block_twin (check
, true);
7967 /* Restart search. */
7968 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7971 /* Continue search. */
7972 sd_iterator_next (&sd_it
);
7975 auto_vec
<rtx_insn
*> priorities_roots
;
7976 clear_priorities (insn
, &priorities_roots
);
7980 rtx_insn
*check
, *twin
;
7983 /* Get the first backward dependency of INSN. */
7984 sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
7985 if (!sd_iterator_cond (&sd_it
, &dep
))
7986 /* INSN has no backward dependencies left. */
7989 gcc_assert ((DEP_STATUS (dep
) & BEGIN_SPEC
) == 0
7990 && (DEP_STATUS (dep
) & BE_IN_SPEC
) != 0
7991 && (DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
7993 check
= DEP_PRO (dep
);
7995 gcc_assert (!IS_SPECULATION_CHECK_P (check
) && !ORIG_PAT (check
)
7996 && QUEUE_INDEX (check
) == QUEUE_NOWHERE
);
7998 rec
= BLOCK_FOR_INSN (check
);
8000 twin
= emit_insn_before (copy_insn (PATTERN (insn
)), BB_END (rec
));
8001 haifa_init_insn (twin
);
8003 sd_copy_back_deps (twin
, insn
, true);
8005 if (sched_verbose
&& spec_info
->dump
)
8006 /* INSN_BB (insn) isn't determined for twin insns yet.
8007 So we can't use current_sched_info->print_insn. */
8008 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8009 INSN_UID (twin
), rec
->index
);
8011 twins
.safe_push (twin
);
8013 /* Add dependences between TWIN and all appropriate
8014 instructions from REC. */
8015 FOR_EACH_DEP (insn
, SD_LIST_SPEC_BACK
, sd_it
, dep
)
8017 rtx_insn
*pro
= DEP_PRO (dep
);
8019 gcc_assert (DEP_TYPE (dep
) == REG_DEP_TRUE
);
8021 /* INSN might have dependencies from the instructions from
8022 several recovery blocks. At this iteration we process those
8023 producers that reside in REC. */
8024 if (BLOCK_FOR_INSN (pro
) == rec
)
8026 dep_def _new_dep
, *new_dep
= &_new_dep
;
8028 init_dep (new_dep
, pro
, twin
, REG_DEP_TRUE
);
8029 sd_add_dep (new_dep
, false);
8033 process_insn_forw_deps_be_in_spec (insn
, twin
, ts
);
8035 /* Remove all dependencies between INSN and insns in REC. */
8036 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8037 sd_iterator_cond (&sd_it
, &dep
);)
8039 rtx_insn
*pro
= DEP_PRO (dep
);
8041 if (BLOCK_FOR_INSN (pro
) == rec
)
8042 sd_delete_dep (sd_it
);
8044 sd_iterator_next (&sd_it
);
8048 /* We couldn't have added the dependencies between INSN and TWINS earlier
8049 because that would make TWINS appear in the INSN_BACK_DEPS (INSN). */
8052 FOR_EACH_VEC_ELT_REVERSE (twins
, i
, twin
)
8054 dep_def _new_dep
, *new_dep
= &_new_dep
;
8056 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8057 sd_add_dep (new_dep
, false);
8060 calc_priorities (priorities_roots
);
8063 /* Extends and fills with zeros (only the new part) array pointed to by P. */
8065 xrecalloc (void *p
, size_t new_nmemb
, size_t old_nmemb
, size_t size
)
8067 gcc_assert (new_nmemb
>= old_nmemb
);
8068 p
= XRESIZEVAR (void, p
, new_nmemb
* size
);
8069 memset (((char *) p
) + old_nmemb
* size
, 0, (new_nmemb
- old_nmemb
) * size
);
8074 Find fallthru edge from PRED. */
8076 find_fallthru_edge_from (basic_block pred
)
8081 succ
= pred
->next_bb
;
8082 gcc_assert (succ
->prev_bb
== pred
);
8084 if (EDGE_COUNT (pred
->succs
) <= EDGE_COUNT (succ
->preds
))
8086 e
= find_fallthru_edge (pred
->succs
);
8090 gcc_assert (e
->dest
== succ
|| e
->dest
->index
== EXIT_BLOCK
);
8096 e
= find_fallthru_edge (succ
->preds
);
8100 gcc_assert (e
->src
== pred
);
8108 /* Extend per basic block data structures. */
8110 sched_extend_bb (void)
8112 /* The following is done to keep current_sched_info->next_tail non null. */
8113 rtx_insn
*end
= BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
);
8114 rtx_insn
*insn
= DEBUG_INSN_P (end
) ? prev_nondebug_insn (end
) : end
;
8115 if (NEXT_INSN (end
) == 0
8118 /* Don't emit a NOTE if it would end up before a BARRIER. */
8119 && !BARRIER_P (next_nondebug_insn (end
))))
8121 rtx_note
*note
= emit_note_after (NOTE_INSN_DELETED
, end
);
8122 /* Make note appear outside BB. */
8123 set_block_for_insn (note
, NULL
);
8124 BB_END (EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
) = end
;
8128 /* Init per basic block data structures. */
8130 sched_init_bbs (void)
8135 /* Initialize BEFORE_RECOVERY variable. */
8137 init_before_recovery (basic_block
*before_recovery_ptr
)
8142 last
= EXIT_BLOCK_PTR_FOR_FN (cfun
)->prev_bb
;
8143 e
= find_fallthru_edge_from (last
);
8147 /* We create two basic blocks:
8148 1. Single instruction block is inserted right after E->SRC
8150 2. Empty block right before EXIT_BLOCK.
8151 Between these two blocks recovery blocks will be emitted. */
8153 basic_block single
, empty
;
8155 /* If the fallthrough edge to exit we've found is from the block we've
8156 created before, don't do anything more. */
8157 if (last
== after_recovery
)
8160 adding_bb_to_current_region_p
= false;
8162 single
= sched_create_empty_bb (last
);
8163 empty
= sched_create_empty_bb (single
);
8165 /* Add new blocks to the root loop. */
8166 if (current_loops
!= NULL
)
8168 add_bb_to_loop (single
, (*current_loops
->larray
)[0]);
8169 add_bb_to_loop (empty
, (*current_loops
->larray
)[0]);
8172 single
->count
= last
->count
;
8173 empty
->count
= last
->count
;
8174 BB_COPY_PARTITION (single
, last
);
8175 BB_COPY_PARTITION (empty
, last
);
8177 redirect_edge_succ (e
, single
);
8178 make_single_succ_edge (single
, empty
, 0);
8179 make_single_succ_edge (empty
, EXIT_BLOCK_PTR_FOR_FN (cfun
),
8182 rtx_code_label
*label
= block_label (empty
);
8183 rtx_jump_insn
*x
= emit_jump_insn_after (targetm
.gen_jump (label
),
8185 JUMP_LABEL (x
) = label
;
8186 LABEL_NUSES (label
)++;
8187 haifa_init_insn (x
);
8189 emit_barrier_after (x
);
8191 sched_init_only_bb (empty
, NULL
);
8192 sched_init_only_bb (single
, NULL
);
8195 adding_bb_to_current_region_p
= true;
8196 before_recovery
= single
;
8197 after_recovery
= empty
;
8199 if (before_recovery_ptr
)
8200 *before_recovery_ptr
= before_recovery
;
8202 if (sched_verbose
>= 2 && spec_info
->dump
)
8203 fprintf (spec_info
->dump
,
8204 ";;\t\tFixed fallthru to EXIT : %d->>%d->%d->>EXIT\n",
8205 last
->index
, single
->index
, empty
->index
);
8208 before_recovery
= last
;
8211 /* Returns new recovery block. */
8213 sched_create_recovery_block (basic_block
*before_recovery_ptr
)
8218 haifa_recovery_bb_recently_added_p
= true;
8219 haifa_recovery_bb_ever_added_p
= true;
8221 init_before_recovery (before_recovery_ptr
);
8223 barrier
= get_last_bb_insn (before_recovery
);
8224 gcc_assert (BARRIER_P (barrier
));
8226 rtx_insn
*label
= emit_label_after (gen_label_rtx (), barrier
);
8228 rec
= create_basic_block (label
, label
, before_recovery
);
8230 /* A recovery block always ends with an unconditional jump. */
8231 emit_barrier_after (BB_END (rec
));
8233 if (BB_PARTITION (before_recovery
) != BB_UNPARTITIONED
)
8234 BB_SET_PARTITION (rec
, BB_COLD_PARTITION
);
8236 if (sched_verbose
&& spec_info
->dump
)
8237 fprintf (spec_info
->dump
, ";;\t\tGenerated recovery block rec%d\n",
8243 /* Create edges: FIRST_BB -> REC; FIRST_BB -> SECOND_BB; REC -> SECOND_BB
8244 and emit necessary jumps. */
8246 sched_create_recovery_edges (basic_block first_bb
, basic_block rec
,
8247 basic_block second_bb
)
8251 /* This is fixing of incoming edge. */
8252 /* ??? Which other flags should be specified? */
8253 if (BB_PARTITION (first_bb
) != BB_PARTITION (rec
))
8254 /* Partition type is the same, if it is "unpartitioned". */
8255 edge_flags
= EDGE_CROSSING
;
8259 edge e2
= single_succ_edge (first_bb
);
8260 edge e
= make_edge (first_bb
, rec
, edge_flags
);
8262 /* TODO: The actual probability can be determined and is computed as
8263 'todo_spec' variable in create_check_block_twin and
8264 in sel-sched.c `check_ds' in create_speculation_check. */
8265 e
->probability
= profile_probability::very_unlikely ();
8266 rec
->count
= e
->count ();
8267 e2
->probability
= e
->probability
.invert ();
8269 rtx_code_label
*label
= block_label (second_bb
);
8270 rtx_jump_insn
*jump
= emit_jump_insn_after (targetm
.gen_jump (label
),
8272 JUMP_LABEL (jump
) = label
;
8273 LABEL_NUSES (label
)++;
8275 if (BB_PARTITION (second_bb
) != BB_PARTITION (rec
))
8276 /* Partition type is the same, if it is "unpartitioned". */
8278 /* Rewritten from cfgrtl.c. */
8279 if (crtl
->has_bb_partition
&& targetm_common
.have_named_sections
)
8281 /* We don't need the same note for the check because
8282 any_condjump_p (check) == true. */
8283 CROSSING_JUMP_P (jump
) = 1;
8285 edge_flags
= EDGE_CROSSING
;
8290 make_single_succ_edge (rec
, second_bb
, edge_flags
);
8291 if (dom_info_available_p (CDI_DOMINATORS
))
8292 set_immediate_dominator (CDI_DOMINATORS
, rec
, first_bb
);
8295 /* This function creates recovery code for INSN. If MUTATE_P is nonzero,
8296 INSN is a simple check, that should be converted to branchy one. */
8298 create_check_block_twin (rtx_insn
*insn
, bool mutate_p
)
8301 rtx_insn
*label
, *check
, *twin
;
8304 sd_iterator_def sd_it
;
8306 dep_def _new_dep
, *new_dep
= &_new_dep
;
8309 gcc_assert (ORIG_PAT (insn
) != NULL_RTX
);
8312 todo_spec
= TODO_SPEC (insn
);
8315 gcc_assert (IS_SPECULATION_SIMPLE_CHECK_P (insn
)
8316 && (TODO_SPEC (insn
) & SPECULATIVE
) == 0);
8318 todo_spec
= CHECK_SPEC (insn
);
8321 todo_spec
&= SPECULATIVE
;
8323 /* Create recovery block. */
8324 if (mutate_p
|| targetm
.sched
.needs_block_p (todo_spec
))
8326 rec
= sched_create_recovery_block (NULL
);
8327 label
= BB_HEAD (rec
);
8331 rec
= EXIT_BLOCK_PTR_FOR_FN (cfun
);
8336 check_pat
= targetm
.sched
.gen_spec_check (insn
, label
, todo_spec
);
8338 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8340 /* To have mem_reg alive at the beginning of second_bb,
8341 we emit check BEFORE insn, so insn after splitting
8342 insn will be at the beginning of second_bb, which will
8343 provide us with the correct life information. */
8344 check
= emit_jump_insn_before (check_pat
, insn
);
8345 JUMP_LABEL (check
) = label
;
8346 LABEL_NUSES (label
)++;
8349 check
= emit_insn_before (check_pat
, insn
);
8351 /* Extend data structures. */
8352 haifa_init_insn (check
);
8354 /* CHECK is being added to current region. Extend ready list. */
8355 gcc_assert (sched_ready_n_insns
!= -1);
8356 sched_extend_ready_list (sched_ready_n_insns
+ 1);
8358 if (current_sched_info
->add_remove_insn
)
8359 current_sched_info
->add_remove_insn (insn
, 0);
8361 RECOVERY_BLOCK (check
) = rec
;
8363 if (sched_verbose
&& spec_info
->dump
)
8364 fprintf (spec_info
->dump
, ";;\t\tGenerated check insn : %s\n",
8365 (*current_sched_info
->print_insn
) (check
, 0));
8367 gcc_assert (ORIG_PAT (insn
));
8369 /* Initialize TWIN (twin is a duplicate of original instruction
8370 in the recovery block). */
8371 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8373 sd_iterator_def sd_it
;
8376 FOR_EACH_DEP (insn
, SD_LIST_RES_BACK
, sd_it
, dep
)
8377 if ((DEP_STATUS (dep
) & DEP_OUTPUT
) != 0)
8379 struct _dep _dep2
, *dep2
= &_dep2
;
8381 init_dep (dep2
, DEP_PRO (dep
), check
, REG_DEP_TRUE
);
8383 sd_add_dep (dep2
, true);
8386 twin
= emit_insn_after (ORIG_PAT (insn
), BB_END (rec
));
8387 haifa_init_insn (twin
);
8389 if (sched_verbose
&& spec_info
->dump
)
8390 /* INSN_BB (insn) isn't determined for twin insns yet.
8391 So we can't use current_sched_info->print_insn. */
8392 fprintf (spec_info
->dump
, ";;\t\tGenerated twin insn : %d/rec%d\n",
8393 INSN_UID (twin
), rec
->index
);
8397 ORIG_PAT (check
) = ORIG_PAT (insn
);
8398 HAS_INTERNAL_DEP (check
) = 1;
8400 /* ??? We probably should change all OUTPUT dependencies to
8404 /* Copy all resolved back dependencies of INSN to TWIN. This will
8405 provide correct value for INSN_TICK (TWIN). */
8406 sd_copy_back_deps (twin
, insn
, true);
8408 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8409 /* In case of branchy check, fix CFG. */
8411 basic_block first_bb
, second_bb
;
8414 first_bb
= BLOCK_FOR_INSN (check
);
8415 second_bb
= sched_split_block (first_bb
, check
);
8417 sched_create_recovery_edges (first_bb
, rec
, second_bb
);
8419 sched_init_only_bb (second_bb
, first_bb
);
8420 sched_init_only_bb (rec
, EXIT_BLOCK_PTR_FOR_FN (cfun
));
8422 jump
= BB_END (rec
);
8423 haifa_init_insn (jump
);
8426 /* Move backward dependences from INSN to CHECK and
8427 move forward dependences from INSN to TWIN. */
8429 /* First, create dependencies between INSN's producers and CHECK & TWIN. */
8430 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8432 rtx_insn
*pro
= DEP_PRO (dep
);
8435 /* If BEGIN_DATA: [insn ~~TRUE~~> producer]:
8436 check --TRUE--> producer ??? or ANTI ???
8437 twin --TRUE--> producer
8438 twin --ANTI--> check
8440 If BEGIN_CONTROL: [insn ~~ANTI~~> producer]:
8441 check --ANTI--> producer
8442 twin --ANTI--> producer
8443 twin --ANTI--> check
8445 If BE_IN_SPEC: [insn ~~TRUE~~> producer]:
8446 check ~~TRUE~~> producer
8447 twin ~~TRUE~~> producer
8448 twin --ANTI--> check */
8450 ds
= DEP_STATUS (dep
);
8452 if (ds
& BEGIN_SPEC
)
8454 gcc_assert (!mutate_p
);
8458 init_dep_1 (new_dep
, pro
, check
, DEP_TYPE (dep
), ds
);
8459 sd_add_dep (new_dep
, false);
8461 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8463 DEP_CON (new_dep
) = twin
;
8464 sd_add_dep (new_dep
, false);
8468 /* Second, remove backward dependencies of INSN. */
8469 for (sd_it
= sd_iterator_start (insn
, SD_LIST_SPEC_BACK
);
8470 sd_iterator_cond (&sd_it
, &dep
);)
8472 if ((DEP_STATUS (dep
) & BEGIN_SPEC
)
8474 /* We can delete this dep because we overcome it with
8475 BEGIN_SPECULATION. */
8476 sd_delete_dep (sd_it
);
8478 sd_iterator_next (&sd_it
);
8481 /* Future Speculations. Determine what BE_IN speculations will be like. */
8484 /* Fields (DONE_SPEC (x) & BEGIN_SPEC) and CHECK_SPEC (x) are set only
8487 gcc_assert (!DONE_SPEC (insn
));
8491 ds_t ts
= TODO_SPEC (insn
);
8493 DONE_SPEC (insn
) = ts
& BEGIN_SPEC
;
8494 CHECK_SPEC (check
) = ts
& BEGIN_SPEC
;
8496 /* Luckiness of future speculations solely depends upon initial
8497 BEGIN speculation. */
8498 if (ts
& BEGIN_DATA
)
8499 fs
= set_dep_weak (fs
, BE_IN_DATA
, get_dep_weak (ts
, BEGIN_DATA
));
8500 if (ts
& BEGIN_CONTROL
)
8501 fs
= set_dep_weak (fs
, BE_IN_CONTROL
,
8502 get_dep_weak (ts
, BEGIN_CONTROL
));
8505 CHECK_SPEC (check
) = CHECK_SPEC (insn
);
8507 /* Future speculations: call the helper. */
8508 process_insn_forw_deps_be_in_spec (insn
, twin
, fs
);
8510 if (rec
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8512 /* Which types of dependencies should we use here is,
8513 generally, machine-dependent question... But, for now,
8518 init_dep (new_dep
, insn
, check
, REG_DEP_TRUE
);
8519 sd_add_dep (new_dep
, false);
8521 init_dep (new_dep
, insn
, twin
, REG_DEP_OUTPUT
);
8522 sd_add_dep (new_dep
, false);
8526 if (spec_info
->dump
)
8527 fprintf (spec_info
->dump
, ";;\t\tRemoved simple check : %s\n",
8528 (*current_sched_info
->print_insn
) (insn
, 0));
8530 /* Remove all dependencies of the INSN. */
8532 sd_it
= sd_iterator_start (insn
, (SD_LIST_FORW
8534 | SD_LIST_RES_BACK
));
8535 while (sd_iterator_cond (&sd_it
, &dep
))
8536 sd_delete_dep (sd_it
);
8539 /* If former check (INSN) already was moved to the ready (or queue)
8540 list, add new check (CHECK) there too. */
8541 if (QUEUE_INDEX (insn
) != QUEUE_NOWHERE
)
8544 /* Remove old check from instruction stream and free its
8546 sched_remove_insn (insn
);
8549 init_dep (new_dep
, check
, twin
, REG_DEP_ANTI
);
8550 sd_add_dep (new_dep
, false);
8554 init_dep_1 (new_dep
, insn
, check
, REG_DEP_TRUE
, DEP_TRUE
| DEP_OUTPUT
);
8555 sd_add_dep (new_dep
, false);
8559 /* Fix priorities. If MUTATE_P is nonzero, this is not necessary,
8560 because it'll be done later in add_to_speculative_block. */
8562 auto_vec
<rtx_insn
*> priorities_roots
;
8564 clear_priorities (twin
, &priorities_roots
);
8565 calc_priorities (priorities_roots
);
8569 /* Removes dependency between instructions in the recovery block REC
8570 and usual region instructions. It keeps inner dependences so it
8571 won't be necessary to recompute them. */
8573 fix_recovery_deps (basic_block rec
)
8575 rtx_insn
*note
, *insn
, *jump
;
8576 auto_vec
<rtx_insn
*, 10> ready_list
;
8577 auto_bitmap in_ready
;
8579 /* NOTE - a basic block note. */
8580 note
= NEXT_INSN (BB_HEAD (rec
));
8581 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8582 insn
= BB_END (rec
);
8583 gcc_assert (JUMP_P (insn
));
8584 insn
= PREV_INSN (insn
);
8588 sd_iterator_def sd_it
;
8591 for (sd_it
= sd_iterator_start (insn
, SD_LIST_FORW
);
8592 sd_iterator_cond (&sd_it
, &dep
);)
8594 rtx_insn
*consumer
= DEP_CON (dep
);
8596 if (BLOCK_FOR_INSN (consumer
) != rec
)
8598 sd_delete_dep (sd_it
);
8600 if (bitmap_set_bit (in_ready
, INSN_LUID (consumer
)))
8601 ready_list
.safe_push (consumer
);
8605 gcc_assert ((DEP_STATUS (dep
) & DEP_TYPES
) == DEP_TRUE
);
8607 sd_iterator_next (&sd_it
);
8611 insn
= PREV_INSN (insn
);
8613 while (insn
!= note
);
8615 /* Try to add instructions to the ready or queue list. */
8618 FOR_EACH_VEC_ELT_REVERSE (ready_list
, i
, temp
)
8621 /* Fixing jump's dependences. */
8622 insn
= BB_HEAD (rec
);
8623 jump
= BB_END (rec
);
8625 gcc_assert (LABEL_P (insn
));
8626 insn
= NEXT_INSN (insn
);
8628 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (insn
));
8629 add_jump_dependencies (insn
, jump
);
8632 /* Change pattern of INSN to NEW_PAT. Invalidate cached haifa
8633 instruction data. */
8635 haifa_change_pattern (rtx_insn
*insn
, rtx new_pat
)
8639 t
= validate_change (insn
, &PATTERN (insn
), new_pat
, 0);
8643 update_insn_after_change (insn
);
8647 /* -1 - can't speculate,
8648 0 - for speculation with REQUEST mode it is OK to use
8649 current instruction pattern,
8650 1 - need to change pattern for *NEW_PAT to be speculative. */
8652 sched_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8654 gcc_assert (current_sched_info
->flags
& DO_SPECULATION
8655 && (request
& SPECULATIVE
)
8656 && sched_insn_is_legitimate_for_speculation_p (insn
, request
));
8658 if ((request
& spec_info
->mask
) != request
)
8661 if (request
& BE_IN_SPEC
8662 && !(request
& BEGIN_SPEC
))
8665 return targetm
.sched
.speculate_insn (insn
, request
, new_pat
);
8669 haifa_speculate_insn (rtx_insn
*insn
, ds_t request
, rtx
*new_pat
)
8671 gcc_assert (sched_deps_info
->generate_spec_deps
8672 && !IS_SPECULATION_CHECK_P (insn
));
8674 if (HAS_INTERNAL_DEP (insn
)
8675 || SCHED_GROUP_P (insn
))
8678 return sched_speculate_insn (insn
, request
, new_pat
);
8681 /* Print some information about block BB, which starts with HEAD and
8682 ends with TAIL, before scheduling it.
8683 I is zero, if scheduler is about to start with the fresh ebb. */
8685 dump_new_block_header (int i
, basic_block bb
, rtx_insn
*head
, rtx_insn
*tail
)
8688 fprintf (sched_dump
,
8689 ";; ======================================================\n");
8691 fprintf (sched_dump
,
8692 ";; =====================ADVANCING TO=====================\n");
8693 fprintf (sched_dump
,
8694 ";; -- basic block %d from %d to %d -- %s reload\n",
8695 bb
->index
, INSN_UID (head
), INSN_UID (tail
),
8696 (reload_completed
? "after" : "before"));
8697 fprintf (sched_dump
,
8698 ";; ======================================================\n");
8699 fprintf (sched_dump
, "\n");
8702 /* Unlink basic block notes and labels and saves them, so they
8703 can be easily restored. We unlink basic block notes in EBB to
8704 provide back-compatibility with the previous code, as target backends
8705 assume, that there'll be only instructions between
8706 current_sched_info->{head and tail}. We restore these notes as soon
8708 FIRST (LAST) is the first (last) basic block in the ebb.
8709 NB: In usual case (FIRST == LAST) nothing is really done. */
8711 unlink_bb_notes (basic_block first
, basic_block last
)
8713 /* We DON'T unlink basic block notes of the first block in the ebb. */
8717 bb_header
= XNEWVEC (rtx_insn
*, last_basic_block_for_fn (cfun
));
8719 /* Make a sentinel. */
8720 if (last
->next_bb
!= EXIT_BLOCK_PTR_FOR_FN (cfun
))
8721 bb_header
[last
->next_bb
->index
] = 0;
8723 first
= first
->next_bb
;
8726 rtx_insn
*prev
, *label
, *note
, *next
;
8728 label
= BB_HEAD (last
);
8729 if (LABEL_P (label
))
8730 note
= NEXT_INSN (label
);
8733 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8735 prev
= PREV_INSN (label
);
8736 next
= NEXT_INSN (note
);
8737 gcc_assert (prev
&& next
);
8739 SET_NEXT_INSN (prev
) = next
;
8740 SET_PREV_INSN (next
) = prev
;
8742 bb_header
[last
->index
] = label
;
8747 last
= last
->prev_bb
;
8752 /* Restore basic block notes.
8753 FIRST is the first basic block in the ebb. */
8755 restore_bb_notes (basic_block first
)
8760 /* We DON'T unlink basic block notes of the first block in the ebb. */
8761 first
= first
->next_bb
;
8762 /* Remember: FIRST is actually a second basic block in the ebb. */
8764 while (first
!= EXIT_BLOCK_PTR_FOR_FN (cfun
)
8765 && bb_header
[first
->index
])
8767 rtx_insn
*prev
, *label
, *note
, *next
;
8769 label
= bb_header
[first
->index
];
8770 prev
= PREV_INSN (label
);
8771 next
= NEXT_INSN (prev
);
8773 if (LABEL_P (label
))
8774 note
= NEXT_INSN (label
);
8777 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (note
));
8779 bb_header
[first
->index
] = 0;
8781 SET_NEXT_INSN (prev
) = label
;
8782 SET_NEXT_INSN (note
) = next
;
8783 SET_PREV_INSN (next
) = note
;
8785 first
= first
->next_bb
;
8793 Fix CFG after both in- and inter-block movement of
8794 control_flow_insn_p JUMP. */
8796 fix_jump_move (rtx_insn
*jump
)
8798 basic_block bb
, jump_bb
, jump_bb_next
;
8800 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8801 jump_bb
= BLOCK_FOR_INSN (jump
);
8802 jump_bb_next
= jump_bb
->next_bb
;
8804 gcc_assert (common_sched_info
->sched_pass_id
== SCHED_EBB_PASS
8805 || IS_SPECULATION_BRANCHY_CHECK_P (jump
));
8807 if (!NOTE_INSN_BASIC_BLOCK_P (BB_END (jump_bb_next
)))
8808 /* if jump_bb_next is not empty. */
8809 BB_END (jump_bb
) = BB_END (jump_bb_next
);
8811 if (BB_END (bb
) != PREV_INSN (jump
))
8812 /* Then there are instruction after jump that should be placed
8814 BB_END (jump_bb_next
) = BB_END (bb
);
8816 /* Otherwise jump_bb_next is empty. */
8817 BB_END (jump_bb_next
) = NEXT_INSN (BB_HEAD (jump_bb_next
));
8819 /* To make assertion in move_insn happy. */
8820 BB_END (bb
) = PREV_INSN (jump
);
8822 update_bb_for_insn (jump_bb_next
);
8825 /* Fix CFG after interblock movement of control_flow_insn_p JUMP. */
8827 move_block_after_check (rtx_insn
*jump
)
8829 basic_block bb
, jump_bb
, jump_bb_next
;
8830 vec
<edge
, va_gc
> *t
;
8832 bb
= BLOCK_FOR_INSN (PREV_INSN (jump
));
8833 jump_bb
= BLOCK_FOR_INSN (jump
);
8834 jump_bb_next
= jump_bb
->next_bb
;
8836 update_bb_for_insn (jump_bb
);
8838 gcc_assert (IS_SPECULATION_CHECK_P (jump
)
8839 || IS_SPECULATION_CHECK_P (BB_END (jump_bb_next
)));
8841 unlink_block (jump_bb_next
);
8842 link_block (jump_bb_next
, bb
);
8846 move_succs (&(jump_bb
->succs
), bb
);
8847 move_succs (&(jump_bb_next
->succs
), jump_bb
);
8848 move_succs (&t
, jump_bb_next
);
8850 df_mark_solutions_dirty ();
8852 common_sched_info
->fix_recovery_cfg
8853 (bb
->index
, jump_bb
->index
, jump_bb_next
->index
);
8856 /* Helper function for move_block_after_check.
8857 This functions attaches edge vector pointed to by SUCCSP to
8860 move_succs (vec
<edge
, va_gc
> **succsp
, basic_block to
)
8865 gcc_assert (to
->succs
== 0);
8867 to
->succs
= *succsp
;
8869 FOR_EACH_EDGE (e
, ei
, to
->succs
)
8875 /* Remove INSN from the instruction stream.
8876 INSN should have any dependencies. */
8878 sched_remove_insn (rtx_insn
*insn
)
8880 sd_finish_insn (insn
);
8882 change_queue_index (insn
, QUEUE_NOWHERE
);
8883 current_sched_info
->add_remove_insn (insn
, 1);
8887 /* Clear priorities of all instructions, that are forward dependent on INSN.
8888 Store in vector pointed to by ROOTS_PTR insns on which priority () should
8889 be invoked to initialize all cleared priorities. */
8891 clear_priorities (rtx_insn
*insn
, rtx_vec_t
*roots_ptr
)
8893 sd_iterator_def sd_it
;
8895 bool insn_is_root_p
= true;
8897 gcc_assert (QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
);
8899 FOR_EACH_DEP (insn
, SD_LIST_BACK
, sd_it
, dep
)
8901 rtx_insn
*pro
= DEP_PRO (dep
);
8903 if (INSN_PRIORITY_STATUS (pro
) >= 0
8904 && QUEUE_INDEX (insn
) != QUEUE_SCHEDULED
)
8906 /* If DEP doesn't contribute to priority then INSN itself should
8907 be added to priority roots. */
8908 if (contributes_to_priority_p (dep
))
8909 insn_is_root_p
= false;
8911 INSN_PRIORITY_STATUS (pro
) = -1;
8912 clear_priorities (pro
, roots_ptr
);
8917 roots_ptr
->safe_push (insn
);
8920 /* Recompute priorities of instructions, whose priorities might have been
8921 changed. ROOTS is a vector of instructions whose priority computation will
8922 trigger initialization of all cleared priorities. */
8924 calc_priorities (rtx_vec_t roots
)
8929 FOR_EACH_VEC_ELT (roots
, i
, insn
)
8934 /* Add dependences between JUMP and other instructions in the recovery
8935 block. INSN is the first insn the recovery block. */
8937 add_jump_dependencies (rtx_insn
*insn
, rtx_insn
*jump
)
8941 insn
= NEXT_INSN (insn
);
8945 if (dep_list_size (insn
, SD_LIST_FORW
) == 0)
8947 dep_def _new_dep
, *new_dep
= &_new_dep
;
8949 init_dep (new_dep
, insn
, jump
, REG_DEP_ANTI
);
8950 sd_add_dep (new_dep
, false);
8955 gcc_assert (!sd_lists_empty_p (jump
, SD_LIST_BACK
));
8958 /* Extend data structures for logical insn UID. */
8960 sched_extend_luids (void)
8962 int new_luids_max_uid
= get_max_uid () + 1;
8964 sched_luids
.safe_grow_cleared (new_luids_max_uid
);
8967 /* Initialize LUID for INSN. */
8969 sched_init_insn_luid (rtx_insn
*insn
)
8971 int i
= INSN_P (insn
) ? 1 : common_sched_info
->luid_for_non_insn (insn
);
8976 luid
= sched_max_luid
;
8977 sched_max_luid
+= i
;
8982 SET_INSN_LUID (insn
, luid
);
8985 /* Initialize luids for BBS.
8986 The hook common_sched_info->luid_for_non_insn () is used to determine
8987 if notes, labels, etc. need luids. */
8989 sched_init_luids (bb_vec_t bbs
)
8994 sched_extend_luids ();
8995 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
8999 FOR_BB_INSNS (bb
, insn
)
9000 sched_init_insn_luid (insn
);
9006 sched_finish_luids (void)
9008 sched_luids
.release ();
9012 /* Return logical uid of INSN. Helpful while debugging. */
9014 insn_luid (rtx_insn
*insn
)
9016 return INSN_LUID (insn
);
9019 /* Extend per insn data in the target. */
9021 sched_extend_target (void)
9023 if (targetm
.sched
.h_i_d_extended
)
9024 targetm
.sched
.h_i_d_extended ();
9027 /* Extend global scheduler structures (those, that live across calls to
9028 schedule_block) to include information about just emitted INSN. */
9032 int reserve
= (get_max_uid () + 1 - h_i_d
.length ());
9034 && ! h_i_d
.space (reserve
))
9036 h_i_d
.safe_grow_cleared (3 * get_max_uid () / 2);
9037 sched_extend_target ();
9041 /* Initialize h_i_d entry of the INSN with default values.
9042 Values, that are not explicitly initialized here, hold zero. */
9044 init_h_i_d (rtx_insn
*insn
)
9046 if (INSN_LUID (insn
) > 0)
9048 INSN_COST (insn
) = -1;
9049 QUEUE_INDEX (insn
) = QUEUE_NOWHERE
;
9050 INSN_TICK (insn
) = INVALID_TICK
;
9051 INSN_EXACT_TICK (insn
) = INVALID_TICK
;
9052 INTER_TICK (insn
) = INVALID_TICK
;
9053 TODO_SPEC (insn
) = HARD_DEP
;
9054 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[0].status
9055 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9056 INSN_AUTOPREF_MULTIPASS_DATA (insn
)[1].status
9057 = AUTOPREF_MULTIPASS_DATA_UNINITIALIZED
;
9061 /* Initialize haifa_insn_data for BBS. */
9063 haifa_init_h_i_d (bb_vec_t bbs
)
9069 FOR_EACH_VEC_ELT (bbs
, i
, bb
)
9073 FOR_BB_INSNS (bb
, insn
)
9078 /* Finalize haifa_insn_data. */
9080 haifa_finish_h_i_d (void)
9083 haifa_insn_data_t data
;
9084 reg_use_data
*use
, *next_use
;
9085 reg_set_data
*set
, *next_set
;
9087 FOR_EACH_VEC_ELT (h_i_d
, i
, data
)
9089 free (data
->max_reg_pressure
);
9090 free (data
->reg_pressure
);
9091 for (use
= data
->reg_use_list
; use
!= NULL
; use
= next_use
)
9093 next_use
= use
->next_insn_use
;
9096 for (set
= data
->reg_set_list
; set
!= NULL
; set
= next_set
)
9098 next_set
= set
->next_insn_set
;
9106 /* Init data for the new insn INSN. */
9108 haifa_init_insn (rtx_insn
*insn
)
9110 gcc_assert (insn
!= NULL
);
9112 sched_extend_luids ();
9113 sched_init_insn_luid (insn
);
9114 sched_extend_target ();
9115 sched_deps_init (false);
9119 if (adding_bb_to_current_region_p
)
9121 sd_init_insn (insn
);
9123 /* Extend dependency caches by one element. */
9124 extend_dependency_caches (1, false);
9126 if (sched_pressure
!= SCHED_PRESSURE_NONE
)
9127 init_insn_reg_pressure_info (insn
);
9130 /* Init data for the new basic block BB which comes after AFTER. */
9132 haifa_init_only_bb (basic_block bb
, basic_block after
)
9134 gcc_assert (bb
!= NULL
);
9138 if (common_sched_info
->add_block
)
9139 /* This changes only data structures of the front-end. */
9140 common_sched_info
->add_block (bb
, after
);
9143 /* A generic version of sched_split_block (). */
9145 sched_split_block_1 (basic_block first_bb
, rtx after
)
9149 e
= split_block (first_bb
, after
);
9150 gcc_assert (e
->src
== first_bb
);
9152 /* sched_split_block emits note if *check == BB_END. Probably it
9153 is better to rip that note off. */
9158 /* A generic version of sched_create_empty_bb (). */
9160 sched_create_empty_bb_1 (basic_block after
)
9162 return create_empty_bb (after
);
9165 /* Insert PAT as an INSN into the schedule and update the necessary data
9166 structures to account for it. */
9168 sched_emit_insn (rtx pat
)
9170 rtx_insn
*insn
= emit_insn_before (pat
, first_nonscheduled_insn ());
9171 haifa_init_insn (insn
);
9173 if (current_sched_info
->add_remove_insn
)
9174 current_sched_info
->add_remove_insn (insn
, 0);
9176 (*current_sched_info
->begin_schedule_ready
) (insn
);
9177 scheduled_insns
.safe_push (insn
);
9179 last_scheduled_insn
= insn
;
9183 /* This function returns a candidate satisfying dispatch constraints from
9187 ready_remove_first_dispatch (struct ready_list
*ready
)
9190 rtx_insn
*insn
= ready_element (ready
, 0);
9192 if (ready
->n_ready
== 1
9194 || INSN_CODE (insn
) < 0
9195 || !active_insn_p (insn
)
9196 || targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9197 return ready_remove_first (ready
);
9199 for (i
= 1; i
< ready
->n_ready
; i
++)
9201 insn
= ready_element (ready
, i
);
9204 || INSN_CODE (insn
) < 0
9205 || !active_insn_p (insn
))
9208 if (targetm
.sched
.dispatch (insn
, FITS_DISPATCH_WINDOW
))
9210 /* Return ith element of ready. */
9211 insn
= ready_remove (ready
, i
);
9216 if (targetm
.sched
.dispatch (NULL
, DISPATCH_VIOLATION
))
9217 return ready_remove_first (ready
);
9219 for (i
= 1; i
< ready
->n_ready
; i
++)
9221 insn
= ready_element (ready
, i
);
9224 || INSN_CODE (insn
) < 0
9225 || !active_insn_p (insn
))
9228 /* Return i-th element of ready. */
9229 if (targetm
.sched
.dispatch (insn
, IS_CMP
))
9230 return ready_remove (ready
, i
);
9233 return ready_remove_first (ready
);
9236 /* Get number of ready insn in the ready list. */
9239 number_in_ready (void)
9241 return ready
.n_ready
;
9244 /* Get number of ready's in the ready list. */
9247 get_ready_element (int i
)
9249 return ready_element (&ready
, i
);
9252 #endif /* INSN_SCHEDULING */