1 /* Passes for transactional memory support.
2 Copyright (C) 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
25 #include "tree-flow.h"
26 #include "tree-pass.h"
27 #include "tree-inline.h"
28 #include "diagnostic-core.h"
31 #include "trans-mem.h"
34 #include "langhooks.h"
35 #include "gimple-pretty-print.h"
39 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 2000 - 1)
40 #define PROB_VERY_LIKELY (PROB_ALWAYS - PROB_VERY_UNLIKELY)
41 #define PROB_UNLIKELY (REG_BR_PROB_BASE / 5 - 1)
42 #define PROB_LIKELY (PROB_ALWAYS - PROB_VERY_LIKELY)
43 #define PROB_ALWAYS (REG_BR_PROB_BASE)
45 #define A_RUNINSTRUMENTEDCODE 0x0001
46 #define A_RUNUNINSTRUMENTEDCODE 0x0002
47 #define A_SAVELIVEVARIABLES 0x0004
48 #define A_RESTORELIVEVARIABLES 0x0008
49 #define A_ABORTTRANSACTION 0x0010
51 #define AR_USERABORT 0x0001
52 #define AR_USERRETRY 0x0002
53 #define AR_TMCONFLICT 0x0004
54 #define AR_EXCEPTIONBLOCKABORT 0x0008
55 #define AR_OUTERABORT 0x0010
57 #define MODE_SERIALIRREVOCABLE 0x0000
60 /* The representation of a transaction changes several times during the
61 lowering process. In the beginning, in the front-end we have the
62 GENERIC tree TRANSACTION_EXPR. For example,
70 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
71 trivially replaced with a GIMPLE_TRANSACTION node.
73 During pass_lower_tm, we examine the body of transactions looking
74 for aborts. Transactions that do not contain an abort may be
75 merged into an outer transaction. We also add a TRY-FINALLY node
76 to arrange for the transaction to be committed on any exit.
78 [??? Think about how this arrangement affects throw-with-commit
79 and throw-with-abort operations. In this case we want the TRY to
80 handle gotos, but not to catch any exceptions because the transaction
81 will already be closed.]
83 GIMPLE_TRANSACTION [label=NULL] {
90 __builtin___tm_abort ();
92 __builtin___tm_commit ();
96 During pass_lower_eh, we create EH regions for the transactions,
97 intermixed with the regular EH stuff. This gives us a nice persistent
98 mapping (all the way through rtl) from transactional memory operation
99 back to the transaction, which allows us to get the abnormal edges
100 correct to model transaction aborts and restarts:
102 GIMPLE_TRANSACTION [label=over]
108 __builtin___tm_abort ();
109 __builtin___tm_commit ();
112 This is the end of all_lowering_passes, and so is what is present
113 during the IPA passes, and through all of the optimization passes.
115 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
116 functions and mark functions for cloning.
118 At the end of gimple optimization, before exiting SSA form,
119 pass_tm_edges replaces statements that perform transactional
120 memory operations with the appropriate TM builtins, and swap
121 out function calls with their transactional clones. At this
122 point we introduce the abnormal transaction restart edges and
123 complete lowering of the GIMPLE_TRANSACTION node.
125 x = __builtin___tm_start (MAY_ABORT);
127 if (x & abort_transaction)
130 t0 = __builtin___tm_load (global);
132 __builtin___tm_store (&global, t1);
134 __builtin___tm_abort ();
135 __builtin___tm_commit ();
139 static void *expand_regions (struct tm_region
*,
140 void *(*callback
)(struct tm_region
*, void *),
144 /* Return the attributes we want to examine for X, or NULL if it's not
145 something we examine. We look at function types, but allow pointers
146 to function types and function decls and peek through. */
149 get_attrs_for (const_tree x
)
151 switch (TREE_CODE (x
))
154 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
161 if (TREE_CODE (x
) != POINTER_TYPE
)
167 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
173 return TYPE_ATTRIBUTES (x
);
177 /* Return true if X has been marked TM_PURE. */
180 is_tm_pure (const_tree x
)
184 switch (TREE_CODE (x
))
195 if (TREE_CODE (x
) != POINTER_TYPE
)
201 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
206 flags
= flags_from_decl_or_type (x
);
207 return (flags
& ECF_TM_PURE
) != 0;
210 /* Return true if X has been marked TM_IRREVOCABLE. */
213 is_tm_irrevocable (tree x
)
215 tree attrs
= get_attrs_for (x
);
217 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
220 /* A call to the irrevocable builtin is by definition,
222 if (TREE_CODE (x
) == ADDR_EXPR
)
223 x
= TREE_OPERAND (x
, 0);
224 if (TREE_CODE (x
) == FUNCTION_DECL
225 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
226 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
232 /* Return true if X has been marked TM_SAFE. */
235 is_tm_safe (const_tree x
)
239 tree attrs
= get_attrs_for (x
);
242 if (lookup_attribute ("transaction_safe", attrs
))
244 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
251 /* Return true if CALL is const, or tm_pure. */
254 is_tm_pure_call (gimple call
)
256 tree fn
= gimple_call_fn (call
);
258 if (TREE_CODE (fn
) == ADDR_EXPR
)
260 fn
= TREE_OPERAND (fn
, 0);
261 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
266 return is_tm_pure (fn
);
269 /* Return true if X has been marked TM_CALLABLE. */
272 is_tm_callable (tree x
)
274 tree attrs
= get_attrs_for (x
);
277 if (lookup_attribute ("transaction_callable", attrs
))
279 if (lookup_attribute ("transaction_safe", attrs
))
281 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
287 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
290 is_tm_may_cancel_outer (tree x
)
292 tree attrs
= get_attrs_for (x
);
294 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
298 /* Return true for built in functions that "end" a transaction. */
301 is_tm_ending_fndecl (tree fndecl
)
303 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
304 switch (DECL_FUNCTION_CODE (fndecl
))
306 case BUILT_IN_TM_COMMIT
:
307 case BUILT_IN_TM_COMMIT_EH
:
308 case BUILT_IN_TM_ABORT
:
309 case BUILT_IN_TM_IRREVOCABLE
:
318 /* Return true if STMT is a TM load. */
321 is_tm_load (gimple stmt
)
325 if (gimple_code (stmt
) != GIMPLE_CALL
)
328 fndecl
= gimple_call_fndecl (stmt
);
329 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
330 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
333 /* Same as above, but for simple TM loads, that is, not the
334 after-write, after-read, etc optimized variants. */
337 is_tm_simple_load (gimple stmt
)
341 if (gimple_code (stmt
) != GIMPLE_CALL
)
344 fndecl
= gimple_call_fndecl (stmt
);
345 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
347 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
348 return (fcode
== BUILT_IN_TM_LOAD_1
349 || fcode
== BUILT_IN_TM_LOAD_2
350 || fcode
== BUILT_IN_TM_LOAD_4
351 || fcode
== BUILT_IN_TM_LOAD_8
352 || fcode
== BUILT_IN_TM_LOAD_FLOAT
353 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
354 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
355 || fcode
== BUILT_IN_TM_LOAD_M64
356 || fcode
== BUILT_IN_TM_LOAD_M128
357 || fcode
== BUILT_IN_TM_LOAD_M256
);
362 /* Return true if STMT is a TM store. */
365 is_tm_store (gimple stmt
)
369 if (gimple_code (stmt
) != GIMPLE_CALL
)
372 fndecl
= gimple_call_fndecl (stmt
);
373 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
374 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
377 /* Same as above, but for simple TM stores, that is, not the
378 after-write, after-read, etc optimized variants. */
381 is_tm_simple_store (gimple stmt
)
385 if (gimple_code (stmt
) != GIMPLE_CALL
)
388 fndecl
= gimple_call_fndecl (stmt
);
389 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
391 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
392 return (fcode
== BUILT_IN_TM_STORE_1
393 || fcode
== BUILT_IN_TM_STORE_2
394 || fcode
== BUILT_IN_TM_STORE_4
395 || fcode
== BUILT_IN_TM_STORE_8
396 || fcode
== BUILT_IN_TM_STORE_FLOAT
397 || fcode
== BUILT_IN_TM_STORE_DOUBLE
398 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
399 || fcode
== BUILT_IN_TM_STORE_M64
400 || fcode
== BUILT_IN_TM_STORE_M128
401 || fcode
== BUILT_IN_TM_STORE_M256
);
406 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
409 is_tm_abort (tree fndecl
)
412 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
413 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
416 /* Build a GENERIC tree for a user abort. This is called by front ends
417 while transforming the __tm_abort statement. */
420 build_tm_abort_call (location_t loc
, bool is_outer
)
422 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
423 build_int_cst (integer_type_node
,
425 | (is_outer
? AR_OUTERABORT
: 0)));
428 /* Common gateing function for several of the TM passes. */
436 /* Map for aribtrary function replacement under TM, as created
437 by the tm_wrap attribute. */
439 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
443 record_tm_replacement (tree from
, tree to
)
445 struct tree_map
**slot
, *h
;
447 /* Do not inline wrapper functions that will get replaced in the TM
450 Suppose you have foo() that will get replaced into tmfoo(). Make
451 sure the inliner doesn't try to outsmart us and inline foo()
452 before we get a chance to do the TM replacement. */
453 DECL_UNINLINABLE (from
) = 1;
455 if (tm_wrap_map
== NULL
)
456 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
458 h
= ggc_alloc_tree_map ();
459 h
->hash
= htab_hash_pointer (from
);
463 slot
= (struct tree_map
**)
464 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
468 /* Return a TM-aware replacement function for DECL. */
471 find_tm_replacement_function (tree fndecl
)
475 struct tree_map
*h
, in
;
477 in
.base
.from
= fndecl
;
478 in
.hash
= htab_hash_pointer (fndecl
);
479 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
484 /* ??? We may well want TM versions of most of the common <string.h>
485 functions. For now, we've already these two defined. */
486 /* Adjust expand_call_tm() attributes as necessary for the cases
488 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
489 switch (DECL_FUNCTION_CODE (fndecl
))
491 case BUILT_IN_MEMCPY
:
492 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
493 case BUILT_IN_MEMMOVE
:
494 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
495 case BUILT_IN_MEMSET
:
496 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
504 /* When appropriate, record TM replacement for memory allocation functions.
506 FROM is the FNDECL to wrap. */
508 tm_malloc_replacement (tree from
)
513 if (TREE_CODE (from
) != FUNCTION_DECL
)
516 /* If we have a previous replacement, the user must be explicitly
517 wrapping malloc/calloc/free. They better know what they're
519 if (find_tm_replacement_function (from
))
522 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
524 if (!strcmp (str
, "malloc"))
525 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
526 else if (!strcmp (str
, "calloc"))
527 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
528 else if (!strcmp (str
, "free"))
529 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
533 TREE_NOTHROW (to
) = 0;
535 record_tm_replacement (from
, to
);
538 /* Diagnostics for tm_safe functions/regions. Called by the front end
539 once we've lowered the function to high-gimple. */
541 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
542 Process exactly one statement. WI->INFO is set to non-null when in
543 the context of a tm_safe function, and null for a __transaction block. */
545 #define DIAG_TM_OUTER 1
546 #define DIAG_TM_SAFE 2
547 #define DIAG_TM_RELAXED 4
551 unsigned int summary_flags
: 8;
552 unsigned int block_flags
: 8;
553 unsigned int func_flags
: 8;
554 unsigned int saw_volatile
: 1;
558 /* Return true if T is a volatile variable of some kind. */
561 volatile_var_p (tree t
)
563 return (SSA_VAR_P (t
)
564 && TREE_THIS_VOLATILE (TREE_TYPE (t
)));
567 /* Tree callback function for diagnose_tm pass. */
570 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
573 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
574 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
576 if (volatile_var_p (*tp
)
577 && d
->block_flags
& DIAG_TM_SAFE
581 error_at (gimple_location (d
->stmt
),
582 "invalid volatile use of %qD inside transaction",
590 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
591 struct walk_stmt_info
*wi
)
593 gimple stmt
= gsi_stmt (*gsi
);
594 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
596 /* Save stmt for use in leaf analysis. */
599 switch (gimple_code (stmt
))
603 tree fn
= gimple_call_fn (stmt
);
605 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
606 && is_tm_may_cancel_outer (fn
))
607 error_at (gimple_location (stmt
),
608 "%<transaction_may_cancel_outer%> function call not within"
609 " outer transaction or %<transaction_may_cancel_outer%>");
611 if (d
->summary_flags
& DIAG_TM_SAFE
)
613 bool is_safe
, direct_call_p
;
616 if (TREE_CODE (fn
) == ADDR_EXPR
617 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
619 direct_call_p
= true;
620 replacement
= TREE_OPERAND (fn
, 0);
621 replacement
= find_tm_replacement_function (replacement
);
627 direct_call_p
= false;
628 replacement
= NULL_TREE
;
631 if (is_tm_safe_or_pure (fn
))
633 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
635 /* A function explicitly marked transaction_callable as
636 opposed to transaction_safe is being defined to be
637 unsafe as part of its ABI, regardless of its contents. */
640 else if (direct_call_p
)
642 if (flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
644 else if (replacement
)
646 /* ??? At present we've been considering replacements
647 merely transaction_callable, and therefore might
648 enter irrevocable. The tm_wrap attribute has not
649 yet made it into the new language spec. */
654 /* ??? Diagnostics for unmarked direct calls moved into
655 the IPA pass. Section 3.2 of the spec details how
656 functions not marked should be considered "implicitly
657 safe" based on having examined the function body. */
663 /* An unmarked indirect call. Consider it unsafe even
664 though optimization may yet figure out how to inline. */
670 if (TREE_CODE (fn
) == ADDR_EXPR
)
671 fn
= TREE_OPERAND (fn
, 0);
672 if (d
->block_flags
& DIAG_TM_SAFE
)
675 error_at (gimple_location (stmt
),
676 "unsafe function call %qD within "
677 "atomic transaction", fn
);
680 if (!DECL_P (fn
) || DECL_NAME (fn
))
681 error_at (gimple_location (stmt
),
682 "unsafe function call %qE within "
683 "atomic transaction", fn
);
685 error_at (gimple_location (stmt
),
686 "unsafe indirect function call within "
687 "atomic transaction");
693 error_at (gimple_location (stmt
),
694 "unsafe function call %qD within "
695 "%<transaction_safe%> function", fn
);
698 if (!DECL_P (fn
) || DECL_NAME (fn
))
699 error_at (gimple_location (stmt
),
700 "unsafe function call %qE within "
701 "%<transaction_safe%> function", fn
);
703 error_at (gimple_location (stmt
),
704 "unsafe indirect function call within "
705 "%<transaction_safe%> function");
714 /* ??? We ought to come up with a way to add attributes to
715 asm statements, and then add "transaction_safe" to it.
716 Either that or get the language spec to resurrect __tm_waiver. */
717 if (d
->block_flags
& DIAG_TM_SAFE
)
718 error_at (gimple_location (stmt
),
719 "asm not allowed in atomic transaction");
720 else if (d
->func_flags
& DIAG_TM_SAFE
)
721 error_at (gimple_location (stmt
),
722 "asm not allowed in %<transaction_safe%> function");
725 case GIMPLE_TRANSACTION
:
727 unsigned char inner_flags
= DIAG_TM_SAFE
;
729 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
731 if (d
->block_flags
& DIAG_TM_SAFE
)
732 error_at (gimple_location (stmt
),
733 "relaxed transaction in atomic transaction");
734 else if (d
->func_flags
& DIAG_TM_SAFE
)
735 error_at (gimple_location (stmt
),
736 "relaxed transaction in %<transaction_safe%> function");
737 inner_flags
= DIAG_TM_RELAXED
;
739 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
742 error_at (gimple_location (stmt
),
743 "outer transaction in transaction");
744 else if (d
->func_flags
& DIAG_TM_OUTER
)
745 error_at (gimple_location (stmt
),
746 "outer transaction in "
747 "%<transaction_may_cancel_outer%> function");
748 else if (d
->func_flags
& DIAG_TM_SAFE
)
749 error_at (gimple_location (stmt
),
750 "outer transaction in %<transaction_safe%> function");
751 inner_flags
|= DIAG_TM_OUTER
;
754 *handled_ops_p
= true;
755 if (gimple_transaction_body (stmt
))
757 struct walk_stmt_info wi_inner
;
758 struct diagnose_tm d_inner
;
760 memset (&d_inner
, 0, sizeof (d_inner
));
761 d_inner
.func_flags
= d
->func_flags
;
762 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
763 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
765 memset (&wi_inner
, 0, sizeof (wi_inner
));
766 wi_inner
.info
= &d_inner
;
768 walk_gimple_seq (gimple_transaction_body (stmt
),
769 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
782 diagnose_tm_blocks (void)
784 struct walk_stmt_info wi
;
785 struct diagnose_tm d
;
787 memset (&d
, 0, sizeof (d
));
788 if (is_tm_may_cancel_outer (current_function_decl
))
789 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
790 else if (is_tm_safe (current_function_decl
))
791 d
.func_flags
= DIAG_TM_SAFE
;
792 d
.summary_flags
= d
.func_flags
;
794 memset (&wi
, 0, sizeof (wi
));
797 walk_gimple_seq (gimple_body (current_function_decl
),
798 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
803 struct gimple_opt_pass pass_diagnose_tm_blocks
=
807 "*diagnose_tm_blocks", /* name */
808 OPTGROUP_NONE
, /* optinfo_flags */
810 diagnose_tm_blocks
, /* execute */
813 0, /* static_pass_number */
814 TV_TRANS_MEM
, /* tv_id */
815 PROP_gimple_any
, /* properties_required */
816 0, /* properties_provided */
817 0, /* properties_destroyed */
818 0, /* todo_flags_start */
819 0, /* todo_flags_finish */
823 /* Instead of instrumenting thread private memory, we save the
824 addresses in a log which we later use to save/restore the addresses
825 upon transaction start/restart.
827 The log is keyed by address, where each element contains individual
828 statements among different code paths that perform the store.
830 This log is later used to generate either plain save/restore of the
831 addresses upon transaction start/restart, or calls to the ITM_L*
834 So for something like:
836 struct large { int x[1000]; };
837 struct large lala = { 0 };
843 We can either save/restore:
846 trxn = _ITM_startTransaction ();
847 if (trxn & a_saveLiveVariables)
848 tmp_lala1 = lala.x[i];
849 else if (a & a_restoreLiveVariables)
850 lala.x[i] = tmp_lala1;
852 or use the logging functions:
855 trxn = _ITM_startTransaction ();
856 _ITM_LU4 (&lala.x[i]);
858 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
859 far up the dominator tree to shadow all of the writes to a given
860 location (thus reducing the total number of logging calls), but not
861 so high as to be called on a path that does not perform a
864 /* One individual log entry. We may have multiple statements for the
865 same location if neither dominate each other (on different
867 typedef struct tm_log_entry
869 /* Address to save. */
871 /* Entry block for the transaction this address occurs in. */
872 basic_block entry_block
;
873 /* Dominating statements the store occurs in. */
875 /* Initially, while we are building the log, we place a nonzero
876 value here to mean that this address *will* be saved with a
877 save/restore sequence. Later, when generating the save sequence
878 we place the SSA temp generated here. */
882 /* The actual log. */
883 static htab_t tm_log
;
885 /* Addresses to log with a save/restore sequence. These should be in
887 static VEC(tree
,heap
) *tm_log_save_addresses
;
889 /* Map for an SSA_NAME originally pointing to a non aliased new piece
890 of memory (malloc, alloc, etc). */
891 static htab_t tm_new_mem_hash
;
893 enum thread_memory_type
897 mem_transaction_local
,
901 typedef struct tm_new_mem_map
903 /* SSA_NAME being dereferenced. */
905 enum thread_memory_type local_new_memory
;
908 /* Htab support. Return hash value for a `tm_log_entry'. */
910 tm_log_hash (const void *p
)
912 const struct tm_log_entry
*log
= (const struct tm_log_entry
*) p
;
913 return iterative_hash_expr (log
->addr
, 0);
916 /* Htab support. Return true if two log entries are the same. */
918 tm_log_eq (const void *p1
, const void *p2
)
920 const struct tm_log_entry
*log1
= (const struct tm_log_entry
*) p1
;
921 const struct tm_log_entry
*log2
= (const struct tm_log_entry
*) p2
;
925 rth: I suggest that we get rid of the component refs etc.
926 I.e. resolve the reference to base + offset.
928 We may need to actually finish a merge with mainline for this,
929 since we'd like to be presented with Richi's MEM_REF_EXPRs more
930 often than not. But in the meantime your tm_log_entry could save
931 the results of get_inner_reference.
933 See: g++.dg/tm/pr46653.C
936 /* Special case plain equality because operand_equal_p() below will
937 return FALSE if the addresses are equal but they have
938 side-effects (e.g. a volatile address). */
939 if (log1
->addr
== log2
->addr
)
942 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
945 /* Htab support. Free one tm_log_entry. */
947 tm_log_free (void *p
)
949 struct tm_log_entry
*lp
= (struct tm_log_entry
*) p
;
950 VEC_free (gimple
, heap
, lp
->stmts
);
954 /* Initialize logging data structures. */
958 tm_log
= htab_create (10, tm_log_hash
, tm_log_eq
, tm_log_free
);
959 tm_new_mem_hash
= htab_create (5, struct_ptr_hash
, struct_ptr_eq
, free
);
960 tm_log_save_addresses
= VEC_alloc (tree
, heap
, 5);
963 /* Free logging data structures. */
967 htab_delete (tm_log
);
968 htab_delete (tm_new_mem_hash
);
969 VEC_free (tree
, heap
, tm_log_save_addresses
);
972 /* Return true if MEM is a transaction invariant memory for the TM
973 region starting at REGION_ENTRY_BLOCK. */
975 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
977 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
978 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
982 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
983 return def_bb
!= region_entry_block
984 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
987 mem
= strip_invariant_refs (mem
);
988 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
991 /* Given an address ADDR in STMT, find it in the memory log or add it,
992 making sure to keep only the addresses highest in the dominator
995 ENTRY_BLOCK is the entry_block for the transaction.
997 If we find the address in the log, make sure it's either the same
998 address, or an equivalent one that dominates ADDR.
1000 If we find the address, but neither ADDR dominates the found
1001 address, nor the found one dominates ADDR, we're on different
1002 execution paths. Add it.
1004 If known, ENTRY_BLOCK is the entry block for the region, otherwise
1007 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
1010 struct tm_log_entry l
, *lp
;
1013 slot
= htab_find_slot (tm_log
, &l
, INSERT
);
1016 tree type
= TREE_TYPE (addr
);
1018 lp
= XNEW (struct tm_log_entry
);
1022 /* Small invariant addresses can be handled as save/restores. */
1024 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1025 && TYPE_SIZE_UNIT (type
) != NULL
1026 && host_integerp (TYPE_SIZE_UNIT (type
), 1)
1027 && (tree_low_cst (TYPE_SIZE_UNIT (type
), 1)
1028 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1029 /* We must be able to copy this type normally. I.e., no
1030 special constructors and the like. */
1031 && !TREE_ADDRESSABLE (type
))
1033 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1035 lp
->entry_block
= entry_block
;
1036 /* Save addresses separately in dominator order so we don't
1037 get confused by overlapping addresses in the save/restore
1039 VEC_safe_push (tree
, heap
, tm_log_save_addresses
, lp
->addr
);
1043 /* Use the logging functions. */
1044 lp
->stmts
= VEC_alloc (gimple
, heap
, 5);
1045 VEC_quick_push (gimple
, lp
->stmts
, stmt
);
1046 lp
->save_var
= NULL
;
1054 lp
= (struct tm_log_entry
*) *slot
;
1056 /* If we're generating a save/restore sequence, we don't care
1057 about statements. */
1061 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, oldstmt
); ++i
)
1063 if (stmt
== oldstmt
)
1065 /* We already have a store to the same address, higher up the
1066 dominator tree. Nothing to do. */
1067 if (dominated_by_p (CDI_DOMINATORS
,
1068 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1070 /* We should be processing blocks in dominator tree order. */
1071 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1072 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1074 /* Store is on a different code path. */
1075 VEC_safe_push (gimple
, heap
, lp
->stmts
, stmt
);
1079 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1080 result, insert the new statements before GSI. */
1083 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1085 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1086 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1088 x
= build_fold_addr_expr (x
);
1089 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1092 /* Instrument one address with the logging functions.
1093 ADDR is the address to save.
1094 STMT is the statement before which to place it. */
1096 tm_log_emit_stmt (tree addr
, gimple stmt
)
1098 tree type
= TREE_TYPE (addr
);
1099 tree size
= TYPE_SIZE_UNIT (type
);
1100 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1102 enum built_in_function code
= BUILT_IN_TM_LOG
;
1104 if (type
== float_type_node
)
1105 code
= BUILT_IN_TM_LOG_FLOAT
;
1106 else if (type
== double_type_node
)
1107 code
= BUILT_IN_TM_LOG_DOUBLE
;
1108 else if (type
== long_double_type_node
)
1109 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1110 else if (host_integerp (size
, 1))
1112 unsigned int n
= tree_low_cst (size
, 1);
1116 code
= BUILT_IN_TM_LOG_1
;
1119 code
= BUILT_IN_TM_LOG_2
;
1122 code
= BUILT_IN_TM_LOG_4
;
1125 code
= BUILT_IN_TM_LOG_8
;
1128 code
= BUILT_IN_TM_LOG
;
1129 if (TREE_CODE (type
) == VECTOR_TYPE
)
1131 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1132 code
= BUILT_IN_TM_LOG_M64
;
1133 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1134 code
= BUILT_IN_TM_LOG_M128
;
1135 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1136 code
= BUILT_IN_TM_LOG_M256
;
1142 addr
= gimplify_addr (&gsi
, addr
);
1143 if (code
== BUILT_IN_TM_LOG
)
1144 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1146 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1147 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1150 /* Go through the log and instrument address that must be instrumented
1151 with the logging functions. Leave the save/restore addresses for
1157 struct tm_log_entry
*lp
;
1159 FOR_EACH_HTAB_ELEMENT (tm_log
, lp
, tm_log_entry_t
, hi
)
1166 fprintf (dump_file
, "TM thread private mem logging: ");
1167 print_generic_expr (dump_file
, lp
->addr
, 0);
1168 fprintf (dump_file
, "\n");
1174 fprintf (dump_file
, "DUMPING to variable\n");
1180 fprintf (dump_file
, "DUMPING with logging functions\n");
1181 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, stmt
); ++i
)
1182 tm_log_emit_stmt (lp
->addr
, stmt
);
1187 /* Emit the save sequence for the corresponding addresses in the log.
1188 ENTRY_BLOCK is the entry block for the transaction.
1189 BB is the basic block to insert the code in. */
1191 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1194 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1196 struct tm_log_entry l
, *lp
;
1198 for (i
= 0; i
< VEC_length (tree
, tm_log_save_addresses
); ++i
)
1200 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1201 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1202 gcc_assert (lp
->save_var
!= NULL
);
1204 /* We only care about variables in the current transaction. */
1205 if (lp
->entry_block
!= entry_block
)
1208 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1210 /* Make sure we can create an SSA_NAME for this type. For
1211 instance, aggregates aren't allowed, in which case the system
1212 will create a VOP for us and everything will just work. */
1213 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1215 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1216 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1219 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1223 /* Emit the restore sequence for the corresponding addresses in the log.
1224 ENTRY_BLOCK is the entry block for the transaction.
1225 BB is the basic block to insert the code in. */
1227 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1230 struct tm_log_entry l
, *lp
;
1231 gimple_stmt_iterator gsi
;
1234 for (i
= VEC_length (tree
, tm_log_save_addresses
) - 1; i
>= 0; i
--)
1236 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1237 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1238 gcc_assert (lp
->save_var
!= NULL
);
1240 /* We only care about variables in the current transaction. */
1241 if (lp
->entry_block
!= entry_block
)
1244 /* Restores are in LIFO order from the saves in case we have
1246 gsi
= gsi_start_bb (bb
);
1248 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1249 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1254 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1255 struct walk_stmt_info
*);
1256 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1257 struct walk_stmt_info
*);
1259 /* Evaluate an address X being dereferenced and determine if it
1260 originally points to a non aliased new chunk of memory (malloc,
1263 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1264 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1265 Return MEM_NON_LOCAL otherwise.
1267 ENTRY_BLOCK is the entry block to the transaction containing the
1268 dereference of X. */
1269 static enum thread_memory_type
1270 thread_private_new_memory (basic_block entry_block
, tree x
)
1273 enum tree_code code
;
1275 tm_new_mem_map_t elt
, *elt_p
;
1277 enum thread_memory_type retval
= mem_transaction_local
;
1280 || TREE_CODE (x
) != SSA_NAME
1281 /* Possible uninitialized use, or a function argument. In
1282 either case, we don't care. */
1283 || SSA_NAME_IS_DEFAULT_DEF (x
))
1284 return mem_non_local
;
1286 /* Look in cache first. */
1288 slot
= htab_find_slot (tm_new_mem_hash
, &elt
, INSERT
);
1289 elt_p
= (tm_new_mem_map_t
*) *slot
;
1291 return elt_p
->local_new_memory
;
1293 /* Optimistically assume the memory is transaction local during
1294 processing. This catches recursion into this variable. */
1295 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1297 elt_p
->local_new_memory
= mem_transaction_local
;
1299 /* Search DEF chain to find the original definition of this address. */
1302 if (ptr_deref_may_alias_global_p (x
))
1304 /* Address escapes. This is not thread-private. */
1305 retval
= mem_non_local
;
1306 goto new_memory_ret
;
1309 stmt
= SSA_NAME_DEF_STMT (x
);
1311 /* If the malloc call is outside the transaction, this is
1313 if (retval
!= mem_thread_local
1314 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1315 retval
= mem_thread_local
;
1317 if (is_gimple_assign (stmt
))
1319 code
= gimple_assign_rhs_code (stmt
);
1320 /* x = foo ==> foo */
1321 if (code
== SSA_NAME
)
1322 x
= gimple_assign_rhs1 (stmt
);
1323 /* x = foo + n ==> foo */
1324 else if (code
== POINTER_PLUS_EXPR
)
1325 x
= gimple_assign_rhs1 (stmt
);
1326 /* x = (cast*) foo ==> foo */
1327 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1328 x
= gimple_assign_rhs1 (stmt
);
1329 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1330 else if (code
== COND_EXPR
)
1332 tree op1
= gimple_assign_rhs2 (stmt
);
1333 tree op2
= gimple_assign_rhs3 (stmt
);
1334 enum thread_memory_type mem
;
1335 retval
= thread_private_new_memory (entry_block
, op1
);
1336 if (retval
== mem_non_local
)
1337 goto new_memory_ret
;
1338 mem
= thread_private_new_memory (entry_block
, op2
);
1339 retval
= MIN (retval
, mem
);
1340 goto new_memory_ret
;
1344 retval
= mem_non_local
;
1345 goto new_memory_ret
;
1350 if (gimple_code (stmt
) == GIMPLE_PHI
)
1353 enum thread_memory_type mem
;
1354 tree phi_result
= gimple_phi_result (stmt
);
1356 /* If any of the ancestors are non-local, we are sure to
1357 be non-local. Otherwise we can avoid doing anything
1358 and inherit what has already been generated. */
1360 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1362 tree op
= PHI_ARG_DEF (stmt
, i
);
1364 /* Exclude self-assignment. */
1365 if (phi_result
== op
)
1368 mem
= thread_private_new_memory (entry_block
, op
);
1369 if (mem
== mem_non_local
)
1372 goto new_memory_ret
;
1374 retval
= MIN (retval
, mem
);
1376 goto new_memory_ret
;
1381 while (TREE_CODE (x
) == SSA_NAME
);
1383 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1384 /* Thread-local or transaction-local. */
1387 retval
= mem_non_local
;
1390 elt_p
->local_new_memory
= retval
;
1394 /* Determine whether X has to be instrumented using a read
1397 ENTRY_BLOCK is the entry block for the region where stmt resides
1398 in. NULL if unknown.
1400 STMT is the statement in which X occurs in. It is used for thread
1401 private memory instrumentation. If no TPM instrumentation is
1402 desired, STMT should be null. */
1404 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1407 while (handled_component_p (x
))
1408 x
= TREE_OPERAND (x
, 0);
1410 switch (TREE_CODE (x
))
1415 enum thread_memory_type ret
;
1417 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1418 if (ret
== mem_non_local
)
1420 if (stmt
&& ret
== mem_thread_local
)
1421 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1422 tm_log_add (entry_block
, orig
, stmt
);
1424 /* Transaction-locals require nothing at all. For malloc, a
1425 transaction restart frees the memory and we reallocate.
1426 For alloca, the stack pointer gets reset by the retry and
1431 case TARGET_MEM_REF
:
1432 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1434 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1435 if (TREE_CODE (x
) == PARM_DECL
)
1437 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1443 if (DECL_BY_REFERENCE (x
))
1445 /* ??? This value is a pointer, but aggregate_value_p has been
1446 jigged to return true which confuses needs_to_live_in_memory.
1447 This ought to be cleaned up generically.
1449 FIXME: Verify this still happens after the next mainline
1450 merge. Testcase ie g++.dg/tm/pr47554.C.
1455 if (is_global_var (x
))
1456 return !TREE_READONLY (x
);
1457 if (/* FIXME: This condition should actually go below in the
1458 tm_log_add() call, however is_call_clobbered() depends on
1459 aliasing info which is not available during
1460 gimplification. Since requires_barrier() gets called
1461 during lower_sequence_tm/gimplification, leave the call
1462 to needs_to_live_in_memory until we eliminate
1463 lower_sequence_tm altogether. */
1464 needs_to_live_in_memory (x
))
1468 /* For local memory that doesn't escape (aka thread private
1469 memory), we can either save the value at the beginning of
1470 the transaction and restore on restart, or call a tm
1471 function to dynamically save and restore on restart
1474 tm_log_add (entry_block
, orig
, stmt
);
1483 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1484 a transaction region. */
1487 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1489 gimple stmt
= gsi_stmt (*gsi
);
1491 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1492 *state
|= GTMA_HAVE_LOAD
;
1493 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1494 *state
|= GTMA_HAVE_STORE
;
1497 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1500 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1502 gimple stmt
= gsi_stmt (*gsi
);
1505 if (is_tm_pure_call (stmt
))
1508 /* Check if this call is a transaction abort. */
1509 fn
= gimple_call_fndecl (stmt
);
1510 if (is_tm_abort (fn
))
1511 *state
|= GTMA_HAVE_ABORT
;
1513 /* Note that something may happen. */
1514 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1517 /* Lower a GIMPLE_TRANSACTION statement. */
1520 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1522 gimple g
, stmt
= gsi_stmt (*gsi
);
1523 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1524 unsigned int this_state
= 0;
1525 struct walk_stmt_info this_wi
;
1527 /* First, lower the body. The scanning that we do inside gives
1528 us some idea of what we're dealing with. */
1529 memset (&this_wi
, 0, sizeof (this_wi
));
1530 this_wi
.info
= (void *) &this_state
;
1531 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1532 lower_sequence_tm
, NULL
, &this_wi
);
1534 /* If there was absolutely nothing transaction related inside the
1535 transaction, we may elide it. Likewise if this is a nested
1536 transaction and does not contain an abort. */
1538 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1541 *outer_state
|= this_state
;
1543 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1545 gimple_transaction_set_body (stmt
, NULL
);
1547 gsi_remove (gsi
, true);
1548 wi
->removed_stmt
= true;
1552 /* Wrap the body of the transaction in a try-finally node so that
1553 the commit call is always properly called. */
1554 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1555 if (flag_exceptions
)
1558 gimple_seq n_seq
, e_seq
;
1560 n_seq
= gimple_seq_alloc_with_stmt (g
);
1563 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1564 1, integer_zero_node
);
1565 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1566 gimple_call_set_lhs (g
, ptr
);
1567 gimple_seq_add_stmt (&e_seq
, g
);
1569 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1571 gimple_seq_add_stmt (&e_seq
, g
);
1573 g
= gimple_build_eh_else (n_seq
, e_seq
);
1576 g
= gimple_build_try (gimple_transaction_body (stmt
),
1577 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1578 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1580 gimple_transaction_set_body (stmt
, NULL
);
1582 /* If the transaction calls abort or if this is an outer transaction,
1583 add an "over" label afterwards. */
1584 if ((this_state
& (GTMA_HAVE_ABORT
))
1585 || (gimple_transaction_subcode(stmt
) & GTMA_IS_OUTER
))
1587 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1588 gimple_transaction_set_label (stmt
, label
);
1589 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1592 /* Record the set of operations found for use later. */
1593 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1594 gimple_transaction_set_subcode (stmt
, this_state
);
1597 /* Iterate through the statements in the sequence, lowering them all
1598 as appropriate for being in a transaction. */
1601 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1602 struct walk_stmt_info
*wi
)
1604 unsigned int *state
= (unsigned int *) wi
->info
;
1605 gimple stmt
= gsi_stmt (*gsi
);
1607 *handled_ops_p
= true;
1608 switch (gimple_code (stmt
))
1611 /* Only memory reads/writes need to be instrumented. */
1612 if (gimple_assign_single_p (stmt
))
1613 examine_assign_tm (state
, gsi
);
1617 examine_call_tm (state
, gsi
);
1621 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1624 case GIMPLE_TRANSACTION
:
1625 lower_transaction (gsi
, wi
);
1629 *handled_ops_p
= !gimple_has_substatements (stmt
);
1636 /* Iterate through the statements in the sequence, lowering them all
1637 as appropriate for being outside of a transaction. */
1640 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1641 struct walk_stmt_info
* wi
)
1643 gimple stmt
= gsi_stmt (*gsi
);
1645 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1647 *handled_ops_p
= true;
1648 lower_transaction (gsi
, wi
);
1651 *handled_ops_p
= !gimple_has_substatements (stmt
);
1656 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1657 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1658 been moved out, and all the data required for constructing a proper
1659 CFG has been recorded. */
1662 execute_lower_tm (void)
1664 struct walk_stmt_info wi
;
1667 /* Transactional clones aren't created until a later pass. */
1668 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1670 body
= gimple_body (current_function_decl
);
1671 memset (&wi
, 0, sizeof (wi
));
1672 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1673 gimple_set_body (current_function_decl
, body
);
1678 struct gimple_opt_pass pass_lower_tm
=
1682 "tmlower", /* name */
1683 OPTGROUP_NONE
, /* optinfo_flags */
1685 execute_lower_tm
, /* execute */
1688 0, /* static_pass_number */
1689 TV_TRANS_MEM
, /* tv_id */
1690 PROP_gimple_lcf
, /* properties_required */
1691 0, /* properties_provided */
1692 0, /* properties_destroyed */
1693 0, /* todo_flags_start */
1694 0, /* todo_flags_finish */
1698 /* Collect region information for each transaction. */
1702 /* Link to the next unnested transaction. */
1703 struct tm_region
*next
;
1705 /* Link to the next inner transaction. */
1706 struct tm_region
*inner
;
1708 /* Link to the next outer transaction. */
1709 struct tm_region
*outer
;
1711 /* The GIMPLE_TRANSACTION statement beginning this transaction.
1712 After TM_MARK, this gets replaced by a call to
1713 BUILT_IN_TM_START. */
1714 gimple transaction_stmt
;
1716 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1717 BUILT_IN_TM_START, this field is true if the transaction is an
1718 outer transaction. */
1719 bool original_transaction_was_outer
;
1721 /* Return value from BUILT_IN_TM_START. */
1724 /* The entry block to this region. This will always be the first
1725 block of the body of the transaction. */
1726 basic_block entry_block
;
1728 /* The first block after an expanded call to _ITM_beginTransaction. */
1729 basic_block restart_block
;
1731 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1732 These blocks are still a part of the region (i.e., the border is
1733 inclusive). Note that this set is only complete for paths in the CFG
1734 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1735 the edge to the "over" label. */
1738 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1742 typedef struct tm_region
*tm_region_p
;
1743 DEF_VEC_P (tm_region_p
);
1744 DEF_VEC_ALLOC_P (tm_region_p
, heap
);
1746 /* True if there are pending edge statements to be committed for the
1747 current function being scanned in the tmmark pass. */
1748 bool pending_edge_inserts_p
;
1750 static struct tm_region
*all_tm_regions
;
1751 static bitmap_obstack tm_obstack
;
1754 /* A subroutine of tm_region_init. Record the existence of the
1755 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1757 static struct tm_region
*
1758 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1760 struct tm_region
*region
;
1762 region
= (struct tm_region
*)
1763 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1767 region
->next
= outer
->inner
;
1768 outer
->inner
= region
;
1772 region
->next
= all_tm_regions
;
1773 all_tm_regions
= region
;
1775 region
->inner
= NULL
;
1776 region
->outer
= outer
;
1778 region
->transaction_stmt
= stmt
;
1779 region
->original_transaction_was_outer
= false;
1780 region
->tm_state
= NULL
;
1782 /* There are either one or two edges out of the block containing
1783 the GIMPLE_TRANSACTION, one to the actual region and one to the
1784 "over" label if the region contains an abort. The former will
1785 always be the one marked FALLTHRU. */
1786 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1788 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1789 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1794 /* A subroutine of tm_region_init. Record all the exit and
1795 irrevocable blocks in BB into the region's exit_blocks and
1796 irr_blocks bitmaps. Returns the new region being scanned. */
1798 static struct tm_region
*
1799 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1801 gimple_stmt_iterator gsi
;
1805 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1808 /* Check to see if this is the end of a region by seeing if it
1809 contains a call to __builtin_tm_commit{,_eh}. Note that the
1810 outermost region for DECL_IS_TM_CLONE need not collect this. */
1811 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1814 if (gimple_code (g
) == GIMPLE_CALL
)
1816 tree fn
= gimple_call_fndecl (g
);
1817 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1819 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1820 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1821 && region
->exit_blocks
)
1823 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1824 region
= region
->outer
;
1827 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1828 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1835 /* Collect all of the transaction regions within the current function
1836 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1837 an "outermost" region for use by tm clones. */
1840 tm_region_init (struct tm_region
*region
)
1846 VEC(basic_block
, heap
) *queue
= NULL
;
1847 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1848 struct tm_region
*old_region
;
1849 VEC(tm_region_p
, heap
) *bb_regions
= NULL
;
1851 all_tm_regions
= region
;
1852 bb
= single_succ (ENTRY_BLOCK_PTR
);
1854 /* We could store this information in bb->aux, but we may get called
1855 through get_all_tm_blocks() from another pass that may be already
1857 VEC_safe_grow_cleared (tm_region_p
, heap
, bb_regions
, last_basic_block
);
1859 VEC_safe_push (basic_block
, heap
, queue
, bb
);
1860 VEC_replace (tm_region_p
, bb_regions
, bb
->index
, region
);
1863 bb
= VEC_pop (basic_block
, queue
);
1864 region
= VEC_index (tm_region_p
, bb_regions
, bb
->index
);
1865 VEC_replace (tm_region_p
, bb_regions
, bb
->index
, NULL
);
1867 /* Record exit and irrevocable blocks. */
1868 region
= tm_region_init_1 (region
, bb
);
1870 /* Check for the last statement in the block beginning a new region. */
1872 old_region
= region
;
1873 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1874 region
= tm_region_init_0 (region
, bb
, g
);
1876 /* Process subsequent blocks. */
1877 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1878 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1880 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1881 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
1883 /* If the current block started a new region, make sure that only
1884 the entry block of the new region is associated with this region.
1885 Other successors are still part of the old region. */
1886 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1887 VEC_replace (tm_region_p
, bb_regions
, e
->dest
->index
, old_region
);
1889 VEC_replace (tm_region_p
, bb_regions
, e
->dest
->index
, region
);
1892 while (!VEC_empty (basic_block
, queue
));
1893 VEC_free (basic_block
, heap
, queue
);
1894 BITMAP_FREE (visited_blocks
);
1895 VEC_free (tm_region_p
, heap
, bb_regions
);
1898 /* The "gate" function for all transactional memory expansion and optimization
1899 passes. We collect region information for each top-level transaction, and
1900 if we don't find any, we skip all of the TM passes. Each region will have
1901 all of the exit blocks recorded, and the originating statement. */
1909 calculate_dominance_info (CDI_DOMINATORS
);
1910 bitmap_obstack_initialize (&tm_obstack
);
1912 /* If the function is a TM_CLONE, then the entire function is the region. */
1913 if (decl_is_tm_clone (current_function_decl
))
1915 struct tm_region
*region
= (struct tm_region
*)
1916 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1917 memset (region
, 0, sizeof (*region
));
1918 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR
);
1919 /* For a clone, the entire function is the region. But even if
1920 we don't need to record any exit blocks, we may need to
1921 record irrevocable blocks. */
1922 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1924 tm_region_init (region
);
1928 tm_region_init (NULL
);
1930 /* If we didn't find any regions, cleanup and skip the whole tree
1931 of tm-related optimizations. */
1932 if (all_tm_regions
== NULL
)
1934 bitmap_obstack_release (&tm_obstack
);
1942 struct gimple_opt_pass pass_tm_init
=
1946 "*tminit", /* name */
1947 OPTGROUP_NONE
, /* optinfo_flags */
1948 gate_tm_init
, /* gate */
1952 0, /* static_pass_number */
1953 TV_TRANS_MEM
, /* tv_id */
1954 PROP_ssa
| PROP_cfg
, /* properties_required */
1955 0, /* properties_provided */
1956 0, /* properties_destroyed */
1957 0, /* todo_flags_start */
1958 0, /* todo_flags_finish */
1962 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
1963 represented by STATE. */
1966 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
1968 if (region
&& region
->transaction_stmt
)
1970 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
1971 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
1975 /* Construct a memory load in a transactional context. Return the
1976 gimple statement performing the load, or NULL if there is no
1977 TM_LOAD builtin of the appropriate size to do the load.
1979 LOC is the location to use for the new statement(s). */
1982 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
1984 enum built_in_function code
= END_BUILTINS
;
1985 tree t
, type
= TREE_TYPE (rhs
), decl
;
1988 if (type
== float_type_node
)
1989 code
= BUILT_IN_TM_LOAD_FLOAT
;
1990 else if (type
== double_type_node
)
1991 code
= BUILT_IN_TM_LOAD_DOUBLE
;
1992 else if (type
== long_double_type_node
)
1993 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
1994 else if (TYPE_SIZE_UNIT (type
) != NULL
1995 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
1997 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2000 code
= BUILT_IN_TM_LOAD_1
;
2003 code
= BUILT_IN_TM_LOAD_2
;
2006 code
= BUILT_IN_TM_LOAD_4
;
2009 code
= BUILT_IN_TM_LOAD_8
;
2014 if (code
== END_BUILTINS
)
2016 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2021 decl
= builtin_decl_explicit (code
);
2023 t
= gimplify_addr (gsi
, rhs
);
2024 gcall
= gimple_build_call (decl
, 1, t
);
2025 gimple_set_location (gcall
, loc
);
2027 t
= TREE_TYPE (TREE_TYPE (decl
));
2028 if (useless_type_conversion_p (type
, t
))
2030 gimple_call_set_lhs (gcall
, lhs
);
2031 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2038 temp
= create_tmp_reg (t
, NULL
);
2039 gimple_call_set_lhs (gcall
, temp
);
2040 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2042 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2043 g
= gimple_build_assign (lhs
, t
);
2044 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2051 /* Similarly for storing TYPE in a transactional context. */
2054 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2056 enum built_in_function code
= END_BUILTINS
;
2057 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2060 if (type
== float_type_node
)
2061 code
= BUILT_IN_TM_STORE_FLOAT
;
2062 else if (type
== double_type_node
)
2063 code
= BUILT_IN_TM_STORE_DOUBLE
;
2064 else if (type
== long_double_type_node
)
2065 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2066 else if (TYPE_SIZE_UNIT (type
) != NULL
2067 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
2069 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2072 code
= BUILT_IN_TM_STORE_1
;
2075 code
= BUILT_IN_TM_STORE_2
;
2078 code
= BUILT_IN_TM_STORE_4
;
2081 code
= BUILT_IN_TM_STORE_8
;
2086 if (code
== END_BUILTINS
)
2088 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2093 fn
= builtin_decl_explicit (code
);
2095 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2097 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2099 /* Handle the easy initialization to zero. */
2100 if (CONSTRUCTOR_ELTS (rhs
) == 0)
2101 rhs
= build_int_cst (simple_type
, 0);
2104 /* ...otherwise punt to the caller and probably use
2105 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2106 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2111 else if (!useless_type_conversion_p (simple_type
, type
))
2116 temp
= create_tmp_reg (simple_type
, NULL
);
2117 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2118 g
= gimple_build_assign (temp
, t
);
2119 gimple_set_location (g
, loc
);
2120 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2125 t
= gimplify_addr (gsi
, lhs
);
2126 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2127 gimple_set_location (gcall
, loc
);
2128 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2134 /* Expand an assignment statement into transactional builtins. */
2137 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2139 gimple stmt
= gsi_stmt (*gsi
);
2140 location_t loc
= gimple_location (stmt
);
2141 tree lhs
= gimple_assign_lhs (stmt
);
2142 tree rhs
= gimple_assign_rhs1 (stmt
);
2143 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2144 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2145 gimple gcall
= NULL
;
2147 if (!load_p
&& !store_p
)
2149 /* Add thread private addresses to log if applicable. */
2150 requires_barrier (region
->entry_block
, lhs
, stmt
);
2155 // Remove original load/store statement.
2156 gsi_remove (gsi
, true);
2158 if (load_p
&& !store_p
)
2160 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2161 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2163 else if (store_p
&& !load_p
)
2165 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2166 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2170 tree lhs_addr
, rhs_addr
, tmp
;
2173 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2175 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2177 /* ??? Figure out if there's any possible overlap between the LHS
2178 and the RHS and if not, use MEMCPY. */
2180 if (load_p
&& is_gimple_reg (lhs
))
2182 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2183 lhs_addr
= build_fold_addr_expr (tmp
);
2188 lhs_addr
= gimplify_addr (gsi
, lhs
);
2190 rhs_addr
= gimplify_addr (gsi
, rhs
);
2191 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2192 3, lhs_addr
, rhs_addr
,
2193 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2194 gimple_set_location (gcall
, loc
);
2195 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2199 gcall
= gimple_build_assign (lhs
, tmp
);
2200 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2204 /* Now that we have the load/store in its instrumented form, add
2205 thread private addresses to the log if applicable. */
2207 requires_barrier (region
->entry_block
, lhs
, gcall
);
2209 // The calls to build_tm_{store,load} above inserted the instrumented
2210 // call into the stream.
2211 // gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2215 /* Expand a call statement as appropriate for a transaction. That is,
2216 either verify that the call does not affect the transaction, or
2217 redirect the call to a clone that handles transactions, or change
2218 the transaction state to IRREVOCABLE. Return true if the call is
2219 one of the builtins that end a transaction. */
2222 expand_call_tm (struct tm_region
*region
,
2223 gimple_stmt_iterator
*gsi
)
2225 gimple stmt
= gsi_stmt (*gsi
);
2226 tree lhs
= gimple_call_lhs (stmt
);
2228 struct cgraph_node
*node
;
2229 bool retval
= false;
2231 fn_decl
= gimple_call_fndecl (stmt
);
2233 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2234 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2235 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2236 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2237 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2239 if (is_tm_pure_call (stmt
))
2243 retval
= is_tm_ending_fndecl (fn_decl
);
2246 /* Assume all non-const/pure calls write to memory, except
2247 transaction ending builtins. */
2248 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2251 /* For indirect calls, we already generated a call into the runtime. */
2254 tree fn
= gimple_call_fn (stmt
);
2256 /* We are guaranteed never to go irrevocable on a safe or pure
2257 call, and the pure call was handled above. */
2258 if (is_tm_safe (fn
))
2261 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2266 node
= cgraph_get_node (fn_decl
);
2267 /* All calls should have cgraph here. */
2270 /* We can have a nodeless call here if some pass after IPA-tm
2271 added uninstrumented calls. For example, loop distribution
2272 can transform certain loop constructs into __builtin_mem*
2273 calls. In this case, see if we have a suitable TM
2274 replacement and fill in the gaps. */
2275 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl
) == BUILT_IN_NORMAL
);
2276 enum built_in_function code
= DECL_FUNCTION_CODE (fn_decl
);
2277 gcc_assert (code
== BUILT_IN_MEMCPY
2278 || code
== BUILT_IN_MEMMOVE
2279 || code
== BUILT_IN_MEMSET
);
2281 tree repl
= find_tm_replacement_function (fn_decl
);
2284 gimple_call_set_fndecl (stmt
, repl
);
2286 node
= cgraph_create_node (repl
);
2287 node
->local
.tm_may_enter_irr
= false;
2288 return expand_call_tm (region
, gsi
);
2292 if (node
->local
.tm_may_enter_irr
)
2293 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2295 if (is_tm_abort (fn_decl
))
2297 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2301 /* Instrument the store if needed.
2303 If the assignment happens inside the function call (return slot
2304 optimization), there is no instrumentation to be done, since
2305 the callee should have done the right thing. */
2306 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2307 && !gimple_call_return_slot_opt_p (stmt
))
2309 tree tmp
= create_tmp_reg (TREE_TYPE (lhs
), NULL
);
2310 location_t loc
= gimple_location (stmt
);
2311 edge fallthru_edge
= NULL
;
2313 /* Remember if the call was going to throw. */
2314 if (stmt_can_throw_internal (stmt
))
2318 basic_block bb
= gimple_bb (stmt
);
2320 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2321 if (e
->flags
& EDGE_FALLTHRU
)
2328 gimple_call_set_lhs (stmt
, tmp
);
2330 stmt
= gimple_build_assign (lhs
, tmp
);
2331 gimple_set_location (stmt
, loc
);
2333 /* We cannot throw in the middle of a BB. If the call was going
2334 to throw, place the instrumentation on the fallthru edge, so
2335 the call remains the last statement in the block. */
2338 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2339 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2340 expand_assign_tm (region
, &fallthru_gsi
);
2341 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2342 pending_edge_inserts_p
= true;
2346 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2347 expand_assign_tm (region
, gsi
);
2350 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2357 /* Expand all statements in BB as appropriate for being inside
2361 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2363 gimple_stmt_iterator gsi
;
2365 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2367 gimple stmt
= gsi_stmt (gsi
);
2368 switch (gimple_code (stmt
))
2371 /* Only memory reads/writes need to be instrumented. */
2372 if (gimple_assign_single_p (stmt
)
2373 && !gimple_clobber_p (stmt
))
2375 expand_assign_tm (region
, &gsi
);
2381 if (expand_call_tm (region
, &gsi
))
2391 if (!gsi_end_p (gsi
))
2396 /* Return the list of basic-blocks in REGION.
2398 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2399 following a TM_IRREVOCABLE call. */
2401 static VEC (basic_block
, heap
) *
2402 get_tm_region_blocks (basic_block entry_block
,
2405 bitmap all_region_blocks
,
2406 bool stop_at_irrevocable_p
)
2408 VEC(basic_block
, heap
) *bbs
= NULL
;
2412 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2415 VEC_safe_push (basic_block
, heap
, bbs
, entry_block
);
2416 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2420 basic_block bb
= VEC_index (basic_block
, bbs
, i
++);
2423 bitmap_bit_p (exit_blocks
, bb
->index
))
2426 if (stop_at_irrevocable_p
2428 && bitmap_bit_p (irr_blocks
, bb
->index
))
2431 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2432 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2434 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2435 VEC_safe_push (basic_block
, heap
, bbs
, e
->dest
);
2438 while (i
< VEC_length (basic_block
, bbs
));
2440 if (all_region_blocks
)
2441 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2443 BITMAP_FREE (visited_blocks
);
2447 // Callback for expand_regions, collect innermost region data for each bb.
2449 collect_bb2reg (struct tm_region
*region
, void *data
)
2451 VEC(tm_region_p
, heap
) *bb2reg
= (VEC(tm_region_p
, heap
) *) data
;
2452 VEC (basic_block
, heap
) *queue
;
2456 queue
= get_tm_region_blocks (region
->entry_block
,
2457 region
->exit_blocks
,
2460 /*stop_at_irr_p=*/false);
2462 // We expect expand_region to perform a post-order traversal of the region
2463 // tree. Therefore the last region seen for any bb is the innermost.
2464 FOR_EACH_VEC_ELT (basic_block
, queue
, i
, bb
)
2465 VEC_replace (tm_region_p
, bb2reg
, bb
->index
, region
);
2467 VEC_free (basic_block
, heap
, queue
);
2471 // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2472 // which a basic block belongs. Note that we only consider the instrumented
2473 // code paths for the region; the uninstrumented code paths are ignored.
2475 // ??? This data is very similar to the bb_regions array that is collected
2476 // during tm_region_init. Or, rather, this data is similar to what could
2477 // be used within tm_region_init. The actual computation in tm_region_init
2478 // begins and ends with bb_regions entirely full of NULL pointers, due to
2479 // the way in which pointers are swapped in and out of the array.
2481 // ??? Our callers expect that blocks are not shared between transactions.
2482 // When the optimizers get too smart, and blocks are shared, then during
2483 // the tm_mark phase we'll add log entries to only one of the two transactions,
2484 // and in the tm_edge phase we'll add edges to the CFG that create invalid
2485 // cycles. The symptom being SSA defs that do not dominate their uses.
2486 // Note that the optimizers were locally correct with their transformation,
2487 // as we have no info within the program that suggests that the blocks cannot
2490 // ??? There is currently a hack inside tree-ssa-pre.c to work around the
2491 // only known instance of this block sharing.
2493 static VEC(tm_region_p
, heap
) *
2494 get_bb_regions_instrumented (void)
2496 unsigned n
= last_basic_block
;
2497 VEC(tm_region_p
, heap
) *ret
;
2499 ret
= VEC_alloc (tm_region_p
, heap
, n
);
2500 VEC_safe_grow_cleared (tm_region_p
, heap
, ret
, n
);
2501 expand_regions (all_tm_regions
, collect_bb2reg
, ret
);
2506 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2510 compute_transaction_bits (void)
2512 struct tm_region
*region
;
2513 VEC (basic_block
, heap
) *queue
;
2517 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2518 certainly don't need it to calculate CDI_DOMINATOR info. */
2522 bb
->flags
&= ~BB_IN_TRANSACTION
;
2524 for (region
= all_tm_regions
; region
; region
= region
->next
)
2526 queue
= get_tm_region_blocks (region
->entry_block
,
2527 region
->exit_blocks
,
2530 /*stop_at_irr_p=*/true);
2531 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2532 bb
->flags
|= BB_IN_TRANSACTION
;
2533 VEC_free (basic_block
, heap
, queue
);
2537 bitmap_obstack_release (&tm_obstack
);
2540 /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2541 call to BUILT_IN_TM_START. */
2544 expand_transaction (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2546 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2547 basic_block transaction_bb
= gimple_bb (region
->transaction_stmt
);
2548 tree tm_state
= region
->tm_state
;
2549 tree tm_state_type
= TREE_TYPE (tm_state
);
2550 edge abort_edge
= NULL
;
2551 edge inst_edge
= NULL
;
2552 edge uninst_edge
= NULL
;
2553 edge fallthru_edge
= NULL
;
2555 // Identify the various successors of the transaction start.
2559 FOR_EACH_EDGE (e
, i
, transaction_bb
->succs
)
2561 if (e
->flags
& EDGE_TM_ABORT
)
2563 else if (e
->flags
& EDGE_TM_UNINSTRUMENTED
)
2567 if (e
->flags
& EDGE_FALLTHRU
)
2572 /* ??? There are plenty of bits here we're not computing. */
2574 int subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2576 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2577 flags
|= PR_DOESGOIRREVOCABLE
;
2578 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2579 flags
|= PR_HASNOIRREVOCABLE
;
2580 /* If the transaction does not have an abort in lexical scope and is not
2581 marked as an outer transaction, then it will never abort. */
2582 if ((subcode
& GTMA_HAVE_ABORT
) == 0 && (subcode
& GTMA_IS_OUTER
) == 0)
2583 flags
|= PR_HASNOABORT
;
2584 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2585 flags
|= PR_READONLY
;
2587 flags
|= PR_INSTRUMENTEDCODE
;
2589 flags
|= PR_UNINSTRUMENTEDCODE
;
2590 if (subcode
& GTMA_IS_OUTER
)
2591 region
->original_transaction_was_outer
= true;
2592 tree t
= build_int_cst (tm_state_type
, flags
);
2593 gimple call
= gimple_build_call (tm_start
, 1, t
);
2594 gimple_call_set_lhs (call
, tm_state
);
2595 gimple_set_location (call
, gimple_location (region
->transaction_stmt
));
2597 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2598 gimple_stmt_iterator gsi
= gsi_last_bb (transaction_bb
);
2599 gcc_assert (gsi_stmt (gsi
) == region
->transaction_stmt
);
2600 gsi_insert_before (&gsi
, call
, GSI_SAME_STMT
);
2601 gsi_remove (&gsi
, true);
2602 region
->transaction_stmt
= call
;
2605 // Generate log saves.
2606 if (!VEC_empty (tree
, tm_log_save_addresses
))
2607 tm_log_emit_saves (region
->entry_block
, transaction_bb
);
2609 // In the beginning, we've no tests to perform on transaction restart.
2610 // Note that after this point, transaction_bb becomes the "most recent
2611 // block containing tests for the transaction".
2612 region
->restart_block
= region
->entry_block
;
2614 // Generate log restores.
2615 if (!VEC_empty (tree
, tm_log_save_addresses
))
2617 basic_block test_bb
= create_empty_bb (transaction_bb
);
2618 basic_block code_bb
= create_empty_bb (test_bb
);
2619 basic_block join_bb
= create_empty_bb (code_bb
);
2620 if (current_loops
&& transaction_bb
->loop_father
)
2622 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2623 add_bb_to_loop (code_bb
, transaction_bb
->loop_father
);
2624 add_bb_to_loop (join_bb
, transaction_bb
->loop_father
);
2626 if (region
->restart_block
== region
->entry_block
)
2627 region
->restart_block
= test_bb
;
2629 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2630 tree t2
= build_int_cst (tm_state_type
, A_RESTORELIVEVARIABLES
);
2631 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2633 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2634 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2636 t2
= build_int_cst (tm_state_type
, 0);
2637 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2638 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2640 tm_log_emit_restores (region
->entry_block
, code_bb
);
2642 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2643 edge et
= make_edge (test_bb
, code_bb
, EDGE_TRUE_VALUE
);
2644 edge ef
= make_edge (test_bb
, join_bb
, EDGE_FALSE_VALUE
);
2645 redirect_edge_pred (fallthru_edge
, join_bb
);
2647 join_bb
->frequency
= test_bb
->frequency
= transaction_bb
->frequency
;
2648 join_bb
->count
= test_bb
->count
= transaction_bb
->count
;
2650 ei
->probability
= PROB_ALWAYS
;
2651 et
->probability
= PROB_LIKELY
;
2652 ef
->probability
= PROB_UNLIKELY
;
2653 et
->count
= apply_probability(test_bb
->count
, et
->probability
);
2654 ef
->count
= apply_probability(test_bb
->count
, ef
->probability
);
2656 code_bb
->count
= et
->count
;
2657 code_bb
->frequency
= EDGE_FREQUENCY (et
);
2659 transaction_bb
= join_bb
;
2662 // If we have an ABORT edge, create a test to perform the abort.
2665 basic_block test_bb
= create_empty_bb (transaction_bb
);
2666 if (current_loops
&& transaction_bb
->loop_father
)
2667 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2668 if (region
->restart_block
== region
->entry_block
)
2669 region
->restart_block
= test_bb
;
2671 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2672 tree t2
= build_int_cst (tm_state_type
, A_ABORTTRANSACTION
);
2673 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2675 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2676 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2678 t2
= build_int_cst (tm_state_type
, 0);
2679 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2680 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2682 edge ei
= make_edge (transaction_bb
, test_bb
, EDGE_FALLTHRU
);
2683 test_bb
->frequency
= transaction_bb
->frequency
;
2684 test_bb
->count
= transaction_bb
->count
;
2685 ei
->probability
= PROB_ALWAYS
;
2687 // Not abort edge. If both are live, chose one at random as we'll
2688 // we'll be fixing that up below.
2689 redirect_edge_pred (fallthru_edge
, test_bb
);
2690 fallthru_edge
->flags
= EDGE_FALSE_VALUE
;
2691 fallthru_edge
->probability
= PROB_VERY_LIKELY
;
2692 fallthru_edge
->count
2693 = apply_probability(test_bb
->count
, fallthru_edge
->probability
);
2696 redirect_edge_pred (abort_edge
, test_bb
);
2697 abort_edge
->flags
= EDGE_TRUE_VALUE
;
2698 abort_edge
->probability
= PROB_VERY_UNLIKELY
;
2700 = apply_probability(test_bb
->count
, abort_edge
->probability
);
2702 transaction_bb
= test_bb
;
2705 // If we have both instrumented and uninstrumented code paths, select one.
2706 if (inst_edge
&& uninst_edge
)
2708 basic_block test_bb
= create_empty_bb (transaction_bb
);
2709 if (current_loops
&& transaction_bb
->loop_father
)
2710 add_bb_to_loop (test_bb
, transaction_bb
->loop_father
);
2711 if (region
->restart_block
== region
->entry_block
)
2712 region
->restart_block
= test_bb
;
2714 tree t1
= create_tmp_reg (tm_state_type
, NULL
);
2715 tree t2
= build_int_cst (tm_state_type
, A_RUNUNINSTRUMENTEDCODE
);
2717 gimple stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
,
2719 gimple_stmt_iterator gsi
= gsi_last_bb (test_bb
);
2720 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2722 t2
= build_int_cst (tm_state_type
, 0);
2723 stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2724 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
2726 // Create the edge into test_bb first, as we want to copy values
2727 // out of the fallthru edge.
2728 edge e
= make_edge (transaction_bb
, test_bb
, fallthru_edge
->flags
);
2729 e
->probability
= fallthru_edge
->probability
;
2730 test_bb
->count
= e
->count
= fallthru_edge
->count
;
2731 test_bb
->frequency
= EDGE_FREQUENCY (e
);
2733 // Now update the edges to the inst/uninist implementations.
2734 // For now assume that the paths are equally likely. When using HTM,
2735 // we'll try the uninst path first and fallback to inst path if htm
2736 // buffers are exceeded. Without HTM we start with the inst path and
2737 // use the uninst path when falling back to serial mode.
2738 redirect_edge_pred (inst_edge
, test_bb
);
2739 inst_edge
->flags
= EDGE_FALSE_VALUE
;
2740 inst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2742 = apply_probability(test_bb
->count
, inst_edge
->probability
);
2744 redirect_edge_pred (uninst_edge
, test_bb
);
2745 uninst_edge
->flags
= EDGE_TRUE_VALUE
;
2746 uninst_edge
->probability
= REG_BR_PROB_BASE
/ 2;
2748 = apply_probability(test_bb
->count
, uninst_edge
->probability
);
2751 // If we have no previous special cases, and we have PHIs at the beginning
2752 // of the atomic region, this means we have a loop at the beginning of the
2753 // atomic region that shares the first block. This can cause problems with
2754 // the transaction restart abnormal edges to be added in the tm_edges pass.
2755 // Solve this by adding a new empty block to receive the abnormal edges.
2756 if (region
->restart_block
== region
->entry_block
2757 && phi_nodes (region
->entry_block
))
2759 basic_block empty_bb
= create_empty_bb (transaction_bb
);
2760 region
->restart_block
= empty_bb
;
2761 if (current_loops
&& transaction_bb
->loop_father
)
2762 add_bb_to_loop (empty_bb
, transaction_bb
->loop_father
);
2764 redirect_edge_pred (fallthru_edge
, empty_bb
);
2765 make_edge (transaction_bb
, empty_bb
, EDGE_FALLTHRU
);
2771 /* Generate the temporary to be used for the return value of
2772 BUILT_IN_TM_START. */
2775 generate_tm_state (struct tm_region
*region
, void *data ATTRIBUTE_UNUSED
)
2777 tree tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2779 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2781 // Reset the subcode, post optimizations. We'll fill this in
2782 // again as we process blocks.
2783 if (region
->exit_blocks
)
2785 unsigned int subcode
2786 = gimple_transaction_subcode (region
->transaction_stmt
);
2788 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2789 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2790 | GTMA_MAY_ENTER_IRREVOCABLE
);
2792 subcode
&= GTMA_DECLARATION_MASK
;
2793 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2799 // Propagate flags from inner transactions outwards.
2801 propagate_tm_flags_out (struct tm_region
*region
)
2805 propagate_tm_flags_out (region
->inner
);
2807 if (region
->outer
&& region
->outer
->transaction_stmt
)
2809 unsigned s
= gimple_transaction_subcode (region
->transaction_stmt
);
2810 s
&= (GTMA_HAVE_ABORT
| GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
2811 | GTMA_MAY_ENTER_IRREVOCABLE
);
2812 s
|= gimple_transaction_subcode (region
->outer
->transaction_stmt
);
2813 gimple_transaction_set_subcode (region
->outer
->transaction_stmt
, s
);
2816 propagate_tm_flags_out (region
->next
);
2819 /* Entry point to the MARK phase of TM expansion. Here we replace
2820 transactional memory statements with calls to builtins, and function
2821 calls with their transactional clones (if available). But we don't
2822 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2825 execute_tm_mark (void)
2827 pending_edge_inserts_p
= false;
2829 expand_regions (all_tm_regions
, generate_tm_state
, NULL
);
2833 VEC(tm_region_p
, heap
) *bb_regions
= get_bb_regions_instrumented ();
2834 struct tm_region
*r
;
2837 // Expand memory operations into calls into the runtime.
2838 // This collects log entries as well.
2839 FOR_EACH_VEC_ELT (tm_region_p
, bb_regions
, i
, r
)
2841 expand_block_tm (r
, BASIC_BLOCK (i
));
2843 // Propagate flags from inner transactions outwards.
2844 propagate_tm_flags_out (all_tm_regions
);
2846 // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
2847 expand_regions (all_tm_regions
, expand_transaction
, NULL
);
2852 if (pending_edge_inserts_p
)
2853 gsi_commit_edge_inserts ();
2854 free_dominance_info (CDI_DOMINATORS
);
2858 struct gimple_opt_pass pass_tm_mark
=
2862 "tmmark", /* name */
2863 OPTGROUP_NONE
, /* optinfo_flags */
2865 execute_tm_mark
, /* execute */
2868 0, /* static_pass_number */
2869 TV_TRANS_MEM
, /* tv_id */
2870 PROP_ssa
| PROP_cfg
, /* properties_required */
2871 0, /* properties_provided */
2872 0, /* properties_destroyed */
2873 0, /* todo_flags_start */
2875 | TODO_verify_ssa
, /* todo_flags_finish */
2880 /* Create an abnormal edge from STMT at iter, splitting the block
2881 as necessary. Adjust *PNEXT as needed for the split block. */
2884 split_bb_make_tm_edge (gimple stmt
, basic_block dest_bb
,
2885 gimple_stmt_iterator iter
, gimple_stmt_iterator
*pnext
)
2887 basic_block bb
= gimple_bb (stmt
);
2888 if (!gsi_one_before_end_p (iter
))
2890 edge e
= split_block (bb
, stmt
);
2891 *pnext
= gsi_start_bb (e
->dest
);
2893 make_edge (bb
, dest_bb
, EDGE_ABNORMAL
);
2895 // Record the need for the edge for the benefit of the rtl passes.
2896 if (cfun
->gimple_df
->tm_restart
== NULL
)
2897 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
2898 struct_ptr_eq
, ggc_free
);
2900 struct tm_restart_node dummy
;
2902 dummy
.label_or_list
= gimple_block_label (dest_bb
);
2904 void **slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
2905 struct tm_restart_node
*n
= (struct tm_restart_node
*) *slot
;
2908 n
= ggc_alloc_tm_restart_node ();
2913 tree old
= n
->label_or_list
;
2914 if (TREE_CODE (old
) == LABEL_DECL
)
2915 old
= tree_cons (NULL
, old
, NULL
);
2916 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
2920 /* Split block BB as necessary for every builtin function we added, and
2921 wire up the abnormal back edges implied by the transaction restart. */
2924 expand_block_edges (struct tm_region
*const region
, basic_block bb
)
2926 gimple_stmt_iterator gsi
, next_gsi
;
2928 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi
= next_gsi
)
2930 gimple stmt
= gsi_stmt (gsi
);
2933 gsi_next (&next_gsi
);
2935 // ??? Shouldn't we split for any non-pure, non-irrevocable function?
2936 if (gimple_code (stmt
) != GIMPLE_CALL
2937 || (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) == 0)
2940 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt
)) == BUILT_IN_TM_ABORT
)
2942 // If we have a ``_transaction_cancel [[outer]]'', there is only
2943 // one abnormal edge: to the transaction marked OUTER.
2944 // All compiler-generated instances of BUILT_IN_TM_ABORT have a
2945 // constant argument, which we can examine here. Users invoking
2946 // TM_ABORT directly get what they deserve.
2947 tree arg
= gimple_call_arg (stmt
, 0);
2948 if (TREE_CODE (arg
) == INTEGER_CST
2949 && (TREE_INT_CST_LOW (arg
) & AR_OUTERABORT
) != 0
2950 && !decl_is_tm_clone (current_function_decl
))
2952 // Find the GTMA_IS_OUTER transaction.
2953 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
2954 if (o
->original_transaction_was_outer
)
2956 split_bb_make_tm_edge (stmt
, o
->restart_block
,
2961 // Otherwise, the front-end should have semantically checked
2962 // outer aborts, but in either case the target region is not
2963 // within this function.
2967 // Non-outer, TM aborts have an abnormal edge to the inner-most
2968 // transaction, the one being aborted;
2969 split_bb_make_tm_edge (stmt
, region
->restart_block
, gsi
, &next_gsi
);
2972 // All TM builtins have an abnormal edge to the outer-most transaction.
2973 // We never restart inner transactions. For tm clones, we know a-priori
2974 // that the outer-most transaction is outside the function.
2975 if (decl_is_tm_clone (current_function_decl
))
2978 if (cfun
->gimple_df
->tm_restart
== NULL
)
2979 cfun
->gimple_df
->tm_restart
2980 = htab_create_ggc (31, struct_ptr_hash
, struct_ptr_eq
, ggc_free
);
2982 // All TM builtins have an abnormal edge to the outer-most transaction.
2983 // We never restart inner transactions.
2984 for (struct tm_region
*o
= region
; o
; o
= o
->outer
)
2987 split_bb_make_tm_edge (stmt
, o
->restart_block
, gsi
, &next_gsi
);
2991 // Delete any tail-call annotation that may have been added.
2992 // The tail-call pass may have mis-identified the commit as being
2993 // a candidate because we had not yet added this restart edge.
2994 gimple_call_set_tail (stmt
, false);
2998 /* Entry point to the final expansion of transactional nodes. */
3001 execute_tm_edges (void)
3003 VEC(tm_region_p
, heap
) *bb_regions
= get_bb_regions_instrumented ();
3004 struct tm_region
*r
;
3007 FOR_EACH_VEC_ELT (tm_region_p
, bb_regions
, i
, r
)
3009 expand_block_edges (r
, BASIC_BLOCK (i
));
3011 VEC_free (tm_region_p
, heap
, bb_regions
);
3013 /* We've got to release the dominance info now, to indicate that it
3014 must be rebuilt completely. Otherwise we'll crash trying to update
3015 the SSA web in the TODO section following this pass. */
3016 free_dominance_info (CDI_DOMINATORS
);
3017 bitmap_obstack_release (&tm_obstack
);
3018 all_tm_regions
= NULL
;
3023 struct gimple_opt_pass pass_tm_edges
=
3027 "tmedge", /* name */
3028 OPTGROUP_NONE
, /* optinfo_flags */
3030 execute_tm_edges
, /* execute */
3033 0, /* static_pass_number */
3034 TV_TRANS_MEM
, /* tv_id */
3035 PROP_ssa
| PROP_cfg
, /* properties_required */
3036 0, /* properties_provided */
3037 0, /* properties_destroyed */
3038 0, /* todo_flags_start */
3040 | TODO_verify_ssa
, /* todo_flags_finish */
3044 /* Helper function for expand_regions. Expand REGION and recurse to
3045 the inner region. Call CALLBACK on each region. CALLBACK returns
3046 NULL to continue the traversal, otherwise a non-null value which
3047 this function will return as well. */
3050 expand_regions_1 (struct tm_region
*region
,
3051 void *(*callback
)(struct tm_region
*, void *),
3054 void *retval
= NULL
;
3055 if (region
->exit_blocks
)
3057 retval
= callback (region
, data
);
3063 retval
= expand_regions (region
->inner
, callback
, data
);
3070 /* Traverse the regions enclosed and including REGION. Execute
3071 CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3072 continue the traversal, otherwise a non-null value which this
3073 function will return as well. */
3076 expand_regions (struct tm_region
*region
,
3077 void *(*callback
)(struct tm_region
*, void *),
3080 void *retval
= NULL
;
3083 retval
= expand_regions_1 (region
, callback
, data
);
3086 region
= region
->next
;
3092 /* A unique TM memory operation. */
3093 typedef struct tm_memop
3095 /* Unique ID that all memory operations to the same location have. */
3096 unsigned int value_id
;
3097 /* Address of load/store. */
3101 /* Sets for solving data flow equations in the memory optimization pass. */
3102 struct tm_memopt_bitmaps
3104 /* Stores available to this BB upon entry. Basically, stores that
3105 dominate this BB. */
3106 bitmap store_avail_in
;
3107 /* Stores available at the end of this BB. */
3108 bitmap store_avail_out
;
3109 bitmap store_antic_in
;
3110 bitmap store_antic_out
;
3111 /* Reads available to this BB upon entry. Basically, reads that
3112 dominate this BB. */
3113 bitmap read_avail_in
;
3114 /* Reads available at the end of this BB. */
3115 bitmap read_avail_out
;
3116 /* Reads performed in this BB. */
3118 /* Writes performed in this BB. */
3121 /* Temporary storage for pass. */
3122 /* Is the current BB in the worklist? */
3123 bool avail_in_worklist_p
;
3124 /* Have we visited this BB? */
3128 static bitmap_obstack tm_memopt_obstack
;
3130 /* Unique counter for TM loads and stores. Loads and stores of the
3131 same address get the same ID. */
3132 static unsigned int tm_memopt_value_id
;
3133 static htab_t tm_memopt_value_numbers
;
3135 #define STORE_AVAIL_IN(BB) \
3136 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3137 #define STORE_AVAIL_OUT(BB) \
3138 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3139 #define STORE_ANTIC_IN(BB) \
3140 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3141 #define STORE_ANTIC_OUT(BB) \
3142 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3143 #define READ_AVAIL_IN(BB) \
3144 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3145 #define READ_AVAIL_OUT(BB) \
3146 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3147 #define READ_LOCAL(BB) \
3148 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3149 #define STORE_LOCAL(BB) \
3150 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3151 #define AVAIL_IN_WORKLIST_P(BB) \
3152 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3153 #define BB_VISITED_P(BB) \
3154 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3156 /* Htab support. Return a hash value for a `tm_memop'. */
3158 tm_memop_hash (const void *p
)
3160 const struct tm_memop
*mem
= (const struct tm_memop
*) p
;
3161 tree addr
= mem
->addr
;
3162 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3163 actually done with operand_equal_p (see tm_memop_eq). */
3164 if (TREE_CODE (addr
) == ADDR_EXPR
)
3165 addr
= TREE_OPERAND (addr
, 0);
3166 return iterative_hash_expr (addr
, 0);
3169 /* Htab support. Return true if two tm_memop's are the same. */
3171 tm_memop_eq (const void *p1
, const void *p2
)
3173 const struct tm_memop
*mem1
= (const struct tm_memop
*) p1
;
3174 const struct tm_memop
*mem2
= (const struct tm_memop
*) p2
;
3176 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
3179 /* Given a TM load/store in STMT, return the value number for the address
3183 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
3185 struct tm_memop tmpmem
, *mem
;
3188 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
3189 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
3190 slot
= htab_find_slot (tm_memopt_value_numbers
, &tmpmem
, op
);
3192 mem
= (struct tm_memop
*) *slot
;
3193 else if (op
== INSERT
)
3195 mem
= XNEW (struct tm_memop
);
3197 mem
->value_id
= tm_memopt_value_id
++;
3198 mem
->addr
= tmpmem
.addr
;
3202 return mem
->value_id
;
3205 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3208 tm_memopt_accumulate_memops (basic_block bb
)
3210 gimple_stmt_iterator gsi
;
3212 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3214 gimple stmt
= gsi_stmt (gsi
);
3218 if (is_tm_store (stmt
))
3219 bits
= STORE_LOCAL (bb
);
3220 else if (is_tm_load (stmt
))
3221 bits
= READ_LOCAL (bb
);
3225 loc
= tm_memopt_value_number (stmt
, INSERT
);
3226 bitmap_set_bit (bits
, loc
);
3229 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
3230 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
3231 gimple_bb (stmt
)->index
);
3232 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
3233 fprintf (dump_file
, "\n");
3238 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3241 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
3245 const char *comma
= "";
3247 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
3248 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
3251 struct tm_memop
*mem
;
3253 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3254 FOR_EACH_HTAB_ELEMENT (tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
3255 if (mem
->value_id
== i
)
3257 gcc_assert (mem
->value_id
== i
);
3258 fprintf (dump_file
, "%s", comma
);
3260 print_generic_expr (dump_file
, mem
->addr
, 0);
3262 fprintf (dump_file
, "]\n");
3265 /* Prettily dump all of the memopt sets in BLOCKS. */
3268 dump_tm_memopt_sets (VEC (basic_block
, heap
) *blocks
)
3273 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3275 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3276 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3277 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3278 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3279 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3280 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3281 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3285 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3288 tm_memopt_compute_avin (basic_block bb
)
3293 /* Seed with the AVOUT of any predecessor. */
3294 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3296 e
= EDGE_PRED (bb
, ix
);
3297 /* Make sure we have already visited this BB, and is thus
3300 If e->src->aux is NULL, this predecessor is actually on an
3301 enclosing transaction. We only care about the current
3302 transaction, so ignore it. */
3303 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3305 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3306 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3311 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3313 e
= EDGE_PRED (bb
, ix
);
3314 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3316 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3317 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3321 BB_VISITED_P (bb
) = true;
3324 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3327 tm_memopt_compute_antin (basic_block bb
)
3332 /* Seed with the ANTIC_OUT of any successor. */
3333 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3335 e
= EDGE_SUCC (bb
, ix
);
3336 /* Make sure we have already visited this BB, and is thus
3338 if (BB_VISITED_P (e
->dest
))
3340 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3345 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3347 e
= EDGE_SUCC (bb
, ix
);
3348 if (BB_VISITED_P (e
->dest
))
3349 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3352 BB_VISITED_P (bb
) = true;
3355 /* Compute the AVAIL sets for every basic block in BLOCKS.
3357 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3359 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3360 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3362 This is basically what we do in lcm's compute_available(), but here
3363 we calculate two sets of sets (one for STOREs and one for READs),
3364 and we work on a region instead of the entire CFG.
3366 REGION is the TM region.
3367 BLOCKS are the basic blocks in the region. */
3370 tm_memopt_compute_available (struct tm_region
*region
,
3371 VEC (basic_block
, heap
) *blocks
)
3374 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3375 unsigned int qlen
, i
;
3379 /* Allocate a worklist array/queue. Entries are only added to the
3380 list if they were not already on the list. So the size is
3381 bounded by the number of basic blocks in the region. */
3382 qlen
= VEC_length (basic_block
, blocks
) - 1;
3383 qin
= qout
= worklist
=
3384 XNEWVEC (basic_block
, qlen
);
3386 /* Put every block in the region on the worklist. */
3387 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3389 /* Seed AVAIL_OUT with the LOCAL set. */
3390 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3391 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3393 AVAIL_IN_WORKLIST_P (bb
) = true;
3394 /* No need to insert the entry block, since it has an AVIN of
3395 null, and an AVOUT that has already been seeded in. */
3396 if (bb
!= region
->entry_block
)
3400 /* The entry block has been initialized with the local sets. */
3401 BB_VISITED_P (region
->entry_block
) = true;
3404 qend
= &worklist
[qlen
];
3406 /* Iterate until the worklist is empty. */
3409 /* Take the first entry off the worklist. */
3416 /* This block can be added to the worklist again if necessary. */
3417 AVAIL_IN_WORKLIST_P (bb
) = false;
3418 tm_memopt_compute_avin (bb
);
3420 /* Note: We do not add the LOCAL sets here because we already
3421 seeded the AVAIL_OUT sets with them. */
3422 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3423 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3425 && (region
->exit_blocks
== NULL
3426 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3427 /* If the out state of this block changed, then we need to add
3428 its successors to the worklist if they are not already in. */
3429 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3430 if (!AVAIL_IN_WORKLIST_P (e
->dest
) && e
->dest
!= EXIT_BLOCK_PTR
)
3433 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3444 dump_tm_memopt_sets (blocks
);
3447 /* Compute ANTIC sets for every basic block in BLOCKS.
3449 We compute STORE_ANTIC_OUT as follows:
3451 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3452 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3454 REGION is the TM region.
3455 BLOCKS are the basic blocks in the region. */
3458 tm_memopt_compute_antic (struct tm_region
*region
,
3459 VEC (basic_block
, heap
) *blocks
)
3462 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3467 /* Allocate a worklist array/queue. Entries are only added to the
3468 list if they were not already on the list. So the size is
3469 bounded by the number of basic blocks in the region. */
3470 qin
= qout
= worklist
=
3471 XNEWVEC (basic_block
, VEC_length (basic_block
, blocks
));
3473 for (qlen
= 0, i
= VEC_length (basic_block
, blocks
) - 1; i
>= 0; --i
)
3475 bb
= VEC_index (basic_block
, blocks
, i
);
3477 /* Seed ANTIC_OUT with the LOCAL set. */
3478 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3480 /* Put every block in the region on the worklist. */
3481 AVAIL_IN_WORKLIST_P (bb
) = true;
3482 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3483 and their ANTIC_OUT has already been seeded in. */
3484 if (region
->exit_blocks
3485 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3492 /* The exit blocks have been initialized with the local sets. */
3493 if (region
->exit_blocks
)
3497 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3498 BB_VISITED_P (BASIC_BLOCK (i
)) = true;
3502 qend
= &worklist
[qlen
];
3504 /* Iterate until the worklist is empty. */
3507 /* Take the first entry off the worklist. */
3514 /* This block can be added to the worklist again if necessary. */
3515 AVAIL_IN_WORKLIST_P (bb
) = false;
3516 tm_memopt_compute_antin (bb
);
3518 /* Note: We do not add the LOCAL sets here because we already
3519 seeded the ANTIC_OUT sets with them. */
3520 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3521 && bb
!= region
->entry_block
)
3522 /* If the out state of this block changed, then we need to add
3523 its predecessors to the worklist if they are not already in. */
3524 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3525 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3528 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3539 dump_tm_memopt_sets (blocks
);
3542 /* Offsets of load variants from TM_LOAD. For example,
3543 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3544 See gtm-builtins.def. */
3545 #define TRANSFORM_RAR 1
3546 #define TRANSFORM_RAW 2
3547 #define TRANSFORM_RFW 3
3548 /* Offsets of store variants from TM_STORE. */
3549 #define TRANSFORM_WAR 1
3550 #define TRANSFORM_WAW 2
3552 /* Inform about a load/store optimization. */
3555 dump_tm_memopt_transform (gimple stmt
)
3559 fprintf (dump_file
, "TM memopt: transforming: ");
3560 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3561 fprintf (dump_file
, "\n");
3565 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3566 by a builtin that is OFFSET entries down in the builtins table in
3567 gtm-builtins.def. */
3570 tm_memopt_transform_stmt (unsigned int offset
,
3572 gimple_stmt_iterator
*gsi
)
3574 tree fn
= gimple_call_fn (stmt
);
3575 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3576 TREE_OPERAND (fn
, 0)
3577 = builtin_decl_explicit ((enum built_in_function
)
3578 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3580 gimple_call_set_fn (stmt
, fn
);
3581 gsi_replace (gsi
, stmt
, true);
3582 dump_tm_memopt_transform (stmt
);
3585 /* Perform the actual TM memory optimization transformations in the
3586 basic blocks in BLOCKS. */
3589 tm_memopt_transform_blocks (VEC (basic_block
, heap
) *blocks
)
3593 gimple_stmt_iterator gsi
;
3595 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3597 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3599 gimple stmt
= gsi_stmt (gsi
);
3600 bitmap read_avail
= READ_AVAIL_IN (bb
);
3601 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3602 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3605 if (is_tm_simple_load (stmt
))
3607 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3608 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3609 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3610 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3612 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3613 bitmap_set_bit (store_avail
, loc
);
3615 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3616 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3618 bitmap_set_bit (read_avail
, loc
);
3620 else if (is_tm_simple_store (stmt
))
3622 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3623 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3624 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3627 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3628 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3629 bitmap_set_bit (store_avail
, loc
);
3636 /* Return a new set of bitmaps for a BB. */
3638 static struct tm_memopt_bitmaps
*
3639 tm_memopt_init_sets (void)
3641 struct tm_memopt_bitmaps
*b
3642 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3643 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3644 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3645 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3646 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3647 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3648 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3649 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3650 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3651 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3655 /* Free sets computed for each BB. */
3658 tm_memopt_free_sets (VEC (basic_block
, heap
) *blocks
)
3663 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3667 /* Clear the visited bit for every basic block in BLOCKS. */
3670 tm_memopt_clear_visited (VEC (basic_block
, heap
) *blocks
)
3675 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3676 BB_VISITED_P (bb
) = false;
3679 /* Replace TM load/stores with hints for the runtime. We handle
3680 things like read-after-write, write-after-read, read-after-read,
3681 read-for-write, etc. */
3684 execute_tm_memopt (void)
3686 struct tm_region
*region
;
3687 VEC (basic_block
, heap
) *bbs
;
3689 tm_memopt_value_id
= 0;
3690 tm_memopt_value_numbers
= htab_create (10, tm_memop_hash
, tm_memop_eq
, free
);
3692 for (region
= all_tm_regions
; region
; region
= region
->next
)
3694 /* All the TM stores/loads in the current region. */
3698 bitmap_obstack_initialize (&tm_memopt_obstack
);
3700 /* Save all BBs for the current region. */
3701 bbs
= get_tm_region_blocks (region
->entry_block
,
3702 region
->exit_blocks
,
3707 /* Collect all the memory operations. */
3708 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
3710 bb
->aux
= tm_memopt_init_sets ();
3711 tm_memopt_accumulate_memops (bb
);
3714 /* Solve data flow equations and transform each block accordingly. */
3715 tm_memopt_clear_visited (bbs
);
3716 tm_memopt_compute_available (region
, bbs
);
3717 tm_memopt_clear_visited (bbs
);
3718 tm_memopt_compute_antic (region
, bbs
);
3719 tm_memopt_transform_blocks (bbs
);
3721 tm_memopt_free_sets (bbs
);
3722 VEC_free (basic_block
, heap
, bbs
);
3723 bitmap_obstack_release (&tm_memopt_obstack
);
3724 htab_empty (tm_memopt_value_numbers
);
3727 htab_delete (tm_memopt_value_numbers
);
3732 gate_tm_memopt (void)
3734 return flag_tm
&& optimize
> 0;
3737 struct gimple_opt_pass pass_tm_memopt
=
3741 "tmmemopt", /* name */
3742 OPTGROUP_NONE
, /* optinfo_flags */
3743 gate_tm_memopt
, /* gate */
3744 execute_tm_memopt
, /* execute */
3747 0, /* static_pass_number */
3748 TV_TRANS_MEM
, /* tv_id */
3749 PROP_ssa
| PROP_cfg
, /* properties_required */
3750 0, /* properties_provided */
3751 0, /* properties_destroyed */
3752 0, /* todo_flags_start */
3753 0, /* todo_flags_finish */
3758 /* Interprocedual analysis for the creation of transactional clones.
3759 The aim of this pass is to find which functions are referenced in
3760 a non-irrevocable transaction context, and for those over which
3761 we have control (or user directive), create a version of the
3762 function which uses only the transactional interface to reference
3763 protected memories. This analysis proceeds in several steps:
3765 (1) Collect the set of all possible transactional clones:
3767 (a) For all local public functions marked tm_callable, push
3768 it onto the tm_callee queue.
3770 (b) For all local functions, scan for calls in transaction blocks.
3771 Push the caller and callee onto the tm_caller and tm_callee
3772 queues. Count the number of callers for each callee.
3774 (c) For each local function on the callee list, assume we will
3775 create a transactional clone. Push *all* calls onto the
3776 callee queues; count the number of clone callers separately
3777 to the number of original callers.
3779 (2) Propagate irrevocable status up the dominator tree:
3781 (a) Any external function on the callee list that is not marked
3782 tm_callable is irrevocable. Push all callers of such onto
3785 (b) For each function on the worklist, mark each block that
3786 contains an irrevocable call. Use the AND operator to
3787 propagate that mark up the dominator tree.
3789 (c) If we reach the entry block for a possible transactional
3790 clone, then the transactional clone is irrevocable, and
3791 we should not create the clone after all. Push all
3792 callers onto the worklist.
3794 (d) Place tm_irrevocable calls at the beginning of the relevant
3795 blocks. Special case here is the entry block for the entire
3796 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
3797 the library to begin the region in serial mode. Decrement
3798 the call count for all callees in the irrevocable region.
3800 (3) Create the transactional clones:
3802 Any tm_callee that still has a non-zero call count is cloned.
3805 /* This structure is stored in the AUX field of each cgraph_node. */
3806 struct tm_ipa_cg_data
3808 /* The clone of the function that got created. */
3809 struct cgraph_node
*clone
;
3811 /* The tm regions in the normal function. */
3812 struct tm_region
*all_tm_regions
;
3814 /* The blocks of the normal/clone functions that contain irrevocable
3815 calls, or blocks that are post-dominated by irrevocable calls. */
3816 bitmap irrevocable_blocks_normal
;
3817 bitmap irrevocable_blocks_clone
;
3819 /* The blocks of the normal function that are involved in transactions. */
3820 bitmap transaction_blocks_normal
;
3822 /* The number of callers to the transactional clone of this function
3823 from normal and transactional clones respectively. */
3824 unsigned tm_callers_normal
;
3825 unsigned tm_callers_clone
;
3827 /* True if all calls to this function's transactional clone
3828 are irrevocable. Also automatically true if the function
3829 has no transactional clone. */
3830 bool is_irrevocable
;
3832 /* Flags indicating the presence of this function in various queues. */
3833 bool in_callee_queue
;
3836 /* Flags indicating the kind of scan desired while in the worklist. */
3837 bool want_irr_scan_normal
;
3840 typedef VEC (cgraph_node_p
, heap
) *cgraph_node_queue
;
3842 /* Return the ipa data associated with NODE, allocating zeroed memory
3843 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
3844 and set *NODE accordingly. */
3846 static struct tm_ipa_cg_data
*
3847 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
3849 struct tm_ipa_cg_data
*d
;
3851 if (traverse_aliases
&& (*node
)->alias
)
3852 *node
= cgraph_get_node ((*node
)->thunk
.alias
);
3854 d
= (struct tm_ipa_cg_data
*) (*node
)->symbol
.aux
;
3858 d
= (struct tm_ipa_cg_data
*)
3859 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
3860 (*node
)->symbol
.aux
= (void *) d
;
3861 memset (d
, 0, sizeof (*d
));
3867 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
3868 it is already present. */
3871 maybe_push_queue (struct cgraph_node
*node
,
3872 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
3877 VEC_safe_push (cgraph_node_p
, heap
, *queue_p
, node
);
3881 /* Duplicate the basic blocks in QUEUE for use in the uninstrumented
3882 code path. QUEUE are the basic blocks inside the transaction
3883 represented in REGION.
3885 Later in split_code_paths() we will add the conditional to choose
3886 between the two alternatives. */
3889 ipa_uninstrument_transaction (struct tm_region
*region
,
3890 VEC (basic_block
, heap
) *queue
)
3892 gimple transaction
= region
->transaction_stmt
;
3893 basic_block transaction_bb
= gimple_bb (transaction
);
3894 int n
= VEC_length (basic_block
, queue
);
3895 basic_block
*new_bbs
= XNEWVEC (basic_block
, n
);
3897 copy_bbs (VEC_address (basic_block
, queue
), n
, new_bbs
,
3898 NULL
, 0, NULL
, NULL
, transaction_bb
);
3899 edge e
= make_edge (transaction_bb
, new_bbs
[0], EDGE_TM_UNINSTRUMENTED
);
3900 add_phi_args_after_copy (new_bbs
, n
, e
);
3902 // Now we will have a GIMPLE_ATOMIC with 3 possible edges out of it.
3903 // a) EDGE_FALLTHRU into the transaction
3904 // b) EDGE_TM_ABORT out of the transaction
3905 // c) EDGE_TM_UNINSTRUMENTED into the uninstrumented blocks.
3910 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
3911 Queue all callees within block BB. */
3914 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
3915 basic_block bb
, bool for_clone
)
3917 gimple_stmt_iterator gsi
;
3919 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3921 gimple stmt
= gsi_stmt (gsi
);
3922 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3924 tree fndecl
= gimple_call_fndecl (stmt
);
3927 struct tm_ipa_cg_data
*d
;
3929 struct cgraph_node
*node
;
3931 if (is_tm_ending_fndecl (fndecl
))
3933 if (find_tm_replacement_function (fndecl
))
3936 node
= cgraph_get_node (fndecl
);
3937 gcc_assert (node
!= NULL
);
3938 d
= get_cg_data (&node
, true);
3940 pcallers
= (for_clone
? &d
->tm_callers_clone
3941 : &d
->tm_callers_normal
);
3944 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
3950 /* Scan all calls in NODE that are within a transaction region,
3951 and push the resulting nodes into the callee queue. */
3954 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
3955 cgraph_node_queue
*callees_p
)
3957 struct tm_region
*r
;
3959 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
3960 d
->all_tm_regions
= all_tm_regions
;
3962 for (r
= all_tm_regions
; r
; r
= r
->next
)
3964 VEC (basic_block
, heap
) *bbs
;
3968 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
3969 d
->transaction_blocks_normal
, false);
3971 // Generate the uninstrumented code path for this transaction.
3972 ipa_uninstrument_transaction (r
, bbs
);
3974 FOR_EACH_VEC_ELT (basic_block
, bbs
, i
, bb
)
3975 ipa_tm_scan_calls_block (callees_p
, bb
, false);
3977 VEC_free (basic_block
, heap
, bbs
);
3980 // ??? copy_bbs should maintain cgraph edges for the blocks as it is
3981 // copying them, rather than forcing us to do this externally.
3982 rebuild_cgraph_edges ();
3984 // ??? In ipa_uninstrument_transaction we don't try to update dominators
3985 // because copy_bbs doesn't return a VEC like iterate_fix_dominators expects.
3986 // Instead, just release dominators here so update_ssa recomputes them.
3987 free_dominance_info (CDI_DOMINATORS
);
3989 // When building the uninstrumented code path, copy_bbs will have invoked
3990 // create_new_def_for starting an "ssa update context". There is only one
3991 // instance of this context, so resolve ssa updates before moving on to
3992 // the next function.
3993 update_ssa (TODO_update_ssa
);
3996 /* Scan all calls in NODE as if this is the transactional clone,
3997 and push the destinations into the callee queue. */
4000 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
4001 cgraph_node_queue
*callees_p
)
4003 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
4006 FOR_EACH_BB_FN (bb
, fn
)
4007 ipa_tm_scan_calls_block (callees_p
, bb
, true);
4010 /* The function NODE has been detected to be irrevocable. Push all
4011 of its callers onto WORKLIST for the purpose of re-scanning them. */
4014 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
4015 cgraph_node_queue
*worklist_p
)
4017 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
4018 struct cgraph_edge
*e
;
4020 d
->is_irrevocable
= true;
4022 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4025 struct cgraph_node
*caller
;
4027 /* Don't examine recursive calls. */
4028 if (e
->caller
== node
)
4030 /* Even if we think we can go irrevocable, believe the user
4032 if (is_tm_safe_or_pure (e
->caller
->symbol
.decl
))
4036 d
= get_cg_data (&caller
, true);
4038 /* Check if the callee is in a transactional region. If so,
4039 schedule the function for normal re-scan as well. */
4040 bb
= gimple_bb (e
->call_stmt
);
4041 gcc_assert (bb
!= NULL
);
4042 if (d
->transaction_blocks_normal
4043 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
4044 d
->want_irr_scan_normal
= true;
4046 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
4050 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4051 within the block is irrevocable. */
4054 ipa_tm_scan_irr_block (basic_block bb
)
4056 gimple_stmt_iterator gsi
;
4059 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4061 gimple stmt
= gsi_stmt (gsi
);
4062 switch (gimple_code (stmt
))
4065 if (gimple_assign_single_p (stmt
))
4067 tree lhs
= gimple_assign_lhs (stmt
);
4068 tree rhs
= gimple_assign_rhs1 (stmt
);
4069 if (volatile_var_p (lhs
) || volatile_var_p (rhs
))
4076 tree lhs
= gimple_call_lhs (stmt
);
4077 if (lhs
&& volatile_var_p (lhs
))
4080 if (is_tm_pure_call (stmt
))
4083 fn
= gimple_call_fn (stmt
);
4085 /* Functions with the attribute are by definition irrevocable. */
4086 if (is_tm_irrevocable (fn
))
4089 /* For direct function calls, go ahead and check for replacement
4090 functions, or transitive irrevocable functions. For indirect
4091 functions, we'll ask the runtime. */
4092 if (TREE_CODE (fn
) == ADDR_EXPR
)
4094 struct tm_ipa_cg_data
*d
;
4095 struct cgraph_node
*node
;
4097 fn
= TREE_OPERAND (fn
, 0);
4098 if (is_tm_ending_fndecl (fn
))
4100 if (find_tm_replacement_function (fn
))
4103 node
= cgraph_get_node(fn
);
4104 d
= get_cg_data (&node
, true);
4106 /* Return true if irrevocable, but above all, believe
4108 if (d
->is_irrevocable
4109 && !is_tm_safe_or_pure (fn
))
4116 /* ??? The Approved Method of indicating that an inline
4117 assembly statement is not relevant to the transaction
4118 is to wrap it in a __tm_waiver block. This is not
4119 yet implemented, so we can't check for it. */
4120 if (is_tm_safe (current_function_decl
))
4122 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
4123 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
4124 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
4136 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4137 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4138 scanning past OLD_IRR or EXIT_BLOCKS. */
4141 ipa_tm_scan_irr_blocks (VEC (basic_block
, heap
) **pqueue
, bitmap new_irr
,
4142 bitmap old_irr
, bitmap exit_blocks
)
4144 bool any_new_irr
= false;
4147 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4151 basic_block bb
= VEC_pop (basic_block
, *pqueue
);
4153 /* Don't re-scan blocks we know already are irrevocable. */
4154 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
4157 if (ipa_tm_scan_irr_block (bb
))
4159 bitmap_set_bit (new_irr
, bb
->index
);
4162 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
4164 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4165 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4167 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4168 VEC_safe_push (basic_block
, heap
, *pqueue
, e
->dest
);
4172 while (!VEC_empty (basic_block
, *pqueue
));
4174 BITMAP_FREE (visited_blocks
);
4179 /* Propagate the irrevocable property both up and down the dominator tree.
4180 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4181 TM regions; OLD_IRR are the results of a previous scan of the dominator
4182 tree which has been fully propagated; NEW_IRR is the set of new blocks
4183 which are gaining the irrevocable property during the current scan. */
4186 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
4187 bitmap old_irr
, bitmap exit_blocks
)
4189 VEC (basic_block
, heap
) *bbs
;
4190 bitmap all_region_blocks
;
4192 /* If this block is in the old set, no need to rescan. */
4193 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
4196 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
4197 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
4198 all_region_blocks
, false);
4201 basic_block bb
= VEC_pop (basic_block
, bbs
);
4202 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
4203 bool all_son_irr
= false;
4207 /* Propagate up. If my children are, I am too, but we must have
4208 at least one child that is. */
4211 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4213 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
4215 all_son_irr
= false;
4223 /* Add block to new_irr if it hasn't already been processed. */
4224 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
4226 bitmap_set_bit (new_irr
, bb
->index
);
4232 /* Propagate down to everyone we immediately dominate. */
4236 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
4238 son
= next_dom_son (CDI_DOMINATORS
, son
))
4240 /* Make sure block is actually in a TM region, and it
4241 isn't already in old_irr. */
4242 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
4243 && bitmap_bit_p (all_region_blocks
, son
->index
))
4244 bitmap_set_bit (new_irr
, son
->index
);
4248 while (!VEC_empty (basic_block
, bbs
));
4250 BITMAP_FREE (all_region_blocks
);
4251 VEC_free (basic_block
, heap
, bbs
);
4255 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
4257 gimple_stmt_iterator gsi
;
4259 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4261 gimple stmt
= gsi_stmt (gsi
);
4262 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
4264 tree fndecl
= gimple_call_fndecl (stmt
);
4267 struct tm_ipa_cg_data
*d
;
4269 struct cgraph_node
*tnode
;
4271 if (is_tm_ending_fndecl (fndecl
))
4273 if (find_tm_replacement_function (fndecl
))
4276 tnode
= cgraph_get_node (fndecl
);
4277 d
= get_cg_data (&tnode
, true);
4279 pcallers
= (for_clone
? &d
->tm_callers_clone
4280 : &d
->tm_callers_normal
);
4282 gcc_assert (*pcallers
> 0);
4289 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4290 as well as other irrevocable actions such as inline assembly. Mark all
4291 such blocks as irrevocable and decrement the number of calls to
4292 transactional clones. Return true if, for the transactional clone, the
4293 entire function is irrevocable. */
4296 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
4298 struct tm_ipa_cg_data
*d
;
4299 bitmap new_irr
, old_irr
;
4300 VEC (basic_block
, heap
) *queue
;
4303 /* Builtin operators (operator new, and such). */
4304 if (DECL_STRUCT_FUNCTION (node
->symbol
.decl
) == NULL
4305 || DECL_STRUCT_FUNCTION (node
->symbol
.decl
)->cfg
== NULL
)
4308 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
4309 calculate_dominance_info (CDI_DOMINATORS
);
4311 d
= get_cg_data (&node
, true);
4312 queue
= VEC_alloc (basic_block
, heap
, 10);
4313 new_irr
= BITMAP_ALLOC (&tm_obstack
);
4315 /* Scan each tm region, propagating irrevocable status through the tree. */
4318 old_irr
= d
->irrevocable_blocks_clone
;
4319 VEC_quick_push (basic_block
, queue
, single_succ (ENTRY_BLOCK_PTR
));
4320 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4322 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR
), new_irr
,
4324 ret
= bitmap_bit_p (new_irr
, single_succ (ENTRY_BLOCK_PTR
)->index
);
4329 struct tm_region
*region
;
4331 old_irr
= d
->irrevocable_blocks_normal
;
4332 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4334 VEC_quick_push (basic_block
, queue
, region
->entry_block
);
4335 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4336 region
->exit_blocks
))
4337 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4338 region
->exit_blocks
);
4342 /* If we found any new irrevocable blocks, reduce the call count for
4343 transactional clones within the irrevocable blocks. Save the new
4344 set of irrevocable blocks for next time. */
4345 if (!bitmap_empty_p (new_irr
))
4347 bitmap_iterator bmi
;
4350 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4351 ipa_tm_decrement_clone_counts (BASIC_BLOCK (i
), for_clone
);
4355 bitmap_ior_into (old_irr
, new_irr
);
4356 BITMAP_FREE (new_irr
);
4359 d
->irrevocable_blocks_clone
= new_irr
;
4361 d
->irrevocable_blocks_normal
= new_irr
;
4363 if (dump_file
&& new_irr
)
4366 bitmap_iterator bmi
;
4369 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4370 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4371 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4375 BITMAP_FREE (new_irr
);
4377 VEC_free (basic_block
, heap
, queue
);
4383 /* Return true if, for the transactional clone of NODE, any call
4384 may enter irrevocable mode. */
4387 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4389 struct tm_ipa_cg_data
*d
;
4393 d
= get_cg_data (&node
, true);
4394 decl
= node
->symbol
.decl
;
4395 flags
= flags_from_decl_or_type (decl
);
4397 /* Handle some TM builtins. Ordinarily these aren't actually generated
4398 at this point, but handling these functions when written in by the
4399 user makes it easier to build unit tests. */
4400 if (flags
& ECF_TM_BUILTIN
)
4403 /* Filter out all functions that are marked. */
4404 if (flags
& ECF_TM_PURE
)
4406 if (is_tm_safe (decl
))
4408 if (is_tm_irrevocable (decl
))
4410 if (is_tm_callable (decl
))
4412 if (find_tm_replacement_function (decl
))
4415 /* If we aren't seeing the final version of the function we don't
4416 know what it will contain at runtime. */
4417 if (cgraph_function_body_availability (node
) < AVAIL_AVAILABLE
)
4420 /* If the function must go irrevocable, then of course true. */
4421 if (d
->is_irrevocable
)
4424 /* If there are any blocks marked irrevocable, then the function
4425 as a whole may enter irrevocable. */
4426 if (d
->irrevocable_blocks_clone
)
4429 /* We may have previously marked this function as tm_may_enter_irr;
4430 see pass_diagnose_tm_blocks. */
4431 if (node
->local
.tm_may_enter_irr
)
4434 /* Recurse on the main body for aliases. In general, this will
4435 result in one of the bits above being set so that we will not
4436 have to recurse next time. */
4438 return ipa_tm_mayenterirr_function (cgraph_get_node (node
->thunk
.alias
));
4440 /* What remains is unmarked local functions without items that force
4441 the function to go irrevocable. */
4445 /* Diagnose calls from transaction_safe functions to unmarked
4446 functions that are determined to not be safe. */
4449 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4451 struct cgraph_edge
*e
;
4453 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4454 if (!is_tm_callable (e
->callee
->symbol
.decl
)
4455 && e
->callee
->local
.tm_may_enter_irr
)
4456 error_at (gimple_location (e
->call_stmt
),
4457 "unsafe function call %qD within "
4458 "%<transaction_safe%> function", e
->callee
->symbol
.decl
);
4461 /* Diagnose call from atomic transactions to unmarked functions
4462 that are determined to not be safe. */
4465 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4466 struct tm_region
*all_tm_regions
)
4468 struct tm_region
*r
;
4470 for (r
= all_tm_regions
; r
; r
= r
->next
)
4471 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4473 /* Atomic transactions can be nested inside relaxed. */
4475 ipa_tm_diagnose_transaction (node
, r
->inner
);
4479 VEC (basic_block
, heap
) *bbs
;
4480 gimple_stmt_iterator gsi
;
4484 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4485 r
->irr_blocks
, NULL
, false);
4487 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
4488 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4490 gimple stmt
= gsi_stmt (gsi
);
4493 if (gimple_code (stmt
) == GIMPLE_ASM
)
4495 error_at (gimple_location (stmt
),
4496 "asm not allowed in atomic transaction");
4500 if (!is_gimple_call (stmt
))
4502 fndecl
= gimple_call_fndecl (stmt
);
4504 /* Indirect function calls have been diagnosed already. */
4508 /* Stop at the end of the transaction. */
4509 if (is_tm_ending_fndecl (fndecl
))
4511 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4516 /* Marked functions have been diagnosed already. */
4517 if (is_tm_pure_call (stmt
))
4519 if (is_tm_callable (fndecl
))
4522 if (cgraph_local_info (fndecl
)->tm_may_enter_irr
)
4523 error_at (gimple_location (stmt
),
4524 "unsafe function call %qD within "
4525 "atomic transaction", fndecl
);
4528 VEC_free (basic_block
, heap
, bbs
);
4532 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4533 OLD_DECL. The returned value is a freshly malloced pointer that
4534 should be freed by the caller. */
4537 tm_mangle (tree old_asm_id
)
4539 const char *old_asm_name
;
4542 struct demangle_component
*dc
;
4545 /* Determine if the symbol is already a valid C++ mangled name. Do this
4546 even for C, which might be interfacing with C++ code via appropriately
4547 ugly identifiers. */
4548 /* ??? We could probably do just as well checking for "_Z" and be done. */
4549 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4550 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4557 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4558 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4562 old_asm_name
+= 2; /* Skip _Z */
4566 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4567 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4568 /* Don't play silly games, you! */
4571 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4572 /* I'd really like to know if we can ever be passed one of
4573 these from the C++ front end. The Logical Thing would
4574 seem that hidden-alias should be outer-most, so that we
4575 get hidden-alias of a transaction-clone and not vice-versa. */
4583 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4587 new_asm_id
= get_identifier (tm_name
);
4594 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4596 cgraph_mark_force_output_node (node
);
4597 /* ??? function_and_variable_visibility will reset
4598 the needed bit, without actually checking. */
4602 /* Callback data for ipa_tm_create_version_alias. */
4603 struct create_version_alias_info
4605 struct cgraph_node
*old_node
;
4609 /* A subroutine of ipa_tm_create_version, called via
4610 cgraph_for_node_and_aliases. Create new tm clones for each of
4611 the existing aliases. */
4613 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4615 struct create_version_alias_info
*info
4616 = (struct create_version_alias_info
*)data
;
4617 tree old_decl
, new_decl
, tm_name
;
4618 struct cgraph_node
*new_node
;
4620 if (!node
->same_body_alias
)
4623 old_decl
= node
->symbol
.decl
;
4624 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4625 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4626 TREE_CODE (old_decl
), tm_name
,
4627 TREE_TYPE (old_decl
));
4629 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4630 SET_DECL_RTL (new_decl
, NULL
);
4632 /* Based loosely on C++'s make_alias_for(). */
4633 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4634 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4635 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4636 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4637 DECL_EXTERNAL (new_decl
) = 0;
4638 DECL_ARTIFICIAL (new_decl
) = 1;
4639 TREE_ADDRESSABLE (new_decl
) = 1;
4640 TREE_USED (new_decl
) = 1;
4641 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4643 /* Perform the same remapping to the comdat group. */
4644 if (DECL_ONE_ONLY (new_decl
))
4645 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4647 new_node
= cgraph_same_body_alias (NULL
, new_decl
, info
->new_decl
);
4648 new_node
->tm_clone
= true;
4649 new_node
->symbol
.externally_visible
= info
->old_node
->symbol
.externally_visible
;
4650 /* ?? Do not traverse aliases here. */
4651 get_cg_data (&node
, false)->clone
= new_node
;
4653 record_tm_clone_pair (old_decl
, new_decl
);
4655 if (info
->old_node
->symbol
.force_output
4656 || ipa_ref_list_first_referring (&info
->old_node
->symbol
.ref_list
))
4657 ipa_tm_mark_force_output_node (new_node
);
4661 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4662 appropriate for the transactional clone. */
4665 ipa_tm_create_version (struct cgraph_node
*old_node
)
4667 tree new_decl
, old_decl
, tm_name
;
4668 struct cgraph_node
*new_node
;
4670 old_decl
= old_node
->symbol
.decl
;
4671 new_decl
= copy_node (old_decl
);
4673 /* DECL_ASSEMBLER_NAME needs to be set before we call
4674 cgraph_copy_node_for_versioning below, because cgraph_node will
4675 fill the assembler_name_hash. */
4676 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4677 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4678 SET_DECL_RTL (new_decl
, NULL
);
4679 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4681 /* Perform the same remapping to the comdat group. */
4682 if (DECL_ONE_ONLY (new_decl
))
4683 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4685 new_node
= cgraph_copy_node_for_versioning (old_node
, new_decl
, NULL
, NULL
);
4686 new_node
->symbol
.externally_visible
= old_node
->symbol
.externally_visible
;
4687 new_node
->lowered
= true;
4688 new_node
->tm_clone
= 1;
4689 get_cg_data (&old_node
, true)->clone
= new_node
;
4691 if (cgraph_function_body_availability (old_node
) >= AVAIL_OVERWRITABLE
)
4693 /* Remap extern inline to static inline. */
4694 /* ??? Is it worth trying to use make_decl_one_only? */
4695 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4697 DECL_EXTERNAL (new_decl
) = 0;
4698 TREE_PUBLIC (new_decl
) = 0;
4699 DECL_WEAK (new_decl
) = 0;
4702 tree_function_versioning (old_decl
, new_decl
, NULL
, false, NULL
, false,
4706 record_tm_clone_pair (old_decl
, new_decl
);
4708 cgraph_call_function_insertion_hooks (new_node
);
4709 if (old_node
->symbol
.force_output
4710 || ipa_ref_list_first_referring (&old_node
->symbol
.ref_list
))
4711 ipa_tm_mark_force_output_node (new_node
);
4713 /* Do the same thing, but for any aliases of the original node. */
4715 struct create_version_alias_info data
;
4716 data
.old_node
= old_node
;
4717 data
.new_decl
= new_decl
;
4718 cgraph_for_node_and_aliases (old_node
, ipa_tm_create_version_alias
,
4723 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4726 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4729 gimple_stmt_iterator gsi
;
4732 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4734 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4735 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4737 split_block_after_labels (bb
);
4738 gsi
= gsi_after_labels (bb
);
4739 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4741 cgraph_create_edge (node
,
4742 cgraph_get_create_node
4743 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4745 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4749 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4752 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4753 struct tm_region
*region
,
4754 gimple_stmt_iterator
*gsi
, gimple stmt
)
4756 tree gettm_fn
, ret
, old_fn
, callfn
;
4760 old_fn
= gimple_call_fn (stmt
);
4762 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
4764 tree fndecl
= TREE_OPERAND (old_fn
, 0);
4765 tree clone
= get_tm_clone_pair (fndecl
);
4767 /* By transforming the call into a TM_GETTMCLONE, we are
4768 technically taking the address of the original function and
4769 its clone. Explain this so inlining will know this function
4771 cgraph_mark_address_taken_node (cgraph_get_node (fndecl
));
4773 cgraph_mark_address_taken_node (cgraph_get_node (clone
));
4776 safe
= is_tm_safe (TREE_TYPE (old_fn
));
4777 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
4778 : BUILT_IN_TM_GETTMCLONE_IRR
);
4779 ret
= create_tmp_var (ptr_type_node
, NULL
);
4782 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4784 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
4785 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
4786 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
4788 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
4789 ret
= make_ssa_name (ret
, g
);
4790 gimple_call_set_lhs (g
, ret
);
4792 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
4794 cgraph_create_edge (node
, cgraph_get_create_node (gettm_fn
), g
, 0,
4795 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4798 /* Cast return value from tm_gettmclone* into appropriate function
4800 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
4801 g2
= gimple_build_assign (callfn
,
4802 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
4803 callfn
= make_ssa_name (callfn
, g2
);
4804 gimple_assign_set_lhs (g2
, callfn
);
4805 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
4807 /* ??? This is a hack to preserve the NOTHROW bit on the call,
4808 which we would have derived from the decl. Failure to save
4809 this bit means we might have to split the basic block. */
4810 if (gimple_call_nothrow_p (stmt
))
4811 gimple_call_set_nothrow (stmt
, true);
4813 gimple_call_set_fn (stmt
, callfn
);
4815 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
4816 for a call statement. Fix it. */
4818 tree lhs
= gimple_call_lhs (stmt
);
4819 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
4821 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
4825 temp
= create_tmp_reg (rettype
, 0);
4826 gimple_call_set_lhs (stmt
, temp
);
4828 g2
= gimple_build_assign (lhs
,
4829 fold_build1 (VIEW_CONVERT_EXPR
,
4830 TREE_TYPE (lhs
), temp
));
4831 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
4840 /* Helper function for ipa_tm_transform_calls*. Given a call
4841 statement in GSI which resides inside transaction REGION, redirect
4842 the call to either its wrapper function, or its clone. */
4845 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
4846 struct tm_region
*region
,
4847 gimple_stmt_iterator
*gsi
,
4848 bool *need_ssa_rename_p
)
4850 gimple stmt
= gsi_stmt (*gsi
);
4851 struct cgraph_node
*new_node
;
4852 struct cgraph_edge
*e
= cgraph_edge (node
, stmt
);
4853 tree fndecl
= gimple_call_fndecl (stmt
);
4855 /* For indirect calls, pass the address through the runtime. */
4858 *need_ssa_rename_p
|=
4859 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4863 /* Handle some TM builtins. Ordinarily these aren't actually generated
4864 at this point, but handling these functions when written in by the
4865 user makes it easier to build unit tests. */
4866 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
4869 /* Fixup recursive calls inside clones. */
4870 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
4871 for recursion but not update the call statements themselves? */
4872 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
4874 gimple_call_set_fndecl (stmt
, current_function_decl
);
4878 /* If there is a replacement, use it. */
4879 fndecl
= find_tm_replacement_function (fndecl
);
4882 new_node
= cgraph_get_create_node (fndecl
);
4884 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
4886 We can't do this earlier in record_tm_replacement because
4887 cgraph_remove_unreachable_nodes is called before we inject
4888 references to the node. Further, we can't do this in some
4889 nice central place in ipa_tm_execute because we don't have
4890 the exact list of wrapper functions that would be used.
4891 Marking more wrappers than necessary results in the creation
4892 of unnecessary cgraph_nodes, which can cause some of the
4893 other IPA passes to crash.
4895 We do need to mark these nodes so that we get the proper
4896 result in expand_call_tm. */
4897 /* ??? This seems broken. How is it that we're marking the
4898 CALLEE as may_enter_irr? Surely we should be marking the
4899 CALLER. Also note that find_tm_replacement_function also
4900 contains mappings into the TM runtime, e.g. memcpy. These
4901 we know won't go irrevocable. */
4902 new_node
->local
.tm_may_enter_irr
= 1;
4906 struct tm_ipa_cg_data
*d
;
4907 struct cgraph_node
*tnode
= e
->callee
;
4909 d
= get_cg_data (&tnode
, true);
4910 new_node
= d
->clone
;
4912 /* As we've already skipped pure calls and appropriate builtins,
4913 and we've already marked irrevocable blocks, if we can't come
4914 up with a static replacement, then ask the runtime. */
4915 if (new_node
== NULL
)
4917 *need_ssa_rename_p
|=
4918 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4922 fndecl
= new_node
->symbol
.decl
;
4925 cgraph_redirect_edge_callee (e
, new_node
);
4926 gimple_call_set_fndecl (stmt
, fndecl
);
4929 /* Helper function for ipa_tm_transform_calls. For a given BB,
4930 install calls to tm_irrevocable when IRR_BLOCKS are reached,
4931 redirect other calls to the generated transactional clone. */
4934 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
4935 basic_block bb
, bitmap irr_blocks
)
4937 gimple_stmt_iterator gsi
;
4938 bool need_ssa_rename
= false;
4940 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4942 ipa_tm_insert_irr_call (node
, region
, bb
);
4946 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4948 gimple stmt
= gsi_stmt (gsi
);
4950 if (!is_gimple_call (stmt
))
4952 if (is_tm_pure_call (stmt
))
4955 /* Redirect edges to the appropriate replacement or clone. */
4956 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
4959 return need_ssa_rename
;
4962 /* Walk the CFG for REGION, beginning at BB. Install calls to
4963 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
4964 the generated transactional clone. */
4967 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
4968 basic_block bb
, bitmap irr_blocks
)
4970 bool need_ssa_rename
= false;
4973 VEC(basic_block
, heap
) *queue
= NULL
;
4974 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4976 VEC_safe_push (basic_block
, heap
, queue
, bb
);
4979 bb
= VEC_pop (basic_block
, queue
);
4982 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
4984 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4987 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
4990 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4991 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4993 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4994 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
4997 while (!VEC_empty (basic_block
, queue
));
4999 VEC_free (basic_block
, heap
, queue
);
5000 BITMAP_FREE (visited_blocks
);
5002 return need_ssa_rename
;
5005 /* Transform the calls within the TM regions within NODE. */
5008 ipa_tm_transform_transaction (struct cgraph_node
*node
)
5010 struct tm_ipa_cg_data
*d
;
5011 struct tm_region
*region
;
5012 bool need_ssa_rename
= false;
5014 d
= get_cg_data (&node
, true);
5016 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
5017 calculate_dominance_info (CDI_DOMINATORS
);
5019 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
5021 /* If we're sure to go irrevocable, don't transform anything. */
5022 if (d
->irrevocable_blocks_normal
5023 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
5024 region
->entry_block
->index
))
5026 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
);
5027 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
5032 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
5033 d
->irrevocable_blocks_normal
);
5036 if (need_ssa_rename
)
5037 update_ssa (TODO_update_ssa_only_virtuals
);
5042 /* Transform the calls within the transactional clone of NODE. */
5045 ipa_tm_transform_clone (struct cgraph_node
*node
)
5047 struct tm_ipa_cg_data
*d
;
5048 bool need_ssa_rename
;
5050 d
= get_cg_data (&node
, true);
5052 /* If this function makes no calls and has no irrevocable blocks,
5053 then there's nothing to do. */
5054 /* ??? Remove non-aborting top-level transactions. */
5055 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
5058 push_cfun (DECL_STRUCT_FUNCTION (d
->clone
->symbol
.decl
));
5059 calculate_dominance_info (CDI_DOMINATORS
);
5062 ipa_tm_transform_calls (d
->clone
, NULL
, single_succ (ENTRY_BLOCK_PTR
),
5063 d
->irrevocable_blocks_clone
);
5065 if (need_ssa_rename
)
5066 update_ssa (TODO_update_ssa_only_virtuals
);
5071 /* Main entry point for the transactional memory IPA pass. */
5074 ipa_tm_execute (void)
5076 cgraph_node_queue tm_callees
= NULL
;
5077 /* List of functions that will go irrevocable. */
5078 cgraph_node_queue irr_worklist
= NULL
;
5080 struct cgraph_node
*node
;
5081 struct tm_ipa_cg_data
*d
;
5082 enum availability a
;
5085 #ifdef ENABLE_CHECKING
5089 bitmap_obstack_initialize (&tm_obstack
);
5090 initialize_original_copy_tables ();
5092 /* For all local functions marked tm_callable, queue them. */
5093 FOR_EACH_DEFINED_FUNCTION (node
)
5094 if (is_tm_callable (node
->symbol
.decl
)
5095 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5097 d
= get_cg_data (&node
, true);
5098 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5101 /* For all local reachable functions... */
5102 FOR_EACH_DEFINED_FUNCTION (node
)
5104 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5106 /* ... marked tm_pure, record that fact for the runtime by
5107 indicating that the pure function is its own tm_callable.
5108 No need to do this if the function's address can't be taken. */
5109 if (is_tm_pure (node
->symbol
.decl
))
5111 if (!node
->local
.local
)
5112 record_tm_clone_pair (node
->symbol
.decl
, node
->symbol
.decl
);
5116 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
5117 calculate_dominance_info (CDI_DOMINATORS
);
5119 tm_region_init (NULL
);
5122 d
= get_cg_data (&node
, true);
5124 /* Scan for calls that are in each transaction, and
5125 generate the uninstrumented code path. */
5126 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
5128 /* Put it in the worklist so we can scan the function
5129 later (ipa_tm_scan_irr_function) and mark the
5130 irrevocable blocks. */
5131 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5132 d
->want_irr_scan_normal
= true;
5138 /* For every local function on the callee list, scan as if we will be
5139 creating a transactional clone, queueing all new functions we find
5141 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
5143 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
5144 a
= cgraph_function_body_availability (node
);
5145 d
= get_cg_data (&node
, true);
5147 /* Put it in the worklist so we can scan the function later
5148 (ipa_tm_scan_irr_function) and mark the irrevocable
5150 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5152 /* Some callees cannot be arbitrarily cloned. These will always be
5153 irrevocable. Mark these now, so that we need not scan them. */
5154 if (is_tm_irrevocable (node
->symbol
.decl
))
5155 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5156 else if (a
<= AVAIL_NOT_AVAILABLE
5157 && !is_tm_safe_or_pure (node
->symbol
.decl
))
5158 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5159 else if (a
>= AVAIL_OVERWRITABLE
)
5161 if (!tree_versionable_function_p (node
->symbol
.decl
))
5162 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5163 else if (!d
->is_irrevocable
)
5165 /* If this is an alias, make sure its base is queued as well.
5166 we need not scan the callees now, as the base will do. */
5169 node
= cgraph_get_node (node
->thunk
.alias
);
5170 d
= get_cg_data (&node
, true);
5171 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
5175 /* Add all nodes called by this function into
5176 tm_callees as well. */
5177 ipa_tm_scan_calls_clone (node
, &tm_callees
);
5182 /* Iterate scans until no more work to be done. Prefer not to use
5183 VEC_pop because the worklist tends to follow a breadth-first
5184 search of the callgraph, which should allow convergance with a
5185 minimum number of scans. But we also don't want the worklist
5186 array to grow without bound, so we shift the array up periodically. */
5187 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
5189 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
5191 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
5195 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
5196 d
= get_cg_data (&node
, true);
5197 d
->in_worklist
= false;
5199 if (d
->want_irr_scan_normal
)
5201 d
->want_irr_scan_normal
= false;
5202 ipa_tm_scan_irr_function (node
, false);
5204 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
5205 ipa_tm_note_irrevocable (node
, &irr_worklist
);
5208 /* For every function on the callee list, collect the tm_may_enter_irr
5210 VEC_truncate (cgraph_node_p
, irr_worklist
, 0);
5211 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
5213 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
5214 if (ipa_tm_mayenterirr_function (node
))
5216 d
= get_cg_data (&node
, true);
5217 gcc_assert (d
->in_worklist
== false);
5218 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
5222 /* Propagate the tm_may_enter_irr bit to callers until stable. */
5223 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
5225 struct cgraph_node
*caller
;
5226 struct cgraph_edge
*e
;
5227 struct ipa_ref
*ref
;
5230 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
5232 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
5236 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
5237 d
= get_cg_data (&node
, true);
5238 d
->in_worklist
= false;
5239 node
->local
.tm_may_enter_irr
= true;
5241 /* Propagate back to normal callers. */
5242 for (e
= node
->callers
; e
; e
= e
->next_caller
)
5245 if (!is_tm_safe_or_pure (caller
->symbol
.decl
)
5246 && !caller
->local
.tm_may_enter_irr
)
5248 d
= get_cg_data (&caller
, true);
5249 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5253 /* Propagate back to referring aliases as well. */
5254 for (j
= 0; ipa_ref_list_referring_iterate (&node
->symbol
.ref_list
, j
, ref
); j
++)
5256 caller
= cgraph (ref
->referring
);
5257 if (ref
->use
== IPA_REF_ALIAS
5258 && !caller
->local
.tm_may_enter_irr
)
5260 /* ?? Do not traverse aliases here. */
5261 d
= get_cg_data (&caller
, false);
5262 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
5267 /* Now validate all tm_safe functions, and all atomic regions in
5269 FOR_EACH_DEFINED_FUNCTION (node
)
5271 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5273 d
= get_cg_data (&node
, true);
5274 if (is_tm_safe (node
->symbol
.decl
))
5275 ipa_tm_diagnose_tm_safe (node
);
5276 else if (d
->all_tm_regions
)
5277 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
5280 /* Create clones. Do those that are not irrevocable and have a
5281 positive call count. Do those publicly visible functions that
5282 the user directed us to clone. */
5283 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
5287 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
5288 if (node
->same_body_alias
)
5291 a
= cgraph_function_body_availability (node
);
5292 d
= get_cg_data (&node
, true);
5294 if (a
<= AVAIL_NOT_AVAILABLE
)
5295 doit
= is_tm_callable (node
->symbol
.decl
);
5296 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->symbol
.decl
))
5298 else if (!d
->is_irrevocable
5299 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
5303 ipa_tm_create_version (node
);
5306 /* Redirect calls to the new clones, and insert irrevocable marks. */
5307 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
5309 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
5312 d
= get_cg_data (&node
, true);
5314 ipa_tm_transform_clone (node
);
5317 FOR_EACH_DEFINED_FUNCTION (node
)
5319 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5321 d
= get_cg_data (&node
, true);
5322 if (d
->all_tm_regions
)
5323 ipa_tm_transform_transaction (node
);
5326 /* Free and clear all data structures. */
5327 VEC_free (cgraph_node_p
, heap
, tm_callees
);
5328 VEC_free (cgraph_node_p
, heap
, irr_worklist
);
5329 bitmap_obstack_release (&tm_obstack
);
5330 free_original_copy_tables ();
5332 FOR_EACH_FUNCTION (node
)
5333 node
->symbol
.aux
= NULL
;
5335 #ifdef ENABLE_CHECKING
5342 struct simple_ipa_opt_pass pass_ipa_tm
=
5347 OPTGROUP_NONE
, /* optinfo_flags */
5349 ipa_tm_execute
, /* execute */
5352 0, /* static_pass_number */
5353 TV_TRANS_MEM
, /* tv_id */
5354 PROP_ssa
| PROP_cfg
, /* properties_required */
5355 0, /* properties_provided */
5356 0, /* properties_destroyed */
5357 0, /* todo_flags_start */
5358 0, /* todo_flags_finish */
5362 #include "gt-trans-mem.h"