1 /* Passes for transactional memory support.
2 Copyright (C) 2008, 2009, 2010, 2011, 2012 Free Software Foundation, Inc.
4 This file is part of GCC.
6 GCC is free software; you can redistribute it and/or modify it under
7 the terms of the GNU General Public License as published by the Free
8 Software Foundation; either version 3, or (at your option) any later
11 GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 You should have received a copy of the GNU General Public License
17 along with GCC; see the file COPYING3. If not see
18 <http://www.gnu.org/licenses/>. */
22 #include "coretypes.h"
25 #include "tree-flow.h"
26 #include "tree-pass.h"
27 #include "tree-inline.h"
28 #include "diagnostic-core.h"
31 #include "trans-mem.h"
34 #include "langhooks.h"
35 #include "tree-pretty-print.h"
36 #include "gimple-pretty-print.h"
40 #define PROB_VERY_UNLIKELY (REG_BR_PROB_BASE / 2000 - 1)
41 #define PROB_ALWAYS (REG_BR_PROB_BASE)
43 #define A_RUNINSTRUMENTEDCODE 0x0001
44 #define A_RUNUNINSTRUMENTEDCODE 0x0002
45 #define A_SAVELIVEVARIABLES 0x0004
46 #define A_RESTORELIVEVARIABLES 0x0008
47 #define A_ABORTTRANSACTION 0x0010
49 #define AR_USERABORT 0x0001
50 #define AR_USERRETRY 0x0002
51 #define AR_TMCONFLICT 0x0004
52 #define AR_EXCEPTIONBLOCKABORT 0x0008
53 #define AR_OUTERABORT 0x0010
55 #define MODE_SERIALIRREVOCABLE 0x0000
58 /* The representation of a transaction changes several times during the
59 lowering process. In the beginning, in the front-end we have the
60 GENERIC tree TRANSACTION_EXPR. For example,
68 During initial gimplification (gimplify.c) the TRANSACTION_EXPR node is
69 trivially replaced with a GIMPLE_TRANSACTION node.
71 During pass_lower_tm, we examine the body of transactions looking
72 for aborts. Transactions that do not contain an abort may be
73 merged into an outer transaction. We also add a TRY-FINALLY node
74 to arrange for the transaction to be committed on any exit.
76 [??? Think about how this arrangement affects throw-with-commit
77 and throw-with-abort operations. In this case we want the TRY to
78 handle gotos, but not to catch any exceptions because the transaction
79 will already be closed.]
81 GIMPLE_TRANSACTION [label=NULL] {
88 __builtin___tm_abort ();
90 __builtin___tm_commit ();
94 During pass_lower_eh, we create EH regions for the transactions,
95 intermixed with the regular EH stuff. This gives us a nice persistent
96 mapping (all the way through rtl) from transactional memory operation
97 back to the transaction, which allows us to get the abnormal edges
98 correct to model transaction aborts and restarts:
100 GIMPLE_TRANSACTION [label=over]
106 __builtin___tm_abort ();
107 __builtin___tm_commit ();
110 This is the end of all_lowering_passes, and so is what is present
111 during the IPA passes, and through all of the optimization passes.
113 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
114 functions and mark functions for cloning.
116 At the end of gimple optimization, before exiting SSA form,
117 pass_tm_edges replaces statements that perform transactional
118 memory operations with the appropriate TM builtins, and swap
119 out function calls with their transactional clones. At this
120 point we introduce the abnormal transaction restart edges and
121 complete lowering of the GIMPLE_TRANSACTION node.
123 x = __builtin___tm_start (MAY_ABORT);
125 if (x & abort_transaction)
128 t0 = __builtin___tm_load (global);
130 __builtin___tm_store (&global, t1);
132 __builtin___tm_abort ();
133 __builtin___tm_commit ();
138 /* Return the attributes we want to examine for X, or NULL if it's not
139 something we examine. We look at function types, but allow pointers
140 to function types and function decls and peek through. */
143 get_attrs_for (const_tree x
)
145 switch (TREE_CODE (x
))
148 return TYPE_ATTRIBUTES (TREE_TYPE (x
));
155 if (TREE_CODE (x
) != POINTER_TYPE
)
161 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
167 return TYPE_ATTRIBUTES (x
);
171 /* Return true if X has been marked TM_PURE. */
174 is_tm_pure (const_tree x
)
178 switch (TREE_CODE (x
))
189 if (TREE_CODE (x
) != POINTER_TYPE
)
195 if (TREE_CODE (x
) != FUNCTION_TYPE
&& TREE_CODE (x
) != METHOD_TYPE
)
200 flags
= flags_from_decl_or_type (x
);
201 return (flags
& ECF_TM_PURE
) != 0;
204 /* Return true if X has been marked TM_IRREVOCABLE. */
207 is_tm_irrevocable (tree x
)
209 tree attrs
= get_attrs_for (x
);
211 if (attrs
&& lookup_attribute ("transaction_unsafe", attrs
))
214 /* A call to the irrevocable builtin is by definition,
216 if (TREE_CODE (x
) == ADDR_EXPR
)
217 x
= TREE_OPERAND (x
, 0);
218 if (TREE_CODE (x
) == FUNCTION_DECL
219 && DECL_BUILT_IN_CLASS (x
) == BUILT_IN_NORMAL
220 && DECL_FUNCTION_CODE (x
) == BUILT_IN_TM_IRREVOCABLE
)
226 /* Return true if X has been marked TM_SAFE. */
229 is_tm_safe (const_tree x
)
233 tree attrs
= get_attrs_for (x
);
236 if (lookup_attribute ("transaction_safe", attrs
))
238 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
245 /* Return true if CALL is const, or tm_pure. */
248 is_tm_pure_call (gimple call
)
250 tree fn
= gimple_call_fn (call
);
252 if (TREE_CODE (fn
) == ADDR_EXPR
)
254 fn
= TREE_OPERAND (fn
, 0);
255 gcc_assert (TREE_CODE (fn
) == FUNCTION_DECL
);
260 return is_tm_pure (fn
);
263 /* Return true if X has been marked TM_CALLABLE. */
266 is_tm_callable (tree x
)
268 tree attrs
= get_attrs_for (x
);
271 if (lookup_attribute ("transaction_callable", attrs
))
273 if (lookup_attribute ("transaction_safe", attrs
))
275 if (lookup_attribute ("transaction_may_cancel_outer", attrs
))
281 /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
284 is_tm_may_cancel_outer (tree x
)
286 tree attrs
= get_attrs_for (x
);
288 return lookup_attribute ("transaction_may_cancel_outer", attrs
) != NULL
;
292 /* Return true for built in functions that "end" a transaction. */
295 is_tm_ending_fndecl (tree fndecl
)
297 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
298 switch (DECL_FUNCTION_CODE (fndecl
))
300 case BUILT_IN_TM_COMMIT
:
301 case BUILT_IN_TM_COMMIT_EH
:
302 case BUILT_IN_TM_ABORT
:
303 case BUILT_IN_TM_IRREVOCABLE
:
312 /* Return true if STMT is a TM load. */
315 is_tm_load (gimple stmt
)
319 if (gimple_code (stmt
) != GIMPLE_CALL
)
322 fndecl
= gimple_call_fndecl (stmt
);
323 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
324 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl
)));
327 /* Same as above, but for simple TM loads, that is, not the
328 after-write, after-read, etc optimized variants. */
331 is_tm_simple_load (gimple stmt
)
335 if (gimple_code (stmt
) != GIMPLE_CALL
)
338 fndecl
= gimple_call_fndecl (stmt
);
339 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
341 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
342 return (fcode
== BUILT_IN_TM_LOAD_1
343 || fcode
== BUILT_IN_TM_LOAD_2
344 || fcode
== BUILT_IN_TM_LOAD_4
345 || fcode
== BUILT_IN_TM_LOAD_8
346 || fcode
== BUILT_IN_TM_LOAD_FLOAT
347 || fcode
== BUILT_IN_TM_LOAD_DOUBLE
348 || fcode
== BUILT_IN_TM_LOAD_LDOUBLE
349 || fcode
== BUILT_IN_TM_LOAD_M64
350 || fcode
== BUILT_IN_TM_LOAD_M128
351 || fcode
== BUILT_IN_TM_LOAD_M256
);
356 /* Return true if STMT is a TM store. */
359 is_tm_store (gimple stmt
)
363 if (gimple_code (stmt
) != GIMPLE_CALL
)
366 fndecl
= gimple_call_fndecl (stmt
);
367 return (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
368 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl
)));
371 /* Same as above, but for simple TM stores, that is, not the
372 after-write, after-read, etc optimized variants. */
375 is_tm_simple_store (gimple stmt
)
379 if (gimple_code (stmt
) != GIMPLE_CALL
)
382 fndecl
= gimple_call_fndecl (stmt
);
383 if (fndecl
&& DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
385 enum built_in_function fcode
= DECL_FUNCTION_CODE (fndecl
);
386 return (fcode
== BUILT_IN_TM_STORE_1
387 || fcode
== BUILT_IN_TM_STORE_2
388 || fcode
== BUILT_IN_TM_STORE_4
389 || fcode
== BUILT_IN_TM_STORE_8
390 || fcode
== BUILT_IN_TM_STORE_FLOAT
391 || fcode
== BUILT_IN_TM_STORE_DOUBLE
392 || fcode
== BUILT_IN_TM_STORE_LDOUBLE
393 || fcode
== BUILT_IN_TM_STORE_M64
394 || fcode
== BUILT_IN_TM_STORE_M128
395 || fcode
== BUILT_IN_TM_STORE_M256
);
400 /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
403 is_tm_abort (tree fndecl
)
406 && DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
407 && DECL_FUNCTION_CODE (fndecl
) == BUILT_IN_TM_ABORT
);
410 /* Build a GENERIC tree for a user abort. This is called by front ends
411 while transforming the __tm_abort statement. */
414 build_tm_abort_call (location_t loc
, bool is_outer
)
416 return build_call_expr_loc (loc
, builtin_decl_explicit (BUILT_IN_TM_ABORT
), 1,
417 build_int_cst (integer_type_node
,
419 | (is_outer
? AR_OUTERABORT
: 0)));
422 /* Common gateing function for several of the TM passes. */
430 /* Map for aribtrary function replacement under TM, as created
431 by the tm_wrap attribute. */
433 static GTY((if_marked ("tree_map_marked_p"), param_is (struct tree_map
)))
437 record_tm_replacement (tree from
, tree to
)
439 struct tree_map
**slot
, *h
;
441 /* Do not inline wrapper functions that will get replaced in the TM
444 Suppose you have foo() that will get replaced into tmfoo(). Make
445 sure the inliner doesn't try to outsmart us and inline foo()
446 before we get a chance to do the TM replacement. */
447 DECL_UNINLINABLE (from
) = 1;
449 if (tm_wrap_map
== NULL
)
450 tm_wrap_map
= htab_create_ggc (32, tree_map_hash
, tree_map_eq
, 0);
452 h
= ggc_alloc_tree_map ();
453 h
->hash
= htab_hash_pointer (from
);
457 slot
= (struct tree_map
**)
458 htab_find_slot_with_hash (tm_wrap_map
, h
, h
->hash
, INSERT
);
462 /* Return a TM-aware replacement function for DECL. */
465 find_tm_replacement_function (tree fndecl
)
469 struct tree_map
*h
, in
;
471 in
.base
.from
= fndecl
;
472 in
.hash
= htab_hash_pointer (fndecl
);
473 h
= (struct tree_map
*) htab_find_with_hash (tm_wrap_map
, &in
, in
.hash
);
478 /* ??? We may well want TM versions of most of the common <string.h>
479 functions. For now, we've already these two defined. */
480 /* Adjust expand_call_tm() attributes as necessary for the cases
482 if (DECL_BUILT_IN_CLASS (fndecl
) == BUILT_IN_NORMAL
)
483 switch (DECL_FUNCTION_CODE (fndecl
))
485 case BUILT_IN_MEMCPY
:
486 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY
);
487 case BUILT_IN_MEMMOVE
:
488 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
);
489 case BUILT_IN_MEMSET
:
490 return builtin_decl_explicit (BUILT_IN_TM_MEMSET
);
498 /* When appropriate, record TM replacement for memory allocation functions.
500 FROM is the FNDECL to wrap. */
502 tm_malloc_replacement (tree from
)
507 if (TREE_CODE (from
) != FUNCTION_DECL
)
510 /* If we have a previous replacement, the user must be explicitly
511 wrapping malloc/calloc/free. They better know what they're
513 if (find_tm_replacement_function (from
))
516 str
= IDENTIFIER_POINTER (DECL_NAME (from
));
518 if (!strcmp (str
, "malloc"))
519 to
= builtin_decl_explicit (BUILT_IN_TM_MALLOC
);
520 else if (!strcmp (str
, "calloc"))
521 to
= builtin_decl_explicit (BUILT_IN_TM_CALLOC
);
522 else if (!strcmp (str
, "free"))
523 to
= builtin_decl_explicit (BUILT_IN_TM_FREE
);
527 TREE_NOTHROW (to
) = 0;
529 record_tm_replacement (from
, to
);
532 /* Diagnostics for tm_safe functions/regions. Called by the front end
533 once we've lowered the function to high-gimple. */
535 /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
536 Process exactly one statement. WI->INFO is set to non-null when in
537 the context of a tm_safe function, and null for a __transaction block. */
539 #define DIAG_TM_OUTER 1
540 #define DIAG_TM_SAFE 2
541 #define DIAG_TM_RELAXED 4
545 unsigned int summary_flags
: 8;
546 unsigned int block_flags
: 8;
547 unsigned int func_flags
: 8;
548 unsigned int saw_volatile
: 1;
552 /* Tree callback function for diagnose_tm pass. */
555 diagnose_tm_1_op (tree
*tp
, int *walk_subtrees ATTRIBUTE_UNUSED
,
558 struct walk_stmt_info
*wi
= (struct walk_stmt_info
*) data
;
559 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
560 enum tree_code code
= TREE_CODE (*tp
);
562 if ((code
== VAR_DECL
563 || code
== RESULT_DECL
564 || code
== PARM_DECL
)
565 && d
->block_flags
& (DIAG_TM_SAFE
| DIAG_TM_RELAXED
)
566 && TREE_THIS_VOLATILE (TREE_TYPE (*tp
))
570 error_at (gimple_location (d
->stmt
),
571 "invalid volatile use of %qD inside transaction",
579 diagnose_tm_1 (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
580 struct walk_stmt_info
*wi
)
582 gimple stmt
= gsi_stmt (*gsi
);
583 struct diagnose_tm
*d
= (struct diagnose_tm
*) wi
->info
;
585 /* Save stmt for use in leaf analysis. */
588 switch (gimple_code (stmt
))
592 tree fn
= gimple_call_fn (stmt
);
594 if ((d
->summary_flags
& DIAG_TM_OUTER
) == 0
595 && is_tm_may_cancel_outer (fn
))
596 error_at (gimple_location (stmt
),
597 "%<transaction_may_cancel_outer%> function call not within"
598 " outer transaction or %<transaction_may_cancel_outer%>");
600 if (d
->summary_flags
& DIAG_TM_SAFE
)
602 bool is_safe
, direct_call_p
;
605 if (TREE_CODE (fn
) == ADDR_EXPR
606 && TREE_CODE (TREE_OPERAND (fn
, 0)) == FUNCTION_DECL
)
608 direct_call_p
= true;
609 replacement
= TREE_OPERAND (fn
, 0);
610 replacement
= find_tm_replacement_function (replacement
);
616 direct_call_p
= false;
617 replacement
= NULL_TREE
;
620 if (is_tm_safe_or_pure (fn
))
622 else if (is_tm_callable (fn
) || is_tm_irrevocable (fn
))
624 /* A function explicitly marked transaction_callable as
625 opposed to transaction_safe is being defined to be
626 unsafe as part of its ABI, regardless of its contents. */
629 else if (direct_call_p
)
631 if (flags_from_decl_or_type (fn
) & ECF_TM_BUILTIN
)
633 else if (replacement
)
635 /* ??? At present we've been considering replacements
636 merely transaction_callable, and therefore might
637 enter irrevocable. The tm_wrap attribute has not
638 yet made it into the new language spec. */
643 /* ??? Diagnostics for unmarked direct calls moved into
644 the IPA pass. Section 3.2 of the spec details how
645 functions not marked should be considered "implicitly
646 safe" based on having examined the function body. */
652 /* An unmarked indirect call. Consider it unsafe even
653 though optimization may yet figure out how to inline. */
659 if (TREE_CODE (fn
) == ADDR_EXPR
)
660 fn
= TREE_OPERAND (fn
, 0);
661 if (d
->block_flags
& DIAG_TM_SAFE
)
664 error_at (gimple_location (stmt
),
665 "unsafe function call %qD within "
666 "atomic transaction", fn
);
669 if (!DECL_P (fn
) || DECL_NAME (fn
))
670 error_at (gimple_location (stmt
),
671 "unsafe function call %qE within "
672 "atomic transaction", fn
);
674 error_at (gimple_location (stmt
),
675 "unsafe indirect function call within "
676 "atomic transaction");
682 error_at (gimple_location (stmt
),
683 "unsafe function call %qD within "
684 "%<transaction_safe%> function", fn
);
687 if (!DECL_P (fn
) || DECL_NAME (fn
))
688 error_at (gimple_location (stmt
),
689 "unsafe function call %qE within "
690 "%<transaction_safe%> function", fn
);
692 error_at (gimple_location (stmt
),
693 "unsafe indirect function call within "
694 "%<transaction_safe%> function");
703 /* ??? We ought to come up with a way to add attributes to
704 asm statements, and then add "transaction_safe" to it.
705 Either that or get the language spec to resurrect __tm_waiver. */
706 if (d
->block_flags
& DIAG_TM_SAFE
)
707 error_at (gimple_location (stmt
),
708 "asm not allowed in atomic transaction");
709 else if (d
->func_flags
& DIAG_TM_SAFE
)
710 error_at (gimple_location (stmt
),
711 "asm not allowed in %<transaction_safe%> function");
714 case GIMPLE_TRANSACTION
:
716 unsigned char inner_flags
= DIAG_TM_SAFE
;
718 if (gimple_transaction_subcode (stmt
) & GTMA_IS_RELAXED
)
720 if (d
->block_flags
& DIAG_TM_SAFE
)
721 error_at (gimple_location (stmt
),
722 "relaxed transaction in atomic transaction");
723 else if (d
->func_flags
& DIAG_TM_SAFE
)
724 error_at (gimple_location (stmt
),
725 "relaxed transaction in %<transaction_safe%> function");
726 inner_flags
= DIAG_TM_RELAXED
;
728 else if (gimple_transaction_subcode (stmt
) & GTMA_IS_OUTER
)
731 error_at (gimple_location (stmt
),
732 "outer transaction in transaction");
733 else if (d
->func_flags
& DIAG_TM_OUTER
)
734 error_at (gimple_location (stmt
),
735 "outer transaction in "
736 "%<transaction_may_cancel_outer%> function");
737 else if (d
->func_flags
& DIAG_TM_SAFE
)
738 error_at (gimple_location (stmt
),
739 "outer transaction in %<transaction_safe%> function");
740 inner_flags
|= DIAG_TM_OUTER
;
743 *handled_ops_p
= true;
744 if (gimple_transaction_body (stmt
))
746 struct walk_stmt_info wi_inner
;
747 struct diagnose_tm d_inner
;
749 memset (&d_inner
, 0, sizeof (d_inner
));
750 d_inner
.func_flags
= d
->func_flags
;
751 d_inner
.block_flags
= d
->block_flags
| inner_flags
;
752 d_inner
.summary_flags
= d_inner
.func_flags
| d_inner
.block_flags
;
754 memset (&wi_inner
, 0, sizeof (wi_inner
));
755 wi_inner
.info
= &d_inner
;
757 walk_gimple_seq (gimple_transaction_body (stmt
),
758 diagnose_tm_1
, diagnose_tm_1_op
, &wi_inner
);
771 diagnose_tm_blocks (void)
773 struct walk_stmt_info wi
;
774 struct diagnose_tm d
;
776 memset (&d
, 0, sizeof (d
));
777 if (is_tm_may_cancel_outer (current_function_decl
))
778 d
.func_flags
= DIAG_TM_OUTER
| DIAG_TM_SAFE
;
779 else if (is_tm_safe (current_function_decl
))
780 d
.func_flags
= DIAG_TM_SAFE
;
781 d
.summary_flags
= d
.func_flags
;
783 memset (&wi
, 0, sizeof (wi
));
786 walk_gimple_seq (gimple_body (current_function_decl
),
787 diagnose_tm_1
, diagnose_tm_1_op
, &wi
);
792 struct gimple_opt_pass pass_diagnose_tm_blocks
=
796 "*diagnose_tm_blocks", /* name */
798 diagnose_tm_blocks
, /* execute */
801 0, /* static_pass_number */
802 TV_TRANS_MEM
, /* tv_id */
803 PROP_gimple_any
, /* properties_required */
804 0, /* properties_provided */
805 0, /* properties_destroyed */
806 0, /* todo_flags_start */
807 0, /* todo_flags_finish */
811 /* Instead of instrumenting thread private memory, we save the
812 addresses in a log which we later use to save/restore the addresses
813 upon transaction start/restart.
815 The log is keyed by address, where each element contains individual
816 statements among different code paths that perform the store.
818 This log is later used to generate either plain save/restore of the
819 addresses upon transaction start/restart, or calls to the ITM_L*
822 So for something like:
824 struct large { int x[1000]; };
825 struct large lala = { 0 };
831 We can either save/restore:
834 trxn = _ITM_startTransaction ();
835 if (trxn & a_saveLiveVariables)
836 tmp_lala1 = lala.x[i];
837 else if (a & a_restoreLiveVariables)
838 lala.x[i] = tmp_lala1;
840 or use the logging functions:
843 trxn = _ITM_startTransaction ();
844 _ITM_LU4 (&lala.x[i]);
846 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
847 far up the dominator tree to shadow all of the writes to a given
848 location (thus reducing the total number of logging calls), but not
849 so high as to be called on a path that does not perform a
852 /* One individual log entry. We may have multiple statements for the
853 same location if neither dominate each other (on different
855 typedef struct tm_log_entry
857 /* Address to save. */
859 /* Entry block for the transaction this address occurs in. */
860 basic_block entry_block
;
861 /* Dominating statements the store occurs in. */
863 /* Initially, while we are building the log, we place a nonzero
864 value here to mean that this address *will* be saved with a
865 save/restore sequence. Later, when generating the save sequence
866 we place the SSA temp generated here. */
870 /* The actual log. */
871 static htab_t tm_log
;
873 /* Addresses to log with a save/restore sequence. These should be in
875 static VEC(tree
,heap
) *tm_log_save_addresses
;
877 /* Map for an SSA_NAME originally pointing to a non aliased new piece
878 of memory (malloc, alloc, etc). */
879 static htab_t tm_new_mem_hash
;
881 enum thread_memory_type
885 mem_transaction_local
,
889 typedef struct tm_new_mem_map
891 /* SSA_NAME being dereferenced. */
893 enum thread_memory_type local_new_memory
;
896 /* Htab support. Return hash value for a `tm_log_entry'. */
898 tm_log_hash (const void *p
)
900 const struct tm_log_entry
*log
= (const struct tm_log_entry
*) p
;
901 return iterative_hash_expr (log
->addr
, 0);
904 /* Htab support. Return true if two log entries are the same. */
906 tm_log_eq (const void *p1
, const void *p2
)
908 const struct tm_log_entry
*log1
= (const struct tm_log_entry
*) p1
;
909 const struct tm_log_entry
*log2
= (const struct tm_log_entry
*) p2
;
913 rth: I suggest that we get rid of the component refs etc.
914 I.e. resolve the reference to base + offset.
916 We may need to actually finish a merge with mainline for this,
917 since we'd like to be presented with Richi's MEM_REF_EXPRs more
918 often than not. But in the meantime your tm_log_entry could save
919 the results of get_inner_reference.
921 See: g++.dg/tm/pr46653.C
924 /* Special case plain equality because operand_equal_p() below will
925 return FALSE if the addresses are equal but they have
926 side-effects (e.g. a volatile address). */
927 if (log1
->addr
== log2
->addr
)
930 return operand_equal_p (log1
->addr
, log2
->addr
, 0);
933 /* Htab support. Free one tm_log_entry. */
935 tm_log_free (void *p
)
937 struct tm_log_entry
*lp
= (struct tm_log_entry
*) p
;
938 VEC_free (gimple
, heap
, lp
->stmts
);
942 /* Initialize logging data structures. */
946 tm_log
= htab_create (10, tm_log_hash
, tm_log_eq
, tm_log_free
);
947 tm_new_mem_hash
= htab_create (5, struct_ptr_hash
, struct_ptr_eq
, free
);
948 tm_log_save_addresses
= VEC_alloc (tree
, heap
, 5);
951 /* Free logging data structures. */
955 htab_delete (tm_log
);
956 htab_delete (tm_new_mem_hash
);
957 VEC_free (tree
, heap
, tm_log_save_addresses
);
960 /* Return true if MEM is a transaction invariant memory for the TM
961 region starting at REGION_ENTRY_BLOCK. */
963 transaction_invariant_address_p (const_tree mem
, basic_block region_entry_block
)
965 if ((TREE_CODE (mem
) == INDIRECT_REF
|| TREE_CODE (mem
) == MEM_REF
)
966 && TREE_CODE (TREE_OPERAND (mem
, 0)) == SSA_NAME
)
970 def_bb
= gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem
, 0)));
971 return def_bb
!= region_entry_block
972 && dominated_by_p (CDI_DOMINATORS
, region_entry_block
, def_bb
);
975 mem
= strip_invariant_refs (mem
);
976 return mem
&& (CONSTANT_CLASS_P (mem
) || decl_address_invariant_p (mem
));
979 /* Given an address ADDR in STMT, find it in the memory log or add it,
980 making sure to keep only the addresses highest in the dominator
983 ENTRY_BLOCK is the entry_block for the transaction.
985 If we find the address in the log, make sure it's either the same
986 address, or an equivalent one that dominates ADDR.
988 If we find the address, but neither ADDR dominates the found
989 address, nor the found one dominates ADDR, we're on different
990 execution paths. Add it.
992 If known, ENTRY_BLOCK is the entry block for the region, otherwise
995 tm_log_add (basic_block entry_block
, tree addr
, gimple stmt
)
998 struct tm_log_entry l
, *lp
;
1001 slot
= htab_find_slot (tm_log
, &l
, INSERT
);
1004 tree type
= TREE_TYPE (addr
);
1006 lp
= XNEW (struct tm_log_entry
);
1010 /* Small invariant addresses can be handled as save/restores. */
1012 && transaction_invariant_address_p (lp
->addr
, entry_block
)
1013 && TYPE_SIZE_UNIT (type
) != NULL
1014 && host_integerp (TYPE_SIZE_UNIT (type
), 1)
1015 && (tree_low_cst (TYPE_SIZE_UNIT (type
), 1)
1016 < PARAM_VALUE (PARAM_TM_MAX_AGGREGATE_SIZE
))
1017 /* We must be able to copy this type normally. I.e., no
1018 special constructors and the like. */
1019 && !TREE_ADDRESSABLE (type
))
1021 lp
->save_var
= create_tmp_reg (TREE_TYPE (lp
->addr
), "tm_save");
1022 add_referenced_var (lp
->save_var
);
1024 lp
->entry_block
= entry_block
;
1025 /* Save addresses separately in dominator order so we don't
1026 get confused by overlapping addresses in the save/restore
1028 VEC_safe_push (tree
, heap
, tm_log_save_addresses
, lp
->addr
);
1032 /* Use the logging functions. */
1033 lp
->stmts
= VEC_alloc (gimple
, heap
, 5);
1034 VEC_quick_push (gimple
, lp
->stmts
, stmt
);
1035 lp
->save_var
= NULL
;
1043 lp
= (struct tm_log_entry
*) *slot
;
1045 /* If we're generating a save/restore sequence, we don't care
1046 about statements. */
1050 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, oldstmt
); ++i
)
1052 if (stmt
== oldstmt
)
1054 /* We already have a store to the same address, higher up the
1055 dominator tree. Nothing to do. */
1056 if (dominated_by_p (CDI_DOMINATORS
,
1057 gimple_bb (stmt
), gimple_bb (oldstmt
)))
1059 /* We should be processing blocks in dominator tree order. */
1060 gcc_assert (!dominated_by_p (CDI_DOMINATORS
,
1061 gimple_bb (oldstmt
), gimple_bb (stmt
)));
1063 /* Store is on a different code path. */
1064 VEC_safe_push (gimple
, heap
, lp
->stmts
, stmt
);
1068 /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1069 result, insert the new statements before GSI. */
1072 gimplify_addr (gimple_stmt_iterator
*gsi
, tree x
)
1074 if (TREE_CODE (x
) == TARGET_MEM_REF
)
1075 x
= tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x
)), x
);
1077 x
= build_fold_addr_expr (x
);
1078 return force_gimple_operand_gsi (gsi
, x
, true, NULL
, true, GSI_SAME_STMT
);
1081 /* Instrument one address with the logging functions.
1082 ADDR is the address to save.
1083 STMT is the statement before which to place it. */
1085 tm_log_emit_stmt (tree addr
, gimple stmt
)
1087 tree type
= TREE_TYPE (addr
);
1088 tree size
= TYPE_SIZE_UNIT (type
);
1089 gimple_stmt_iterator gsi
= gsi_for_stmt (stmt
);
1091 enum built_in_function code
= BUILT_IN_TM_LOG
;
1093 if (type
== float_type_node
)
1094 code
= BUILT_IN_TM_LOG_FLOAT
;
1095 else if (type
== double_type_node
)
1096 code
= BUILT_IN_TM_LOG_DOUBLE
;
1097 else if (type
== long_double_type_node
)
1098 code
= BUILT_IN_TM_LOG_LDOUBLE
;
1099 else if (host_integerp (size
, 1))
1101 unsigned int n
= tree_low_cst (size
, 1);
1105 code
= BUILT_IN_TM_LOG_1
;
1108 code
= BUILT_IN_TM_LOG_2
;
1111 code
= BUILT_IN_TM_LOG_4
;
1114 code
= BUILT_IN_TM_LOG_8
;
1117 code
= BUILT_IN_TM_LOG
;
1118 if (TREE_CODE (type
) == VECTOR_TYPE
)
1120 if (n
== 8 && builtin_decl_explicit (BUILT_IN_TM_LOG_M64
))
1121 code
= BUILT_IN_TM_LOG_M64
;
1122 else if (n
== 16 && builtin_decl_explicit (BUILT_IN_TM_LOG_M128
))
1123 code
= BUILT_IN_TM_LOG_M128
;
1124 else if (n
== 32 && builtin_decl_explicit (BUILT_IN_TM_LOG_M256
))
1125 code
= BUILT_IN_TM_LOG_M256
;
1131 addr
= gimplify_addr (&gsi
, addr
);
1132 if (code
== BUILT_IN_TM_LOG
)
1133 log
= gimple_build_call (builtin_decl_explicit (code
), 2, addr
, size
);
1135 log
= gimple_build_call (builtin_decl_explicit (code
), 1, addr
);
1136 gsi_insert_before (&gsi
, log
, GSI_SAME_STMT
);
1139 /* Go through the log and instrument address that must be instrumented
1140 with the logging functions. Leave the save/restore addresses for
1146 struct tm_log_entry
*lp
;
1148 FOR_EACH_HTAB_ELEMENT (tm_log
, lp
, tm_log_entry_t
, hi
)
1155 fprintf (dump_file
, "TM thread private mem logging: ");
1156 print_generic_expr (dump_file
, lp
->addr
, 0);
1157 fprintf (dump_file
, "\n");
1163 fprintf (dump_file
, "DUMPING to variable\n");
1169 fprintf (dump_file
, "DUMPING with logging functions\n");
1170 for (i
= 0; VEC_iterate (gimple
, lp
->stmts
, i
, stmt
); ++i
)
1171 tm_log_emit_stmt (lp
->addr
, stmt
);
1176 /* Emit the save sequence for the corresponding addresses in the log.
1177 ENTRY_BLOCK is the entry block for the transaction.
1178 BB is the basic block to insert the code in. */
1180 tm_log_emit_saves (basic_block entry_block
, basic_block bb
)
1183 gimple_stmt_iterator gsi
= gsi_last_bb (bb
);
1185 struct tm_log_entry l
, *lp
;
1187 for (i
= 0; i
< VEC_length (tree
, tm_log_save_addresses
); ++i
)
1189 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1190 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1191 gcc_assert (lp
->save_var
!= NULL
);
1193 /* We only care about variables in the current transaction. */
1194 if (lp
->entry_block
!= entry_block
)
1197 stmt
= gimple_build_assign (lp
->save_var
, unshare_expr (lp
->addr
));
1199 /* Make sure we can create an SSA_NAME for this type. For
1200 instance, aggregates aren't allowed, in which case the system
1201 will create a VOP for us and everything will just work. */
1202 if (is_gimple_reg_type (TREE_TYPE (lp
->save_var
)))
1204 lp
->save_var
= make_ssa_name (lp
->save_var
, stmt
);
1205 gimple_assign_set_lhs (stmt
, lp
->save_var
);
1208 gsi_insert_before (&gsi
, stmt
, GSI_SAME_STMT
);
1212 /* Emit the restore sequence for the corresponding addresses in the log.
1213 ENTRY_BLOCK is the entry block for the transaction.
1214 BB is the basic block to insert the code in. */
1216 tm_log_emit_restores (basic_block entry_block
, basic_block bb
)
1219 struct tm_log_entry l
, *lp
;
1220 gimple_stmt_iterator gsi
;
1223 for (i
= VEC_length (tree
, tm_log_save_addresses
) - 1; i
>= 0; i
--)
1225 l
.addr
= VEC_index (tree
, tm_log_save_addresses
, i
);
1226 lp
= (struct tm_log_entry
*) *htab_find_slot (tm_log
, &l
, NO_INSERT
);
1227 gcc_assert (lp
->save_var
!= NULL
);
1229 /* We only care about variables in the current transaction. */
1230 if (lp
->entry_block
!= entry_block
)
1233 /* Restores are in LIFO order from the saves in case we have
1235 gsi
= gsi_start_bb (bb
);
1237 stmt
= gimple_build_assign (unshare_expr (lp
->addr
), lp
->save_var
);
1238 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1242 /* Emit the checks for performing either a save or a restore sequence.
1244 TRXN_PROP is either A_SAVELIVEVARIABLES or A_RESTORELIVEVARIABLES.
1246 The code sequence is inserted in a new basic block created in
1247 END_BB which is inserted between BEFORE_BB and the destination of
1250 STATUS is the return value from _ITM_beginTransaction.
1251 ENTRY_BLOCK is the entry block for the transaction.
1252 EMITF is a callback to emit the actual save/restore code.
1254 The basic block containing the conditional checking for TRXN_PROP
1257 tm_log_emit_save_or_restores (basic_block entry_block
,
1260 void (*emitf
)(basic_block
, basic_block
),
1261 basic_block before_bb
,
1263 basic_block
*end_bb
)
1265 basic_block cond_bb
, code_bb
;
1266 gimple cond_stmt
, stmt
;
1267 gimple_stmt_iterator gsi
;
1269 int old_flags
= fallthru_edge
->flags
;
1271 cond_bb
= create_empty_bb (before_bb
);
1272 code_bb
= create_empty_bb (cond_bb
);
1273 *end_bb
= create_empty_bb (code_bb
);
1274 if (current_loops
&& before_bb
->loop_father
)
1276 add_bb_to_loop (cond_bb
, before_bb
->loop_father
);
1277 add_bb_to_loop (code_bb
, before_bb
->loop_father
);
1278 add_bb_to_loop (*end_bb
, before_bb
->loop_father
);
1280 redirect_edge_pred (fallthru_edge
, *end_bb
);
1281 fallthru_edge
->flags
= EDGE_FALLTHRU
;
1282 make_edge (before_bb
, cond_bb
, old_flags
);
1284 set_immediate_dominator (CDI_DOMINATORS
, cond_bb
, before_bb
);
1285 set_immediate_dominator (CDI_DOMINATORS
, code_bb
, cond_bb
);
1287 gsi
= gsi_last_bb (cond_bb
);
1289 /* t1 = status & A_{property}. */
1290 t1
= make_rename_temp (TREE_TYPE (status
), NULL
);
1291 t2
= build_int_cst (TREE_TYPE (status
), trxn_prop
);
1292 stmt
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
, status
, t2
);
1293 gsi_insert_after (&gsi
, stmt
, GSI_CONTINUE_LINKING
);
1296 t2
= build_int_cst (TREE_TYPE (status
), 0);
1297 cond_stmt
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
1298 gsi_insert_after (&gsi
, cond_stmt
, GSI_CONTINUE_LINKING
);
1300 emitf (entry_block
, code_bb
);
1302 make_edge (cond_bb
, code_bb
, EDGE_TRUE_VALUE
);
1303 make_edge (cond_bb
, *end_bb
, EDGE_FALSE_VALUE
);
1304 make_edge (code_bb
, *end_bb
, EDGE_FALLTHRU
);
1309 static tree
lower_sequence_tm (gimple_stmt_iterator
*, bool *,
1310 struct walk_stmt_info
*);
1311 static tree
lower_sequence_no_tm (gimple_stmt_iterator
*, bool *,
1312 struct walk_stmt_info
*);
1314 /* Evaluate an address X being dereferenced and determine if it
1315 originally points to a non aliased new chunk of memory (malloc,
1318 Return MEM_THREAD_LOCAL if it points to a thread-local address.
1319 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1320 Return MEM_NON_LOCAL otherwise.
1322 ENTRY_BLOCK is the entry block to the transaction containing the
1323 dereference of X. */
1324 static enum thread_memory_type
1325 thread_private_new_memory (basic_block entry_block
, tree x
)
1328 enum tree_code code
;
1330 tm_new_mem_map_t elt
, *elt_p
;
1332 enum thread_memory_type retval
= mem_transaction_local
;
1335 || TREE_CODE (x
) != SSA_NAME
1336 /* Possible uninitialized use, or a function argument. In
1337 either case, we don't care. */
1338 || SSA_NAME_IS_DEFAULT_DEF (x
))
1339 return mem_non_local
;
1341 /* Look in cache first. */
1343 slot
= htab_find_slot (tm_new_mem_hash
, &elt
, INSERT
);
1344 elt_p
= (tm_new_mem_map_t
*) *slot
;
1346 return elt_p
->local_new_memory
;
1348 /* Optimistically assume the memory is transaction local during
1349 processing. This catches recursion into this variable. */
1350 *slot
= elt_p
= XNEW (tm_new_mem_map_t
);
1352 elt_p
->local_new_memory
= mem_transaction_local
;
1354 /* Search DEF chain to find the original definition of this address. */
1357 if (ptr_deref_may_alias_global_p (x
))
1359 /* Address escapes. This is not thread-private. */
1360 retval
= mem_non_local
;
1361 goto new_memory_ret
;
1364 stmt
= SSA_NAME_DEF_STMT (x
);
1366 /* If the malloc call is outside the transaction, this is
1368 if (retval
!= mem_thread_local
1369 && !dominated_by_p (CDI_DOMINATORS
, gimple_bb (stmt
), entry_block
))
1370 retval
= mem_thread_local
;
1372 if (is_gimple_assign (stmt
))
1374 code
= gimple_assign_rhs_code (stmt
);
1375 /* x = foo ==> foo */
1376 if (code
== SSA_NAME
)
1377 x
= gimple_assign_rhs1 (stmt
);
1378 /* x = foo + n ==> foo */
1379 else if (code
== POINTER_PLUS_EXPR
)
1380 x
= gimple_assign_rhs1 (stmt
);
1381 /* x = (cast*) foo ==> foo */
1382 else if (code
== VIEW_CONVERT_EXPR
|| code
== NOP_EXPR
)
1383 x
= gimple_assign_rhs1 (stmt
);
1386 retval
= mem_non_local
;
1387 goto new_memory_ret
;
1392 if (gimple_code (stmt
) == GIMPLE_PHI
)
1395 enum thread_memory_type mem
;
1396 tree phi_result
= gimple_phi_result (stmt
);
1398 /* If any of the ancestors are non-local, we are sure to
1399 be non-local. Otherwise we can avoid doing anything
1400 and inherit what has already been generated. */
1402 for (i
= 0; i
< gimple_phi_num_args (stmt
); ++i
)
1404 tree op
= PHI_ARG_DEF (stmt
, i
);
1406 /* Exclude self-assignment. */
1407 if (phi_result
== op
)
1410 mem
= thread_private_new_memory (entry_block
, op
);
1411 if (mem
== mem_non_local
)
1414 goto new_memory_ret
;
1416 retval
= MIN (retval
, mem
);
1418 goto new_memory_ret
;
1423 while (TREE_CODE (x
) == SSA_NAME
);
1425 if (stmt
&& is_gimple_call (stmt
) && gimple_call_flags (stmt
) & ECF_MALLOC
)
1426 /* Thread-local or transaction-local. */
1429 retval
= mem_non_local
;
1432 elt_p
->local_new_memory
= retval
;
1436 /* Determine whether X has to be instrumented using a read
1439 ENTRY_BLOCK is the entry block for the region where stmt resides
1440 in. NULL if unknown.
1442 STMT is the statement in which X occurs in. It is used for thread
1443 private memory instrumentation. If no TPM instrumentation is
1444 desired, STMT should be null. */
1446 requires_barrier (basic_block entry_block
, tree x
, gimple stmt
)
1449 while (handled_component_p (x
))
1450 x
= TREE_OPERAND (x
, 0);
1452 switch (TREE_CODE (x
))
1457 enum thread_memory_type ret
;
1459 ret
= thread_private_new_memory (entry_block
, TREE_OPERAND (x
, 0));
1460 if (ret
== mem_non_local
)
1462 if (stmt
&& ret
== mem_thread_local
)
1463 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1464 tm_log_add (entry_block
, orig
, stmt
);
1466 /* Transaction-locals require nothing at all. For malloc, a
1467 transaction restart frees the memory and we reallocate.
1468 For alloca, the stack pointer gets reset by the retry and
1473 case TARGET_MEM_REF
:
1474 if (TREE_CODE (TMR_BASE (x
)) != ADDR_EXPR
)
1476 x
= TREE_OPERAND (TMR_BASE (x
), 0);
1477 if (TREE_CODE (x
) == PARM_DECL
)
1479 gcc_assert (TREE_CODE (x
) == VAR_DECL
);
1485 if (DECL_BY_REFERENCE (x
))
1487 /* ??? This value is a pointer, but aggregate_value_p has been
1488 jigged to return true which confuses needs_to_live_in_memory.
1489 This ought to be cleaned up generically.
1491 FIXME: Verify this still happens after the next mainline
1492 merge. Testcase ie g++.dg/tm/pr47554.C.
1497 if (is_global_var (x
))
1498 return !TREE_READONLY (x
);
1499 if (/* FIXME: This condition should actually go below in the
1500 tm_log_add() call, however is_call_clobbered() depends on
1501 aliasing info which is not available during
1502 gimplification. Since requires_barrier() gets called
1503 during lower_sequence_tm/gimplification, leave the call
1504 to needs_to_live_in_memory until we eliminate
1505 lower_sequence_tm altogether. */
1506 needs_to_live_in_memory (x
))
1510 /* For local memory that doesn't escape (aka thread private
1511 memory), we can either save the value at the beginning of
1512 the transaction and restore on restart, or call a tm
1513 function to dynamically save and restore on restart
1516 tm_log_add (entry_block
, orig
, stmt
);
1525 /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1526 a transaction region. */
1529 examine_assign_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1531 gimple stmt
= gsi_stmt (*gsi
);
1533 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_rhs1 (stmt
), NULL
))
1534 *state
|= GTMA_HAVE_LOAD
;
1535 if (requires_barrier (/*entry_block=*/NULL
, gimple_assign_lhs (stmt
), NULL
))
1536 *state
|= GTMA_HAVE_STORE
;
1539 /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1542 examine_call_tm (unsigned *state
, gimple_stmt_iterator
*gsi
)
1544 gimple stmt
= gsi_stmt (*gsi
);
1547 if (is_tm_pure_call (stmt
))
1550 /* Check if this call is a transaction abort. */
1551 fn
= gimple_call_fndecl (stmt
);
1552 if (is_tm_abort (fn
))
1553 *state
|= GTMA_HAVE_ABORT
;
1555 /* Note that something may happen. */
1556 *state
|= GTMA_HAVE_LOAD
| GTMA_HAVE_STORE
;
1559 /* Lower a GIMPLE_TRANSACTION statement. */
1562 lower_transaction (gimple_stmt_iterator
*gsi
, struct walk_stmt_info
*wi
)
1564 gimple g
, stmt
= gsi_stmt (*gsi
);
1565 unsigned int *outer_state
= (unsigned int *) wi
->info
;
1566 unsigned int this_state
= 0;
1567 struct walk_stmt_info this_wi
;
1569 /* First, lower the body. The scanning that we do inside gives
1570 us some idea of what we're dealing with. */
1571 memset (&this_wi
, 0, sizeof (this_wi
));
1572 this_wi
.info
= (void *) &this_state
;
1573 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt
),
1574 lower_sequence_tm
, NULL
, &this_wi
);
1576 /* If there was absolutely nothing transaction related inside the
1577 transaction, we may elide it. Likewise if this is a nested
1578 transaction and does not contain an abort. */
1580 || (!(this_state
& GTMA_HAVE_ABORT
) && outer_state
!= NULL
))
1583 *outer_state
|= this_state
;
1585 gsi_insert_seq_before (gsi
, gimple_transaction_body (stmt
),
1587 gimple_transaction_set_body (stmt
, NULL
);
1589 gsi_remove (gsi
, true);
1590 wi
->removed_stmt
= true;
1594 /* Wrap the body of the transaction in a try-finally node so that
1595 the commit call is always properly called. */
1596 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT
), 0);
1597 if (flag_exceptions
)
1600 gimple_seq n_seq
, e_seq
;
1602 n_seq
= gimple_seq_alloc_with_stmt (g
);
1605 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER
),
1606 1, integer_zero_node
);
1607 ptr
= create_tmp_var (ptr_type_node
, NULL
);
1608 gimple_call_set_lhs (g
, ptr
);
1609 gimple_seq_add_stmt (&e_seq
, g
);
1611 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH
),
1613 gimple_seq_add_stmt (&e_seq
, g
);
1615 g
= gimple_build_eh_else (n_seq
, e_seq
);
1618 g
= gimple_build_try (gimple_transaction_body (stmt
),
1619 gimple_seq_alloc_with_stmt (g
), GIMPLE_TRY_FINALLY
);
1620 gsi_insert_after (gsi
, g
, GSI_CONTINUE_LINKING
);
1622 gimple_transaction_set_body (stmt
, NULL
);
1624 /* If the transaction calls abort or if this is an outer transaction,
1625 add an "over" label afterwards. */
1626 if ((this_state
& (GTMA_HAVE_ABORT
))
1627 || (gimple_transaction_subcode(stmt
) & GTMA_IS_OUTER
))
1629 tree label
= create_artificial_label (UNKNOWN_LOCATION
);
1630 gimple_transaction_set_label (stmt
, label
);
1631 gsi_insert_after (gsi
, gimple_build_label (label
), GSI_CONTINUE_LINKING
);
1634 /* Record the set of operations found for use later. */
1635 this_state
|= gimple_transaction_subcode (stmt
) & GTMA_DECLARATION_MASK
;
1636 gimple_transaction_set_subcode (stmt
, this_state
);
1639 /* Iterate through the statements in the sequence, lowering them all
1640 as appropriate for being in a transaction. */
1643 lower_sequence_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1644 struct walk_stmt_info
*wi
)
1646 unsigned int *state
= (unsigned int *) wi
->info
;
1647 gimple stmt
= gsi_stmt (*gsi
);
1649 *handled_ops_p
= true;
1650 switch (gimple_code (stmt
))
1653 /* Only memory reads/writes need to be instrumented. */
1654 if (gimple_assign_single_p (stmt
))
1655 examine_assign_tm (state
, gsi
);
1659 examine_call_tm (state
, gsi
);
1663 *state
|= GTMA_MAY_ENTER_IRREVOCABLE
;
1666 case GIMPLE_TRANSACTION
:
1667 lower_transaction (gsi
, wi
);
1671 *handled_ops_p
= !gimple_has_substatements (stmt
);
1678 /* Iterate through the statements in the sequence, lowering them all
1679 as appropriate for being outside of a transaction. */
1682 lower_sequence_no_tm (gimple_stmt_iterator
*gsi
, bool *handled_ops_p
,
1683 struct walk_stmt_info
* wi
)
1685 gimple stmt
= gsi_stmt (*gsi
);
1687 if (gimple_code (stmt
) == GIMPLE_TRANSACTION
)
1689 *handled_ops_p
= true;
1690 lower_transaction (gsi
, wi
);
1693 *handled_ops_p
= !gimple_has_substatements (stmt
);
1698 /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1699 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1700 been moved out, and all the data required for constructing a proper
1701 CFG has been recorded. */
1704 execute_lower_tm (void)
1706 struct walk_stmt_info wi
;
1709 /* Transactional clones aren't created until a later pass. */
1710 gcc_assert (!decl_is_tm_clone (current_function_decl
));
1712 body
= gimple_body (current_function_decl
);
1713 memset (&wi
, 0, sizeof (wi
));
1714 walk_gimple_seq_mod (&body
, lower_sequence_no_tm
, NULL
, &wi
);
1715 gimple_set_body (current_function_decl
, body
);
1720 struct gimple_opt_pass pass_lower_tm
=
1724 "tmlower", /* name */
1726 execute_lower_tm
, /* execute */
1729 0, /* static_pass_number */
1730 TV_TRANS_MEM
, /* tv_id */
1731 PROP_gimple_lcf
, /* properties_required */
1732 0, /* properties_provided */
1733 0, /* properties_destroyed */
1734 0, /* todo_flags_start */
1735 0, /* todo_flags_finish */
1739 /* Collect region information for each transaction. */
1743 /* Link to the next unnested transaction. */
1744 struct tm_region
*next
;
1746 /* Link to the next inner transaction. */
1747 struct tm_region
*inner
;
1749 /* Link to the next outer transaction. */
1750 struct tm_region
*outer
;
1752 /* The GIMPLE_TRANSACTION statement beginning this transaction. */
1753 gimple transaction_stmt
;
1755 /* The entry block to this region. */
1756 basic_block entry_block
;
1758 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1759 These blocks are still a part of the region (i.e., the border is
1760 inclusive). Note that this set is only complete for paths in the CFG
1761 starting at ENTRY_BLOCK, and that there is no exit block recorded for
1762 the edge to the "over" label. */
1765 /* The set of all blocks that have an TM_IRREVOCABLE call. */
1769 typedef struct tm_region
*tm_region_p
;
1770 DEF_VEC_P (tm_region_p
);
1771 DEF_VEC_ALLOC_P (tm_region_p
, heap
);
1773 /* True if there are pending edge statements to be committed for the
1774 current function being scanned in the tmmark pass. */
1775 bool pending_edge_inserts_p
;
1777 static struct tm_region
*all_tm_regions
;
1778 static bitmap_obstack tm_obstack
;
1781 /* A subroutine of tm_region_init. Record the existence of the
1782 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1784 static struct tm_region
*
1785 tm_region_init_0 (struct tm_region
*outer
, basic_block bb
, gimple stmt
)
1787 struct tm_region
*region
;
1789 region
= (struct tm_region
*)
1790 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1794 region
->next
= outer
->inner
;
1795 outer
->inner
= region
;
1799 region
->next
= all_tm_regions
;
1800 all_tm_regions
= region
;
1802 region
->inner
= NULL
;
1803 region
->outer
= outer
;
1805 region
->transaction_stmt
= stmt
;
1807 /* There are either one or two edges out of the block containing
1808 the GIMPLE_TRANSACTION, one to the actual region and one to the
1809 "over" label if the region contains an abort. The former will
1810 always be the one marked FALLTHRU. */
1811 region
->entry_block
= FALLTHRU_EDGE (bb
)->dest
;
1813 region
->exit_blocks
= BITMAP_ALLOC (&tm_obstack
);
1814 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1819 /* A subroutine of tm_region_init. Record all the exit and
1820 irrevocable blocks in BB into the region's exit_blocks and
1821 irr_blocks bitmaps. Returns the new region being scanned. */
1823 static struct tm_region
*
1824 tm_region_init_1 (struct tm_region
*region
, basic_block bb
)
1826 gimple_stmt_iterator gsi
;
1830 || (!region
->irr_blocks
&& !region
->exit_blocks
))
1833 /* Check to see if this is the end of a region by seeing if it
1834 contains a call to __builtin_tm_commit{,_eh}. Note that the
1835 outermost region for DECL_IS_TM_CLONE need not collect this. */
1836 for (gsi
= gsi_last_bb (bb
); !gsi_end_p (gsi
); gsi_prev (&gsi
))
1839 if (gimple_code (g
) == GIMPLE_CALL
)
1841 tree fn
= gimple_call_fndecl (g
);
1842 if (fn
&& DECL_BUILT_IN_CLASS (fn
) == BUILT_IN_NORMAL
)
1844 if ((DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT
1845 || DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_COMMIT_EH
)
1846 && region
->exit_blocks
)
1848 bitmap_set_bit (region
->exit_blocks
, bb
->index
);
1849 region
= region
->outer
;
1852 if (DECL_FUNCTION_CODE (fn
) == BUILT_IN_TM_IRREVOCABLE
)
1853 bitmap_set_bit (region
->irr_blocks
, bb
->index
);
1860 /* Collect all of the transaction regions within the current function
1861 and record them in ALL_TM_REGIONS. The REGION parameter may specify
1862 an "outermost" region for use by tm clones. */
1865 tm_region_init (struct tm_region
*region
)
1871 VEC(basic_block
, heap
) *queue
= NULL
;
1872 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
1873 struct tm_region
*old_region
;
1874 VEC(tm_region_p
, heap
) *bb_regions
= NULL
;
1876 all_tm_regions
= region
;
1877 bb
= single_succ (ENTRY_BLOCK_PTR
);
1879 /* We could store this information in bb->aux, but we may get called
1880 through get_all_tm_blocks() from another pass that may be already
1882 VEC_safe_grow_cleared (tm_region_p
, heap
, bb_regions
, last_basic_block
);
1884 VEC_safe_push (basic_block
, heap
, queue
, bb
);
1885 VEC_replace (tm_region_p
, bb_regions
, bb
->index
, region
);
1888 bb
= VEC_pop (basic_block
, queue
);
1889 region
= VEC_index (tm_region_p
, bb_regions
, bb
->index
);
1890 VEC_replace (tm_region_p
, bb_regions
, bb
->index
, NULL
);
1892 /* Record exit and irrevocable blocks. */
1893 region
= tm_region_init_1 (region
, bb
);
1895 /* Check for the last statement in the block beginning a new region. */
1897 old_region
= region
;
1898 if (g
&& gimple_code (g
) == GIMPLE_TRANSACTION
)
1899 region
= tm_region_init_0 (region
, bb
, g
);
1901 /* Process subsequent blocks. */
1902 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
1903 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
1905 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
1906 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
1908 /* If the current block started a new region, make sure that only
1909 the entry block of the new region is associated with this region.
1910 Other successors are still part of the old region. */
1911 if (old_region
!= region
&& e
->dest
!= region
->entry_block
)
1912 VEC_replace (tm_region_p
, bb_regions
, e
->dest
->index
, old_region
);
1914 VEC_replace (tm_region_p
, bb_regions
, e
->dest
->index
, region
);
1917 while (!VEC_empty (basic_block
, queue
));
1918 VEC_free (basic_block
, heap
, queue
);
1919 BITMAP_FREE (visited_blocks
);
1920 VEC_free (tm_region_p
, heap
, bb_regions
);
1923 /* The "gate" function for all transactional memory expansion and optimization
1924 passes. We collect region information for each top-level transaction, and
1925 if we don't find any, we skip all of the TM passes. Each region will have
1926 all of the exit blocks recorded, and the originating statement. */
1934 calculate_dominance_info (CDI_DOMINATORS
);
1935 bitmap_obstack_initialize (&tm_obstack
);
1937 /* If the function is a TM_CLONE, then the entire function is the region. */
1938 if (decl_is_tm_clone (current_function_decl
))
1940 struct tm_region
*region
= (struct tm_region
*)
1941 obstack_alloc (&tm_obstack
.obstack
, sizeof (struct tm_region
));
1942 memset (region
, 0, sizeof (*region
));
1943 region
->entry_block
= single_succ (ENTRY_BLOCK_PTR
);
1944 /* For a clone, the entire function is the region. But even if
1945 we don't need to record any exit blocks, we may need to
1946 record irrevocable blocks. */
1947 region
->irr_blocks
= BITMAP_ALLOC (&tm_obstack
);
1949 tm_region_init (region
);
1953 tm_region_init (NULL
);
1955 /* If we didn't find any regions, cleanup and skip the whole tree
1956 of tm-related optimizations. */
1957 if (all_tm_regions
== NULL
)
1959 bitmap_obstack_release (&tm_obstack
);
1967 struct gimple_opt_pass pass_tm_init
=
1971 "*tminit", /* name */
1972 gate_tm_init
, /* gate */
1976 0, /* static_pass_number */
1977 TV_TRANS_MEM
, /* tv_id */
1978 PROP_ssa
| PROP_cfg
, /* properties_required */
1979 0, /* properties_provided */
1980 0, /* properties_destroyed */
1981 0, /* todo_flags_start */
1982 0, /* todo_flags_finish */
1986 /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
1987 represented by STATE. */
1990 transaction_subcode_ior (struct tm_region
*region
, unsigned flags
)
1992 if (region
&& region
->transaction_stmt
)
1994 flags
|= gimple_transaction_subcode (region
->transaction_stmt
);
1995 gimple_transaction_set_subcode (region
->transaction_stmt
, flags
);
1999 /* Construct a memory load in a transactional context. Return the
2000 gimple statement performing the load, or NULL if there is no
2001 TM_LOAD builtin of the appropriate size to do the load.
2003 LOC is the location to use for the new statement(s). */
2006 build_tm_load (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2008 enum built_in_function code
= END_BUILTINS
;
2009 tree t
, type
= TREE_TYPE (rhs
), decl
;
2012 if (type
== float_type_node
)
2013 code
= BUILT_IN_TM_LOAD_FLOAT
;
2014 else if (type
== double_type_node
)
2015 code
= BUILT_IN_TM_LOAD_DOUBLE
;
2016 else if (type
== long_double_type_node
)
2017 code
= BUILT_IN_TM_LOAD_LDOUBLE
;
2018 else if (TYPE_SIZE_UNIT (type
) != NULL
2019 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
2021 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2024 code
= BUILT_IN_TM_LOAD_1
;
2027 code
= BUILT_IN_TM_LOAD_2
;
2030 code
= BUILT_IN_TM_LOAD_4
;
2033 code
= BUILT_IN_TM_LOAD_8
;
2038 if (code
== END_BUILTINS
)
2040 decl
= targetm
.vectorize
.builtin_tm_load (type
);
2045 decl
= builtin_decl_explicit (code
);
2047 t
= gimplify_addr (gsi
, rhs
);
2048 gcall
= gimple_build_call (decl
, 1, t
);
2049 gimple_set_location (gcall
, loc
);
2051 t
= TREE_TYPE (TREE_TYPE (decl
));
2052 if (useless_type_conversion_p (type
, t
))
2054 gimple_call_set_lhs (gcall
, lhs
);
2055 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2062 temp
= make_rename_temp (t
, NULL
);
2063 gimple_call_set_lhs (gcall
, temp
);
2064 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2066 t
= fold_build1 (VIEW_CONVERT_EXPR
, type
, temp
);
2067 g
= gimple_build_assign (lhs
, t
);
2068 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2075 /* Similarly for storing TYPE in a transactional context. */
2078 build_tm_store (location_t loc
, tree lhs
, tree rhs
, gimple_stmt_iterator
*gsi
)
2080 enum built_in_function code
= END_BUILTINS
;
2081 tree t
, fn
, type
= TREE_TYPE (rhs
), simple_type
;
2084 if (type
== float_type_node
)
2085 code
= BUILT_IN_TM_STORE_FLOAT
;
2086 else if (type
== double_type_node
)
2087 code
= BUILT_IN_TM_STORE_DOUBLE
;
2088 else if (type
== long_double_type_node
)
2089 code
= BUILT_IN_TM_STORE_LDOUBLE
;
2090 else if (TYPE_SIZE_UNIT (type
) != NULL
2091 && host_integerp (TYPE_SIZE_UNIT (type
), 1))
2093 switch (tree_low_cst (TYPE_SIZE_UNIT (type
), 1))
2096 code
= BUILT_IN_TM_STORE_1
;
2099 code
= BUILT_IN_TM_STORE_2
;
2102 code
= BUILT_IN_TM_STORE_4
;
2105 code
= BUILT_IN_TM_STORE_8
;
2110 if (code
== END_BUILTINS
)
2112 fn
= targetm
.vectorize
.builtin_tm_store (type
);
2117 fn
= builtin_decl_explicit (code
);
2119 simple_type
= TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn
))));
2121 if (TREE_CODE (rhs
) == CONSTRUCTOR
)
2123 /* Handle the easy initialization to zero. */
2124 if (CONSTRUCTOR_ELTS (rhs
) == 0)
2125 rhs
= build_int_cst (simple_type
, 0);
2128 /* ...otherwise punt to the caller and probably use
2129 BUILT_IN_TM_MEMMOVE, because we can't wrap a
2130 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2135 else if (!useless_type_conversion_p (simple_type
, type
))
2140 temp
= make_rename_temp (simple_type
, NULL
);
2141 t
= fold_build1 (VIEW_CONVERT_EXPR
, simple_type
, rhs
);
2142 g
= gimple_build_assign (temp
, t
);
2143 gimple_set_location (g
, loc
);
2144 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
2149 t
= gimplify_addr (gsi
, lhs
);
2150 gcall
= gimple_build_call (fn
, 2, t
, rhs
);
2151 gimple_set_location (gcall
, loc
);
2152 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2158 /* Expand an assignment statement into transactional builtins. */
2161 expand_assign_tm (struct tm_region
*region
, gimple_stmt_iterator
*gsi
)
2163 gimple stmt
= gsi_stmt (*gsi
);
2164 location_t loc
= gimple_location (stmt
);
2165 tree lhs
= gimple_assign_lhs (stmt
);
2166 tree rhs
= gimple_assign_rhs1 (stmt
);
2167 bool store_p
= requires_barrier (region
->entry_block
, lhs
, NULL
);
2168 bool load_p
= requires_barrier (region
->entry_block
, rhs
, NULL
);
2169 gimple gcall
= NULL
;
2171 if (!load_p
&& !store_p
)
2173 /* Add thread private addresses to log if applicable. */
2174 requires_barrier (region
->entry_block
, lhs
, stmt
);
2179 gsi_remove (gsi
, true);
2181 if (load_p
&& !store_p
)
2183 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2184 gcall
= build_tm_load (loc
, lhs
, rhs
, gsi
);
2186 else if (store_p
&& !load_p
)
2188 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2189 gcall
= build_tm_store (loc
, lhs
, rhs
, gsi
);
2193 tree lhs_addr
, rhs_addr
, tmp
;
2196 transaction_subcode_ior (region
, GTMA_HAVE_LOAD
);
2198 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2200 /* ??? Figure out if there's any possible overlap between the LHS
2201 and the RHS and if not, use MEMCPY. */
2203 if (load_p
&& is_gimple_reg (lhs
))
2205 tmp
= create_tmp_var (TREE_TYPE (lhs
), NULL
);
2206 lhs_addr
= build_fold_addr_expr (tmp
);
2211 lhs_addr
= gimplify_addr (gsi
, lhs
);
2213 rhs_addr
= gimplify_addr (gsi
, rhs
);
2214 gcall
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
),
2215 3, lhs_addr
, rhs_addr
,
2216 TYPE_SIZE_UNIT (TREE_TYPE (lhs
)));
2217 gimple_set_location (gcall
, loc
);
2218 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2222 gcall
= gimple_build_assign (lhs
, tmp
);
2223 gsi_insert_before (gsi
, gcall
, GSI_SAME_STMT
);
2227 /* Now that we have the load/store in its instrumented form, add
2228 thread private addresses to the log if applicable. */
2230 requires_barrier (region
->entry_block
, lhs
, gcall
);
2232 /* add_stmt_to_tm_region (region, gcall); */
2236 /* Expand a call statement as appropriate for a transaction. That is,
2237 either verify that the call does not affect the transaction, or
2238 redirect the call to a clone that handles transactions, or change
2239 the transaction state to IRREVOCABLE. Return true if the call is
2240 one of the builtins that end a transaction. */
2243 expand_call_tm (struct tm_region
*region
,
2244 gimple_stmt_iterator
*gsi
)
2246 gimple stmt
= gsi_stmt (*gsi
);
2247 tree lhs
= gimple_call_lhs (stmt
);
2249 struct cgraph_node
*node
;
2250 bool retval
= false;
2252 fn_decl
= gimple_call_fndecl (stmt
);
2254 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMCPY
)
2255 || fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMMOVE
))
2256 transaction_subcode_ior (region
, GTMA_HAVE_STORE
| GTMA_HAVE_LOAD
);
2257 if (fn_decl
== builtin_decl_explicit (BUILT_IN_TM_MEMSET
))
2258 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2260 if (is_tm_pure_call (stmt
))
2264 retval
= is_tm_ending_fndecl (fn_decl
);
2267 /* Assume all non-const/pure calls write to memory, except
2268 transaction ending builtins. */
2269 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2272 /* For indirect calls, we already generated a call into the runtime. */
2275 tree fn
= gimple_call_fn (stmt
);
2277 /* We are guaranteed never to go irrevocable on a safe or pure
2278 call, and the pure call was handled above. */
2279 if (is_tm_safe (fn
))
2282 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2287 node
= cgraph_get_node (fn_decl
);
2288 /* All calls should have cgraph here. */
2290 if (node
->local
.tm_may_enter_irr
)
2291 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
2293 if (is_tm_abort (fn_decl
))
2295 transaction_subcode_ior (region
, GTMA_HAVE_ABORT
);
2299 /* Instrument the store if needed.
2301 If the assignment happens inside the function call (return slot
2302 optimization), there is no instrumentation to be done, since
2303 the callee should have done the right thing. */
2304 if (lhs
&& requires_barrier (region
->entry_block
, lhs
, stmt
)
2305 && !gimple_call_return_slot_opt_p (stmt
))
2307 tree tmp
= make_rename_temp (TREE_TYPE (lhs
), NULL
);
2308 location_t loc
= gimple_location (stmt
);
2309 edge fallthru_edge
= NULL
;
2311 /* Remember if the call was going to throw. */
2312 if (stmt_can_throw_internal (stmt
))
2316 basic_block bb
= gimple_bb (stmt
);
2318 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2319 if (e
->flags
& EDGE_FALLTHRU
)
2326 gimple_call_set_lhs (stmt
, tmp
);
2328 stmt
= gimple_build_assign (lhs
, tmp
);
2329 gimple_set_location (stmt
, loc
);
2331 /* We cannot throw in the middle of a BB. If the call was going
2332 to throw, place the instrumentation on the fallthru edge, so
2333 the call remains the last statement in the block. */
2336 gimple_seq fallthru_seq
= gimple_seq_alloc_with_stmt (stmt
);
2337 gimple_stmt_iterator fallthru_gsi
= gsi_start (fallthru_seq
);
2338 expand_assign_tm (region
, &fallthru_gsi
);
2339 gsi_insert_seq_on_edge (fallthru_edge
, fallthru_seq
);
2340 pending_edge_inserts_p
= true;
2344 gsi_insert_after (gsi
, stmt
, GSI_CONTINUE_LINKING
);
2345 expand_assign_tm (region
, gsi
);
2348 transaction_subcode_ior (region
, GTMA_HAVE_STORE
);
2355 /* Expand all statements in BB as appropriate for being inside
2359 expand_block_tm (struct tm_region
*region
, basic_block bb
)
2361 gimple_stmt_iterator gsi
;
2363 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2365 gimple stmt
= gsi_stmt (gsi
);
2366 switch (gimple_code (stmt
))
2369 /* Only memory reads/writes need to be instrumented. */
2370 if (gimple_assign_single_p (stmt
)
2371 && !gimple_clobber_p (stmt
))
2373 expand_assign_tm (region
, &gsi
);
2379 if (expand_call_tm (region
, &gsi
))
2389 if (!gsi_end_p (gsi
))
2394 /* Return the list of basic-blocks in REGION.
2396 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2397 following a TM_IRREVOCABLE call. */
2399 static VEC (basic_block
, heap
) *
2400 get_tm_region_blocks (basic_block entry_block
,
2403 bitmap all_region_blocks
,
2404 bool stop_at_irrevocable_p
)
2406 VEC(basic_block
, heap
) *bbs
= NULL
;
2410 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
2413 VEC_safe_push (basic_block
, heap
, bbs
, entry_block
);
2414 bitmap_set_bit (visited_blocks
, entry_block
->index
);
2418 basic_block bb
= VEC_index (basic_block
, bbs
, i
++);
2421 bitmap_bit_p (exit_blocks
, bb
->index
))
2424 if (stop_at_irrevocable_p
2426 && bitmap_bit_p (irr_blocks
, bb
->index
))
2429 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
2430 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
2432 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
2433 VEC_safe_push (basic_block
, heap
, bbs
, e
->dest
);
2436 while (i
< VEC_length (basic_block
, bbs
));
2438 if (all_region_blocks
)
2439 bitmap_ior_into (all_region_blocks
, visited_blocks
);
2441 BITMAP_FREE (visited_blocks
);
2445 /* Set the IN_TRANSACTION for all gimple statements that appear in a
2449 compute_transaction_bits (void)
2451 struct tm_region
*region
;
2452 VEC (basic_block
, heap
) *queue
;
2456 /* ?? Perhaps we need to abstract gate_tm_init further, because we
2457 certainly don't need it to calculate CDI_DOMINATOR info. */
2461 bb
->flags
&= ~BB_IN_TRANSACTION
;
2463 for (region
= all_tm_regions
; region
; region
= region
->next
)
2465 queue
= get_tm_region_blocks (region
->entry_block
,
2466 region
->exit_blocks
,
2469 /*stop_at_irr_p=*/true);
2470 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2471 bb
->flags
|= BB_IN_TRANSACTION
;
2472 VEC_free (basic_block
, heap
, queue
);
2476 bitmap_obstack_release (&tm_obstack
);
2479 /* Entry point to the MARK phase of TM expansion. Here we replace
2480 transactional memory statements with calls to builtins, and function
2481 calls with their transactional clones (if available). But we don't
2482 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
2485 execute_tm_mark (void)
2487 struct tm_region
*region
;
2489 VEC (basic_block
, heap
) *queue
;
2492 queue
= VEC_alloc (basic_block
, heap
, 10);
2493 pending_edge_inserts_p
= false;
2495 for (region
= all_tm_regions
; region
; region
= region
->next
)
2498 /* If we have a transaction... */
2499 if (region
->exit_blocks
)
2501 unsigned int subcode
2502 = gimple_transaction_subcode (region
->transaction_stmt
);
2504 /* Collect a new SUBCODE set, now that optimizations are done... */
2505 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2506 subcode
&= (GTMA_DECLARATION_MASK
| GTMA_DOES_GO_IRREVOCABLE
2507 | GTMA_MAY_ENTER_IRREVOCABLE
);
2509 subcode
&= GTMA_DECLARATION_MASK
;
2510 gimple_transaction_set_subcode (region
->transaction_stmt
, subcode
);
2513 queue
= get_tm_region_blocks (region
->entry_block
,
2514 region
->exit_blocks
,
2517 /*stop_at_irr_p=*/true);
2518 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2519 expand_block_tm (region
, bb
);
2520 VEC_free (basic_block
, heap
, queue
);
2525 if (pending_edge_inserts_p
)
2526 gsi_commit_edge_inserts ();
2530 struct gimple_opt_pass pass_tm_mark
=
2534 "tmmark", /* name */
2536 execute_tm_mark
, /* execute */
2539 0, /* static_pass_number */
2540 TV_TRANS_MEM
, /* tv_id */
2541 PROP_ssa
| PROP_cfg
, /* properties_required */
2542 0, /* properties_provided */
2543 0, /* properties_destroyed */
2544 0, /* todo_flags_start */
2546 | TODO_verify_ssa
, /* todo_flags_finish */
2550 /* Create an abnormal call edge from BB to the first block of the region
2551 represented by STATE. Also record the edge in the TM_RESTART map. */
2554 make_tm_edge (gimple stmt
, basic_block bb
, struct tm_region
*region
)
2557 struct tm_restart_node
*n
, dummy
;
2559 if (cfun
->gimple_df
->tm_restart
== NULL
)
2560 cfun
->gimple_df
->tm_restart
= htab_create_ggc (31, struct_ptr_hash
,
2561 struct_ptr_eq
, ggc_free
);
2564 dummy
.label_or_list
= gimple_block_label (region
->entry_block
);
2565 slot
= htab_find_slot (cfun
->gimple_df
->tm_restart
, &dummy
, INSERT
);
2566 n
= (struct tm_restart_node
*) *slot
;
2569 n
= ggc_alloc_tm_restart_node ();
2574 tree old
= n
->label_or_list
;
2575 if (TREE_CODE (old
) == LABEL_DECL
)
2576 old
= tree_cons (NULL
, old
, NULL
);
2577 n
->label_or_list
= tree_cons (NULL
, dummy
.label_or_list
, old
);
2580 make_edge (bb
, region
->entry_block
, EDGE_ABNORMAL
| EDGE_ABNORMAL_CALL
);
2584 /* Split block BB as necessary for every builtin function we added, and
2585 wire up the abnormal back edges implied by the transaction restart. */
2588 expand_block_edges (struct tm_region
*region
, basic_block bb
)
2590 gimple_stmt_iterator gsi
;
2592 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); )
2594 bool do_next
= true;
2595 gimple stmt
= gsi_stmt (gsi
);
2597 /* ??? TM_COMMIT (and any other tm builtin function) in a nested
2598 transaction has an abnormal edge back to the outer-most transaction
2599 (there are no nested retries), while a TM_ABORT also has an abnormal
2600 backedge to the inner-most transaction. We haven't actually saved
2601 the inner-most transaction here. We should be able to get to it
2602 via the region_nr saved on STMT, and read the transaction_stmt from
2603 that, and find the first region block from there. */
2604 /* ??? Shouldn't we split for any non-pure, non-irrevocable function? */
2605 if (gimple_code (stmt
) == GIMPLE_CALL
2606 && (gimple_call_flags (stmt
) & ECF_TM_BUILTIN
) != 0)
2608 if (gsi_one_before_end_p (gsi
))
2609 make_tm_edge (stmt
, bb
, region
);
2612 edge e
= split_block (bb
, stmt
);
2613 make_tm_edge (stmt
, bb
, region
);
2615 gsi
= gsi_start_bb (bb
);
2619 /* Delete any tail-call annotation that may have been added.
2620 The tail-call pass may have mis-identified the commit as being
2621 a candidate because we had not yet added this restart edge. */
2622 gimple_call_set_tail (stmt
, false);
2630 /* Expand the GIMPLE_TRANSACTION statement into the STM library call. */
2633 expand_transaction (struct tm_region
*region
)
2635 tree status
, tm_start
;
2636 basic_block atomic_bb
, slice_bb
;
2637 gimple_stmt_iterator gsi
;
2642 tm_start
= builtin_decl_explicit (BUILT_IN_TM_START
);
2643 status
= make_rename_temp (TREE_TYPE (TREE_TYPE (tm_start
)), "tm_state");
2645 /* ??? There are plenty of bits here we're not computing. */
2646 subcode
= gimple_transaction_subcode (region
->transaction_stmt
);
2647 if (subcode
& GTMA_DOES_GO_IRREVOCABLE
)
2648 flags
= PR_DOESGOIRREVOCABLE
| PR_UNINSTRUMENTEDCODE
;
2650 flags
= PR_INSTRUMENTEDCODE
;
2651 if ((subcode
& GTMA_MAY_ENTER_IRREVOCABLE
) == 0)
2652 flags
|= PR_HASNOIRREVOCABLE
;
2653 /* If the transaction does not have an abort in lexical scope and is not
2654 marked as an outer transaction, then it will never abort. */
2655 if ((subcode
& GTMA_HAVE_ABORT
) == 0
2656 && (subcode
& GTMA_IS_OUTER
) == 0)
2657 flags
|= PR_HASNOABORT
;
2658 if ((subcode
& GTMA_HAVE_STORE
) == 0)
2659 flags
|= PR_READONLY
;
2660 t2
= build_int_cst (TREE_TYPE (status
), flags
);
2661 g
= gimple_build_call (tm_start
, 1, t2
);
2662 gimple_call_set_lhs (g
, status
);
2663 gimple_set_location (g
, gimple_location (region
->transaction_stmt
));
2665 atomic_bb
= gimple_bb (region
->transaction_stmt
);
2667 if (!VEC_empty (tree
, tm_log_save_addresses
))
2668 tm_log_emit_saves (region
->entry_block
, atomic_bb
);
2670 gsi
= gsi_last_bb (atomic_bb
);
2671 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
2672 gsi_remove (&gsi
, true);
2674 if (!VEC_empty (tree
, tm_log_save_addresses
))
2675 region
->entry_block
=
2676 tm_log_emit_save_or_restores (region
->entry_block
,
2677 A_RESTORELIVEVARIABLES
,
2679 tm_log_emit_restores
,
2681 FALLTHRU_EDGE (atomic_bb
),
2684 slice_bb
= atomic_bb
;
2686 /* If we have an ABORT statement, create a test following the start
2687 call to perform the abort. */
2688 if (gimple_transaction_label (region
->transaction_stmt
))
2691 basic_block test_bb
;
2693 test_bb
= create_empty_bb (slice_bb
);
2694 if (current_loops
&& slice_bb
->loop_father
)
2695 add_bb_to_loop (test_bb
, slice_bb
->loop_father
);
2696 if (VEC_empty (tree
, tm_log_save_addresses
))
2697 region
->entry_block
= test_bb
;
2698 gsi
= gsi_last_bb (test_bb
);
2700 t1
= make_rename_temp (TREE_TYPE (status
), NULL
);
2701 t2
= build_int_cst (TREE_TYPE (status
), A_ABORTTRANSACTION
);
2702 g
= gimple_build_assign_with_ops (BIT_AND_EXPR
, t1
, status
, t2
);
2703 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
2705 t2
= build_int_cst (TREE_TYPE (status
), 0);
2706 g
= gimple_build_cond (NE_EXPR
, t1
, t2
, NULL
, NULL
);
2707 gsi_insert_after (&gsi
, g
, GSI_CONTINUE_LINKING
);
2709 e
= FALLTHRU_EDGE (slice_bb
);
2710 redirect_edge_pred (e
, test_bb
);
2711 e
->flags
= EDGE_FALSE_VALUE
;
2712 e
->probability
= PROB_ALWAYS
- PROB_VERY_UNLIKELY
;
2714 e
= BRANCH_EDGE (atomic_bb
);
2715 redirect_edge_pred (e
, test_bb
);
2716 e
->flags
= EDGE_TRUE_VALUE
;
2717 e
->probability
= PROB_VERY_UNLIKELY
;
2719 e
= make_edge (slice_bb
, test_bb
, EDGE_FALLTHRU
);
2722 /* If we've no abort, but we do have PHIs at the beginning of the atomic
2723 region, that means we've a loop at the beginning of the atomic region
2724 that shares the first block. This can cause problems with the abnormal
2725 edges we're about to add for the transaction restart. Solve this by
2726 adding a new empty block to receive the abnormal edges. */
2727 else if (phi_nodes (region
->entry_block
))
2730 basic_block empty_bb
;
2732 region
->entry_block
= empty_bb
= create_empty_bb (atomic_bb
);
2733 if (current_loops
&& atomic_bb
->loop_father
)
2734 add_bb_to_loop (empty_bb
, atomic_bb
->loop_father
);
2736 e
= FALLTHRU_EDGE (atomic_bb
);
2737 redirect_edge_pred (e
, empty_bb
);
2739 e
= make_edge (atomic_bb
, empty_bb
, EDGE_FALLTHRU
);
2742 /* The GIMPLE_TRANSACTION statement no longer exists. */
2743 region
->transaction_stmt
= NULL
;
2746 static void expand_regions (struct tm_region
*);
2748 /* Helper function for expand_regions. Expand REGION and recurse to
2749 the inner region. */
2752 expand_regions_1 (struct tm_region
*region
)
2754 if (region
->exit_blocks
)
2758 VEC (basic_block
, heap
) *queue
;
2760 /* Collect the set of blocks in this region. Do this before
2761 splitting edges, so that we don't have to play with the
2762 dominator tree in the middle. */
2763 queue
= get_tm_region_blocks (region
->entry_block
,
2764 region
->exit_blocks
,
2767 /*stop_at_irr_p=*/false);
2768 expand_transaction (region
);
2769 for (i
= 0; VEC_iterate (basic_block
, queue
, i
, bb
); ++i
)
2770 expand_block_edges (region
, bb
);
2771 VEC_free (basic_block
, heap
, queue
);
2774 expand_regions (region
->inner
);
2777 /* Expand regions starting at REGION. */
2780 expand_regions (struct tm_region
*region
)
2784 expand_regions_1 (region
);
2785 region
= region
->next
;
2789 /* Entry point to the final expansion of transactional nodes. */
2792 execute_tm_edges (void)
2794 expand_regions (all_tm_regions
);
2797 /* We've got to release the dominance info now, to indicate that it
2798 must be rebuilt completely. Otherwise we'll crash trying to update
2799 the SSA web in the TODO section following this pass. */
2800 free_dominance_info (CDI_DOMINATORS
);
2801 bitmap_obstack_release (&tm_obstack
);
2802 all_tm_regions
= NULL
;
2807 struct gimple_opt_pass pass_tm_edges
=
2811 "tmedge", /* name */
2813 execute_tm_edges
, /* execute */
2816 0, /* static_pass_number */
2817 TV_TRANS_MEM
, /* tv_id */
2818 PROP_ssa
| PROP_cfg
, /* properties_required */
2819 0, /* properties_provided */
2820 0, /* properties_destroyed */
2821 0, /* todo_flags_start */
2823 | TODO_verify_ssa
, /* todo_flags_finish */
2827 /* A unique TM memory operation. */
2828 typedef struct tm_memop
2830 /* Unique ID that all memory operations to the same location have. */
2831 unsigned int value_id
;
2832 /* Address of load/store. */
2836 /* Sets for solving data flow equations in the memory optimization pass. */
2837 struct tm_memopt_bitmaps
2839 /* Stores available to this BB upon entry. Basically, stores that
2840 dominate this BB. */
2841 bitmap store_avail_in
;
2842 /* Stores available at the end of this BB. */
2843 bitmap store_avail_out
;
2844 bitmap store_antic_in
;
2845 bitmap store_antic_out
;
2846 /* Reads available to this BB upon entry. Basically, reads that
2847 dominate this BB. */
2848 bitmap read_avail_in
;
2849 /* Reads available at the end of this BB. */
2850 bitmap read_avail_out
;
2851 /* Reads performed in this BB. */
2853 /* Writes performed in this BB. */
2856 /* Temporary storage for pass. */
2857 /* Is the current BB in the worklist? */
2858 bool avail_in_worklist_p
;
2859 /* Have we visited this BB? */
2863 static bitmap_obstack tm_memopt_obstack
;
2865 /* Unique counter for TM loads and stores. Loads and stores of the
2866 same address get the same ID. */
2867 static unsigned int tm_memopt_value_id
;
2868 static htab_t tm_memopt_value_numbers
;
2870 #define STORE_AVAIL_IN(BB) \
2871 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
2872 #define STORE_AVAIL_OUT(BB) \
2873 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
2874 #define STORE_ANTIC_IN(BB) \
2875 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
2876 #define STORE_ANTIC_OUT(BB) \
2877 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
2878 #define READ_AVAIL_IN(BB) \
2879 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
2880 #define READ_AVAIL_OUT(BB) \
2881 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
2882 #define READ_LOCAL(BB) \
2883 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
2884 #define STORE_LOCAL(BB) \
2885 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
2886 #define AVAIL_IN_WORKLIST_P(BB) \
2887 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
2888 #define BB_VISITED_P(BB) \
2889 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
2891 /* Htab support. Return a hash value for a `tm_memop'. */
2893 tm_memop_hash (const void *p
)
2895 const struct tm_memop
*mem
= (const struct tm_memop
*) p
;
2896 tree addr
= mem
->addr
;
2897 /* We drill down to the SSA_NAME/DECL for the hash, but equality is
2898 actually done with operand_equal_p (see tm_memop_eq). */
2899 if (TREE_CODE (addr
) == ADDR_EXPR
)
2900 addr
= TREE_OPERAND (addr
, 0);
2901 return iterative_hash_expr (addr
, 0);
2904 /* Htab support. Return true if two tm_memop's are the same. */
2906 tm_memop_eq (const void *p1
, const void *p2
)
2908 const struct tm_memop
*mem1
= (const struct tm_memop
*) p1
;
2909 const struct tm_memop
*mem2
= (const struct tm_memop
*) p2
;
2911 return operand_equal_p (mem1
->addr
, mem2
->addr
, 0);
2914 /* Given a TM load/store in STMT, return the value number for the address
2918 tm_memopt_value_number (gimple stmt
, enum insert_option op
)
2920 struct tm_memop tmpmem
, *mem
;
2923 gcc_assert (is_tm_load (stmt
) || is_tm_store (stmt
));
2924 tmpmem
.addr
= gimple_call_arg (stmt
, 0);
2925 slot
= htab_find_slot (tm_memopt_value_numbers
, &tmpmem
, op
);
2927 mem
= (struct tm_memop
*) *slot
;
2928 else if (op
== INSERT
)
2930 mem
= XNEW (struct tm_memop
);
2932 mem
->value_id
= tm_memopt_value_id
++;
2933 mem
->addr
= tmpmem
.addr
;
2937 return mem
->value_id
;
2940 /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
2943 tm_memopt_accumulate_memops (basic_block bb
)
2945 gimple_stmt_iterator gsi
;
2947 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
2949 gimple stmt
= gsi_stmt (gsi
);
2953 if (is_tm_store (stmt
))
2954 bits
= STORE_LOCAL (bb
);
2955 else if (is_tm_load (stmt
))
2956 bits
= READ_LOCAL (bb
);
2960 loc
= tm_memopt_value_number (stmt
, INSERT
);
2961 bitmap_set_bit (bits
, loc
);
2964 fprintf (dump_file
, "TM memopt (%s): value num=%d, BB=%d, addr=",
2965 is_tm_load (stmt
) ? "LOAD" : "STORE", loc
,
2966 gimple_bb (stmt
)->index
);
2967 print_generic_expr (dump_file
, gimple_call_arg (stmt
, 0), 0);
2968 fprintf (dump_file
, "\n");
2973 /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
2976 dump_tm_memopt_set (const char *set_name
, bitmap bits
)
2980 const char *comma
= "";
2982 fprintf (dump_file
, "TM memopt: %s: [", set_name
);
2983 EXECUTE_IF_SET_IN_BITMAP (bits
, 0, i
, bi
)
2986 struct tm_memop
*mem
;
2988 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
2989 FOR_EACH_HTAB_ELEMENT (tm_memopt_value_numbers
, mem
, tm_memop_t
, hi
)
2990 if (mem
->value_id
== i
)
2992 gcc_assert (mem
->value_id
== i
);
2993 fprintf (dump_file
, "%s", comma
);
2995 print_generic_expr (dump_file
, mem
->addr
, 0);
2997 fprintf (dump_file
, "]\n");
3000 /* Prettily dump all of the memopt sets in BLOCKS. */
3003 dump_tm_memopt_sets (VEC (basic_block
, heap
) *blocks
)
3008 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3010 fprintf (dump_file
, "------------BB %d---------\n", bb
->index
);
3011 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb
));
3012 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb
));
3013 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb
));
3014 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb
));
3015 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb
));
3016 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb
));
3020 /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3023 tm_memopt_compute_avin (basic_block bb
)
3028 /* Seed with the AVOUT of any predecessor. */
3029 for (ix
= 0; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3031 e
= EDGE_PRED (bb
, ix
);
3032 /* Make sure we have already visited this BB, and is thus
3035 If e->src->aux is NULL, this predecessor is actually on an
3036 enclosing transaction. We only care about the current
3037 transaction, so ignore it. */
3038 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3040 bitmap_copy (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3041 bitmap_copy (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3046 for (; ix
< EDGE_COUNT (bb
->preds
); ix
++)
3048 e
= EDGE_PRED (bb
, ix
);
3049 if (e
->src
->aux
&& BB_VISITED_P (e
->src
))
3051 bitmap_and_into (STORE_AVAIL_IN (bb
), STORE_AVAIL_OUT (e
->src
));
3052 bitmap_and_into (READ_AVAIL_IN (bb
), READ_AVAIL_OUT (e
->src
));
3056 BB_VISITED_P (bb
) = true;
3059 /* Compute the STORE_ANTIC_IN for the basic block BB. */
3062 tm_memopt_compute_antin (basic_block bb
)
3067 /* Seed with the ANTIC_OUT of any successor. */
3068 for (ix
= 0; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3070 e
= EDGE_SUCC (bb
, ix
);
3071 /* Make sure we have already visited this BB, and is thus
3073 if (BB_VISITED_P (e
->dest
))
3075 bitmap_copy (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3080 for (; ix
< EDGE_COUNT (bb
->succs
); ix
++)
3082 e
= EDGE_SUCC (bb
, ix
);
3083 if (BB_VISITED_P (e
->dest
))
3084 bitmap_and_into (STORE_ANTIC_IN (bb
), STORE_ANTIC_OUT (e
->dest
));
3087 BB_VISITED_P (bb
) = true;
3090 /* Compute the AVAIL sets for every basic block in BLOCKS.
3092 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3094 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3095 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3097 This is basically what we do in lcm's compute_available(), but here
3098 we calculate two sets of sets (one for STOREs and one for READs),
3099 and we work on a region instead of the entire CFG.
3101 REGION is the TM region.
3102 BLOCKS are the basic blocks in the region. */
3105 tm_memopt_compute_available (struct tm_region
*region
,
3106 VEC (basic_block
, heap
) *blocks
)
3109 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3110 unsigned int qlen
, i
;
3114 /* Allocate a worklist array/queue. Entries are only added to the
3115 list if they were not already on the list. So the size is
3116 bounded by the number of basic blocks in the region. */
3117 qlen
= VEC_length (basic_block
, blocks
) - 1;
3118 qin
= qout
= worklist
=
3119 XNEWVEC (basic_block
, qlen
);
3121 /* Put every block in the region on the worklist. */
3122 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3124 /* Seed AVAIL_OUT with the LOCAL set. */
3125 bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_LOCAL (bb
));
3126 bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_LOCAL (bb
));
3128 AVAIL_IN_WORKLIST_P (bb
) = true;
3129 /* No need to insert the entry block, since it has an AVIN of
3130 null, and an AVOUT that has already been seeded in. */
3131 if (bb
!= region
->entry_block
)
3135 /* The entry block has been initialized with the local sets. */
3136 BB_VISITED_P (region
->entry_block
) = true;
3139 qend
= &worklist
[qlen
];
3141 /* Iterate until the worklist is empty. */
3144 /* Take the first entry off the worklist. */
3151 /* This block can be added to the worklist again if necessary. */
3152 AVAIL_IN_WORKLIST_P (bb
) = false;
3153 tm_memopt_compute_avin (bb
);
3155 /* Note: We do not add the LOCAL sets here because we already
3156 seeded the AVAIL_OUT sets with them. */
3157 changed
= bitmap_ior_into (STORE_AVAIL_OUT (bb
), STORE_AVAIL_IN (bb
));
3158 changed
|= bitmap_ior_into (READ_AVAIL_OUT (bb
), READ_AVAIL_IN (bb
));
3160 && (region
->exit_blocks
== NULL
3161 || !bitmap_bit_p (region
->exit_blocks
, bb
->index
)))
3162 /* If the out state of this block changed, then we need to add
3163 its successors to the worklist if they are not already in. */
3164 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3165 if (!AVAIL_IN_WORKLIST_P (e
->dest
) && e
->dest
!= EXIT_BLOCK_PTR
)
3168 AVAIL_IN_WORKLIST_P (e
->dest
) = true;
3179 dump_tm_memopt_sets (blocks
);
3182 /* Compute ANTIC sets for every basic block in BLOCKS.
3184 We compute STORE_ANTIC_OUT as follows:
3186 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3187 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3189 REGION is the TM region.
3190 BLOCKS are the basic blocks in the region. */
3193 tm_memopt_compute_antic (struct tm_region
*region
,
3194 VEC (basic_block
, heap
) *blocks
)
3197 basic_block
*worklist
, *qin
, *qout
, *qend
, bb
;
3202 /* Allocate a worklist array/queue. Entries are only added to the
3203 list if they were not already on the list. So the size is
3204 bounded by the number of basic blocks in the region. */
3205 qin
= qout
= worklist
=
3206 XNEWVEC (basic_block
, VEC_length (basic_block
, blocks
));
3208 for (qlen
= 0, i
= VEC_length (basic_block
, blocks
) - 1; i
>= 0; --i
)
3210 bb
= VEC_index (basic_block
, blocks
, i
);
3212 /* Seed ANTIC_OUT with the LOCAL set. */
3213 bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_LOCAL (bb
));
3215 /* Put every block in the region on the worklist. */
3216 AVAIL_IN_WORKLIST_P (bb
) = true;
3217 /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3218 and their ANTIC_OUT has already been seeded in. */
3219 if (region
->exit_blocks
3220 && !bitmap_bit_p (region
->exit_blocks
, bb
->index
))
3227 /* The exit blocks have been initialized with the local sets. */
3228 if (region
->exit_blocks
)
3232 EXECUTE_IF_SET_IN_BITMAP (region
->exit_blocks
, 0, i
, bi
)
3233 BB_VISITED_P (BASIC_BLOCK (i
)) = true;
3237 qend
= &worklist
[qlen
];
3239 /* Iterate until the worklist is empty. */
3242 /* Take the first entry off the worklist. */
3249 /* This block can be added to the worklist again if necessary. */
3250 AVAIL_IN_WORKLIST_P (bb
) = false;
3251 tm_memopt_compute_antin (bb
);
3253 /* Note: We do not add the LOCAL sets here because we already
3254 seeded the ANTIC_OUT sets with them. */
3255 if (bitmap_ior_into (STORE_ANTIC_OUT (bb
), STORE_ANTIC_IN (bb
))
3256 && bb
!= region
->entry_block
)
3257 /* If the out state of this block changed, then we need to add
3258 its predecessors to the worklist if they are not already in. */
3259 FOR_EACH_EDGE (e
, ei
, bb
->preds
)
3260 if (!AVAIL_IN_WORKLIST_P (e
->src
))
3263 AVAIL_IN_WORKLIST_P (e
->src
) = true;
3274 dump_tm_memopt_sets (blocks
);
3277 /* Offsets of load variants from TM_LOAD. For example,
3278 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3279 See gtm-builtins.def. */
3280 #define TRANSFORM_RAR 1
3281 #define TRANSFORM_RAW 2
3282 #define TRANSFORM_RFW 3
3283 /* Offsets of store variants from TM_STORE. */
3284 #define TRANSFORM_WAR 1
3285 #define TRANSFORM_WAW 2
3287 /* Inform about a load/store optimization. */
3290 dump_tm_memopt_transform (gimple stmt
)
3294 fprintf (dump_file
, "TM memopt: transforming: ");
3295 print_gimple_stmt (dump_file
, stmt
, 0, 0);
3296 fprintf (dump_file
, "\n");
3300 /* Perform a read/write optimization. Replaces the TM builtin in STMT
3301 by a builtin that is OFFSET entries down in the builtins table in
3302 gtm-builtins.def. */
3305 tm_memopt_transform_stmt (unsigned int offset
,
3307 gimple_stmt_iterator
*gsi
)
3309 tree fn
= gimple_call_fn (stmt
);
3310 gcc_assert (TREE_CODE (fn
) == ADDR_EXPR
);
3311 TREE_OPERAND (fn
, 0)
3312 = builtin_decl_explicit ((enum built_in_function
)
3313 (DECL_FUNCTION_CODE (TREE_OPERAND (fn
, 0))
3315 gimple_call_set_fn (stmt
, fn
);
3316 gsi_replace (gsi
, stmt
, true);
3317 dump_tm_memopt_transform (stmt
);
3320 /* Perform the actual TM memory optimization transformations in the
3321 basic blocks in BLOCKS. */
3324 tm_memopt_transform_blocks (VEC (basic_block
, heap
) *blocks
)
3328 gimple_stmt_iterator gsi
;
3330 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3332 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3334 gimple stmt
= gsi_stmt (gsi
);
3335 bitmap read_avail
= READ_AVAIL_IN (bb
);
3336 bitmap store_avail
= STORE_AVAIL_IN (bb
);
3337 bitmap store_antic
= STORE_ANTIC_OUT (bb
);
3340 if (is_tm_simple_load (stmt
))
3342 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3343 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3344 tm_memopt_transform_stmt (TRANSFORM_RAW
, stmt
, &gsi
);
3345 else if (store_antic
&& bitmap_bit_p (store_antic
, loc
))
3347 tm_memopt_transform_stmt (TRANSFORM_RFW
, stmt
, &gsi
);
3348 bitmap_set_bit (store_avail
, loc
);
3350 else if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3351 tm_memopt_transform_stmt (TRANSFORM_RAR
, stmt
, &gsi
);
3353 bitmap_set_bit (read_avail
, loc
);
3355 else if (is_tm_simple_store (stmt
))
3357 loc
= tm_memopt_value_number (stmt
, NO_INSERT
);
3358 if (store_avail
&& bitmap_bit_p (store_avail
, loc
))
3359 tm_memopt_transform_stmt (TRANSFORM_WAW
, stmt
, &gsi
);
3362 if (read_avail
&& bitmap_bit_p (read_avail
, loc
))
3363 tm_memopt_transform_stmt (TRANSFORM_WAR
, stmt
, &gsi
);
3364 bitmap_set_bit (store_avail
, loc
);
3371 /* Return a new set of bitmaps for a BB. */
3373 static struct tm_memopt_bitmaps
*
3374 tm_memopt_init_sets (void)
3376 struct tm_memopt_bitmaps
*b
3377 = XOBNEW (&tm_memopt_obstack
.obstack
, struct tm_memopt_bitmaps
);
3378 b
->store_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3379 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3380 b
->store_antic_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3381 b
->store_antic_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3382 b
->store_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3383 b
->read_avail_in
= BITMAP_ALLOC (&tm_memopt_obstack
);
3384 b
->read_avail_out
= BITMAP_ALLOC (&tm_memopt_obstack
);
3385 b
->read_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3386 b
->store_local
= BITMAP_ALLOC (&tm_memopt_obstack
);
3390 /* Free sets computed for each BB. */
3393 tm_memopt_free_sets (VEC (basic_block
, heap
) *blocks
)
3398 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3402 /* Clear the visited bit for every basic block in BLOCKS. */
3405 tm_memopt_clear_visited (VEC (basic_block
, heap
) *blocks
)
3410 for (i
= 0; VEC_iterate (basic_block
, blocks
, i
, bb
); ++i
)
3411 BB_VISITED_P (bb
) = false;
3414 /* Replace TM load/stores with hints for the runtime. We handle
3415 things like read-after-write, write-after-read, read-after-read,
3416 read-for-write, etc. */
3419 execute_tm_memopt (void)
3421 struct tm_region
*region
;
3422 VEC (basic_block
, heap
) *bbs
;
3424 tm_memopt_value_id
= 0;
3425 tm_memopt_value_numbers
= htab_create (10, tm_memop_hash
, tm_memop_eq
, free
);
3427 for (region
= all_tm_regions
; region
; region
= region
->next
)
3429 /* All the TM stores/loads in the current region. */
3433 bitmap_obstack_initialize (&tm_memopt_obstack
);
3435 /* Save all BBs for the current region. */
3436 bbs
= get_tm_region_blocks (region
->entry_block
,
3437 region
->exit_blocks
,
3442 /* Collect all the memory operations. */
3443 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
3445 bb
->aux
= tm_memopt_init_sets ();
3446 tm_memopt_accumulate_memops (bb
);
3449 /* Solve data flow equations and transform each block accordingly. */
3450 tm_memopt_clear_visited (bbs
);
3451 tm_memopt_compute_available (region
, bbs
);
3452 tm_memopt_clear_visited (bbs
);
3453 tm_memopt_compute_antic (region
, bbs
);
3454 tm_memopt_transform_blocks (bbs
);
3456 tm_memopt_free_sets (bbs
);
3457 VEC_free (basic_block
, heap
, bbs
);
3458 bitmap_obstack_release (&tm_memopt_obstack
);
3459 htab_empty (tm_memopt_value_numbers
);
3462 htab_delete (tm_memopt_value_numbers
);
3467 gate_tm_memopt (void)
3469 return flag_tm
&& optimize
> 0;
3472 struct gimple_opt_pass pass_tm_memopt
=
3476 "tmmemopt", /* name */
3477 gate_tm_memopt
, /* gate */
3478 execute_tm_memopt
, /* execute */
3481 0, /* static_pass_number */
3482 TV_TRANS_MEM
, /* tv_id */
3483 PROP_ssa
| PROP_cfg
, /* properties_required */
3484 0, /* properties_provided */
3485 0, /* properties_destroyed */
3486 0, /* todo_flags_start */
3487 0, /* todo_flags_finish */
3492 /* Interprocedual analysis for the creation of transactional clones.
3493 The aim of this pass is to find which functions are referenced in
3494 a non-irrevocable transaction context, and for those over which
3495 we have control (or user directive), create a version of the
3496 function which uses only the transactional interface to reference
3497 protected memories. This analysis proceeds in several steps:
3499 (1) Collect the set of all possible transactional clones:
3501 (a) For all local public functions marked tm_callable, push
3502 it onto the tm_callee queue.
3504 (b) For all local functions, scan for calls in transaction blocks.
3505 Push the caller and callee onto the tm_caller and tm_callee
3506 queues. Count the number of callers for each callee.
3508 (c) For each local function on the callee list, assume we will
3509 create a transactional clone. Push *all* calls onto the
3510 callee queues; count the number of clone callers separately
3511 to the number of original callers.
3513 (2) Propagate irrevocable status up the dominator tree:
3515 (a) Any external function on the callee list that is not marked
3516 tm_callable is irrevocable. Push all callers of such onto
3519 (b) For each function on the worklist, mark each block that
3520 contains an irrevocable call. Use the AND operator to
3521 propagate that mark up the dominator tree.
3523 (c) If we reach the entry block for a possible transactional
3524 clone, then the transactional clone is irrevocable, and
3525 we should not create the clone after all. Push all
3526 callers onto the worklist.
3528 (d) Place tm_irrevocable calls at the beginning of the relevant
3529 blocks. Special case here is the entry block for the entire
3530 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
3531 the library to begin the region in serial mode. Decrement
3532 the call count for all callees in the irrevocable region.
3534 (3) Create the transactional clones:
3536 Any tm_callee that still has a non-zero call count is cloned.
3539 /* This structure is stored in the AUX field of each cgraph_node. */
3540 struct tm_ipa_cg_data
3542 /* The clone of the function that got created. */
3543 struct cgraph_node
*clone
;
3545 /* The tm regions in the normal function. */
3546 struct tm_region
*all_tm_regions
;
3548 /* The blocks of the normal/clone functions that contain irrevocable
3549 calls, or blocks that are post-dominated by irrevocable calls. */
3550 bitmap irrevocable_blocks_normal
;
3551 bitmap irrevocable_blocks_clone
;
3553 /* The blocks of the normal function that are involved in transactions. */
3554 bitmap transaction_blocks_normal
;
3556 /* The number of callers to the transactional clone of this function
3557 from normal and transactional clones respectively. */
3558 unsigned tm_callers_normal
;
3559 unsigned tm_callers_clone
;
3561 /* True if all calls to this function's transactional clone
3562 are irrevocable. Also automatically true if the function
3563 has no transactional clone. */
3564 bool is_irrevocable
;
3566 /* Flags indicating the presence of this function in various queues. */
3567 bool in_callee_queue
;
3570 /* Flags indicating the kind of scan desired while in the worklist. */
3571 bool want_irr_scan_normal
;
3574 typedef struct cgraph_node
*cgraph_node_p
;
3576 DEF_VEC_P (cgraph_node_p
);
3577 DEF_VEC_ALLOC_P (cgraph_node_p
, heap
);
3579 typedef VEC (cgraph_node_p
, heap
) *cgraph_node_queue
;
3581 /* Return the ipa data associated with NODE, allocating zeroed memory
3582 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
3583 and set *NODE accordingly. */
3585 static struct tm_ipa_cg_data
*
3586 get_cg_data (struct cgraph_node
**node
, bool traverse_aliases
)
3588 struct tm_ipa_cg_data
*d
;
3590 if (traverse_aliases
&& (*node
)->alias
)
3591 *node
= cgraph_get_node ((*node
)->thunk
.alias
);
3593 d
= (struct tm_ipa_cg_data
*) (*node
)->symbol
.aux
;
3597 d
= (struct tm_ipa_cg_data
*)
3598 obstack_alloc (&tm_obstack
.obstack
, sizeof (*d
));
3599 (*node
)->symbol
.aux
= (void *) d
;
3600 memset (d
, 0, sizeof (*d
));
3606 /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
3607 it is already present. */
3610 maybe_push_queue (struct cgraph_node
*node
,
3611 cgraph_node_queue
*queue_p
, bool *in_queue_p
)
3616 VEC_safe_push (cgraph_node_p
, heap
, *queue_p
, node
);
3620 /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
3621 Queue all callees within block BB. */
3624 ipa_tm_scan_calls_block (cgraph_node_queue
*callees_p
,
3625 basic_block bb
, bool for_clone
)
3627 gimple_stmt_iterator gsi
;
3629 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3631 gimple stmt
= gsi_stmt (gsi
);
3632 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3634 tree fndecl
= gimple_call_fndecl (stmt
);
3637 struct tm_ipa_cg_data
*d
;
3639 struct cgraph_node
*node
;
3641 if (is_tm_ending_fndecl (fndecl
))
3643 if (find_tm_replacement_function (fndecl
))
3646 node
= cgraph_get_node (fndecl
);
3647 gcc_assert (node
!= NULL
);
3648 d
= get_cg_data (&node
, true);
3650 pcallers
= (for_clone
? &d
->tm_callers_clone
3651 : &d
->tm_callers_normal
);
3654 maybe_push_queue (node
, callees_p
, &d
->in_callee_queue
);
3660 /* Scan all calls in NODE that are within a transaction region,
3661 and push the resulting nodes into the callee queue. */
3664 ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data
*d
,
3665 cgraph_node_queue
*callees_p
)
3667 struct tm_region
*r
;
3669 d
->transaction_blocks_normal
= BITMAP_ALLOC (&tm_obstack
);
3670 d
->all_tm_regions
= all_tm_regions
;
3672 for (r
= all_tm_regions
; r
; r
= r
->next
)
3674 VEC (basic_block
, heap
) *bbs
;
3678 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
, NULL
,
3679 d
->transaction_blocks_normal
, false);
3681 FOR_EACH_VEC_ELT (basic_block
, bbs
, i
, bb
)
3682 ipa_tm_scan_calls_block (callees_p
, bb
, false);
3684 VEC_free (basic_block
, heap
, bbs
);
3688 /* Scan all calls in NODE as if this is the transactional clone,
3689 and push the destinations into the callee queue. */
3692 ipa_tm_scan_calls_clone (struct cgraph_node
*node
,
3693 cgraph_node_queue
*callees_p
)
3695 struct function
*fn
= DECL_STRUCT_FUNCTION (node
->symbol
.decl
);
3698 FOR_EACH_BB_FN (bb
, fn
)
3699 ipa_tm_scan_calls_block (callees_p
, bb
, true);
3702 /* The function NODE has been detected to be irrevocable. Push all
3703 of its callers onto WORKLIST for the purpose of re-scanning them. */
3706 ipa_tm_note_irrevocable (struct cgraph_node
*node
,
3707 cgraph_node_queue
*worklist_p
)
3709 struct tm_ipa_cg_data
*d
= get_cg_data (&node
, true);
3710 struct cgraph_edge
*e
;
3712 d
->is_irrevocable
= true;
3714 for (e
= node
->callers
; e
; e
= e
->next_caller
)
3717 struct cgraph_node
*caller
;
3719 /* Don't examine recursive calls. */
3720 if (e
->caller
== node
)
3722 /* Even if we think we can go irrevocable, believe the user
3724 if (is_tm_safe_or_pure (e
->caller
->symbol
.decl
))
3728 d
= get_cg_data (&caller
, true);
3730 /* Check if the callee is in a transactional region. If so,
3731 schedule the function for normal re-scan as well. */
3732 bb
= gimple_bb (e
->call_stmt
);
3733 gcc_assert (bb
!= NULL
);
3734 if (d
->transaction_blocks_normal
3735 && bitmap_bit_p (d
->transaction_blocks_normal
, bb
->index
))
3736 d
->want_irr_scan_normal
= true;
3738 maybe_push_queue (caller
, worklist_p
, &d
->in_worklist
);
3742 /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
3743 within the block is irrevocable. */
3746 ipa_tm_scan_irr_block (basic_block bb
)
3748 gimple_stmt_iterator gsi
;
3751 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3753 gimple stmt
= gsi_stmt (gsi
);
3754 switch (gimple_code (stmt
))
3757 if (is_tm_pure_call (stmt
))
3760 fn
= gimple_call_fn (stmt
);
3762 /* Functions with the attribute are by definition irrevocable. */
3763 if (is_tm_irrevocable (fn
))
3766 /* For direct function calls, go ahead and check for replacement
3767 functions, or transitive irrevocable functions. For indirect
3768 functions, we'll ask the runtime. */
3769 if (TREE_CODE (fn
) == ADDR_EXPR
)
3771 struct tm_ipa_cg_data
*d
;
3772 struct cgraph_node
*node
;
3774 fn
= TREE_OPERAND (fn
, 0);
3775 if (is_tm_ending_fndecl (fn
))
3777 if (find_tm_replacement_function (fn
))
3780 node
= cgraph_get_node(fn
);
3781 d
= get_cg_data (&node
, true);
3783 /* Return true if irrevocable, but above all, believe
3785 if (d
->is_irrevocable
3786 && !is_tm_safe_or_pure (fn
))
3792 /* ??? The Approved Method of indicating that an inline
3793 assembly statement is not relevant to the transaction
3794 is to wrap it in a __tm_waiver block. This is not
3795 yet implemented, so we can't check for it. */
3796 if (is_tm_safe (current_function_decl
))
3798 tree t
= build1 (NOP_EXPR
, void_type_node
, size_zero_node
);
3799 SET_EXPR_LOCATION (t
, gimple_location (stmt
));
3800 TREE_BLOCK (t
) = gimple_block (stmt
);
3801 error ("%Kasm not allowed in %<transaction_safe%> function", t
);
3813 /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
3814 for new irrevocable blocks, marking them in NEW_IRR. Don't bother
3815 scanning past OLD_IRR or EXIT_BLOCKS. */
3818 ipa_tm_scan_irr_blocks (VEC (basic_block
, heap
) **pqueue
, bitmap new_irr
,
3819 bitmap old_irr
, bitmap exit_blocks
)
3821 bool any_new_irr
= false;
3824 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
3828 basic_block bb
= VEC_pop (basic_block
, *pqueue
);
3830 /* Don't re-scan blocks we know already are irrevocable. */
3831 if (old_irr
&& bitmap_bit_p (old_irr
, bb
->index
))
3834 if (ipa_tm_scan_irr_block (bb
))
3836 bitmap_set_bit (new_irr
, bb
->index
);
3839 else if (exit_blocks
== NULL
|| !bitmap_bit_p (exit_blocks
, bb
->index
))
3841 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3842 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
3844 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
3845 VEC_safe_push (basic_block
, heap
, *pqueue
, e
->dest
);
3849 while (!VEC_empty (basic_block
, *pqueue
));
3851 BITMAP_FREE (visited_blocks
);
3856 /* Propagate the irrevocable property both up and down the dominator tree.
3857 BB is the current block being scanned; EXIT_BLOCKS are the edges of the
3858 TM regions; OLD_IRR are the results of a previous scan of the dominator
3859 tree which has been fully propagated; NEW_IRR is the set of new blocks
3860 which are gaining the irrevocable property during the current scan. */
3863 ipa_tm_propagate_irr (basic_block entry_block
, bitmap new_irr
,
3864 bitmap old_irr
, bitmap exit_blocks
)
3866 VEC (basic_block
, heap
) *bbs
;
3867 bitmap all_region_blocks
;
3869 /* If this block is in the old set, no need to rescan. */
3870 if (old_irr
&& bitmap_bit_p (old_irr
, entry_block
->index
))
3873 all_region_blocks
= BITMAP_ALLOC (&tm_obstack
);
3874 bbs
= get_tm_region_blocks (entry_block
, exit_blocks
, NULL
,
3875 all_region_blocks
, false);
3878 basic_block bb
= VEC_pop (basic_block
, bbs
);
3879 bool this_irr
= bitmap_bit_p (new_irr
, bb
->index
);
3880 bool all_son_irr
= false;
3884 /* Propagate up. If my children are, I am too, but we must have
3885 at least one child that is. */
3888 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
3890 if (!bitmap_bit_p (new_irr
, e
->dest
->index
))
3892 all_son_irr
= false;
3900 /* Add block to new_irr if it hasn't already been processed. */
3901 if (!old_irr
|| !bitmap_bit_p (old_irr
, bb
->index
))
3903 bitmap_set_bit (new_irr
, bb
->index
);
3909 /* Propagate down to everyone we immediately dominate. */
3913 for (son
= first_dom_son (CDI_DOMINATORS
, bb
);
3915 son
= next_dom_son (CDI_DOMINATORS
, son
))
3917 /* Make sure block is actually in a TM region, and it
3918 isn't already in old_irr. */
3919 if ((!old_irr
|| !bitmap_bit_p (old_irr
, son
->index
))
3920 && bitmap_bit_p (all_region_blocks
, son
->index
))
3921 bitmap_set_bit (new_irr
, son
->index
);
3925 while (!VEC_empty (basic_block
, bbs
));
3927 BITMAP_FREE (all_region_blocks
);
3928 VEC_free (basic_block
, heap
, bbs
);
3932 ipa_tm_decrement_clone_counts (basic_block bb
, bool for_clone
)
3934 gimple_stmt_iterator gsi
;
3936 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
3938 gimple stmt
= gsi_stmt (gsi
);
3939 if (is_gimple_call (stmt
) && !is_tm_pure_call (stmt
))
3941 tree fndecl
= gimple_call_fndecl (stmt
);
3944 struct tm_ipa_cg_data
*d
;
3946 struct cgraph_node
*tnode
;
3948 if (is_tm_ending_fndecl (fndecl
))
3950 if (find_tm_replacement_function (fndecl
))
3953 tnode
= cgraph_get_node (fndecl
);
3954 d
= get_cg_data (&tnode
, true);
3956 pcallers
= (for_clone
? &d
->tm_callers_clone
3957 : &d
->tm_callers_normal
);
3959 gcc_assert (*pcallers
> 0);
3966 /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
3967 as well as other irrevocable actions such as inline assembly. Mark all
3968 such blocks as irrevocable and decrement the number of calls to
3969 transactional clones. Return true if, for the transactional clone, the
3970 entire function is irrevocable. */
3973 ipa_tm_scan_irr_function (struct cgraph_node
*node
, bool for_clone
)
3975 struct tm_ipa_cg_data
*d
;
3976 bitmap new_irr
, old_irr
;
3977 VEC (basic_block
, heap
) *queue
;
3980 /* Builtin operators (operator new, and such). */
3981 if (DECL_STRUCT_FUNCTION (node
->symbol
.decl
) == NULL
3982 || DECL_STRUCT_FUNCTION (node
->symbol
.decl
)->cfg
== NULL
)
3985 current_function_decl
= node
->symbol
.decl
;
3986 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
3987 calculate_dominance_info (CDI_DOMINATORS
);
3989 d
= get_cg_data (&node
, true);
3990 queue
= VEC_alloc (basic_block
, heap
, 10);
3991 new_irr
= BITMAP_ALLOC (&tm_obstack
);
3993 /* Scan each tm region, propagating irrevocable status through the tree. */
3996 old_irr
= d
->irrevocable_blocks_clone
;
3997 VEC_quick_push (basic_block
, queue
, single_succ (ENTRY_BLOCK_PTR
));
3998 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
, NULL
))
4000 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR
), new_irr
,
4002 ret
= bitmap_bit_p (new_irr
, single_succ (ENTRY_BLOCK_PTR
)->index
);
4007 struct tm_region
*region
;
4009 old_irr
= d
->irrevocable_blocks_normal
;
4010 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4012 VEC_quick_push (basic_block
, queue
, region
->entry_block
);
4013 if (ipa_tm_scan_irr_blocks (&queue
, new_irr
, old_irr
,
4014 region
->exit_blocks
))
4015 ipa_tm_propagate_irr (region
->entry_block
, new_irr
, old_irr
,
4016 region
->exit_blocks
);
4020 /* If we found any new irrevocable blocks, reduce the call count for
4021 transactional clones within the irrevocable blocks. Save the new
4022 set of irrevocable blocks for next time. */
4023 if (!bitmap_empty_p (new_irr
))
4025 bitmap_iterator bmi
;
4028 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4029 ipa_tm_decrement_clone_counts (BASIC_BLOCK (i
), for_clone
);
4033 bitmap_ior_into (old_irr
, new_irr
);
4034 BITMAP_FREE (new_irr
);
4037 d
->irrevocable_blocks_clone
= new_irr
;
4039 d
->irrevocable_blocks_normal
= new_irr
;
4041 if (dump_file
&& new_irr
)
4044 bitmap_iterator bmi
;
4047 dname
= lang_hooks
.decl_printable_name (current_function_decl
, 2);
4048 EXECUTE_IF_SET_IN_BITMAP (new_irr
, 0, i
, bmi
)
4049 fprintf (dump_file
, "%s: bb %d goes irrevocable\n", dname
, i
);
4053 BITMAP_FREE (new_irr
);
4055 VEC_free (basic_block
, heap
, queue
);
4057 current_function_decl
= NULL
;
4062 /* Return true if, for the transactional clone of NODE, any call
4063 may enter irrevocable mode. */
4066 ipa_tm_mayenterirr_function (struct cgraph_node
*node
)
4068 struct tm_ipa_cg_data
*d
;
4072 d
= get_cg_data (&node
, true);
4073 decl
= node
->symbol
.decl
;
4074 flags
= flags_from_decl_or_type (decl
);
4076 /* Handle some TM builtins. Ordinarily these aren't actually generated
4077 at this point, but handling these functions when written in by the
4078 user makes it easier to build unit tests. */
4079 if (flags
& ECF_TM_BUILTIN
)
4082 /* Filter out all functions that are marked. */
4083 if (flags
& ECF_TM_PURE
)
4085 if (is_tm_safe (decl
))
4087 if (is_tm_irrevocable (decl
))
4089 if (is_tm_callable (decl
))
4091 if (find_tm_replacement_function (decl
))
4094 /* If we aren't seeing the final version of the function we don't
4095 know what it will contain at runtime. */
4096 if (cgraph_function_body_availability (node
) < AVAIL_AVAILABLE
)
4099 /* If the function must go irrevocable, then of course true. */
4100 if (d
->is_irrevocable
)
4103 /* If there are any blocks marked irrevocable, then the function
4104 as a whole may enter irrevocable. */
4105 if (d
->irrevocable_blocks_clone
)
4108 /* We may have previously marked this function as tm_may_enter_irr;
4109 see pass_diagnose_tm_blocks. */
4110 if (node
->local
.tm_may_enter_irr
)
4113 /* Recurse on the main body for aliases. In general, this will
4114 result in one of the bits above being set so that we will not
4115 have to recurse next time. */
4117 return ipa_tm_mayenterirr_function (cgraph_get_node (node
->thunk
.alias
));
4119 /* What remains is unmarked local functions without items that force
4120 the function to go irrevocable. */
4124 /* Diagnose calls from transaction_safe functions to unmarked
4125 functions that are determined to not be safe. */
4128 ipa_tm_diagnose_tm_safe (struct cgraph_node
*node
)
4130 struct cgraph_edge
*e
;
4132 for (e
= node
->callees
; e
; e
= e
->next_callee
)
4133 if (!is_tm_callable (e
->callee
->symbol
.decl
)
4134 && e
->callee
->local
.tm_may_enter_irr
)
4135 error_at (gimple_location (e
->call_stmt
),
4136 "unsafe function call %qD within "
4137 "%<transaction_safe%> function", e
->callee
->symbol
.decl
);
4140 /* Diagnose call from atomic transactions to unmarked functions
4141 that are determined to not be safe. */
4144 ipa_tm_diagnose_transaction (struct cgraph_node
*node
,
4145 struct tm_region
*all_tm_regions
)
4147 struct tm_region
*r
;
4149 for (r
= all_tm_regions
; r
; r
= r
->next
)
4150 if (gimple_transaction_subcode (r
->transaction_stmt
) & GTMA_IS_RELAXED
)
4152 /* Atomic transactions can be nested inside relaxed. */
4154 ipa_tm_diagnose_transaction (node
, r
->inner
);
4158 VEC (basic_block
, heap
) *bbs
;
4159 gimple_stmt_iterator gsi
;
4163 bbs
= get_tm_region_blocks (r
->entry_block
, r
->exit_blocks
,
4164 r
->irr_blocks
, NULL
, false);
4166 for (i
= 0; VEC_iterate (basic_block
, bbs
, i
, bb
); ++i
)
4167 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4169 gimple stmt
= gsi_stmt (gsi
);
4172 if (gimple_code (stmt
) == GIMPLE_ASM
)
4174 error_at (gimple_location (stmt
),
4175 "asm not allowed in atomic transaction");
4179 if (!is_gimple_call (stmt
))
4181 fndecl
= gimple_call_fndecl (stmt
);
4183 /* Indirect function calls have been diagnosed already. */
4187 /* Stop at the end of the transaction. */
4188 if (is_tm_ending_fndecl (fndecl
))
4190 if (bitmap_bit_p (r
->exit_blocks
, bb
->index
))
4195 /* Marked functions have been diagnosed already. */
4196 if (is_tm_pure_call (stmt
))
4198 if (is_tm_callable (fndecl
))
4201 if (cgraph_local_info (fndecl
)->tm_may_enter_irr
)
4202 error_at (gimple_location (stmt
),
4203 "unsafe function call %qD within "
4204 "atomic transaction", fndecl
);
4207 VEC_free (basic_block
, heap
, bbs
);
4211 /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4212 OLD_DECL. The returned value is a freshly malloced pointer that
4213 should be freed by the caller. */
4216 tm_mangle (tree old_asm_id
)
4218 const char *old_asm_name
;
4221 struct demangle_component
*dc
;
4224 /* Determine if the symbol is already a valid C++ mangled name. Do this
4225 even for C, which might be interfacing with C++ code via appropriately
4226 ugly identifiers. */
4227 /* ??? We could probably do just as well checking for "_Z" and be done. */
4228 old_asm_name
= IDENTIFIER_POINTER (old_asm_id
);
4229 dc
= cplus_demangle_v3_components (old_asm_name
, DMGL_NO_OPTS
, &alloc
);
4236 sprintf (length
, "%u", IDENTIFIER_LENGTH (old_asm_id
));
4237 tm_name
= concat ("_ZGTt", length
, old_asm_name
, NULL
);
4241 old_asm_name
+= 2; /* Skip _Z */
4245 case DEMANGLE_COMPONENT_TRANSACTION_CLONE
:
4246 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE
:
4247 /* Don't play silly games, you! */
4250 case DEMANGLE_COMPONENT_HIDDEN_ALIAS
:
4251 /* I'd really like to know if we can ever be passed one of
4252 these from the C++ front end. The Logical Thing would
4253 seem that hidden-alias should be outer-most, so that we
4254 get hidden-alias of a transaction-clone and not vice-versa. */
4262 tm_name
= concat ("_ZGTt", old_asm_name
, NULL
);
4266 new_asm_id
= get_identifier (tm_name
);
4273 ipa_tm_mark_force_output_node (struct cgraph_node
*node
)
4275 cgraph_mark_force_output_node (node
);
4276 /* ??? function_and_variable_visibility will reset
4277 the needed bit, without actually checking. */
4281 /* Callback data for ipa_tm_create_version_alias. */
4282 struct create_version_alias_info
4284 struct cgraph_node
*old_node
;
4288 /* A subroutine of ipa_tm_create_version, called via
4289 cgraph_for_node_and_aliases. Create new tm clones for each of
4290 the existing aliases. */
4292 ipa_tm_create_version_alias (struct cgraph_node
*node
, void *data
)
4294 struct create_version_alias_info
*info
4295 = (struct create_version_alias_info
*)data
;
4296 tree old_decl
, new_decl
, tm_name
;
4297 struct cgraph_node
*new_node
;
4299 if (!node
->same_body_alias
)
4302 old_decl
= node
->symbol
.decl
;
4303 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4304 new_decl
= build_decl (DECL_SOURCE_LOCATION (old_decl
),
4305 TREE_CODE (old_decl
), tm_name
,
4306 TREE_TYPE (old_decl
));
4308 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4309 SET_DECL_RTL (new_decl
, NULL
);
4311 /* Based loosely on C++'s make_alias_for(). */
4312 TREE_PUBLIC (new_decl
) = TREE_PUBLIC (old_decl
);
4313 DECL_CONTEXT (new_decl
) = DECL_CONTEXT (old_decl
);
4314 DECL_LANG_SPECIFIC (new_decl
) = DECL_LANG_SPECIFIC (old_decl
);
4315 TREE_READONLY (new_decl
) = TREE_READONLY (old_decl
);
4316 DECL_EXTERNAL (new_decl
) = 0;
4317 DECL_ARTIFICIAL (new_decl
) = 1;
4318 TREE_ADDRESSABLE (new_decl
) = 1;
4319 TREE_USED (new_decl
) = 1;
4320 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4322 /* Perform the same remapping to the comdat group. */
4323 if (DECL_ONE_ONLY (new_decl
))
4324 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4326 new_node
= cgraph_same_body_alias (NULL
, new_decl
, info
->new_decl
);
4327 new_node
->tm_clone
= true;
4328 new_node
->symbol
.externally_visible
= info
->old_node
->symbol
.externally_visible
;
4329 /* ?? Do not traverse aliases here. */
4330 get_cg_data (&node
, false)->clone
= new_node
;
4332 record_tm_clone_pair (old_decl
, new_decl
);
4334 if (info
->old_node
->symbol
.force_output
4335 || ipa_ref_list_first_referring (&info
->old_node
->symbol
.ref_list
))
4336 ipa_tm_mark_force_output_node (new_node
);
4340 /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4341 appropriate for the transactional clone. */
4344 ipa_tm_create_version (struct cgraph_node
*old_node
)
4346 tree new_decl
, old_decl
, tm_name
;
4347 struct cgraph_node
*new_node
;
4349 old_decl
= old_node
->symbol
.decl
;
4350 new_decl
= copy_node (old_decl
);
4352 /* DECL_ASSEMBLER_NAME needs to be set before we call
4353 cgraph_copy_node_for_versioning below, because cgraph_node will
4354 fill the assembler_name_hash. */
4355 tm_name
= tm_mangle (DECL_ASSEMBLER_NAME (old_decl
));
4356 SET_DECL_ASSEMBLER_NAME (new_decl
, tm_name
);
4357 SET_DECL_RTL (new_decl
, NULL
);
4358 TREE_SYMBOL_REFERENCED (tm_name
) = 1;
4360 /* Perform the same remapping to the comdat group. */
4361 if (DECL_ONE_ONLY (new_decl
))
4362 DECL_COMDAT_GROUP (new_decl
) = tm_mangle (DECL_COMDAT_GROUP (old_decl
));
4364 new_node
= cgraph_copy_node_for_versioning (old_node
, new_decl
, NULL
, NULL
);
4365 new_node
->symbol
.externally_visible
= old_node
->symbol
.externally_visible
;
4366 new_node
->lowered
= true;
4367 new_node
->tm_clone
= 1;
4368 get_cg_data (&old_node
, true)->clone
= new_node
;
4370 if (cgraph_function_body_availability (old_node
) >= AVAIL_OVERWRITABLE
)
4372 /* Remap extern inline to static inline. */
4373 /* ??? Is it worth trying to use make_decl_one_only? */
4374 if (DECL_DECLARED_INLINE_P (new_decl
) && DECL_EXTERNAL (new_decl
))
4376 DECL_EXTERNAL (new_decl
) = 0;
4377 TREE_PUBLIC (new_decl
) = 0;
4378 DECL_WEAK (new_decl
) = 0;
4381 tree_function_versioning (old_decl
, new_decl
, NULL
, false, NULL
, false,
4385 record_tm_clone_pair (old_decl
, new_decl
);
4387 cgraph_call_function_insertion_hooks (new_node
);
4388 if (old_node
->symbol
.force_output
4389 || ipa_ref_list_first_referring (&old_node
->symbol
.ref_list
))
4390 ipa_tm_mark_force_output_node (new_node
);
4392 /* Do the same thing, but for any aliases of the original node. */
4394 struct create_version_alias_info data
;
4395 data
.old_node
= old_node
;
4396 data
.new_decl
= new_decl
;
4397 cgraph_for_node_and_aliases (old_node
, ipa_tm_create_version_alias
,
4402 /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
4405 ipa_tm_insert_irr_call (struct cgraph_node
*node
, struct tm_region
*region
,
4408 gimple_stmt_iterator gsi
;
4411 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4413 g
= gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
),
4414 1, build_int_cst (NULL_TREE
, MODE_SERIALIRREVOCABLE
));
4416 split_block_after_labels (bb
);
4417 gsi
= gsi_after_labels (bb
);
4418 gsi_insert_before (&gsi
, g
, GSI_SAME_STMT
);
4420 cgraph_create_edge (node
,
4421 cgraph_get_create_node
4422 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE
)),
4424 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4428 /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
4431 ipa_tm_insert_gettmclone_call (struct cgraph_node
*node
,
4432 struct tm_region
*region
,
4433 gimple_stmt_iterator
*gsi
, gimple stmt
)
4435 tree gettm_fn
, ret
, old_fn
, callfn
;
4439 old_fn
= gimple_call_fn (stmt
);
4441 if (TREE_CODE (old_fn
) == ADDR_EXPR
)
4443 tree fndecl
= TREE_OPERAND (old_fn
, 0);
4444 tree clone
= get_tm_clone_pair (fndecl
);
4446 /* By transforming the call into a TM_GETTMCLONE, we are
4447 technically taking the address of the original function and
4448 its clone. Explain this so inlining will know this function
4450 cgraph_mark_address_taken_node (cgraph_get_node (fndecl
));
4452 cgraph_mark_address_taken_node (cgraph_get_node (clone
));
4455 safe
= is_tm_safe (TREE_TYPE (old_fn
));
4456 gettm_fn
= builtin_decl_explicit (safe
? BUILT_IN_TM_GETTMCLONE_SAFE
4457 : BUILT_IN_TM_GETTMCLONE_IRR
);
4458 ret
= create_tmp_var (ptr_type_node
, NULL
);
4459 add_referenced_var (ret
);
4462 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4464 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
4465 if (TREE_CODE (old_fn
) == OBJ_TYPE_REF
)
4466 old_fn
= OBJ_TYPE_REF_EXPR (old_fn
);
4468 g
= gimple_build_call (gettm_fn
, 1, old_fn
);
4469 ret
= make_ssa_name (ret
, g
);
4470 gimple_call_set_lhs (g
, ret
);
4472 gsi_insert_before (gsi
, g
, GSI_SAME_STMT
);
4474 cgraph_create_edge (node
, cgraph_get_create_node (gettm_fn
), g
, 0,
4475 compute_call_stmt_bb_frequency (node
->symbol
.decl
,
4478 /* Cast return value from tm_gettmclone* into appropriate function
4480 callfn
= create_tmp_var (TREE_TYPE (old_fn
), NULL
);
4481 add_referenced_var (callfn
);
4482 g2
= gimple_build_assign (callfn
,
4483 fold_build1 (NOP_EXPR
, TREE_TYPE (callfn
), ret
));
4484 callfn
= make_ssa_name (callfn
, g2
);
4485 gimple_assign_set_lhs (g2
, callfn
);
4486 gsi_insert_before (gsi
, g2
, GSI_SAME_STMT
);
4488 /* ??? This is a hack to preserve the NOTHROW bit on the call,
4489 which we would have derived from the decl. Failure to save
4490 this bit means we might have to split the basic block. */
4491 if (gimple_call_nothrow_p (stmt
))
4492 gimple_call_set_nothrow (stmt
, true);
4494 gimple_call_set_fn (stmt
, callfn
);
4496 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
4497 for a call statement. Fix it. */
4499 tree lhs
= gimple_call_lhs (stmt
);
4500 tree rettype
= TREE_TYPE (gimple_call_fntype (stmt
));
4502 && !useless_type_conversion_p (TREE_TYPE (lhs
), rettype
))
4506 temp
= make_rename_temp (rettype
, 0);
4507 gimple_call_set_lhs (stmt
, temp
);
4509 g2
= gimple_build_assign (lhs
,
4510 fold_build1 (VIEW_CONVERT_EXPR
,
4511 TREE_TYPE (lhs
), temp
));
4512 gsi_insert_after (gsi
, g2
, GSI_SAME_STMT
);
4521 /* Helper function for ipa_tm_transform_calls*. Given a call
4522 statement in GSI which resides inside transaction REGION, redirect
4523 the call to either its wrapper function, or its clone. */
4526 ipa_tm_transform_calls_redirect (struct cgraph_node
*node
,
4527 struct tm_region
*region
,
4528 gimple_stmt_iterator
*gsi
,
4529 bool *need_ssa_rename_p
)
4531 gimple stmt
= gsi_stmt (*gsi
);
4532 struct cgraph_node
*new_node
;
4533 struct cgraph_edge
*e
= cgraph_edge (node
, stmt
);
4534 tree fndecl
= gimple_call_fndecl (stmt
);
4536 /* For indirect calls, pass the address through the runtime. */
4539 *need_ssa_rename_p
|=
4540 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4544 /* Handle some TM builtins. Ordinarily these aren't actually generated
4545 at this point, but handling these functions when written in by the
4546 user makes it easier to build unit tests. */
4547 if (flags_from_decl_or_type (fndecl
) & ECF_TM_BUILTIN
)
4550 /* Fixup recursive calls inside clones. */
4551 /* ??? Why did cgraph_copy_node_for_versioning update the call edges
4552 for recursion but not update the call statements themselves? */
4553 if (e
->caller
== e
->callee
&& decl_is_tm_clone (current_function_decl
))
4555 gimple_call_set_fndecl (stmt
, current_function_decl
);
4559 /* If there is a replacement, use it. */
4560 fndecl
= find_tm_replacement_function (fndecl
);
4563 new_node
= cgraph_get_create_node (fndecl
);
4565 /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
4567 We can't do this earlier in record_tm_replacement because
4568 cgraph_remove_unreachable_nodes is called before we inject
4569 references to the node. Further, we can't do this in some
4570 nice central place in ipa_tm_execute because we don't have
4571 the exact list of wrapper functions that would be used.
4572 Marking more wrappers than necessary results in the creation
4573 of unnecessary cgraph_nodes, which can cause some of the
4574 other IPA passes to crash.
4576 We do need to mark these nodes so that we get the proper
4577 result in expand_call_tm. */
4578 /* ??? This seems broken. How is it that we're marking the
4579 CALLEE as may_enter_irr? Surely we should be marking the
4580 CALLER. Also note that find_tm_replacement_function also
4581 contains mappings into the TM runtime, e.g. memcpy. These
4582 we know won't go irrevocable. */
4583 new_node
->local
.tm_may_enter_irr
= 1;
4587 struct tm_ipa_cg_data
*d
;
4588 struct cgraph_node
*tnode
= e
->callee
;
4590 d
= get_cg_data (&tnode
, true);
4591 new_node
= d
->clone
;
4593 /* As we've already skipped pure calls and appropriate builtins,
4594 and we've already marked irrevocable blocks, if we can't come
4595 up with a static replacement, then ask the runtime. */
4596 if (new_node
== NULL
)
4598 *need_ssa_rename_p
|=
4599 ipa_tm_insert_gettmclone_call (node
, region
, gsi
, stmt
);
4603 fndecl
= new_node
->symbol
.decl
;
4606 cgraph_redirect_edge_callee (e
, new_node
);
4607 gimple_call_set_fndecl (stmt
, fndecl
);
4610 /* Helper function for ipa_tm_transform_calls. For a given BB,
4611 install calls to tm_irrevocable when IRR_BLOCKS are reached,
4612 redirect other calls to the generated transactional clone. */
4615 ipa_tm_transform_calls_1 (struct cgraph_node
*node
, struct tm_region
*region
,
4616 basic_block bb
, bitmap irr_blocks
)
4618 gimple_stmt_iterator gsi
;
4619 bool need_ssa_rename
= false;
4621 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4623 ipa_tm_insert_irr_call (node
, region
, bb
);
4627 for (gsi
= gsi_start_bb (bb
); !gsi_end_p (gsi
); gsi_next (&gsi
))
4629 gimple stmt
= gsi_stmt (gsi
);
4631 if (!is_gimple_call (stmt
))
4633 if (is_tm_pure_call (stmt
))
4636 /* Redirect edges to the appropriate replacement or clone. */
4637 ipa_tm_transform_calls_redirect (node
, region
, &gsi
, &need_ssa_rename
);
4640 return need_ssa_rename
;
4643 /* Walk the CFG for REGION, beginning at BB. Install calls to
4644 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
4645 the generated transactional clone. */
4648 ipa_tm_transform_calls (struct cgraph_node
*node
, struct tm_region
*region
,
4649 basic_block bb
, bitmap irr_blocks
)
4651 bool need_ssa_rename
= false;
4654 VEC(basic_block
, heap
) *queue
= NULL
;
4655 bitmap visited_blocks
= BITMAP_ALLOC (NULL
);
4657 VEC_safe_push (basic_block
, heap
, queue
, bb
);
4660 bb
= VEC_pop (basic_block
, queue
);
4663 ipa_tm_transform_calls_1 (node
, region
, bb
, irr_blocks
);
4665 if (irr_blocks
&& bitmap_bit_p (irr_blocks
, bb
->index
))
4668 if (region
&& bitmap_bit_p (region
->exit_blocks
, bb
->index
))
4671 FOR_EACH_EDGE (e
, ei
, bb
->succs
)
4672 if (!bitmap_bit_p (visited_blocks
, e
->dest
->index
))
4674 bitmap_set_bit (visited_blocks
, e
->dest
->index
);
4675 VEC_safe_push (basic_block
, heap
, queue
, e
->dest
);
4678 while (!VEC_empty (basic_block
, queue
));
4680 VEC_free (basic_block
, heap
, queue
);
4681 BITMAP_FREE (visited_blocks
);
4683 return need_ssa_rename
;
4686 /* Transform the calls within the TM regions within NODE. */
4689 ipa_tm_transform_transaction (struct cgraph_node
*node
)
4691 struct tm_ipa_cg_data
*d
;
4692 struct tm_region
*region
;
4693 bool need_ssa_rename
= false;
4695 d
= get_cg_data (&node
, true);
4697 current_function_decl
= node
->symbol
.decl
;
4698 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
4699 calculate_dominance_info (CDI_DOMINATORS
);
4701 for (region
= d
->all_tm_regions
; region
; region
= region
->next
)
4703 /* If we're sure to go irrevocable, don't transform anything. */
4704 if (d
->irrevocable_blocks_normal
4705 && bitmap_bit_p (d
->irrevocable_blocks_normal
,
4706 region
->entry_block
->index
))
4708 transaction_subcode_ior (region
, GTMA_DOES_GO_IRREVOCABLE
);
4709 transaction_subcode_ior (region
, GTMA_MAY_ENTER_IRREVOCABLE
);
4714 ipa_tm_transform_calls (node
, region
, region
->entry_block
,
4715 d
->irrevocable_blocks_normal
);
4718 if (need_ssa_rename
)
4719 update_ssa (TODO_update_ssa_only_virtuals
);
4722 current_function_decl
= NULL
;
4725 /* Transform the calls within the transactional clone of NODE. */
4728 ipa_tm_transform_clone (struct cgraph_node
*node
)
4730 struct tm_ipa_cg_data
*d
;
4731 bool need_ssa_rename
;
4733 d
= get_cg_data (&node
, true);
4735 /* If this function makes no calls and has no irrevocable blocks,
4736 then there's nothing to do. */
4737 /* ??? Remove non-aborting top-level transactions. */
4738 if (!node
->callees
&& !node
->indirect_calls
&& !d
->irrevocable_blocks_clone
)
4741 current_function_decl
= d
->clone
->symbol
.decl
;
4742 push_cfun (DECL_STRUCT_FUNCTION (current_function_decl
));
4743 calculate_dominance_info (CDI_DOMINATORS
);
4746 ipa_tm_transform_calls (d
->clone
, NULL
, single_succ (ENTRY_BLOCK_PTR
),
4747 d
->irrevocable_blocks_clone
);
4749 if (need_ssa_rename
)
4750 update_ssa (TODO_update_ssa_only_virtuals
);
4753 current_function_decl
= NULL
;
4756 /* Main entry point for the transactional memory IPA pass. */
4759 ipa_tm_execute (void)
4761 cgraph_node_queue tm_callees
= NULL
;
4762 /* List of functions that will go irrevocable. */
4763 cgraph_node_queue irr_worklist
= NULL
;
4765 struct cgraph_node
*node
;
4766 struct tm_ipa_cg_data
*d
;
4767 enum availability a
;
4770 #ifdef ENABLE_CHECKING
4774 bitmap_obstack_initialize (&tm_obstack
);
4776 /* For all local functions marked tm_callable, queue them. */
4777 FOR_EACH_DEFINED_FUNCTION (node
)
4778 if (is_tm_callable (node
->symbol
.decl
)
4779 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4781 d
= get_cg_data (&node
, true);
4782 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
4785 /* For all local reachable functions... */
4786 FOR_EACH_DEFINED_FUNCTION (node
)
4788 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4790 /* ... marked tm_pure, record that fact for the runtime by
4791 indicating that the pure function is its own tm_callable.
4792 No need to do this if the function's address can't be taken. */
4793 if (is_tm_pure (node
->symbol
.decl
))
4795 if (!node
->local
.local
)
4796 record_tm_clone_pair (node
->symbol
.decl
, node
->symbol
.decl
);
4800 current_function_decl
= node
->symbol
.decl
;
4801 push_cfun (DECL_STRUCT_FUNCTION (node
->symbol
.decl
));
4802 calculate_dominance_info (CDI_DOMINATORS
);
4804 tm_region_init (NULL
);
4807 d
= get_cg_data (&node
, true);
4809 /* Scan for calls that are in each transaction. */
4810 ipa_tm_scan_calls_transaction (d
, &tm_callees
);
4812 /* Put it in the worklist so we can scan the function
4813 later (ipa_tm_scan_irr_function) and mark the
4814 irrevocable blocks. */
4815 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4816 d
->want_irr_scan_normal
= true;
4820 current_function_decl
= NULL
;
4823 /* For every local function on the callee list, scan as if we will be
4824 creating a transactional clone, queueing all new functions we find
4826 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4828 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4829 a
= cgraph_function_body_availability (node
);
4830 d
= get_cg_data (&node
, true);
4832 /* Put it in the worklist so we can scan the function later
4833 (ipa_tm_scan_irr_function) and mark the irrevocable
4835 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4837 /* Some callees cannot be arbitrarily cloned. These will always be
4838 irrevocable. Mark these now, so that we need not scan them. */
4839 if (is_tm_irrevocable (node
->symbol
.decl
))
4840 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4841 else if (a
<= AVAIL_NOT_AVAILABLE
4842 && !is_tm_safe_or_pure (node
->symbol
.decl
))
4843 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4844 else if (a
>= AVAIL_OVERWRITABLE
)
4846 if (!tree_versionable_function_p (node
->symbol
.decl
))
4847 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4848 else if (!d
->is_irrevocable
)
4850 /* If this is an alias, make sure its base is queued as well.
4851 we need not scan the callees now, as the base will do. */
4854 node
= cgraph_get_node (node
->thunk
.alias
);
4855 d
= get_cg_data (&node
, true);
4856 maybe_push_queue (node
, &tm_callees
, &d
->in_callee_queue
);
4860 /* Add all nodes called by this function into
4861 tm_callees as well. */
4862 ipa_tm_scan_calls_clone (node
, &tm_callees
);
4867 /* Iterate scans until no more work to be done. Prefer not to use
4868 VEC_pop because the worklist tends to follow a breadth-first
4869 search of the callgraph, which should allow convergance with a
4870 minimum number of scans. But we also don't want the worklist
4871 array to grow without bound, so we shift the array up periodically. */
4872 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
4874 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
4876 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
4880 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
4881 d
= get_cg_data (&node
, true);
4882 d
->in_worklist
= false;
4884 if (d
->want_irr_scan_normal
)
4886 d
->want_irr_scan_normal
= false;
4887 ipa_tm_scan_irr_function (node
, false);
4889 if (d
->in_callee_queue
&& ipa_tm_scan_irr_function (node
, true))
4890 ipa_tm_note_irrevocable (node
, &irr_worklist
);
4893 /* For every function on the callee list, collect the tm_may_enter_irr
4895 VEC_truncate (cgraph_node_p
, irr_worklist
, 0);
4896 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4898 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4899 if (ipa_tm_mayenterirr_function (node
))
4901 d
= get_cg_data (&node
, true);
4902 gcc_assert (d
->in_worklist
== false);
4903 maybe_push_queue (node
, &irr_worklist
, &d
->in_worklist
);
4907 /* Propagate the tm_may_enter_irr bit to callers until stable. */
4908 for (i
= 0; i
< VEC_length (cgraph_node_p
, irr_worklist
); ++i
)
4910 struct cgraph_node
*caller
;
4911 struct cgraph_edge
*e
;
4912 struct ipa_ref
*ref
;
4915 if (i
> 256 && i
== VEC_length (cgraph_node_p
, irr_worklist
) / 8)
4917 VEC_block_remove (cgraph_node_p
, irr_worklist
, 0, i
);
4921 node
= VEC_index (cgraph_node_p
, irr_worklist
, i
);
4922 d
= get_cg_data (&node
, true);
4923 d
->in_worklist
= false;
4924 node
->local
.tm_may_enter_irr
= true;
4926 /* Propagate back to normal callers. */
4927 for (e
= node
->callers
; e
; e
= e
->next_caller
)
4930 if (!is_tm_safe_or_pure (caller
->symbol
.decl
)
4931 && !caller
->local
.tm_may_enter_irr
)
4933 d
= get_cg_data (&caller
, true);
4934 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
4938 /* Propagate back to referring aliases as well. */
4939 for (j
= 0; ipa_ref_list_referring_iterate (&node
->symbol
.ref_list
, j
, ref
); j
++)
4941 caller
= cgraph (ref
->referring
);
4942 if (ref
->use
== IPA_REF_ALIAS
4943 && !caller
->local
.tm_may_enter_irr
)
4945 /* ?? Do not traverse aliases here. */
4946 d
= get_cg_data (&caller
, false);
4947 maybe_push_queue (caller
, &irr_worklist
, &d
->in_worklist
);
4952 /* Now validate all tm_safe functions, and all atomic regions in
4954 FOR_EACH_DEFINED_FUNCTION (node
)
4956 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
4958 d
= get_cg_data (&node
, true);
4959 if (is_tm_safe (node
->symbol
.decl
))
4960 ipa_tm_diagnose_tm_safe (node
);
4961 else if (d
->all_tm_regions
)
4962 ipa_tm_diagnose_transaction (node
, d
->all_tm_regions
);
4965 /* Create clones. Do those that are not irrevocable and have a
4966 positive call count. Do those publicly visible functions that
4967 the user directed us to clone. */
4968 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4972 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4973 if (node
->same_body_alias
)
4976 a
= cgraph_function_body_availability (node
);
4977 d
= get_cg_data (&node
, true);
4979 if (a
<= AVAIL_NOT_AVAILABLE
)
4980 doit
= is_tm_callable (node
->symbol
.decl
);
4981 else if (a
<= AVAIL_AVAILABLE
&& is_tm_callable (node
->symbol
.decl
))
4983 else if (!d
->is_irrevocable
4984 && d
->tm_callers_normal
+ d
->tm_callers_clone
> 0)
4988 ipa_tm_create_version (node
);
4991 /* Redirect calls to the new clones, and insert irrevocable marks. */
4992 for (i
= 0; i
< VEC_length (cgraph_node_p
, tm_callees
); ++i
)
4994 node
= VEC_index (cgraph_node_p
, tm_callees
, i
);
4997 d
= get_cg_data (&node
, true);
4999 ipa_tm_transform_clone (node
);
5002 FOR_EACH_DEFINED_FUNCTION (node
)
5004 && cgraph_function_body_availability (node
) >= AVAIL_OVERWRITABLE
)
5006 d
= get_cg_data (&node
, true);
5007 if (d
->all_tm_regions
)
5008 ipa_tm_transform_transaction (node
);
5011 /* Free and clear all data structures. */
5012 VEC_free (cgraph_node_p
, heap
, tm_callees
);
5013 VEC_free (cgraph_node_p
, heap
, irr_worklist
);
5014 bitmap_obstack_release (&tm_obstack
);
5016 FOR_EACH_FUNCTION (node
)
5017 node
->symbol
.aux
= NULL
;
5019 #ifdef ENABLE_CHECKING
5026 struct simple_ipa_opt_pass pass_ipa_tm
=
5032 ipa_tm_execute
, /* execute */
5035 0, /* static_pass_number */
5036 TV_TRANS_MEM
, /* tv_id */
5037 PROP_ssa
| PROP_cfg
, /* properties_required */
5038 0, /* properties_provided */
5039 0, /* properties_destroyed */
5040 0, /* todo_flags_start */
5041 0, /* todo_flags_finish */
5045 #include "gt-trans-mem.h"