1/* Passes for transactional memory support. 2 Copyright (C) 2008-2022 Free Software Foundation, Inc. 3 Contributed by Richard Henderson <rth@redhat.com> 4 and Aldy Hernandez <aldyh@redhat.com>. 5 6 This file is part of GCC. 7 8 GCC is free software; you can redistribute it and/or modify it under 9 the terms of the GNU General Public License as published by the Free 10 Software Foundation; either version 3, or (at your option) any later 11 version. 12 13 GCC is distributed in the hope that it will be useful, but WITHOUT ANY 14 WARRANTY; without even the implied warranty of MERCHANTABILITY or 15 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 16 for more details. 17 18 You should have received a copy of the GNU General Public License 19 along with GCC; see the file COPYING3. If not see 20 <http://www.gnu.org/licenses/>. */ 21 22#include "config.h" 23#include "system.h" 24#include "coretypes.h" 25#include "backend.h" 26#include "target.h" 27#include "rtl.h" 28#include "tree.h" 29#include "gimple.h" 30#include "cfghooks.h" 31#include "tree-pass.h" 32#include "ssa.h" 33#include "cgraph.h" 34#include "gimple-pretty-print.h" 35#include "diagnostic-core.h" 36#include "fold-const.h" 37#include "tree-eh.h" 38#include "calls.h" 39#include "gimplify.h" 40#include "gimple-iterator.h" 41#include "gimplify-me.h" 42#include "gimple-walk.h" 43#include "tree-cfg.h" 44#include "tree-into-ssa.h" 45#include "tree-inline.h" 46#include "demangle.h" 47#include "output.h" 48#include "trans-mem.h" 49#include "langhooks.h" 50#include "cfgloop.h" 51#include "tree-ssa-address.h" 52#include "stringpool.h" 53#include "attribs.h" 54#include "alloc-pool.h" 55#include "symbol-summary.h" 56#include "symtab-thunks.h" 57 58#define A_RUNINSTRUMENTEDCODE 0x0001 59#define A_RUNUNINSTRUMENTEDCODE 0x0002 60#define A_SAVELIVEVARIABLES 0x0004 61#define A_RESTORELIVEVARIABLES 0x0008 62#define A_ABORTTRANSACTION 0x0010 63 64#define AR_USERABORT 0x0001 65#define AR_USERRETRY 0x0002 66#define AR_TMCONFLICT 0x0004 67#define AR_EXCEPTIONBLOCKABORT 0x0008 68#define AR_OUTERABORT 0x0010 69 70#define MODE_SERIALIRREVOCABLE 0x0000 71 72 73/* The representation of a transaction changes several times during the 74 lowering process. In the beginning, in the front-end we have the 75 GENERIC tree TRANSACTION_EXPR. For example, 76 77 __transaction { 78 local++; 79 if (++global == 10) 80 __tm_abort; 81 } 82 83 During initial gimplification (gimplify.cc) the TRANSACTION_EXPR node is 84 trivially replaced with a GIMPLE_TRANSACTION node. 85 86 During pass_lower_tm, we examine the body of transactions looking 87 for aborts. Transactions that do not contain an abort may be 88 merged into an outer transaction. We also add a TRY-FINALLY node 89 to arrange for the transaction to be committed on any exit. 90 91 [??? Think about how this arrangement affects throw-with-commit 92 and throw-with-abort operations. In this case we want the TRY to 93 handle gotos, but not to catch any exceptions because the transaction 94 will already be closed.] 95 96 GIMPLE_TRANSACTION [label=NULL] { 97 try { 98 local = local + 1; 99 t0 = global; 100 t1 = t0 + 1; 101 global = t1; 102 if (t1 == 10) 103 __builtin___tm_abort (); 104 } finally { 105 __builtin___tm_commit (); 106 } 107 } 108 109 During pass_lower_eh, we create EH regions for the transactions, 110 intermixed with the regular EH stuff. This gives us a nice persistent 111 mapping (all the way through rtl) from transactional memory operation 112 back to the transaction, which allows us to get the abnormal edges 113 correct to model transaction aborts and restarts: 114 115 GIMPLE_TRANSACTION [label=over] 116 local = local + 1; 117 t0 = global; 118 t1 = t0 + 1; 119 global = t1; 120 if (t1 == 10) 121 __builtin___tm_abort (); 122 __builtin___tm_commit (); 123 over: 124 125 This is the end of all_lowering_passes, and so is what is present 126 during the IPA passes, and through all of the optimization passes. 127 128 During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all 129 functions and mark functions for cloning. 130 131 At the end of gimple optimization, before exiting SSA form, 132 pass_tm_edges replaces statements that perform transactional 133 memory operations with the appropriate TM builtins, and swap 134 out function calls with their transactional clones. At this 135 point we introduce the abnormal transaction restart edges and 136 complete lowering of the GIMPLE_TRANSACTION node. 137 138 x = __builtin___tm_start (MAY_ABORT); 139 eh_label: 140 if (x & abort_transaction) 141 goto over; 142 local = local + 1; 143 t0 = __builtin___tm_load (global); 144 t1 = t0 + 1; 145 __builtin___tm_store (&global, t1); 146 if (t1 == 10) 147 __builtin___tm_abort (); 148 __builtin___tm_commit (); 149 over: 150*/ 151 152static void *expand_regions (struct tm_region *, 153 void *(*callback)(struct tm_region *, void *), 154 void *, bool); 155 156 157/* Return the attributes we want to examine for X, or NULL if it's not 158 something we examine. We look at function types, but allow pointers 159 to function types and function decls and peek through. */ 160 161static tree 162get_attrs_for (const_tree x) 163{ 164 if (x == NULL_TREE) 165 return NULL_TREE; 166 167 switch (TREE_CODE (x)) 168 { 169 case FUNCTION_DECL: 170 return TYPE_ATTRIBUTES (TREE_TYPE (x)); 171 172 default: 173 if (TYPE_P (x)) 174 return NULL_TREE; 175 x = TREE_TYPE (x); 176 if (TREE_CODE (x) != POINTER_TYPE) 177 return NULL_TREE; 178 /* FALLTHRU */ 179 180 case POINTER_TYPE: 181 x = TREE_TYPE (x); 182 if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE) 183 return NULL_TREE; 184 /* FALLTHRU */ 185 186 case FUNCTION_TYPE: 187 case METHOD_TYPE: 188 return TYPE_ATTRIBUTES (x); 189 } 190} 191 192/* Return true if X has been marked TM_PURE. */ 193 194bool 195is_tm_pure (const_tree x) 196{ 197 unsigned flags; 198 199 switch (TREE_CODE (x)) 200 { 201 case FUNCTION_DECL: 202 case FUNCTION_TYPE: 203 case METHOD_TYPE: 204 break; 205 206 default: 207 if (TYPE_P (x)) 208 return false; 209 x = TREE_TYPE (x); 210 if (TREE_CODE (x) != POINTER_TYPE) 211 return false; 212 /* FALLTHRU */ 213 214 case POINTER_TYPE: 215 x = TREE_TYPE (x); 216 if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE) 217 return false; 218 break; 219 } 220 221 flags = flags_from_decl_or_type (x); 222 return (flags & ECF_TM_PURE) != 0; 223} 224 225/* Return true if X has been marked TM_IRREVOCABLE. */ 226 227static bool 228is_tm_irrevocable (tree x) 229{ 230 tree attrs = get_attrs_for (x); 231 232 if (attrs && lookup_attribute ("transaction_unsafe", attrs)) 233 return true; 234 235 /* A call to the irrevocable builtin is by definition, 236 irrevocable. */ 237 if (TREE_CODE (x) == ADDR_EXPR) 238 x = TREE_OPERAND (x, 0); 239 if (TREE_CODE (x) == FUNCTION_DECL 240 && fndecl_built_in_p (x, BUILT_IN_TM_IRREVOCABLE)) 241 return true; 242 243 return false; 244} 245 246/* Return true if X has been marked TM_SAFE. */ 247 248bool 249is_tm_safe (const_tree x) 250{ 251 if (flag_tm) 252 { 253 tree attrs = get_attrs_for (x); 254 if (attrs) 255 { 256 if (lookup_attribute ("transaction_safe", attrs)) 257 return true; 258 if (lookup_attribute ("transaction_may_cancel_outer", attrs)) 259 return true; 260 } 261 } 262 return false; 263} 264 265/* Return true if CALL is const, or tm_pure. */ 266 267static bool 268is_tm_pure_call (gimple *call) 269{ 270 return (gimple_call_flags (call) & (ECF_CONST | ECF_TM_PURE)) != 0; 271} 272 273/* Return true if X has been marked TM_CALLABLE. */ 274 275static bool 276is_tm_callable (tree x) 277{ 278 tree attrs = get_attrs_for (x); 279 if (attrs) 280 { 281 if (lookup_attribute ("transaction_callable", attrs)) 282 return true; 283 if (lookup_attribute ("transaction_safe", attrs)) 284 return true; 285 if (lookup_attribute ("transaction_may_cancel_outer", attrs)) 286 return true; 287 } 288 return false; 289} 290 291/* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */ 292 293bool 294is_tm_may_cancel_outer (tree x) 295{ 296 tree attrs = get_attrs_for (x); 297 if (attrs) 298 return lookup_attribute ("transaction_may_cancel_outer", attrs) != NULL; 299 return false; 300} 301 302/* Return true for built in functions that "end" a transaction. */ 303 304bool 305is_tm_ending_fndecl (tree fndecl) 306{ 307 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 308 switch (DECL_FUNCTION_CODE (fndecl)) 309 { 310 case BUILT_IN_TM_COMMIT: 311 case BUILT_IN_TM_COMMIT_EH: 312 case BUILT_IN_TM_ABORT: 313 case BUILT_IN_TM_IRREVOCABLE: 314 return true; 315 default: 316 break; 317 } 318 319 return false; 320} 321 322/* Return true if STMT is a built in function call that "ends" a 323 transaction. */ 324 325bool 326is_tm_ending (gimple *stmt) 327{ 328 tree fndecl; 329 330 if (gimple_code (stmt) != GIMPLE_CALL) 331 return false; 332 333 fndecl = gimple_call_fndecl (stmt); 334 return (fndecl != NULL_TREE 335 && is_tm_ending_fndecl (fndecl)); 336} 337 338/* Return true if STMT is a TM load. */ 339 340static bool 341is_tm_load (gimple *stmt) 342{ 343 tree fndecl; 344 345 if (gimple_code (stmt) != GIMPLE_CALL) 346 return false; 347 348 fndecl = gimple_call_fndecl (stmt); 349 return (fndecl 350 && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL) 351 && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl))); 352} 353 354/* Same as above, but for simple TM loads, that is, not the 355 after-write, after-read, etc optimized variants. */ 356 357static bool 358is_tm_simple_load (gimple *stmt) 359{ 360 tree fndecl; 361 362 if (gimple_code (stmt) != GIMPLE_CALL) 363 return false; 364 365 fndecl = gimple_call_fndecl (stmt); 366 if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL)) 367 { 368 enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl); 369 return (fcode == BUILT_IN_TM_LOAD_1 370 || fcode == BUILT_IN_TM_LOAD_2 371 || fcode == BUILT_IN_TM_LOAD_4 372 || fcode == BUILT_IN_TM_LOAD_8 373 || fcode == BUILT_IN_TM_LOAD_FLOAT 374 || fcode == BUILT_IN_TM_LOAD_DOUBLE 375 || fcode == BUILT_IN_TM_LOAD_LDOUBLE 376 || fcode == BUILT_IN_TM_LOAD_M64 377 || fcode == BUILT_IN_TM_LOAD_M128 378 || fcode == BUILT_IN_TM_LOAD_M256); 379 } 380 return false; 381} 382 383/* Return true if STMT is a TM store. */ 384 385static bool 386is_tm_store (gimple *stmt) 387{ 388 tree fndecl; 389 390 if (gimple_code (stmt) != GIMPLE_CALL) 391 return false; 392 393 fndecl = gimple_call_fndecl (stmt); 394 return (fndecl 395 && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL) 396 && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl))); 397} 398 399/* Same as above, but for simple TM stores, that is, not the 400 after-write, after-read, etc optimized variants. */ 401 402static bool 403is_tm_simple_store (gimple *stmt) 404{ 405 tree fndecl; 406 407 if (gimple_code (stmt) != GIMPLE_CALL) 408 return false; 409 410 fndecl = gimple_call_fndecl (stmt); 411 if (fndecl 412 && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL)) 413 { 414 enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl); 415 return (fcode == BUILT_IN_TM_STORE_1 416 || fcode == BUILT_IN_TM_STORE_2 417 || fcode == BUILT_IN_TM_STORE_4 418 || fcode == BUILT_IN_TM_STORE_8 419 || fcode == BUILT_IN_TM_STORE_FLOAT 420 || fcode == BUILT_IN_TM_STORE_DOUBLE 421 || fcode == BUILT_IN_TM_STORE_LDOUBLE 422 || fcode == BUILT_IN_TM_STORE_M64 423 || fcode == BUILT_IN_TM_STORE_M128 424 || fcode == BUILT_IN_TM_STORE_M256); 425 } 426 return false; 427} 428 429/* Return true if FNDECL is BUILT_IN_TM_ABORT. */ 430 431static bool 432is_tm_abort (tree fndecl) 433{ 434 return (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_TM_ABORT)); 435} 436 437/* Build a GENERIC tree for a user abort. This is called by front ends 438 while transforming the __tm_abort statement. */ 439 440tree 441build_tm_abort_call (location_t loc, bool is_outer) 442{ 443 return build_call_expr_loc (loc, builtin_decl_explicit (BUILT_IN_TM_ABORT), 1, 444 build_int_cst (integer_type_node, 445 AR_USERABORT 446 | (is_outer ? AR_OUTERABORT : 0))); 447} 448 449/* Map for arbitrary function replacement under TM, as created 450 by the tm_wrap attribute. */ 451 452struct tm_wrapper_hasher : ggc_cache_ptr_hash<tree_map> 453{ 454 static inline hashval_t hash (tree_map *m) { return m->hash; } 455 static inline bool 456 equal (tree_map *a, tree_map *b) 457 { 458 return a->base.from == b->base.from; 459 } 460 461 static int 462 keep_cache_entry (tree_map *&m) 463 { 464 return ggc_marked_p (m->base.from); 465 } 466}; 467 468static GTY((cache)) hash_table<tm_wrapper_hasher> *tm_wrap_map; 469 470void 471record_tm_replacement (tree from, tree to) 472{ 473 struct tree_map **slot, *h; 474 475 /* Do not inline wrapper functions that will get replaced in the TM 476 pass. 477 478 Suppose you have foo() that will get replaced into tmfoo(). Make 479 sure the inliner doesn't try to outsmart us and inline foo() 480 before we get a chance to do the TM replacement. */ 481 DECL_UNINLINABLE (from) = 1; 482 483 if (tm_wrap_map == NULL) 484 tm_wrap_map = hash_table<tm_wrapper_hasher>::create_ggc (32); 485 486 h = ggc_alloc<tree_map> (); 487 h->hash = htab_hash_pointer (from); 488 h->base.from = from; 489 h->to = to; 490 491 slot = tm_wrap_map->find_slot_with_hash (h, h->hash, INSERT); 492 *slot = h; 493} 494 495/* Return a TM-aware replacement function for DECL. */ 496 497static tree 498find_tm_replacement_function (tree fndecl) 499{ 500 if (tm_wrap_map) 501 { 502 struct tree_map *h, in; 503 504 in.base.from = fndecl; 505 in.hash = htab_hash_pointer (fndecl); 506 h = tm_wrap_map->find_with_hash (&in, in.hash); 507 if (h) 508 return h->to; 509 } 510 511 /* ??? We may well want TM versions of most of the common <string.h> 512 functions. For now, we've already these two defined. */ 513 /* Adjust expand_call_tm() attributes as necessary for the cases 514 handled here: */ 515 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 516 switch (DECL_FUNCTION_CODE (fndecl)) 517 { 518 case BUILT_IN_MEMCPY: 519 return builtin_decl_explicit (BUILT_IN_TM_MEMCPY); 520 case BUILT_IN_MEMMOVE: 521 return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE); 522 case BUILT_IN_MEMSET: 523 return builtin_decl_explicit (BUILT_IN_TM_MEMSET); 524 default: 525 return NULL; 526 } 527 528 return NULL; 529} 530 531/* When appropriate, record TM replacement for memory allocation functions. 532 533 FROM is the FNDECL to wrap. */ 534void 535tm_malloc_replacement (tree from) 536{ 537 const char *str; 538 tree to; 539 540 if (TREE_CODE (from) != FUNCTION_DECL) 541 return; 542 543 /* If we have a previous replacement, the user must be explicitly 544 wrapping malloc/calloc/free. They better know what they're 545 doing... */ 546 if (find_tm_replacement_function (from)) 547 return; 548 549 str = IDENTIFIER_POINTER (DECL_NAME (from)); 550 551 if (!strcmp (str, "malloc")) 552 to = builtin_decl_explicit (BUILT_IN_TM_MALLOC); 553 else if (!strcmp (str, "calloc")) 554 to = builtin_decl_explicit (BUILT_IN_TM_CALLOC); 555 else if (!strcmp (str, "free")) 556 to = builtin_decl_explicit (BUILT_IN_TM_FREE); 557 else 558 return; 559 560 TREE_NOTHROW (to) = 0; 561 562 record_tm_replacement (from, to); 563} 564 565/* Diagnostics for tm_safe functions/regions. Called by the front end 566 once we've lowered the function to high-gimple. */ 567 568/* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq. 569 Process exactly one statement. WI->INFO is set to non-null when in 570 the context of a tm_safe function, and null for a __transaction block. */ 571 572#define DIAG_TM_OUTER 1 573#define DIAG_TM_SAFE 2 574#define DIAG_TM_RELAXED 4 575 576struct diagnose_tm 577{ 578 unsigned int summary_flags : 8; 579 unsigned int block_flags : 8; 580 unsigned int func_flags : 8; 581 unsigned int saw_volatile : 1; 582 gimple *stmt; 583}; 584 585/* Return true if T is a volatile lvalue of some kind. */ 586 587static bool 588volatile_lvalue_p (tree t) 589{ 590 return ((SSA_VAR_P (t) || REFERENCE_CLASS_P (t)) 591 && TREE_THIS_VOLATILE (TREE_TYPE (t))); 592} 593 594/* Tree callback function for diagnose_tm pass. */ 595 596static tree 597diagnose_tm_1_op (tree *tp, int *walk_subtrees, void *data) 598{ 599 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 600 struct diagnose_tm *d = (struct diagnose_tm *) wi->info; 601 602 if (TYPE_P (*tp)) 603 *walk_subtrees = false; 604 else if (volatile_lvalue_p (*tp) 605 && !d->saw_volatile) 606 { 607 d->saw_volatile = 1; 608 if (d->block_flags & DIAG_TM_SAFE) 609 error_at (gimple_location (d->stmt), 610 "invalid use of volatile lvalue inside transaction"); 611 else if (d->func_flags & DIAG_TM_SAFE) 612 error_at (gimple_location (d->stmt), 613 "invalid use of volatile lvalue inside %<transaction_safe%> " 614 "function"); 615 } 616 617 return NULL_TREE; 618} 619 620static inline bool 621is_tm_safe_or_pure (const_tree x) 622{ 623 return is_tm_safe (x) || is_tm_pure (x); 624} 625 626static tree 627diagnose_tm_1 (gimple_stmt_iterator *gsi, bool *handled_ops_p, 628 struct walk_stmt_info *wi) 629{ 630 gimple *stmt = gsi_stmt (*gsi); 631 struct diagnose_tm *d = (struct diagnose_tm *) wi->info; 632 633 /* Save stmt for use in leaf analysis. */ 634 d->stmt = stmt; 635 636 switch (gimple_code (stmt)) 637 { 638 case GIMPLE_CALL: 639 { 640 tree fn = gimple_call_fn (stmt); 641 642 if ((d->summary_flags & DIAG_TM_OUTER) == 0 643 && is_tm_may_cancel_outer (fn)) 644 error_at (gimple_location (stmt), 645 "%<transaction_may_cancel_outer%> function call not within" 646 " outer transaction or %<transaction_may_cancel_outer%>"); 647 648 if (d->summary_flags & DIAG_TM_SAFE) 649 { 650 bool is_safe, direct_call_p; 651 tree replacement; 652 653 if (TREE_CODE (fn) == ADDR_EXPR 654 && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL) 655 { 656 direct_call_p = true; 657 replacement = TREE_OPERAND (fn, 0); 658 replacement = find_tm_replacement_function (replacement); 659 if (replacement) 660 fn = replacement; 661 } 662 else 663 { 664 direct_call_p = false; 665 replacement = NULL_TREE; 666 } 667 668 if (is_tm_safe_or_pure (fn)) 669 is_safe = true; 670 else if (is_tm_callable (fn) || is_tm_irrevocable (fn)) 671 { 672 /* A function explicitly marked transaction_callable as 673 opposed to transaction_safe is being defined to be 674 unsafe as part of its ABI, regardless of its contents. */ 675 is_safe = false; 676 } 677 else if (direct_call_p) 678 { 679 if (IS_TYPE_OR_DECL_P (fn) 680 && flags_from_decl_or_type (fn) & ECF_TM_BUILTIN) 681 is_safe = true; 682 else if (replacement) 683 { 684 /* ??? At present we've been considering replacements 685 merely transaction_callable, and therefore might 686 enter irrevocable. The tm_wrap attribute has not 687 yet made it into the new language spec. */ 688 is_safe = false; 689 } 690 else 691 { 692 /* ??? Diagnostics for unmarked direct calls moved into 693 the IPA pass. Section 3.2 of the spec details how 694 functions not marked should be considered "implicitly 695 safe" based on having examined the function body. */ 696 is_safe = true; 697 } 698 } 699 else 700 { 701 /* An unmarked indirect call. Consider it unsafe even 702 though optimization may yet figure out how to inline. */ 703 is_safe = false; 704 } 705 706 if (!is_safe) 707 { 708 if (TREE_CODE (fn) == ADDR_EXPR) 709 fn = TREE_OPERAND (fn, 0); 710 if (d->block_flags & DIAG_TM_SAFE) 711 { 712 if (direct_call_p) 713 error_at (gimple_location (stmt), 714 "unsafe function call %qD within " 715 "atomic transaction", fn); 716 else 717 { 718 if ((!DECL_P (fn) || DECL_NAME (fn)) 719 && TREE_CODE (fn) != SSA_NAME) 720 error_at (gimple_location (stmt), 721 "unsafe function call %qE within " 722 "atomic transaction", fn); 723 else 724 error_at (gimple_location (stmt), 725 "unsafe indirect function call within " 726 "atomic transaction"); 727 } 728 } 729 else 730 { 731 if (direct_call_p) 732 error_at (gimple_location (stmt), 733 "unsafe function call %qD within " 734 "%<transaction_safe%> function", fn); 735 else 736 { 737 if ((!DECL_P (fn) || DECL_NAME (fn)) 738 && TREE_CODE (fn) != SSA_NAME) 739 error_at (gimple_location (stmt), 740 "unsafe function call %qE within " 741 "%<transaction_safe%> function", fn); 742 else 743 error_at (gimple_location (stmt), 744 "unsafe indirect function call within " 745 "%<transaction_safe%> function"); 746 } 747 } 748 } 749 } 750 } 751 break; 752 753 case GIMPLE_ASM: 754 /* ??? We ought to come up with a way to add attributes to 755 asm statements, and then add "transaction_safe" to it. 756 Either that or get the language spec to resurrect __tm_waiver. */ 757 if (d->block_flags & DIAG_TM_SAFE) 758 error_at (gimple_location (stmt), 759 "%<asm%> not allowed in atomic transaction"); 760 else if (d->func_flags & DIAG_TM_SAFE) 761 error_at (gimple_location (stmt), 762 "%<asm%> not allowed in %<transaction_safe%> function"); 763 break; 764 765 case GIMPLE_TRANSACTION: 766 { 767 gtransaction *trans_stmt = as_a <gtransaction *> (stmt); 768 unsigned char inner_flags = DIAG_TM_SAFE; 769 770 if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_RELAXED) 771 { 772 if (d->block_flags & DIAG_TM_SAFE) 773 error_at (gimple_location (stmt), 774 "relaxed transaction in atomic transaction"); 775 else if (d->func_flags & DIAG_TM_SAFE) 776 error_at (gimple_location (stmt), 777 "relaxed transaction in %<transaction_safe%> function"); 778 inner_flags = DIAG_TM_RELAXED; 779 } 780 else if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_OUTER) 781 { 782 if (d->block_flags) 783 error_at (gimple_location (stmt), 784 "outer transaction in transaction"); 785 else if (d->func_flags & DIAG_TM_OUTER) 786 error_at (gimple_location (stmt), 787 "outer transaction in " 788 "%<transaction_may_cancel_outer%> function"); 789 else if (d->func_flags & DIAG_TM_SAFE) 790 error_at (gimple_location (stmt), 791 "outer transaction in %<transaction_safe%> function"); 792 inner_flags |= DIAG_TM_OUTER; 793 } 794 795 *handled_ops_p = true; 796 if (gimple_transaction_body (trans_stmt)) 797 { 798 struct walk_stmt_info wi_inner; 799 struct diagnose_tm d_inner; 800 801 memset (&d_inner, 0, sizeof (d_inner)); 802 d_inner.func_flags = d->func_flags; 803 d_inner.block_flags = d->block_flags | inner_flags; 804 d_inner.summary_flags = d_inner.func_flags | d_inner.block_flags; 805 806 memset (&wi_inner, 0, sizeof (wi_inner)); 807 wi_inner.info = &d_inner; 808 809 walk_gimple_seq (gimple_transaction_body (trans_stmt), 810 diagnose_tm_1, diagnose_tm_1_op, &wi_inner); 811 } 812 } 813 break; 814 815 default: 816 break; 817 } 818 819 return NULL_TREE; 820} 821 822static unsigned int 823diagnose_tm_blocks (void) 824{ 825 struct walk_stmt_info wi; 826 struct diagnose_tm d; 827 828 memset (&d, 0, sizeof (d)); 829 if (is_tm_may_cancel_outer (current_function_decl)) 830 d.func_flags = DIAG_TM_OUTER | DIAG_TM_SAFE; 831 else if (is_tm_safe (current_function_decl)) 832 d.func_flags = DIAG_TM_SAFE; 833 d.summary_flags = d.func_flags; 834 835 memset (&wi, 0, sizeof (wi)); 836 wi.info = &d; 837 838 walk_gimple_seq (gimple_body (current_function_decl), 839 diagnose_tm_1, diagnose_tm_1_op, &wi); 840 841 return 0; 842} 843 844namespace { 845 846const pass_data pass_data_diagnose_tm_blocks = 847{ 848 GIMPLE_PASS, /* type */ 849 "*diagnose_tm_blocks", /* name */ 850 OPTGROUP_NONE, /* optinfo_flags */ 851 TV_TRANS_MEM, /* tv_id */ 852 PROP_gimple_any, /* properties_required */ 853 0, /* properties_provided */ 854 0, /* properties_destroyed */ 855 0, /* todo_flags_start */ 856 0, /* todo_flags_finish */ 857}; 858 859class pass_diagnose_tm_blocks : public gimple_opt_pass 860{ 861public: 862 pass_diagnose_tm_blocks (gcc::context *ctxt) 863 : gimple_opt_pass (pass_data_diagnose_tm_blocks, ctxt) 864 {} 865 866 /* opt_pass methods: */ 867 virtual bool gate (function *) { return flag_tm; } 868 virtual unsigned int execute (function *) { return diagnose_tm_blocks (); } 869 870}; // class pass_diagnose_tm_blocks 871 872} // anon namespace 873 874gimple_opt_pass * 875make_pass_diagnose_tm_blocks (gcc::context *ctxt) 876{ 877 return new pass_diagnose_tm_blocks (ctxt); 878} 879 880/* Instead of instrumenting thread private memory, we save the 881 addresses in a log which we later use to save/restore the addresses 882 upon transaction start/restart. 883 884 The log is keyed by address, where each element contains individual 885 statements among different code paths that perform the store. 886 887 This log is later used to generate either plain save/restore of the 888 addresses upon transaction start/restart, or calls to the ITM_L* 889 logging functions. 890 891 So for something like: 892 893 struct large { int x[1000]; }; 894 struct large lala = { 0 }; 895 __transaction { 896 lala.x[i] = 123; 897 ... 898 } 899 900 We can either save/restore: 901 902 lala = { 0 }; 903 trxn = _ITM_startTransaction (); 904 if (trxn & a_saveLiveVariables) 905 tmp_lala1 = lala.x[i]; 906 else if (a & a_restoreLiveVariables) 907 lala.x[i] = tmp_lala1; 908 909 or use the logging functions: 910 911 lala = { 0 }; 912 trxn = _ITM_startTransaction (); 913 _ITM_LU4 (&lala.x[i]); 914 915 Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as 916 far up the dominator tree to shadow all of the writes to a given 917 location (thus reducing the total number of logging calls), but not 918 so high as to be called on a path that does not perform a 919 write. */ 920 921/* One individual log entry. We may have multiple statements for the 922 same location if neither dominate each other (on different 923 execution paths). */ 924struct tm_log_entry 925{ 926 /* Address to save. */ 927 tree addr; 928 /* Entry block for the transaction this address occurs in. */ 929 basic_block entry_block; 930 /* Dominating statements the store occurs in. */ 931 vec<gimple *> stmts; 932 /* Initially, while we are building the log, we place a nonzero 933 value here to mean that this address *will* be saved with a 934 save/restore sequence. Later, when generating the save sequence 935 we place the SSA temp generated here. */ 936 tree save_var; 937}; 938 939 940/* Log entry hashtable helpers. */ 941 942struct log_entry_hasher : pointer_hash <tm_log_entry> 943{ 944 static inline hashval_t hash (const tm_log_entry *); 945 static inline bool equal (const tm_log_entry *, const tm_log_entry *); 946 static inline void remove (tm_log_entry *); 947}; 948 949/* Htab support. Return hash value for a `tm_log_entry'. */ 950inline hashval_t 951log_entry_hasher::hash (const tm_log_entry *log) 952{ 953 return iterative_hash_expr (log->addr, 0); 954} 955 956/* Htab support. Return true if two log entries are the same. */ 957inline bool 958log_entry_hasher::equal (const tm_log_entry *log1, const tm_log_entry *log2) 959{ 960 /* FIXME: 961 962 rth: I suggest that we get rid of the component refs etc. 963 I.e. resolve the reference to base + offset. 964 965 We may need to actually finish a merge with mainline for this, 966 since we'd like to be presented with Richi's MEM_REF_EXPRs more 967 often than not. But in the meantime your tm_log_entry could save 968 the results of get_inner_reference. 969 970 See: g++.dg/tm/pr46653.C 971 */ 972 973 /* Special case plain equality because operand_equal_p() below will 974 return FALSE if the addresses are equal but they have 975 side-effects (e.g. a volatile address). */ 976 if (log1->addr == log2->addr) 977 return true; 978 979 return operand_equal_p (log1->addr, log2->addr, 0); 980} 981 982/* Htab support. Free one tm_log_entry. */ 983inline void 984log_entry_hasher::remove (tm_log_entry *lp) 985{ 986 lp->stmts.release (); 987 free (lp); 988} 989 990 991/* The actual log. */ 992static hash_table<log_entry_hasher> *tm_log; 993 994/* Addresses to log with a save/restore sequence. These should be in 995 dominator order. */ 996static vec<tree> tm_log_save_addresses; 997 998enum thread_memory_type 999 { 1000 mem_non_local = 0, 1001 mem_thread_local, 1002 mem_transaction_local, 1003 mem_max 1004 }; 1005 1006struct tm_new_mem_map 1007{ 1008 /* SSA_NAME being dereferenced. */ 1009 tree val; 1010 enum thread_memory_type local_new_memory; 1011}; 1012 1013/* Hashtable helpers. */ 1014 1015struct tm_mem_map_hasher : free_ptr_hash <tm_new_mem_map> 1016{ 1017 static inline hashval_t hash (const tm_new_mem_map *); 1018 static inline bool equal (const tm_new_mem_map *, const tm_new_mem_map *); 1019}; 1020 1021inline hashval_t 1022tm_mem_map_hasher::hash (const tm_new_mem_map *v) 1023{ 1024 return (intptr_t)v->val >> 4; 1025} 1026 1027inline bool 1028tm_mem_map_hasher::equal (const tm_new_mem_map *v, const tm_new_mem_map *c) 1029{ 1030 return v->val == c->val; 1031} 1032 1033/* Map for an SSA_NAME originally pointing to a non aliased new piece 1034 of memory (malloc, alloc, etc). */ 1035static hash_table<tm_mem_map_hasher> *tm_new_mem_hash; 1036 1037/* Initialize logging data structures. */ 1038static void 1039tm_log_init (void) 1040{ 1041 tm_log = new hash_table<log_entry_hasher> (10); 1042 tm_new_mem_hash = new hash_table<tm_mem_map_hasher> (5); 1043 tm_log_save_addresses.create (5); 1044} 1045 1046/* Free logging data structures. */ 1047static void 1048tm_log_delete (void) 1049{ 1050 delete tm_log; 1051 tm_log = NULL; 1052 delete tm_new_mem_hash; 1053 tm_new_mem_hash = NULL; 1054 tm_log_save_addresses.release (); 1055} 1056 1057/* Return true if MEM is a transaction invariant memory for the TM 1058 region starting at REGION_ENTRY_BLOCK. */ 1059static bool 1060transaction_invariant_address_p (const_tree mem, basic_block region_entry_block) 1061{ 1062 if ((TREE_CODE (mem) == INDIRECT_REF || TREE_CODE (mem) == MEM_REF) 1063 && TREE_CODE (TREE_OPERAND (mem, 0)) == SSA_NAME) 1064 { 1065 basic_block def_bb; 1066 1067 def_bb = gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem, 0))); 1068 return def_bb != region_entry_block 1069 && dominated_by_p (CDI_DOMINATORS, region_entry_block, def_bb); 1070 } 1071 1072 mem = strip_invariant_refs (mem); 1073 return mem && (CONSTANT_CLASS_P (mem) || decl_address_invariant_p (mem)); 1074} 1075 1076/* Given an address ADDR in STMT, find it in the memory log or add it, 1077 making sure to keep only the addresses highest in the dominator 1078 tree. 1079 1080 ENTRY_BLOCK is the entry_block for the transaction. 1081 1082 If we find the address in the log, make sure it's either the same 1083 address, or an equivalent one that dominates ADDR. 1084 1085 If we find the address, but neither ADDR dominates the found 1086 address, nor the found one dominates ADDR, we're on different 1087 execution paths. Add it. 1088 1089 If known, ENTRY_BLOCK is the entry block for the region, otherwise 1090 NULL. */ 1091static void 1092tm_log_add (basic_block entry_block, tree addr, gimple *stmt) 1093{ 1094 tm_log_entry **slot; 1095 struct tm_log_entry l, *lp; 1096 1097 l.addr = addr; 1098 slot = tm_log->find_slot (&l, INSERT); 1099 if (!*slot) 1100 { 1101 tree type = TREE_TYPE (addr); 1102 1103 lp = XNEW (struct tm_log_entry); 1104 lp->addr = addr; 1105 *slot = lp; 1106 1107 /* Small invariant addresses can be handled as save/restores. */ 1108 if (entry_block 1109 && transaction_invariant_address_p (lp->addr, entry_block) 1110 && TYPE_SIZE_UNIT (type) != NULL 1111 && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)) 1112 && ((HOST_WIDE_INT) tree_to_uhwi (TYPE_SIZE_UNIT (type)) 1113 < param_tm_max_aggregate_size) 1114 /* We must be able to copy this type normally. I.e., no 1115 special constructors and the like. */ 1116 && !TREE_ADDRESSABLE (type)) 1117 { 1118 lp->save_var = create_tmp_reg (TREE_TYPE (lp->addr), "tm_save"); 1119 lp->stmts.create (0); 1120 lp->entry_block = entry_block; 1121 /* Save addresses separately in dominator order so we don't 1122 get confused by overlapping addresses in the save/restore 1123 sequence. */ 1124 tm_log_save_addresses.safe_push (lp->addr); 1125 } 1126 else 1127 { 1128 /* Use the logging functions. */ 1129 lp->stmts.create (5); 1130 lp->stmts.quick_push (stmt); 1131 lp->save_var = NULL; 1132 } 1133 } 1134 else 1135 { 1136 size_t i; 1137 gimple *oldstmt; 1138 1139 lp = *slot; 1140 1141 /* If we're generating a save/restore sequence, we don't care 1142 about statements. */ 1143 if (lp->save_var) 1144 return; 1145 1146 for (i = 0; lp->stmts.iterate (i, &oldstmt); ++i) 1147 { 1148 if (stmt == oldstmt) 1149 return; 1150 /* We already have a store to the same address, higher up the 1151 dominator tree. Nothing to do. */ 1152 if (dominated_by_p (CDI_DOMINATORS, 1153 gimple_bb (stmt), gimple_bb (oldstmt))) 1154 return; 1155 /* We should be processing blocks in dominator tree order. */ 1156 gcc_assert (!dominated_by_p (CDI_DOMINATORS, 1157 gimple_bb (oldstmt), gimple_bb (stmt))); 1158 } 1159 /* Store is on a different code path. */ 1160 lp->stmts.safe_push (stmt); 1161 } 1162} 1163 1164/* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME 1165 result, insert the new statements before GSI. */ 1166 1167static tree 1168gimplify_addr (gimple_stmt_iterator *gsi, tree x) 1169{ 1170 if (TREE_CODE (x) == TARGET_MEM_REF) 1171 x = tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x)), x); 1172 else 1173 x = build_fold_addr_expr (x); 1174 return force_gimple_operand_gsi (gsi, x, true, NULL, true, GSI_SAME_STMT); 1175} 1176 1177/* Instrument one address with the logging functions. 1178 ADDR is the address to save. 1179 STMT is the statement before which to place it. */ 1180static void 1181tm_log_emit_stmt (tree addr, gimple *stmt) 1182{ 1183 tree type = TREE_TYPE (addr); 1184 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 1185 gimple *log; 1186 enum built_in_function code = BUILT_IN_TM_LOG; 1187 1188 if (type == float_type_node) 1189 code = BUILT_IN_TM_LOG_FLOAT; 1190 else if (type == double_type_node) 1191 code = BUILT_IN_TM_LOG_DOUBLE; 1192 else if (type == long_double_type_node) 1193 code = BUILT_IN_TM_LOG_LDOUBLE; 1194 else if (TYPE_SIZE (type) != NULL 1195 && tree_fits_uhwi_p (TYPE_SIZE (type))) 1196 { 1197 unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type)); 1198 1199 if (TREE_CODE (type) == VECTOR_TYPE) 1200 { 1201 switch (type_size) 1202 { 1203 case 64: 1204 code = BUILT_IN_TM_LOG_M64; 1205 break; 1206 case 128: 1207 code = BUILT_IN_TM_LOG_M128; 1208 break; 1209 case 256: 1210 code = BUILT_IN_TM_LOG_M256; 1211 break; 1212 default: 1213 goto unhandled_vec; 1214 } 1215 if (!builtin_decl_explicit_p (code)) 1216 goto unhandled_vec; 1217 } 1218 else 1219 { 1220 unhandled_vec: 1221 switch (type_size) 1222 { 1223 case 8: 1224 code = BUILT_IN_TM_LOG_1; 1225 break; 1226 case 16: 1227 code = BUILT_IN_TM_LOG_2; 1228 break; 1229 case 32: 1230 code = BUILT_IN_TM_LOG_4; 1231 break; 1232 case 64: 1233 code = BUILT_IN_TM_LOG_8; 1234 break; 1235 } 1236 } 1237 } 1238 1239 if (code != BUILT_IN_TM_LOG && !builtin_decl_explicit_p (code)) 1240 code = BUILT_IN_TM_LOG; 1241 tree decl = builtin_decl_explicit (code); 1242 1243 addr = gimplify_addr (&gsi, addr); 1244 if (code == BUILT_IN_TM_LOG) 1245 log = gimple_build_call (decl, 2, addr, TYPE_SIZE_UNIT (type)); 1246 else 1247 log = gimple_build_call (decl, 1, addr); 1248 gsi_insert_before (&gsi, log, GSI_SAME_STMT); 1249} 1250 1251/* Go through the log and instrument address that must be instrumented 1252 with the logging functions. Leave the save/restore addresses for 1253 later. */ 1254static void 1255tm_log_emit (void) 1256{ 1257 hash_table<log_entry_hasher>::iterator hi; 1258 struct tm_log_entry *lp; 1259 1260 FOR_EACH_HASH_TABLE_ELEMENT (*tm_log, lp, tm_log_entry_t, hi) 1261 { 1262 size_t i; 1263 gimple *stmt; 1264 1265 if (dump_file) 1266 { 1267 fprintf (dump_file, "TM thread private mem logging: "); 1268 print_generic_expr (dump_file, lp->addr); 1269 fprintf (dump_file, "\n"); 1270 } 1271 1272 if (lp->save_var) 1273 { 1274 if (dump_file) 1275 fprintf (dump_file, "DUMPING to variable\n"); 1276 continue; 1277 } 1278 else 1279 { 1280 if (dump_file) 1281 fprintf (dump_file, "DUMPING with logging functions\n"); 1282 for (i = 0; lp->stmts.iterate (i, &stmt); ++i) 1283 tm_log_emit_stmt (lp->addr, stmt); 1284 } 1285 } 1286} 1287 1288/* Emit the save sequence for the corresponding addresses in the log. 1289 ENTRY_BLOCK is the entry block for the transaction. 1290 BB is the basic block to insert the code in. */ 1291static void 1292tm_log_emit_saves (basic_block entry_block, basic_block bb) 1293{ 1294 size_t i; 1295 gimple_stmt_iterator gsi = gsi_last_bb (bb); 1296 gimple *stmt; 1297 struct tm_log_entry l, *lp; 1298 1299 for (i = 0; i < tm_log_save_addresses.length (); ++i) 1300 { 1301 l.addr = tm_log_save_addresses[i]; 1302 lp = *(tm_log->find_slot (&l, NO_INSERT)); 1303 gcc_assert (lp->save_var != NULL); 1304 1305 /* We only care about variables in the current transaction. */ 1306 if (lp->entry_block != entry_block) 1307 continue; 1308 1309 stmt = gimple_build_assign (lp->save_var, unshare_expr (lp->addr)); 1310 1311 /* Make sure we can create an SSA_NAME for this type. For 1312 instance, aggregates aren't allowed, in which case the system 1313 will create a VOP for us and everything will just work. */ 1314 if (is_gimple_reg_type (TREE_TYPE (lp->save_var))) 1315 { 1316 lp->save_var = make_ssa_name (lp->save_var, stmt); 1317 gimple_assign_set_lhs (stmt, lp->save_var); 1318 } 1319 1320 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 1321 } 1322} 1323 1324/* Emit the restore sequence for the corresponding addresses in the log. 1325 ENTRY_BLOCK is the entry block for the transaction. 1326 BB is the basic block to insert the code in. */ 1327static void 1328tm_log_emit_restores (basic_block entry_block, basic_block bb) 1329{ 1330 int i; 1331 struct tm_log_entry l, *lp; 1332 gimple_stmt_iterator gsi; 1333 gimple *stmt; 1334 1335 for (i = tm_log_save_addresses.length () - 1; i >= 0; i--) 1336 { 1337 l.addr = tm_log_save_addresses[i]; 1338 lp = *(tm_log->find_slot (&l, NO_INSERT)); 1339 gcc_assert (lp->save_var != NULL); 1340 1341 /* We only care about variables in the current transaction. */ 1342 if (lp->entry_block != entry_block) 1343 continue; 1344 1345 /* Restores are in LIFO order from the saves in case we have 1346 overlaps. */ 1347 gsi = gsi_start_bb (bb); 1348 1349 stmt = gimple_build_assign (unshare_expr (lp->addr), lp->save_var); 1350 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 1351 } 1352} 1353 1354 1355static tree lower_sequence_tm (gimple_stmt_iterator *, bool *, 1356 struct walk_stmt_info *); 1357static tree lower_sequence_no_tm (gimple_stmt_iterator *, bool *, 1358 struct walk_stmt_info *); 1359 1360/* Evaluate an address X being dereferenced and determine if it 1361 originally points to a non aliased new chunk of memory (malloc, 1362 alloca, etc). 1363 1364 Return MEM_THREAD_LOCAL if it points to a thread-local address. 1365 Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address. 1366 Return MEM_NON_LOCAL otherwise. 1367 1368 ENTRY_BLOCK is the entry block to the transaction containing the 1369 dereference of X. */ 1370static enum thread_memory_type 1371thread_private_new_memory (basic_block entry_block, tree x) 1372{ 1373 gimple *stmt = NULL; 1374 enum tree_code code; 1375 tm_new_mem_map **slot; 1376 tm_new_mem_map elt, *elt_p; 1377 tree val = x; 1378 enum thread_memory_type retval = mem_transaction_local; 1379 1380 if (!entry_block 1381 || TREE_CODE (x) != SSA_NAME 1382 /* Possible uninitialized use, or a function argument. In 1383 either case, we don't care. */ 1384 || SSA_NAME_IS_DEFAULT_DEF (x)) 1385 return mem_non_local; 1386 1387 /* Look in cache first. */ 1388 elt.val = x; 1389 slot = tm_new_mem_hash->find_slot (&elt, INSERT); 1390 elt_p = *slot; 1391 if (elt_p) 1392 return elt_p->local_new_memory; 1393 1394 /* Optimistically assume the memory is transaction local during 1395 processing. This catches recursion into this variable. */ 1396 *slot = elt_p = XNEW (tm_new_mem_map); 1397 elt_p->val = val; 1398 elt_p->local_new_memory = mem_transaction_local; 1399 1400 /* Search DEF chain to find the original definition of this address. */ 1401 do 1402 { 1403 if (ptr_deref_may_alias_global_p (x, true)) 1404 { 1405 /* Address escapes. This is not thread-private. */ 1406 retval = mem_non_local; 1407 goto new_memory_ret; 1408 } 1409 1410 stmt = SSA_NAME_DEF_STMT (x); 1411 1412 /* If the malloc call is outside the transaction, this is 1413 thread-local. */ 1414 if (retval != mem_thread_local 1415 && !dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt), entry_block)) 1416 retval = mem_thread_local; 1417 1418 if (is_gimple_assign (stmt)) 1419 { 1420 code = gimple_assign_rhs_code (stmt); 1421 /* x = foo ==> foo */ 1422 if (code == SSA_NAME) 1423 x = gimple_assign_rhs1 (stmt); 1424 /* x = foo + n ==> foo */ 1425 else if (code == POINTER_PLUS_EXPR) 1426 x = gimple_assign_rhs1 (stmt); 1427 /* x = (cast*) foo ==> foo */ 1428 else if (code == VIEW_CONVERT_EXPR || CONVERT_EXPR_CODE_P (code)) 1429 x = gimple_assign_rhs1 (stmt); 1430 /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */ 1431 else if (code == COND_EXPR) 1432 { 1433 tree op1 = gimple_assign_rhs2 (stmt); 1434 tree op2 = gimple_assign_rhs3 (stmt); 1435 enum thread_memory_type mem; 1436 retval = thread_private_new_memory (entry_block, op1); 1437 if (retval == mem_non_local) 1438 goto new_memory_ret; 1439 mem = thread_private_new_memory (entry_block, op2); 1440 retval = MIN (retval, mem); 1441 goto new_memory_ret; 1442 } 1443 else 1444 { 1445 retval = mem_non_local; 1446 goto new_memory_ret; 1447 } 1448 } 1449 else 1450 { 1451 if (gimple_code (stmt) == GIMPLE_PHI) 1452 { 1453 unsigned int i; 1454 enum thread_memory_type mem; 1455 tree phi_result = gimple_phi_result (stmt); 1456 1457 /* If any of the ancestors are non-local, we are sure to 1458 be non-local. Otherwise we can avoid doing anything 1459 and inherit what has already been generated. */ 1460 retval = mem_max; 1461 for (i = 0; i < gimple_phi_num_args (stmt); ++i) 1462 { 1463 tree op = PHI_ARG_DEF (stmt, i); 1464 1465 /* Exclude self-assignment. */ 1466 if (phi_result == op) 1467 continue; 1468 1469 mem = thread_private_new_memory (entry_block, op); 1470 if (mem == mem_non_local) 1471 { 1472 retval = mem; 1473 goto new_memory_ret; 1474 } 1475 retval = MIN (retval, mem); 1476 } 1477 goto new_memory_ret; 1478 } 1479 break; 1480 } 1481 } 1482 while (TREE_CODE (x) == SSA_NAME); 1483 1484 if (stmt && is_gimple_call (stmt) && gimple_call_flags (stmt) & ECF_MALLOC) 1485 /* Thread-local or transaction-local. */ 1486 ; 1487 else 1488 retval = mem_non_local; 1489 1490 new_memory_ret: 1491 elt_p->local_new_memory = retval; 1492 return retval; 1493} 1494 1495/* Determine whether X has to be instrumented using a read 1496 or write barrier. 1497 1498 ENTRY_BLOCK is the entry block for the region where stmt resides 1499 in. NULL if unknown. 1500 1501 STMT is the statement in which X occurs in. It is used for thread 1502 private memory instrumentation. If no TPM instrumentation is 1503 desired, STMT should be null. */ 1504static bool 1505requires_barrier (basic_block entry_block, tree x, gimple *stmt) 1506{ 1507 tree orig = x; 1508 while (handled_component_p (x)) 1509 x = TREE_OPERAND (x, 0); 1510 1511 switch (TREE_CODE (x)) 1512 { 1513 case INDIRECT_REF: 1514 case MEM_REF: 1515 { 1516 enum thread_memory_type ret; 1517 1518 ret = thread_private_new_memory (entry_block, TREE_OPERAND (x, 0)); 1519 if (ret == mem_non_local) 1520 return true; 1521 if (stmt && ret == mem_thread_local) 1522 /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */ 1523 tm_log_add (entry_block, orig, stmt); 1524 1525 /* Transaction-locals require nothing at all. For malloc, a 1526 transaction restart frees the memory and we reallocate. 1527 For alloca, the stack pointer gets reset by the retry and 1528 we reallocate. */ 1529 return false; 1530 } 1531 1532 case TARGET_MEM_REF: 1533 if (TREE_CODE (TMR_BASE (x)) != ADDR_EXPR) 1534 return true; 1535 x = TREE_OPERAND (TMR_BASE (x), 0); 1536 if (TREE_CODE (x) == PARM_DECL) 1537 return false; 1538 gcc_assert (VAR_P (x)); 1539 /* FALLTHRU */ 1540 1541 case PARM_DECL: 1542 case RESULT_DECL: 1543 case VAR_DECL: 1544 if (DECL_BY_REFERENCE (x)) 1545 { 1546 /* ??? This value is a pointer, but aggregate_value_p has been 1547 jigged to return true which confuses needs_to_live_in_memory. 1548 This ought to be cleaned up generically. 1549 1550 FIXME: Verify this still happens after the next mainline 1551 merge. Testcase ie g++.dg/tm/pr47554.C. 1552 */ 1553 return false; 1554 } 1555 1556 if (is_global_var (x)) 1557 return !TREE_READONLY (x); 1558 if (/* FIXME: This condition should actually go below in the 1559 tm_log_add() call, however is_call_clobbered() depends on 1560 aliasing info which is not available during 1561 gimplification. Since requires_barrier() gets called 1562 during lower_sequence_tm/gimplification, leave the call 1563 to needs_to_live_in_memory until we eliminate 1564 lower_sequence_tm altogether. */ 1565 needs_to_live_in_memory (x)) 1566 return true; 1567 else 1568 { 1569 /* For local memory that doesn't escape (aka thread private 1570 memory), we can either save the value at the beginning of 1571 the transaction and restore on restart, or call a tm 1572 function to dynamically save and restore on restart 1573 (ITM_L*). */ 1574 if (stmt) 1575 tm_log_add (entry_block, orig, stmt); 1576 return false; 1577 } 1578 1579 default: 1580 return false; 1581 } 1582} 1583 1584/* Mark the GIMPLE_ASSIGN statement as appropriate for being inside 1585 a transaction region. */ 1586 1587static void 1588examine_assign_tm (unsigned *state, gimple_stmt_iterator *gsi) 1589{ 1590 gimple *stmt = gsi_stmt (*gsi); 1591 1592 if (requires_barrier (/*entry_block=*/NULL, gimple_assign_rhs1 (stmt), NULL)) 1593 *state |= GTMA_HAVE_LOAD; 1594 if (requires_barrier (/*entry_block=*/NULL, gimple_assign_lhs (stmt), NULL)) 1595 *state |= GTMA_HAVE_STORE; 1596} 1597 1598/* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */ 1599 1600static void 1601examine_call_tm (unsigned *state, gimple_stmt_iterator *gsi) 1602{ 1603 gimple *stmt = gsi_stmt (*gsi); 1604 tree fn; 1605 1606 if (is_tm_pure_call (stmt)) 1607 return; 1608 1609 /* Check if this call is a transaction abort. */ 1610 fn = gimple_call_fndecl (stmt); 1611 if (is_tm_abort (fn)) 1612 *state |= GTMA_HAVE_ABORT; 1613 1614 /* Note that something may happen. */ 1615 *state |= GTMA_HAVE_LOAD | GTMA_HAVE_STORE; 1616} 1617 1618/* Iterate through the statements in the sequence, moving labels 1619 (and thus edges) of transactions from "label_norm" to "label_uninst". */ 1620 1621static tree 1622make_tm_uninst (gimple_stmt_iterator *gsi, bool *handled_ops_p, 1623 struct walk_stmt_info *) 1624{ 1625 gimple *stmt = gsi_stmt (*gsi); 1626 1627 if (gtransaction *txn = dyn_cast <gtransaction *> (stmt)) 1628 { 1629 *handled_ops_p = true; 1630 txn->label_uninst = txn->label_norm; 1631 txn->label_norm = NULL; 1632 } 1633 else 1634 *handled_ops_p = !gimple_has_substatements (stmt); 1635 1636 return NULL_TREE; 1637} 1638 1639/* Lower a GIMPLE_TRANSACTION statement. */ 1640 1641static void 1642lower_transaction (gimple_stmt_iterator *gsi, struct walk_stmt_info *wi) 1643{ 1644 gimple *g; 1645 gtransaction *stmt = as_a <gtransaction *> (gsi_stmt (*gsi)); 1646 unsigned int *outer_state = (unsigned int *) wi->info; 1647 unsigned int this_state = 0; 1648 struct walk_stmt_info this_wi; 1649 1650 /* First, lower the body. The scanning that we do inside gives 1651 us some idea of what we're dealing with. */ 1652 memset (&this_wi, 0, sizeof (this_wi)); 1653 this_wi.info = (void *) &this_state; 1654 walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt), 1655 lower_sequence_tm, NULL, &this_wi); 1656 1657 /* If there was absolutely nothing transaction related inside the 1658 transaction, we may elide it. Likewise if this is a nested 1659 transaction and does not contain an abort. */ 1660 if (this_state == 0 1661 || (!(this_state & GTMA_HAVE_ABORT) && outer_state != NULL)) 1662 { 1663 if (outer_state) 1664 *outer_state |= this_state; 1665 1666 gsi_insert_seq_before (gsi, gimple_transaction_body (stmt), 1667 GSI_SAME_STMT); 1668 gimple_transaction_set_body (stmt, NULL); 1669 1670 gsi_remove (gsi, true); 1671 wi->removed_stmt = true; 1672 return; 1673 } 1674 1675 /* Wrap the body of the transaction in a try-finally node so that 1676 the commit call is always properly called. */ 1677 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT), 0); 1678 if (flag_exceptions) 1679 { 1680 tree ptr; 1681 gimple_seq n_seq, e_seq; 1682 1683 n_seq = gimple_seq_alloc_with_stmt (g); 1684 e_seq = NULL; 1685 1686 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER), 1687 1, integer_zero_node); 1688 ptr = create_tmp_var (ptr_type_node); 1689 gimple_call_set_lhs (g, ptr); 1690 gimple_seq_add_stmt (&e_seq, g); 1691 1692 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH), 1693 1, ptr); 1694 gimple_seq_add_stmt (&e_seq, g); 1695 1696 g = gimple_build_eh_else (n_seq, e_seq); 1697 } 1698 1699 g = gimple_build_try (gimple_transaction_body (stmt), 1700 gimple_seq_alloc_with_stmt (g), GIMPLE_TRY_FINALLY); 1701 1702 /* For a (potentially) outer transaction, create two paths. */ 1703 gimple_seq uninst = NULL; 1704 if (outer_state == NULL) 1705 { 1706 uninst = copy_gimple_seq_and_replace_locals (g); 1707 /* In the uninstrumented copy, reset inner transactions to have only 1708 an uninstrumented code path. */ 1709 memset (&this_wi, 0, sizeof (this_wi)); 1710 walk_gimple_seq (uninst, make_tm_uninst, NULL, &this_wi); 1711 } 1712 1713 tree label1 = create_artificial_label (UNKNOWN_LOCATION); 1714 gsi_insert_after (gsi, gimple_build_label (label1), GSI_CONTINUE_LINKING); 1715 gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING); 1716 gimple_transaction_set_label_norm (stmt, label1); 1717 1718 /* If the transaction calls abort or if this is an outer transaction, 1719 add an "over" label afterwards. */ 1720 tree label3 = NULL; 1721 if ((this_state & GTMA_HAVE_ABORT) 1722 || outer_state == NULL 1723 || (gimple_transaction_subcode (stmt) & GTMA_IS_OUTER)) 1724 { 1725 label3 = create_artificial_label (UNKNOWN_LOCATION); 1726 gimple_transaction_set_label_over (stmt, label3); 1727 } 1728 1729 if (uninst != NULL) 1730 { 1731 gsi_insert_after (gsi, gimple_build_goto (label3), GSI_CONTINUE_LINKING); 1732 1733 tree label2 = create_artificial_label (UNKNOWN_LOCATION); 1734 gsi_insert_after (gsi, gimple_build_label (label2), GSI_CONTINUE_LINKING); 1735 gsi_insert_seq_after (gsi, uninst, GSI_CONTINUE_LINKING); 1736 gimple_transaction_set_label_uninst (stmt, label2); 1737 } 1738 1739 if (label3 != NULL) 1740 gsi_insert_after (gsi, gimple_build_label (label3), GSI_CONTINUE_LINKING); 1741 1742 gimple_transaction_set_body (stmt, NULL); 1743 1744 /* Record the set of operations found for use later. */ 1745 this_state |= gimple_transaction_subcode (stmt) & GTMA_DECLARATION_MASK; 1746 gimple_transaction_set_subcode (stmt, this_state); 1747} 1748 1749/* Iterate through the statements in the sequence, lowering them all 1750 as appropriate for being in a transaction. */ 1751 1752static tree 1753lower_sequence_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p, 1754 struct walk_stmt_info *wi) 1755{ 1756 unsigned int *state = (unsigned int *) wi->info; 1757 gimple *stmt = gsi_stmt (*gsi); 1758 1759 *handled_ops_p = true; 1760 switch (gimple_code (stmt)) 1761 { 1762 case GIMPLE_ASSIGN: 1763 /* Only memory reads/writes need to be instrumented. */ 1764 if (gimple_assign_single_p (stmt)) 1765 examine_assign_tm (state, gsi); 1766 break; 1767 1768 case GIMPLE_CALL: 1769 examine_call_tm (state, gsi); 1770 break; 1771 1772 case GIMPLE_ASM: 1773 *state |= GTMA_MAY_ENTER_IRREVOCABLE; 1774 break; 1775 1776 case GIMPLE_TRANSACTION: 1777 lower_transaction (gsi, wi); 1778 break; 1779 1780 default: 1781 *handled_ops_p = !gimple_has_substatements (stmt); 1782 break; 1783 } 1784 1785 return NULL_TREE; 1786} 1787 1788/* Iterate through the statements in the sequence, lowering them all 1789 as appropriate for being outside of a transaction. */ 1790 1791static tree 1792lower_sequence_no_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p, 1793 struct walk_stmt_info * wi) 1794{ 1795 gimple *stmt = gsi_stmt (*gsi); 1796 1797 if (gimple_code (stmt) == GIMPLE_TRANSACTION) 1798 { 1799 *handled_ops_p = true; 1800 lower_transaction (gsi, wi); 1801 } 1802 else 1803 *handled_ops_p = !gimple_has_substatements (stmt); 1804 1805 return NULL_TREE; 1806} 1807 1808/* Main entry point for flattening GIMPLE_TRANSACTION constructs. After 1809 this, GIMPLE_TRANSACTION nodes still exist, but the nested body has 1810 been moved out, and all the data required for constructing a proper 1811 CFG has been recorded. */ 1812 1813static unsigned int 1814execute_lower_tm (void) 1815{ 1816 struct walk_stmt_info wi; 1817 gimple_seq body; 1818 1819 /* Transactional clones aren't created until a later pass. */ 1820 gcc_assert (!decl_is_tm_clone (current_function_decl)); 1821 1822 body = gimple_body (current_function_decl); 1823 memset (&wi, 0, sizeof (wi)); 1824 walk_gimple_seq_mod (&body, lower_sequence_no_tm, NULL, &wi); 1825 gimple_set_body (current_function_decl, body); 1826 1827 return 0; 1828} 1829 1830namespace { 1831 1832const pass_data pass_data_lower_tm = 1833{ 1834 GIMPLE_PASS, /* type */ 1835 "tmlower", /* name */ 1836 OPTGROUP_NONE, /* optinfo_flags */ 1837 TV_TRANS_MEM, /* tv_id */ 1838 PROP_gimple_lcf, /* properties_required */ 1839 0, /* properties_provided */ 1840 0, /* properties_destroyed */ 1841 0, /* todo_flags_start */ 1842 0, /* todo_flags_finish */ 1843}; 1844 1845class pass_lower_tm : public gimple_opt_pass 1846{ 1847public: 1848 pass_lower_tm (gcc::context *ctxt) 1849 : gimple_opt_pass (pass_data_lower_tm, ctxt) 1850 {} 1851 1852 /* opt_pass methods: */ 1853 virtual bool gate (function *) { return flag_tm; } 1854 virtual unsigned int execute (function *) { return execute_lower_tm (); } 1855 1856}; // class pass_lower_tm 1857 1858} // anon namespace 1859 1860gimple_opt_pass * 1861make_pass_lower_tm (gcc::context *ctxt) 1862{ 1863 return new pass_lower_tm (ctxt); 1864} 1865 1866/* Collect region information for each transaction. */ 1867 1868struct tm_region 1869{ 1870public: 1871 1872 /* The field "transaction_stmt" is initially a gtransaction *, 1873 but eventually gets lowered to a gcall *(to BUILT_IN_TM_START). 1874 1875 Helper method to get it as a gtransaction *, with code-checking 1876 in a checked-build. */ 1877 1878 gtransaction * 1879 get_transaction_stmt () const 1880 { 1881 return as_a <gtransaction *> (transaction_stmt); 1882 } 1883 1884public: 1885 1886 /* Link to the next unnested transaction. */ 1887 struct tm_region *next; 1888 1889 /* Link to the next inner transaction. */ 1890 struct tm_region *inner; 1891 1892 /* Link to the next outer transaction. */ 1893 struct tm_region *outer; 1894 1895 /* The GIMPLE_TRANSACTION statement beginning this transaction. 1896 After TM_MARK, this gets replaced by a call to 1897 BUILT_IN_TM_START. 1898 Hence this will be either a gtransaction *or a gcall *. */ 1899 gimple *transaction_stmt; 1900 1901 /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to 1902 BUILT_IN_TM_START, this field is true if the transaction is an 1903 outer transaction. */ 1904 bool original_transaction_was_outer; 1905 1906 /* Return value from BUILT_IN_TM_START. */ 1907 tree tm_state; 1908 1909 /* The entry block to this region. This will always be the first 1910 block of the body of the transaction. */ 1911 basic_block entry_block; 1912 1913 /* The first block after an expanded call to _ITM_beginTransaction. */ 1914 basic_block restart_block; 1915 1916 /* The set of all blocks that end the region; NULL if only EXIT_BLOCK. 1917 These blocks are still a part of the region (i.e., the border is 1918 inclusive). Note that this set is only complete for paths in the CFG 1919 starting at ENTRY_BLOCK, and that there is no exit block recorded for 1920 the edge to the "over" label. */ 1921 bitmap exit_blocks; 1922 1923 /* The set of all blocks that have an TM_IRREVOCABLE call. */ 1924 bitmap irr_blocks; 1925}; 1926 1927/* True if there are pending edge statements to be committed for the 1928 current function being scanned in the tmmark pass. */ 1929bool pending_edge_inserts_p; 1930 1931static struct tm_region *all_tm_regions; 1932static bitmap_obstack tm_obstack; 1933 1934 1935/* A subroutine of tm_region_init. Record the existence of the 1936 GIMPLE_TRANSACTION statement in a tree of tm_region elements. */ 1937 1938static struct tm_region * 1939tm_region_init_0 (struct tm_region *outer, basic_block bb, 1940 gtransaction *stmt) 1941{ 1942 struct tm_region *region; 1943 1944 region = (struct tm_region *) 1945 obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region)); 1946 1947 if (outer) 1948 { 1949 region->next = outer->inner; 1950 outer->inner = region; 1951 } 1952 else 1953 { 1954 region->next = all_tm_regions; 1955 all_tm_regions = region; 1956 } 1957 region->inner = NULL; 1958 region->outer = outer; 1959 1960 region->transaction_stmt = stmt; 1961 region->original_transaction_was_outer = false; 1962 region->tm_state = NULL; 1963 1964 /* There are either one or two edges out of the block containing 1965 the GIMPLE_TRANSACTION, one to the actual region and one to the 1966 "over" label if the region contains an abort. The former will 1967 always be the one marked FALLTHRU. */ 1968 region->entry_block = FALLTHRU_EDGE (bb)->dest; 1969 1970 region->exit_blocks = BITMAP_ALLOC (&tm_obstack); 1971 region->irr_blocks = BITMAP_ALLOC (&tm_obstack); 1972 1973 return region; 1974} 1975 1976/* A subroutine of tm_region_init. Record all the exit and 1977 irrevocable blocks in BB into the region's exit_blocks and 1978 irr_blocks bitmaps. Returns the new region being scanned. */ 1979 1980static struct tm_region * 1981tm_region_init_1 (struct tm_region *region, basic_block bb) 1982{ 1983 gimple_stmt_iterator gsi; 1984 gimple *g; 1985 1986 if (!region 1987 || (!region->irr_blocks && !region->exit_blocks)) 1988 return region; 1989 1990 /* Check to see if this is the end of a region by seeing if it 1991 contains a call to __builtin_tm_commit{,_eh}. Note that the 1992 outermost region for DECL_IS_TM_CLONE need not collect this. */ 1993 for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi)) 1994 { 1995 g = gsi_stmt (gsi); 1996 if (gimple_code (g) == GIMPLE_CALL) 1997 { 1998 tree fn = gimple_call_fndecl (g); 1999 if (fn && fndecl_built_in_p (fn, BUILT_IN_NORMAL)) 2000 { 2001 if ((DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT 2002 || DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT_EH) 2003 && region->exit_blocks) 2004 { 2005 bitmap_set_bit (region->exit_blocks, bb->index); 2006 region = region->outer; 2007 break; 2008 } 2009 if (DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_IRREVOCABLE) 2010 bitmap_set_bit (region->irr_blocks, bb->index); 2011 } 2012 } 2013 } 2014 return region; 2015} 2016 2017/* Collect all of the transaction regions within the current function 2018 and record them in ALL_TM_REGIONS. The REGION parameter may specify 2019 an "outermost" region for use by tm clones. */ 2020 2021static void 2022tm_region_init (struct tm_region *region) 2023{ 2024 gimple *g; 2025 edge_iterator ei; 2026 edge e; 2027 basic_block bb; 2028 auto_vec<basic_block> queue; 2029 bitmap visited_blocks = BITMAP_ALLOC (NULL); 2030 struct tm_region *old_region; 2031 auto_vec<tm_region *> bb_regions; 2032 2033 /* We could store this information in bb->aux, but we may get called 2034 through get_all_tm_blocks() from another pass that may be already 2035 using bb->aux. */ 2036 bb_regions.safe_grow_cleared (last_basic_block_for_fn (cfun), true); 2037 2038 all_tm_regions = region; 2039 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 2040 queue.safe_push (bb); 2041 bitmap_set_bit (visited_blocks, bb->index); 2042 bb_regions[bb->index] = region; 2043 2044 do 2045 { 2046 bb = queue.pop (); 2047 region = bb_regions[bb->index]; 2048 bb_regions[bb->index] = NULL; 2049 2050 /* Record exit and irrevocable blocks. */ 2051 region = tm_region_init_1 (region, bb); 2052 2053 /* Check for the last statement in the block beginning a new region. */ 2054 g = last_stmt (bb); 2055 old_region = region; 2056 if (g) 2057 if (gtransaction *trans_stmt = dyn_cast <gtransaction *> (g)) 2058 region = tm_region_init_0 (region, bb, trans_stmt); 2059 2060 /* Process subsequent blocks. */ 2061 FOR_EACH_EDGE (e, ei, bb->succs) 2062 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 2063 { 2064 bitmap_set_bit (visited_blocks, e->dest->index); 2065 queue.safe_push (e->dest); 2066 2067 /* If the current block started a new region, make sure that only 2068 the entry block of the new region is associated with this region. 2069 Other successors are still part of the old region. */ 2070 if (old_region != region && e->dest != region->entry_block) 2071 bb_regions[e->dest->index] = old_region; 2072 else 2073 bb_regions[e->dest->index] = region; 2074 } 2075 } 2076 while (!queue.is_empty ()); 2077 BITMAP_FREE (visited_blocks); 2078} 2079 2080/* The "gate" function for all transactional memory expansion and optimization 2081 passes. We collect region information for each top-level transaction, and 2082 if we don't find any, we skip all of the TM passes. Each region will have 2083 all of the exit blocks recorded, and the originating statement. */ 2084 2085static bool 2086gate_tm_init (void) 2087{ 2088 if (!flag_tm) 2089 return false; 2090 2091 calculate_dominance_info (CDI_DOMINATORS); 2092 bitmap_obstack_initialize (&tm_obstack); 2093 2094 /* If the function is a TM_CLONE, then the entire function is the region. */ 2095 if (decl_is_tm_clone (current_function_decl)) 2096 { 2097 struct tm_region *region = (struct tm_region *) 2098 obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region)); 2099 memset (region, 0, sizeof (*region)); 2100 region->entry_block = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 2101 /* For a clone, the entire function is the region. But even if 2102 we don't need to record any exit blocks, we may need to 2103 record irrevocable blocks. */ 2104 region->irr_blocks = BITMAP_ALLOC (&tm_obstack); 2105 2106 tm_region_init (region); 2107 } 2108 else 2109 { 2110 tm_region_init (NULL); 2111 2112 /* If we didn't find any regions, cleanup and skip the whole tree 2113 of tm-related optimizations. */ 2114 if (all_tm_regions == NULL) 2115 { 2116 bitmap_obstack_release (&tm_obstack); 2117 return false; 2118 } 2119 } 2120 2121 return true; 2122} 2123 2124namespace { 2125 2126const pass_data pass_data_tm_init = 2127{ 2128 GIMPLE_PASS, /* type */ 2129 "*tminit", /* name */ 2130 OPTGROUP_NONE, /* optinfo_flags */ 2131 TV_TRANS_MEM, /* tv_id */ 2132 ( PROP_ssa | PROP_cfg ), /* properties_required */ 2133 0, /* properties_provided */ 2134 0, /* properties_destroyed */ 2135 0, /* todo_flags_start */ 2136 0, /* todo_flags_finish */ 2137}; 2138 2139class pass_tm_init : public gimple_opt_pass 2140{ 2141public: 2142 pass_tm_init (gcc::context *ctxt) 2143 : gimple_opt_pass (pass_data_tm_init, ctxt) 2144 {} 2145 2146 /* opt_pass methods: */ 2147 virtual bool gate (function *) { return gate_tm_init (); } 2148 2149}; // class pass_tm_init 2150 2151} // anon namespace 2152 2153gimple_opt_pass * 2154make_pass_tm_init (gcc::context *ctxt) 2155{ 2156 return new pass_tm_init (ctxt); 2157} 2158 2159/* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region 2160 represented by STATE. */ 2161 2162static inline void 2163transaction_subcode_ior (struct tm_region *region, unsigned flags) 2164{ 2165 if (region && region->transaction_stmt) 2166 { 2167 gtransaction *transaction_stmt = region->get_transaction_stmt (); 2168 flags |= gimple_transaction_subcode (transaction_stmt); 2169 gimple_transaction_set_subcode (transaction_stmt, flags); 2170 } 2171} 2172 2173/* Construct a memory load in a transactional context. Return the 2174 gimple statement performing the load, or NULL if there is no 2175 TM_LOAD builtin of the appropriate size to do the load. 2176 2177 LOC is the location to use for the new statement(s). */ 2178 2179static gcall * 2180build_tm_load (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi) 2181{ 2182 tree t, type = TREE_TYPE (rhs); 2183 gcall *gcall; 2184 2185 built_in_function code; 2186 if (type == float_type_node) 2187 code = BUILT_IN_TM_LOAD_FLOAT; 2188 else if (type == double_type_node) 2189 code = BUILT_IN_TM_LOAD_DOUBLE; 2190 else if (type == long_double_type_node) 2191 code = BUILT_IN_TM_LOAD_LDOUBLE; 2192 else 2193 { 2194 if (TYPE_SIZE (type) == NULL || !tree_fits_uhwi_p (TYPE_SIZE (type))) 2195 return NULL; 2196 unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type)); 2197 2198 if (TREE_CODE (type) == VECTOR_TYPE) 2199 { 2200 switch (type_size) 2201 { 2202 case 64: 2203 code = BUILT_IN_TM_LOAD_M64; 2204 break; 2205 case 128: 2206 code = BUILT_IN_TM_LOAD_M128; 2207 break; 2208 case 256: 2209 code = BUILT_IN_TM_LOAD_M256; 2210 break; 2211 default: 2212 goto unhandled_vec; 2213 } 2214 if (!builtin_decl_explicit_p (code)) 2215 goto unhandled_vec; 2216 } 2217 else 2218 { 2219 unhandled_vec: 2220 switch (type_size) 2221 { 2222 case 8: 2223 code = BUILT_IN_TM_LOAD_1; 2224 break; 2225 case 16: 2226 code = BUILT_IN_TM_LOAD_2; 2227 break; 2228 case 32: 2229 code = BUILT_IN_TM_LOAD_4; 2230 break; 2231 case 64: 2232 code = BUILT_IN_TM_LOAD_8; 2233 break; 2234 default: 2235 return NULL; 2236 } 2237 } 2238 } 2239 2240 tree decl = builtin_decl_explicit (code); 2241 gcc_assert (decl); 2242 2243 t = gimplify_addr (gsi, rhs); 2244 gcall = gimple_build_call (decl, 1, t); 2245 gimple_set_location (gcall, loc); 2246 2247 t = TREE_TYPE (TREE_TYPE (decl)); 2248 if (useless_type_conversion_p (type, t)) 2249 { 2250 gimple_call_set_lhs (gcall, lhs); 2251 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2252 } 2253 else 2254 { 2255 gimple *g; 2256 tree temp; 2257 2258 temp = create_tmp_reg (t); 2259 gimple_call_set_lhs (gcall, temp); 2260 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2261 2262 t = fold_build1 (VIEW_CONVERT_EXPR, type, temp); 2263 g = gimple_build_assign (lhs, t); 2264 gsi_insert_before (gsi, g, GSI_SAME_STMT); 2265 } 2266 2267 return gcall; 2268} 2269 2270 2271/* Similarly for storing TYPE in a transactional context. */ 2272 2273static gcall * 2274build_tm_store (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi) 2275{ 2276 tree t, fn, type = TREE_TYPE (rhs), simple_type; 2277 gcall *gcall; 2278 2279 built_in_function code; 2280 if (type == float_type_node) 2281 code = BUILT_IN_TM_STORE_FLOAT; 2282 else if (type == double_type_node) 2283 code = BUILT_IN_TM_STORE_DOUBLE; 2284 else if (type == long_double_type_node) 2285 code = BUILT_IN_TM_STORE_LDOUBLE; 2286 else 2287 { 2288 if (TYPE_SIZE (type) == NULL || !tree_fits_uhwi_p (TYPE_SIZE (type))) 2289 return NULL; 2290 unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type)); 2291 2292 if (TREE_CODE (type) == VECTOR_TYPE) 2293 { 2294 switch (type_size) 2295 { 2296 case 64: 2297 code = BUILT_IN_TM_STORE_M64; 2298 break; 2299 case 128: 2300 code = BUILT_IN_TM_STORE_M128; 2301 break; 2302 case 256: 2303 code = BUILT_IN_TM_STORE_M256; 2304 break; 2305 default: 2306 goto unhandled_vec; 2307 } 2308 if (!builtin_decl_explicit_p (code)) 2309 goto unhandled_vec; 2310 } 2311 else 2312 { 2313 unhandled_vec: 2314 switch (type_size) 2315 { 2316 case 8: 2317 code = BUILT_IN_TM_STORE_1; 2318 break; 2319 case 16: 2320 code = BUILT_IN_TM_STORE_2; 2321 break; 2322 case 32: 2323 code = BUILT_IN_TM_STORE_4; 2324 break; 2325 case 64: 2326 code = BUILT_IN_TM_STORE_8; 2327 break; 2328 default: 2329 return NULL; 2330 } 2331 } 2332 } 2333 2334 fn = builtin_decl_explicit (code); 2335 gcc_assert (fn); 2336 2337 simple_type = TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn)))); 2338 2339 if (TREE_CODE (rhs) == CONSTRUCTOR) 2340 { 2341 /* Handle the easy initialization to zero. */ 2342 if (!CONSTRUCTOR_ELTS (rhs)) 2343 rhs = build_int_cst (simple_type, 0); 2344 else 2345 { 2346 /* ...otherwise punt to the caller and probably use 2347 BUILT_IN_TM_MEMMOVE, because we can't wrap a 2348 VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce 2349 valid gimple. */ 2350 return NULL; 2351 } 2352 } 2353 else if (!useless_type_conversion_p (simple_type, type)) 2354 { 2355 gimple *g; 2356 tree temp; 2357 2358 temp = create_tmp_reg (simple_type); 2359 t = fold_build1 (VIEW_CONVERT_EXPR, simple_type, rhs); 2360 g = gimple_build_assign (temp, t); 2361 gimple_set_location (g, loc); 2362 gsi_insert_before (gsi, g, GSI_SAME_STMT); 2363 2364 rhs = temp; 2365 } 2366 2367 t = gimplify_addr (gsi, lhs); 2368 gcall = gimple_build_call (fn, 2, t, rhs); 2369 gimple_set_location (gcall, loc); 2370 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2371 2372 return gcall; 2373} 2374 2375 2376/* Expand an assignment statement into transactional builtins. */ 2377 2378static void 2379expand_assign_tm (struct tm_region *region, gimple_stmt_iterator *gsi) 2380{ 2381 gimple *stmt = gsi_stmt (*gsi); 2382 location_t loc = gimple_location (stmt); 2383 tree lhs = gimple_assign_lhs (stmt); 2384 tree rhs = gimple_assign_rhs1 (stmt); 2385 bool store_p = requires_barrier (region->entry_block, lhs, NULL); 2386 bool load_p = requires_barrier (region->entry_block, rhs, NULL); 2387 gimple *gcall = NULL; 2388 2389 if (!load_p && !store_p) 2390 { 2391 /* Add thread private addresses to log if applicable. */ 2392 requires_barrier (region->entry_block, lhs, stmt); 2393 gsi_next (gsi); 2394 return; 2395 } 2396 2397 if (load_p) 2398 transaction_subcode_ior (region, GTMA_HAVE_LOAD); 2399 if (store_p) 2400 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2401 2402 // Remove original load/store statement. 2403 gsi_remove (gsi, true); 2404 2405 // Attempt to use a simple load/store helper function. 2406 if (load_p && !store_p) 2407 gcall = build_tm_load (loc, lhs, rhs, gsi); 2408 else if (store_p && !load_p) 2409 gcall = build_tm_store (loc, lhs, rhs, gsi); 2410 2411 // If gcall has not been set, then we do not have a simple helper 2412 // function available for the type. This may be true of larger 2413 // structures, vectors, and non-standard float types. 2414 if (!gcall) 2415 { 2416 tree lhs_addr, rhs_addr, ltmp = NULL, copy_fn; 2417 2418 // If this is a type that we couldn't handle above, but it's 2419 // in a register, we must spill it to memory for the copy. 2420 if (is_gimple_reg (lhs)) 2421 { 2422 ltmp = create_tmp_var (TREE_TYPE (lhs)); 2423 lhs_addr = build_fold_addr_expr (ltmp); 2424 } 2425 else 2426 lhs_addr = gimplify_addr (gsi, lhs); 2427 if (is_gimple_reg (rhs)) 2428 { 2429 tree rtmp = create_tmp_var (TREE_TYPE (rhs)); 2430 TREE_ADDRESSABLE (rtmp) = 1; 2431 rhs_addr = build_fold_addr_expr (rtmp); 2432 gcall = gimple_build_assign (rtmp, rhs); 2433 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2434 } 2435 else 2436 rhs_addr = gimplify_addr (gsi, rhs); 2437 2438 // Choose the appropriate memory transfer function. 2439 if (load_p && store_p) 2440 { 2441 // ??? Figure out if there's any possible overlap between 2442 // the LHS and the RHS and if not, use MEMCPY. 2443 copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMMOVE); 2444 } 2445 else if (load_p) 2446 { 2447 // Note that the store is non-transactional and cannot overlap. 2448 copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RTWN); 2449 } 2450 else 2451 { 2452 // Note that the load is non-transactional and cannot overlap. 2453 copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RNWT); 2454 } 2455 2456 gcall = gimple_build_call (copy_fn, 3, lhs_addr, rhs_addr, 2457 TYPE_SIZE_UNIT (TREE_TYPE (lhs))); 2458 gimple_set_location (gcall, loc); 2459 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2460 2461 if (ltmp) 2462 { 2463 gcall = gimple_build_assign (lhs, ltmp); 2464 gsi_insert_before (gsi, gcall, GSI_SAME_STMT); 2465 } 2466 } 2467 2468 // Now that we have the load/store in its instrumented form, add 2469 // thread private addresses to the log if applicable. 2470 if (!store_p) 2471 requires_barrier (region->entry_block, lhs, gcall); 2472} 2473 2474 2475/* Expand a call statement as appropriate for a transaction. That is, 2476 either verify that the call does not affect the transaction, or 2477 redirect the call to a clone that handles transactions, or change 2478 the transaction state to IRREVOCABLE. Return true if the call is 2479 one of the builtins that end a transaction. */ 2480 2481static bool 2482expand_call_tm (struct tm_region *region, 2483 gimple_stmt_iterator *gsi) 2484{ 2485 gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi)); 2486 tree lhs = gimple_call_lhs (stmt); 2487 tree fn_decl; 2488 struct cgraph_node *node; 2489 bool retval = false; 2490 2491 fn_decl = gimple_call_fndecl (stmt); 2492 2493 if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMCPY) 2494 || fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMMOVE)) 2495 transaction_subcode_ior (region, GTMA_HAVE_STORE | GTMA_HAVE_LOAD); 2496 if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMSET)) 2497 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2498 2499 if (is_tm_pure_call (stmt)) 2500 return false; 2501 2502 if (fn_decl) 2503 retval = is_tm_ending_fndecl (fn_decl); 2504 if (!retval) 2505 { 2506 /* Assume all non-const/pure calls write to memory, except 2507 transaction ending builtins. */ 2508 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2509 } 2510 2511 /* For indirect calls, we already generated a call into the runtime. */ 2512 if (!fn_decl) 2513 { 2514 tree fn = gimple_call_fn (stmt); 2515 2516 /* We are guaranteed never to go irrevocable on a safe or pure 2517 call, and the pure call was handled above. */ 2518 if (is_tm_safe (fn)) 2519 return false; 2520 else 2521 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 2522 2523 return false; 2524 } 2525 2526 node = cgraph_node::get (fn_decl); 2527 /* All calls should have cgraph here. */ 2528 if (!node) 2529 { 2530 /* We can have a nodeless call here if some pass after IPA-tm 2531 added uninstrumented calls. For example, loop distribution 2532 can transform certain loop constructs into __builtin_mem* 2533 calls. In this case, see if we have a suitable TM 2534 replacement and fill in the gaps. */ 2535 gcc_assert (DECL_BUILT_IN_CLASS (fn_decl) == BUILT_IN_NORMAL); 2536 enum built_in_function code = DECL_FUNCTION_CODE (fn_decl); 2537 gcc_assert (code == BUILT_IN_MEMCPY 2538 || code == BUILT_IN_MEMMOVE 2539 || code == BUILT_IN_MEMSET); 2540 2541 tree repl = find_tm_replacement_function (fn_decl); 2542 if (repl) 2543 { 2544 gimple_call_set_fndecl (stmt, repl); 2545 update_stmt (stmt); 2546 node = cgraph_node::create (repl); 2547 node->tm_may_enter_irr = false; 2548 return expand_call_tm (region, gsi); 2549 } 2550 gcc_unreachable (); 2551 } 2552 if (node->tm_may_enter_irr) 2553 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 2554 2555 if (is_tm_abort (fn_decl)) 2556 { 2557 transaction_subcode_ior (region, GTMA_HAVE_ABORT); 2558 return true; 2559 } 2560 2561 /* Instrument the store if needed. 2562 2563 If the assignment happens inside the function call (return slot 2564 optimization), there is no instrumentation to be done, since 2565 the callee should have done the right thing. */ 2566 if (lhs && requires_barrier (region->entry_block, lhs, stmt) 2567 && !gimple_call_return_slot_opt_p (stmt)) 2568 { 2569 tree tmp = create_tmp_reg (TREE_TYPE (lhs)); 2570 location_t loc = gimple_location (stmt); 2571 edge fallthru_edge = NULL; 2572 gassign *assign_stmt; 2573 2574 /* Remember if the call was going to throw. */ 2575 if (stmt_can_throw_internal (cfun, stmt)) 2576 { 2577 edge_iterator ei; 2578 edge e; 2579 basic_block bb = gimple_bb (stmt); 2580 2581 FOR_EACH_EDGE (e, ei, bb->succs) 2582 if (e->flags & EDGE_FALLTHRU) 2583 { 2584 fallthru_edge = e; 2585 break; 2586 } 2587 } 2588 2589 gimple_call_set_lhs (stmt, tmp); 2590 update_stmt (stmt); 2591 assign_stmt = gimple_build_assign (lhs, tmp); 2592 gimple_set_location (assign_stmt, loc); 2593 2594 /* We cannot throw in the middle of a BB. If the call was going 2595 to throw, place the instrumentation on the fallthru edge, so 2596 the call remains the last statement in the block. */ 2597 if (fallthru_edge) 2598 { 2599 gimple_seq fallthru_seq = gimple_seq_alloc_with_stmt (assign_stmt); 2600 gimple_stmt_iterator fallthru_gsi = gsi_start (fallthru_seq); 2601 expand_assign_tm (region, &fallthru_gsi); 2602 gsi_insert_seq_on_edge (fallthru_edge, fallthru_seq); 2603 pending_edge_inserts_p = true; 2604 } 2605 else 2606 { 2607 gsi_insert_after (gsi, assign_stmt, GSI_CONTINUE_LINKING); 2608 expand_assign_tm (region, gsi); 2609 } 2610 2611 transaction_subcode_ior (region, GTMA_HAVE_STORE); 2612 } 2613 2614 return retval; 2615} 2616 2617 2618/* Expand all statements in BB as appropriate for being inside 2619 a transaction. */ 2620 2621static void 2622expand_block_tm (struct tm_region *region, basic_block bb) 2623{ 2624 gimple_stmt_iterator gsi; 2625 2626 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); ) 2627 { 2628 gimple *stmt = gsi_stmt (gsi); 2629 switch (gimple_code (stmt)) 2630 { 2631 case GIMPLE_ASSIGN: 2632 /* Only memory reads/writes need to be instrumented. */ 2633 if (gimple_assign_single_p (stmt) 2634 && !gimple_clobber_p (stmt)) 2635 { 2636 expand_assign_tm (region, &gsi); 2637 continue; 2638 } 2639 break; 2640 2641 case GIMPLE_CALL: 2642 if (expand_call_tm (region, &gsi)) 2643 return; 2644 break; 2645 2646 case GIMPLE_ASM: 2647 gcc_unreachable (); 2648 2649 default: 2650 break; 2651 } 2652 if (!gsi_end_p (gsi)) 2653 gsi_next (&gsi); 2654 } 2655} 2656 2657/* Return the list of basic-blocks in REGION. 2658 2659 STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks 2660 following a TM_IRREVOCABLE call. 2661 2662 INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the 2663 uninstrumented code path blocks in the list of basic blocks 2664 returned, false otherwise. */ 2665 2666static vec<basic_block> 2667get_tm_region_blocks (basic_block entry_block, 2668 bitmap exit_blocks, 2669 bitmap irr_blocks, 2670 bitmap all_region_blocks, 2671 bool stop_at_irrevocable_p, 2672 bool include_uninstrumented_p = true) 2673{ 2674 vec<basic_block> bbs = vNULL; 2675 unsigned i; 2676 edge e; 2677 edge_iterator ei; 2678 bitmap visited_blocks = BITMAP_ALLOC (NULL); 2679 2680 i = 0; 2681 bbs.safe_push (entry_block); 2682 bitmap_set_bit (visited_blocks, entry_block->index); 2683 2684 do 2685 { 2686 basic_block bb = bbs[i++]; 2687 2688 if (exit_blocks && 2689 bitmap_bit_p (exit_blocks, bb->index)) 2690 continue; 2691 2692 if (stop_at_irrevocable_p 2693 && irr_blocks 2694 && bitmap_bit_p (irr_blocks, bb->index)) 2695 continue; 2696 2697 FOR_EACH_EDGE (e, ei, bb->succs) 2698 if ((include_uninstrumented_p 2699 || !(e->flags & EDGE_TM_UNINSTRUMENTED)) 2700 && !bitmap_bit_p (visited_blocks, e->dest->index)) 2701 { 2702 bitmap_set_bit (visited_blocks, e->dest->index); 2703 bbs.safe_push (e->dest); 2704 } 2705 } 2706 while (i < bbs.length ()); 2707 2708 if (all_region_blocks) 2709 bitmap_ior_into (all_region_blocks, visited_blocks); 2710 2711 BITMAP_FREE (visited_blocks); 2712 return bbs; 2713} 2714 2715// Callback data for collect_bb2reg. 2716struct bb2reg_stuff 2717{ 2718 vec<tm_region *> *bb2reg; 2719 bool include_uninstrumented_p; 2720}; 2721 2722// Callback for expand_regions, collect innermost region data for each bb. 2723static void * 2724collect_bb2reg (struct tm_region *region, void *data) 2725{ 2726 struct bb2reg_stuff *stuff = (struct bb2reg_stuff *)data; 2727 vec<tm_region *> *bb2reg = stuff->bb2reg; 2728 vec<basic_block> queue; 2729 unsigned int i; 2730 basic_block bb; 2731 2732 queue = get_tm_region_blocks (region->entry_block, 2733 region->exit_blocks, 2734 region->irr_blocks, 2735 NULL, 2736 /*stop_at_irr_p=*/true, 2737 stuff->include_uninstrumented_p); 2738 2739 // We expect expand_region to perform a post-order traversal of the region 2740 // tree. Therefore the last region seen for any bb is the innermost. 2741 FOR_EACH_VEC_ELT (queue, i, bb) 2742 (*bb2reg)[bb->index] = region; 2743 2744 queue.release (); 2745 return NULL; 2746} 2747 2748// Returns a vector, indexed by BB->INDEX, of the innermost tm_region to 2749// which a basic block belongs. Note that we only consider the instrumented 2750// code paths for the region; the uninstrumented code paths are ignored if 2751// INCLUDE_UNINSTRUMENTED_P is false. 2752// 2753// ??? This data is very similar to the bb_regions array that is collected 2754// during tm_region_init. Or, rather, this data is similar to what could 2755// be used within tm_region_init. The actual computation in tm_region_init 2756// begins and ends with bb_regions entirely full of NULL pointers, due to 2757// the way in which pointers are swapped in and out of the array. 2758// 2759// ??? Our callers expect that blocks are not shared between transactions. 2760// When the optimizers get too smart, and blocks are shared, then during 2761// the tm_mark phase we'll add log entries to only one of the two transactions, 2762// and in the tm_edge phase we'll add edges to the CFG that create invalid 2763// cycles. The symptom being SSA defs that do not dominate their uses. 2764// Note that the optimizers were locally correct with their transformation, 2765// as we have no info within the program that suggests that the blocks cannot 2766// be shared. 2767// 2768// ??? There is currently a hack inside tree-ssa-pre.cc to work around the 2769// only known instance of this block sharing. 2770 2771static vec<tm_region *> 2772get_bb_regions_instrumented (bool traverse_clones, 2773 bool include_uninstrumented_p) 2774{ 2775 unsigned n = last_basic_block_for_fn (cfun); 2776 struct bb2reg_stuff stuff; 2777 vec<tm_region *> ret; 2778 2779 ret.create (n); 2780 ret.safe_grow_cleared (n, true); 2781 stuff.bb2reg = &ret; 2782 stuff.include_uninstrumented_p = include_uninstrumented_p; 2783 expand_regions (all_tm_regions, collect_bb2reg, &stuff, traverse_clones); 2784 2785 return ret; 2786} 2787 2788/* Set the IN_TRANSACTION for all gimple statements that appear in a 2789 transaction. */ 2790 2791void 2792compute_transaction_bits (void) 2793{ 2794 struct tm_region *region; 2795 vec<basic_block> queue; 2796 unsigned int i; 2797 basic_block bb; 2798 2799 /* ?? Perhaps we need to abstract gate_tm_init further, because we 2800 certainly don't need it to calculate CDI_DOMINATOR info. */ 2801 gate_tm_init (); 2802 2803 FOR_EACH_BB_FN (bb, cfun) 2804 bb->flags &= ~BB_IN_TRANSACTION; 2805 2806 for (region = all_tm_regions; region; region = region->next) 2807 { 2808 queue = get_tm_region_blocks (region->entry_block, 2809 region->exit_blocks, 2810 region->irr_blocks, 2811 NULL, 2812 /*stop_at_irr_p=*/true); 2813 for (i = 0; queue.iterate (i, &bb); ++i) 2814 bb->flags |= BB_IN_TRANSACTION; 2815 queue.release (); 2816 } 2817 2818 if (all_tm_regions) 2819 bitmap_obstack_release (&tm_obstack); 2820} 2821 2822/* Replace the GIMPLE_TRANSACTION in this region with the corresponding 2823 call to BUILT_IN_TM_START. */ 2824 2825static void * 2826expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED) 2827{ 2828 tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START); 2829 basic_block transaction_bb = gimple_bb (region->transaction_stmt); 2830 tree tm_state = region->tm_state; 2831 tree tm_state_type = TREE_TYPE (tm_state); 2832 edge abort_edge = NULL; 2833 edge inst_edge = NULL; 2834 edge uninst_edge = NULL; 2835 edge fallthru_edge = NULL; 2836 2837 // Identify the various successors of the transaction start. 2838 { 2839 edge_iterator i; 2840 edge e; 2841 FOR_EACH_EDGE (e, i, transaction_bb->succs) 2842 { 2843 if (e->flags & EDGE_TM_ABORT) 2844 abort_edge = e; 2845 else if (e->flags & EDGE_TM_UNINSTRUMENTED) 2846 uninst_edge = e; 2847 else 2848 inst_edge = e; 2849 if (e->flags & EDGE_FALLTHRU) 2850 fallthru_edge = e; 2851 } 2852 } 2853 2854 /* ??? There are plenty of bits here we're not computing. */ 2855 { 2856 int subcode = gimple_transaction_subcode (region->get_transaction_stmt ()); 2857 int flags = 0; 2858 if (subcode & GTMA_DOES_GO_IRREVOCABLE) 2859 flags |= PR_DOESGOIRREVOCABLE; 2860 if ((subcode & GTMA_MAY_ENTER_IRREVOCABLE) == 0) 2861 flags |= PR_HASNOIRREVOCABLE; 2862 /* If the transaction does not have an abort in lexical scope and is not 2863 marked as an outer transaction, then it will never abort. */ 2864 if ((subcode & GTMA_HAVE_ABORT) == 0 && (subcode & GTMA_IS_OUTER) == 0) 2865 flags |= PR_HASNOABORT; 2866 if ((subcode & GTMA_HAVE_STORE) == 0) 2867 flags |= PR_READONLY; 2868 if (inst_edge && !(subcode & GTMA_HAS_NO_INSTRUMENTATION)) 2869 flags |= PR_INSTRUMENTEDCODE; 2870 if (uninst_edge) 2871 flags |= PR_UNINSTRUMENTEDCODE; 2872 if (subcode & GTMA_IS_OUTER) 2873 region->original_transaction_was_outer = true; 2874 tree t = build_int_cst (tm_state_type, flags); 2875 gcall *call = gimple_build_call (tm_start, 1, t); 2876 gimple_call_set_lhs (call, tm_state); 2877 gimple_set_location (call, gimple_location (region->transaction_stmt)); 2878 2879 // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START. 2880 gimple_stmt_iterator gsi = gsi_last_bb (transaction_bb); 2881 gcc_assert (gsi_stmt (gsi) == region->transaction_stmt); 2882 gsi_insert_before (&gsi, call, GSI_SAME_STMT); 2883 gsi_remove (&gsi, true); 2884 region->transaction_stmt = call; 2885 } 2886 2887 // Generate log saves. 2888 if (!tm_log_save_addresses.is_empty ()) 2889 tm_log_emit_saves (region->entry_block, transaction_bb); 2890 2891 // In the beginning, we've no tests to perform on transaction restart. 2892 // Note that after this point, transaction_bb becomes the "most recent 2893 // block containing tests for the transaction". 2894 region->restart_block = region->entry_block; 2895 2896 // Generate log restores. 2897 if (!tm_log_save_addresses.is_empty ()) 2898 { 2899 basic_block test_bb = create_empty_bb (transaction_bb); 2900 basic_block code_bb = create_empty_bb (test_bb); 2901 basic_block join_bb = create_empty_bb (code_bb); 2902 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2903 add_bb_to_loop (code_bb, transaction_bb->loop_father); 2904 add_bb_to_loop (join_bb, transaction_bb->loop_father); 2905 if (region->restart_block == region->entry_block) 2906 region->restart_block = test_bb; 2907 2908 tree t1 = create_tmp_reg (tm_state_type); 2909 tree t2 = build_int_cst (tm_state_type, A_RESTORELIVEVARIABLES); 2910 gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2911 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2912 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2913 2914 t2 = build_int_cst (tm_state_type, 0); 2915 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2916 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2917 2918 tm_log_emit_restores (region->entry_block, code_bb); 2919 2920 edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU); 2921 edge et = make_edge (test_bb, code_bb, EDGE_TRUE_VALUE); 2922 edge ef = make_edge (test_bb, join_bb, EDGE_FALSE_VALUE); 2923 redirect_edge_pred (fallthru_edge, join_bb); 2924 2925 join_bb->count = test_bb->count = transaction_bb->count; 2926 2927 ei->probability = profile_probability::always (); 2928 et->probability = profile_probability::likely (); 2929 ef->probability = profile_probability::unlikely (); 2930 2931 code_bb->count = et->count (); 2932 2933 transaction_bb = join_bb; 2934 } 2935 2936 // If we have an ABORT edge, create a test to perform the abort. 2937 if (abort_edge) 2938 { 2939 basic_block test_bb = create_empty_bb (transaction_bb); 2940 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2941 if (region->restart_block == region->entry_block) 2942 region->restart_block = test_bb; 2943 2944 tree t1 = create_tmp_reg (tm_state_type); 2945 tree t2 = build_int_cst (tm_state_type, A_ABORTTRANSACTION); 2946 gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2947 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2948 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2949 2950 t2 = build_int_cst (tm_state_type, 0); 2951 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2952 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2953 2954 edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU); 2955 test_bb->count = transaction_bb->count; 2956 ei->probability = profile_probability::always (); 2957 2958 // Not abort edge. If both are live, chose one at random as we'll 2959 // we'll be fixing that up below. 2960 redirect_edge_pred (fallthru_edge, test_bb); 2961 fallthru_edge->flags = EDGE_FALSE_VALUE; 2962 fallthru_edge->probability = profile_probability::very_likely (); 2963 2964 // Abort/over edge. 2965 redirect_edge_pred (abort_edge, test_bb); 2966 abort_edge->flags = EDGE_TRUE_VALUE; 2967 abort_edge->probability = profile_probability::unlikely (); 2968 2969 transaction_bb = test_bb; 2970 } 2971 2972 // If we have both instrumented and uninstrumented code paths, select one. 2973 if (inst_edge && uninst_edge) 2974 { 2975 basic_block test_bb = create_empty_bb (transaction_bb); 2976 add_bb_to_loop (test_bb, transaction_bb->loop_father); 2977 if (region->restart_block == region->entry_block) 2978 region->restart_block = test_bb; 2979 2980 tree t1 = create_tmp_reg (tm_state_type); 2981 tree t2 = build_int_cst (tm_state_type, A_RUNUNINSTRUMENTEDCODE); 2982 2983 gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2); 2984 gimple_stmt_iterator gsi = gsi_last_bb (test_bb); 2985 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2986 2987 t2 = build_int_cst (tm_state_type, 0); 2988 stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL); 2989 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 2990 2991 // Create the edge into test_bb first, as we want to copy values 2992 // out of the fallthru edge. 2993 edge e = make_edge (transaction_bb, test_bb, fallthru_edge->flags); 2994 e->probability = fallthru_edge->probability; 2995 test_bb->count = fallthru_edge->count (); 2996 2997 // Now update the edges to the inst/uninist implementations. 2998 // For now assume that the paths are equally likely. When using HTM, 2999 // we'll try the uninst path first and fallback to inst path if htm 3000 // buffers are exceeded. Without HTM we start with the inst path and 3001 // use the uninst path when falling back to serial mode. 3002 redirect_edge_pred (inst_edge, test_bb); 3003 inst_edge->flags = EDGE_FALSE_VALUE; 3004 inst_edge->probability = profile_probability::even (); 3005 3006 redirect_edge_pred (uninst_edge, test_bb); 3007 uninst_edge->flags = EDGE_TRUE_VALUE; 3008 uninst_edge->probability = profile_probability::even (); 3009 } 3010 3011 // If we have no previous special cases, and we have PHIs at the beginning 3012 // of the atomic region, this means we have a loop at the beginning of the 3013 // atomic region that shares the first block. This can cause problems with 3014 // the transaction restart abnormal edges to be added in the tm_edges pass. 3015 // Solve this by adding a new empty block to receive the abnormal edges. 3016 if (region->restart_block == region->entry_block 3017 && phi_nodes (region->entry_block)) 3018 { 3019 basic_block empty_bb = create_empty_bb (transaction_bb); 3020 region->restart_block = empty_bb; 3021 add_bb_to_loop (empty_bb, transaction_bb->loop_father); 3022 3023 redirect_edge_pred (fallthru_edge, empty_bb); 3024 make_edge (transaction_bb, empty_bb, EDGE_FALLTHRU); 3025 } 3026 3027 return NULL; 3028} 3029 3030/* Generate the temporary to be used for the return value of 3031 BUILT_IN_TM_START. */ 3032 3033static void * 3034generate_tm_state (struct tm_region *region, void *data ATTRIBUTE_UNUSED) 3035{ 3036 tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START); 3037 region->tm_state = 3038 create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start)), "tm_state"); 3039 3040 // Reset the subcode, post optimizations. We'll fill this in 3041 // again as we process blocks. 3042 if (region->exit_blocks) 3043 { 3044 gtransaction *transaction_stmt = region->get_transaction_stmt (); 3045 unsigned int subcode = gimple_transaction_subcode (transaction_stmt); 3046 3047 if (subcode & GTMA_DOES_GO_IRREVOCABLE) 3048 subcode &= (GTMA_DECLARATION_MASK | GTMA_DOES_GO_IRREVOCABLE 3049 | GTMA_MAY_ENTER_IRREVOCABLE 3050 | GTMA_HAS_NO_INSTRUMENTATION); 3051 else 3052 subcode &= GTMA_DECLARATION_MASK; 3053 gimple_transaction_set_subcode (transaction_stmt, subcode); 3054 } 3055 3056 return NULL; 3057} 3058 3059// Propagate flags from inner transactions outwards. 3060static void 3061propagate_tm_flags_out (struct tm_region *region) 3062{ 3063 if (region == NULL) 3064 return; 3065 propagate_tm_flags_out (region->inner); 3066 3067 if (region->outer && region->outer->transaction_stmt) 3068 { 3069 unsigned s 3070 = gimple_transaction_subcode (region->get_transaction_stmt ()); 3071 s &= (GTMA_HAVE_ABORT | GTMA_HAVE_LOAD | GTMA_HAVE_STORE 3072 | GTMA_MAY_ENTER_IRREVOCABLE); 3073 s |= gimple_transaction_subcode (region->outer->get_transaction_stmt ()); 3074 gimple_transaction_set_subcode (region->outer->get_transaction_stmt (), 3075 s); 3076 } 3077 3078 propagate_tm_flags_out (region->next); 3079} 3080 3081/* Entry point to the MARK phase of TM expansion. Here we replace 3082 transactional memory statements with calls to builtins, and function 3083 calls with their transactional clones (if available). But we don't 3084 yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */ 3085 3086static unsigned int 3087execute_tm_mark (void) 3088{ 3089 pending_edge_inserts_p = false; 3090 3091 expand_regions (all_tm_regions, generate_tm_state, NULL, 3092 /*traverse_clones=*/true); 3093 3094 tm_log_init (); 3095 3096 vec<tm_region *> bb_regions 3097 = get_bb_regions_instrumented (/*traverse_clones=*/true, 3098 /*include_uninstrumented_p=*/false); 3099 struct tm_region *r; 3100 unsigned i; 3101 3102 // Expand memory operations into calls into the runtime. 3103 // This collects log entries as well. 3104 FOR_EACH_VEC_ELT (bb_regions, i, r) 3105 { 3106 if (r != NULL) 3107 { 3108 if (r->transaction_stmt) 3109 { 3110 unsigned sub 3111 = gimple_transaction_subcode (r->get_transaction_stmt ()); 3112 3113 /* If we're sure to go irrevocable, there won't be 3114 anything to expand, since the run-time will go 3115 irrevocable right away. */ 3116 if (sub & GTMA_DOES_GO_IRREVOCABLE 3117 && sub & GTMA_MAY_ENTER_IRREVOCABLE) 3118 continue; 3119 } 3120 expand_block_tm (r, BASIC_BLOCK_FOR_FN (cfun, i)); 3121 } 3122 } 3123 3124 bb_regions.release (); 3125 3126 // Propagate flags from inner transactions outwards. 3127 propagate_tm_flags_out (all_tm_regions); 3128 3129 // Expand GIMPLE_TRANSACTIONs into calls into the runtime. 3130 expand_regions (all_tm_regions, expand_transaction, NULL, 3131 /*traverse_clones=*/false); 3132 3133 tm_log_emit (); 3134 tm_log_delete (); 3135 3136 if (pending_edge_inserts_p) 3137 gsi_commit_edge_inserts (); 3138 free_dominance_info (CDI_DOMINATORS); 3139 return 0; 3140} 3141 3142namespace { 3143 3144const pass_data pass_data_tm_mark = 3145{ 3146 GIMPLE_PASS, /* type */ 3147 "tmmark", /* name */ 3148 OPTGROUP_NONE, /* optinfo_flags */ 3149 TV_TRANS_MEM, /* tv_id */ 3150 ( PROP_ssa | PROP_cfg ), /* properties_required */ 3151 0, /* properties_provided */ 3152 0, /* properties_destroyed */ 3153 0, /* todo_flags_start */ 3154 TODO_update_ssa, /* todo_flags_finish */ 3155}; 3156 3157class pass_tm_mark : public gimple_opt_pass 3158{ 3159public: 3160 pass_tm_mark (gcc::context *ctxt) 3161 : gimple_opt_pass (pass_data_tm_mark, ctxt) 3162 {} 3163 3164 /* opt_pass methods: */ 3165 virtual unsigned int execute (function *) { return execute_tm_mark (); } 3166 3167}; // class pass_tm_mark 3168 3169} // anon namespace 3170 3171gimple_opt_pass * 3172make_pass_tm_mark (gcc::context *ctxt) 3173{ 3174 return new pass_tm_mark (ctxt); 3175} 3176 3177 3178/* Create an abnormal edge from STMT at iter, splitting the block 3179 as necessary. Adjust *PNEXT as needed for the split block. */ 3180 3181static inline void 3182split_bb_make_tm_edge (gimple *stmt, basic_block dest_bb, 3183 gimple_stmt_iterator iter, gimple_stmt_iterator *pnext) 3184{ 3185 basic_block bb = gimple_bb (stmt); 3186 if (!gsi_one_before_end_p (iter)) 3187 { 3188 edge e = split_block (bb, stmt); 3189 *pnext = gsi_start_bb (e->dest); 3190 } 3191 edge e = make_edge (bb, dest_bb, EDGE_ABNORMAL); 3192 if (e) 3193 e->probability = profile_probability::guessed_never (); 3194 3195 // Record the need for the edge for the benefit of the rtl passes. 3196 if (cfun->gimple_df->tm_restart == NULL) 3197 cfun->gimple_df->tm_restart 3198 = hash_table<tm_restart_hasher>::create_ggc (31); 3199 3200 struct tm_restart_node dummy; 3201 dummy.stmt = stmt; 3202 dummy.label_or_list = gimple_block_label (dest_bb); 3203 3204 tm_restart_node **slot = cfun->gimple_df->tm_restart->find_slot (&dummy, 3205 INSERT); 3206 struct tm_restart_node *n = *slot; 3207 if (n == NULL) 3208 { 3209 n = ggc_alloc<tm_restart_node> (); 3210 *n = dummy; 3211 } 3212 else 3213 { 3214 tree old = n->label_or_list; 3215 if (TREE_CODE (old) == LABEL_DECL) 3216 old = tree_cons (NULL, old, NULL); 3217 n->label_or_list = tree_cons (NULL, dummy.label_or_list, old); 3218 } 3219} 3220 3221/* Split block BB as necessary for every builtin function we added, and 3222 wire up the abnormal back edges implied by the transaction restart. */ 3223 3224static void 3225expand_block_edges (struct tm_region *const region, basic_block bb) 3226{ 3227 gimple_stmt_iterator gsi, next_gsi; 3228 3229 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi = next_gsi) 3230 { 3231 gimple *stmt = gsi_stmt (gsi); 3232 gcall *call_stmt; 3233 3234 next_gsi = gsi; 3235 gsi_next (&next_gsi); 3236 3237 // ??? Shouldn't we split for any non-pure, non-irrevocable function? 3238 call_stmt = dyn_cast <gcall *> (stmt); 3239 if ((!call_stmt) 3240 || (gimple_call_flags (call_stmt) & ECF_TM_BUILTIN) == 0) 3241 continue; 3242 3243 if (gimple_call_builtin_p (call_stmt, BUILT_IN_TM_ABORT)) 3244 { 3245 // If we have a ``_transaction_cancel [[outer]]'', there is only 3246 // one abnormal edge: to the transaction marked OUTER. 3247 // All compiler-generated instances of BUILT_IN_TM_ABORT have a 3248 // constant argument, which we can examine here. Users invoking 3249 // TM_ABORT directly get what they deserve. 3250 tree arg = gimple_call_arg (call_stmt, 0); 3251 if (TREE_CODE (arg) == INTEGER_CST 3252 && (TREE_INT_CST_LOW (arg) & AR_OUTERABORT) != 0 3253 && !decl_is_tm_clone (current_function_decl)) 3254 { 3255 // Find the GTMA_IS_OUTER transaction. 3256 for (struct tm_region *o = region; o; o = o->outer) 3257 if (o->original_transaction_was_outer) 3258 { 3259 split_bb_make_tm_edge (call_stmt, o->restart_block, 3260 gsi, &next_gsi); 3261 break; 3262 } 3263 3264 // Otherwise, the front-end should have semantically checked 3265 // outer aborts, but in either case the target region is not 3266 // within this function. 3267 continue; 3268 } 3269 3270 // Non-outer, TM aborts have an abnormal edge to the inner-most 3271 // transaction, the one being aborted; 3272 split_bb_make_tm_edge (call_stmt, region->restart_block, gsi, 3273 &next_gsi); 3274 } 3275 3276 // All TM builtins have an abnormal edge to the outer-most transaction. 3277 // We never restart inner transactions. For tm clones, we know a-priori 3278 // that the outer-most transaction is outside the function. 3279 if (decl_is_tm_clone (current_function_decl)) 3280 continue; 3281 3282 if (cfun->gimple_df->tm_restart == NULL) 3283 cfun->gimple_df->tm_restart 3284 = hash_table<tm_restart_hasher>::create_ggc (31); 3285 3286 // All TM builtins have an abnormal edge to the outer-most transaction. 3287 // We never restart inner transactions. 3288 for (struct tm_region *o = region; o; o = o->outer) 3289 if (!o->outer) 3290 { 3291 split_bb_make_tm_edge (call_stmt, o->restart_block, gsi, &next_gsi); 3292 break; 3293 } 3294 3295 // Delete any tail-call annotation that may have been added. 3296 // The tail-call pass may have mis-identified the commit as being 3297 // a candidate because we had not yet added this restart edge. 3298 gimple_call_set_tail (call_stmt, false); 3299 } 3300} 3301 3302/* Entry point to the final expansion of transactional nodes. */ 3303 3304namespace { 3305 3306const pass_data pass_data_tm_edges = 3307{ 3308 GIMPLE_PASS, /* type */ 3309 "tmedge", /* name */ 3310 OPTGROUP_NONE, /* optinfo_flags */ 3311 TV_TRANS_MEM, /* tv_id */ 3312 ( PROP_ssa | PROP_cfg ), /* properties_required */ 3313 0, /* properties_provided */ 3314 0, /* properties_destroyed */ 3315 0, /* todo_flags_start */ 3316 TODO_update_ssa, /* todo_flags_finish */ 3317}; 3318 3319class pass_tm_edges : public gimple_opt_pass 3320{ 3321public: 3322 pass_tm_edges (gcc::context *ctxt) 3323 : gimple_opt_pass (pass_data_tm_edges, ctxt) 3324 {} 3325 3326 /* opt_pass methods: */ 3327 virtual unsigned int execute (function *); 3328 3329}; // class pass_tm_edges 3330 3331unsigned int 3332pass_tm_edges::execute (function *fun) 3333{ 3334 vec<tm_region *> bb_regions 3335 = get_bb_regions_instrumented (/*traverse_clones=*/false, 3336 /*include_uninstrumented_p=*/true); 3337 struct tm_region *r; 3338 unsigned i; 3339 3340 FOR_EACH_VEC_ELT (bb_regions, i, r) 3341 if (r != NULL) 3342 expand_block_edges (r, BASIC_BLOCK_FOR_FN (fun, i)); 3343 3344 bb_regions.release (); 3345 3346 /* We've got to release the dominance info now, to indicate that it 3347 must be rebuilt completely. Otherwise we'll crash trying to update 3348 the SSA web in the TODO section following this pass. */ 3349 free_dominance_info (CDI_DOMINATORS); 3350 /* We'ge also wrecked loops badly with inserting of abnormal edges. */ 3351 loops_state_set (LOOPS_NEED_FIXUP); 3352 bitmap_obstack_release (&tm_obstack); 3353 all_tm_regions = NULL; 3354 3355 return 0; 3356} 3357 3358} // anon namespace 3359 3360gimple_opt_pass * 3361make_pass_tm_edges (gcc::context *ctxt) 3362{ 3363 return new pass_tm_edges (ctxt); 3364} 3365 3366/* Helper function for expand_regions. Expand REGION and recurse to 3367 the inner region. Call CALLBACK on each region. CALLBACK returns 3368 NULL to continue the traversal, otherwise a non-null value which 3369 this function will return as well. TRAVERSE_CLONES is true if we 3370 should traverse transactional clones. */ 3371 3372static void * 3373expand_regions_1 (struct tm_region *region, 3374 void *(*callback)(struct tm_region *, void *), 3375 void *data, 3376 bool traverse_clones) 3377{ 3378 void *retval = NULL; 3379 if (region->exit_blocks 3380 || (traverse_clones && decl_is_tm_clone (current_function_decl))) 3381 { 3382 retval = callback (region, data); 3383 if (retval) 3384 return retval; 3385 } 3386 if (region->inner) 3387 { 3388 retval = expand_regions (region->inner, callback, data, traverse_clones); 3389 if (retval) 3390 return retval; 3391 } 3392 return retval; 3393} 3394 3395/* Traverse the regions enclosed and including REGION. Execute 3396 CALLBACK for each region, passing DATA. CALLBACK returns NULL to 3397 continue the traversal, otherwise a non-null value which this 3398 function will return as well. TRAVERSE_CLONES is true if we should 3399 traverse transactional clones. */ 3400 3401static void * 3402expand_regions (struct tm_region *region, 3403 void *(*callback)(struct tm_region *, void *), 3404 void *data, 3405 bool traverse_clones) 3406{ 3407 void *retval = NULL; 3408 while (region) 3409 { 3410 retval = expand_regions_1 (region, callback, data, traverse_clones); 3411 if (retval) 3412 return retval; 3413 region = region->next; 3414 } 3415 return retval; 3416} 3417 3418 3419/* A unique TM memory operation. */ 3420struct tm_memop 3421{ 3422 /* Unique ID that all memory operations to the same location have. */ 3423 unsigned int value_id; 3424 /* Address of load/store. */ 3425 tree addr; 3426}; 3427 3428/* TM memory operation hashtable helpers. */ 3429 3430struct tm_memop_hasher : free_ptr_hash <tm_memop> 3431{ 3432 static inline hashval_t hash (const tm_memop *); 3433 static inline bool equal (const tm_memop *, const tm_memop *); 3434}; 3435 3436/* Htab support. Return a hash value for a `tm_memop'. */ 3437inline hashval_t 3438tm_memop_hasher::hash (const tm_memop *mem) 3439{ 3440 tree addr = mem->addr; 3441 /* We drill down to the SSA_NAME/DECL for the hash, but equality is 3442 actually done with operand_equal_p (see tm_memop_eq). */ 3443 if (TREE_CODE (addr) == ADDR_EXPR) 3444 addr = TREE_OPERAND (addr, 0); 3445 return iterative_hash_expr (addr, 0); 3446} 3447 3448/* Htab support. Return true if two tm_memop's are the same. */ 3449inline bool 3450tm_memop_hasher::equal (const tm_memop *mem1, const tm_memop *mem2) 3451{ 3452 return operand_equal_p (mem1->addr, mem2->addr, 0); 3453} 3454 3455/* Sets for solving data flow equations in the memory optimization pass. */ 3456struct tm_memopt_bitmaps 3457{ 3458 /* Stores available to this BB upon entry. Basically, stores that 3459 dominate this BB. */ 3460 bitmap store_avail_in; 3461 /* Stores available at the end of this BB. */ 3462 bitmap store_avail_out; 3463 bitmap store_antic_in; 3464 bitmap store_antic_out; 3465 /* Reads available to this BB upon entry. Basically, reads that 3466 dominate this BB. */ 3467 bitmap read_avail_in; 3468 /* Reads available at the end of this BB. */ 3469 bitmap read_avail_out; 3470 /* Reads performed in this BB. */ 3471 bitmap read_local; 3472 /* Writes performed in this BB. */ 3473 bitmap store_local; 3474 3475 /* Temporary storage for pass. */ 3476 /* Is the current BB in the worklist? */ 3477 bool avail_in_worklist_p; 3478 /* Have we visited this BB? */ 3479 bool visited_p; 3480}; 3481 3482static bitmap_obstack tm_memopt_obstack; 3483 3484/* Unique counter for TM loads and stores. Loads and stores of the 3485 same address get the same ID. */ 3486static unsigned int tm_memopt_value_id; 3487static hash_table<tm_memop_hasher> *tm_memopt_value_numbers; 3488 3489#define STORE_AVAIL_IN(BB) \ 3490 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in 3491#define STORE_AVAIL_OUT(BB) \ 3492 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out 3493#define STORE_ANTIC_IN(BB) \ 3494 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in 3495#define STORE_ANTIC_OUT(BB) \ 3496 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out 3497#define READ_AVAIL_IN(BB) \ 3498 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in 3499#define READ_AVAIL_OUT(BB) \ 3500 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out 3501#define READ_LOCAL(BB) \ 3502 ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local 3503#define STORE_LOCAL(BB) \ 3504 ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local 3505#define AVAIL_IN_WORKLIST_P(BB) \ 3506 ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p 3507#define BB_VISITED_P(BB) \ 3508 ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p 3509 3510/* Given a TM load/store in STMT, return the value number for the address 3511 it accesses. */ 3512 3513static unsigned int 3514tm_memopt_value_number (gimple *stmt, enum insert_option op) 3515{ 3516 struct tm_memop tmpmem, *mem; 3517 tm_memop **slot; 3518 3519 gcc_assert (is_tm_load (stmt) || is_tm_store (stmt)); 3520 tmpmem.addr = gimple_call_arg (stmt, 0); 3521 slot = tm_memopt_value_numbers->find_slot (&tmpmem, op); 3522 if (*slot) 3523 mem = *slot; 3524 else if (op == INSERT) 3525 { 3526 mem = XNEW (struct tm_memop); 3527 *slot = mem; 3528 mem->value_id = tm_memopt_value_id++; 3529 mem->addr = tmpmem.addr; 3530 } 3531 else 3532 gcc_unreachable (); 3533 return mem->value_id; 3534} 3535 3536/* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */ 3537 3538static void 3539tm_memopt_accumulate_memops (basic_block bb) 3540{ 3541 gimple_stmt_iterator gsi; 3542 3543 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 3544 { 3545 gimple *stmt = gsi_stmt (gsi); 3546 bitmap bits; 3547 unsigned int loc; 3548 3549 if (is_tm_store (stmt)) 3550 bits = STORE_LOCAL (bb); 3551 else if (is_tm_load (stmt)) 3552 bits = READ_LOCAL (bb); 3553 else 3554 continue; 3555 3556 loc = tm_memopt_value_number (stmt, INSERT); 3557 bitmap_set_bit (bits, loc); 3558 if (dump_file) 3559 { 3560 fprintf (dump_file, "TM memopt (%s): value num=%d, BB=%d, addr=", 3561 is_tm_load (stmt) ? "LOAD" : "STORE", loc, 3562 gimple_bb (stmt)->index); 3563 print_generic_expr (dump_file, gimple_call_arg (stmt, 0)); 3564 fprintf (dump_file, "\n"); 3565 } 3566 } 3567} 3568 3569/* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */ 3570 3571static void 3572dump_tm_memopt_set (const char *set_name, bitmap bits) 3573{ 3574 unsigned i; 3575 bitmap_iterator bi; 3576 const char *comma = ""; 3577 3578 fprintf (dump_file, "TM memopt: %s: [", set_name); 3579 EXECUTE_IF_SET_IN_BITMAP (bits, 0, i, bi) 3580 { 3581 hash_table<tm_memop_hasher>::iterator hi; 3582 struct tm_memop *mem = NULL; 3583 3584 /* Yeah, yeah, yeah. Whatever. This is just for debugging. */ 3585 FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers, mem, tm_memop_t, hi) 3586 if (mem->value_id == i) 3587 break; 3588 gcc_assert (mem->value_id == i); 3589 fprintf (dump_file, "%s", comma); 3590 comma = ", "; 3591 print_generic_expr (dump_file, mem->addr); 3592 } 3593 fprintf (dump_file, "]\n"); 3594} 3595 3596/* Prettily dump all of the memopt sets in BLOCKS. */ 3597 3598static void 3599dump_tm_memopt_sets (vec<basic_block> blocks) 3600{ 3601 size_t i; 3602 basic_block bb; 3603 3604 for (i = 0; blocks.iterate (i, &bb); ++i) 3605 { 3606 fprintf (dump_file, "------------BB %d---------\n", bb->index); 3607 dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb)); 3608 dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb)); 3609 dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb)); 3610 dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb)); 3611 dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb)); 3612 dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb)); 3613 } 3614} 3615 3616/* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */ 3617 3618static void 3619tm_memopt_compute_avin (basic_block bb) 3620{ 3621 edge e; 3622 unsigned ix; 3623 3624 /* Seed with the AVOUT of any predecessor. */ 3625 for (ix = 0; ix < EDGE_COUNT (bb->preds); ix++) 3626 { 3627 e = EDGE_PRED (bb, ix); 3628 /* Make sure we have already visited this BB, and is thus 3629 initialized. 3630 3631 If e->src->aux is NULL, this predecessor is actually on an 3632 enclosing transaction. We only care about the current 3633 transaction, so ignore it. */ 3634 if (e->src->aux && BB_VISITED_P (e->src)) 3635 { 3636 bitmap_copy (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src)); 3637 bitmap_copy (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src)); 3638 break; 3639 } 3640 } 3641 3642 for (; ix < EDGE_COUNT (bb->preds); ix++) 3643 { 3644 e = EDGE_PRED (bb, ix); 3645 if (e->src->aux && BB_VISITED_P (e->src)) 3646 { 3647 bitmap_and_into (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src)); 3648 bitmap_and_into (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src)); 3649 } 3650 } 3651 3652 BB_VISITED_P (bb) = true; 3653} 3654 3655/* Compute the STORE_ANTIC_IN for the basic block BB. */ 3656 3657static void 3658tm_memopt_compute_antin (basic_block bb) 3659{ 3660 edge e; 3661 unsigned ix; 3662 3663 /* Seed with the ANTIC_OUT of any successor. */ 3664 for (ix = 0; ix < EDGE_COUNT (bb->succs); ix++) 3665 { 3666 e = EDGE_SUCC (bb, ix); 3667 /* Make sure we have already visited this BB, and is thus 3668 initialized. */ 3669 if (BB_VISITED_P (e->dest)) 3670 { 3671 bitmap_copy (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest)); 3672 break; 3673 } 3674 } 3675 3676 for (; ix < EDGE_COUNT (bb->succs); ix++) 3677 { 3678 e = EDGE_SUCC (bb, ix); 3679 if (BB_VISITED_P (e->dest)) 3680 bitmap_and_into (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest)); 3681 } 3682 3683 BB_VISITED_P (bb) = true; 3684} 3685 3686/* Compute the AVAIL sets for every basic block in BLOCKS. 3687 3688 We compute {STORE,READ}_AVAIL_{OUT,IN} as follows: 3689 3690 AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb]) 3691 AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors]) 3692 3693 This is basically what we do in lcm's compute_available(), but here 3694 we calculate two sets of sets (one for STOREs and one for READs), 3695 and we work on a region instead of the entire CFG. 3696 3697 REGION is the TM region. 3698 BLOCKS are the basic blocks in the region. */ 3699 3700static void 3701tm_memopt_compute_available (struct tm_region *region, 3702 vec<basic_block> blocks) 3703{ 3704 edge e; 3705 basic_block *worklist, *qin, *qout, *qend, bb; 3706 unsigned int qlen, i; 3707 edge_iterator ei; 3708 bool changed; 3709 3710 /* Allocate a worklist array/queue. Entries are only added to the 3711 list if they were not already on the list. So the size is 3712 bounded by the number of basic blocks in the region. */ 3713 gcc_assert (!blocks.is_empty ()); 3714 qlen = blocks.length () - 1; 3715 qin = qout = worklist = XNEWVEC (basic_block, qlen); 3716 3717 /* Put every block in the region on the worklist. */ 3718 for (i = 0; blocks.iterate (i, &bb); ++i) 3719 { 3720 /* Seed AVAIL_OUT with the LOCAL set. */ 3721 bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_LOCAL (bb)); 3722 bitmap_ior_into (READ_AVAIL_OUT (bb), READ_LOCAL (bb)); 3723 3724 AVAIL_IN_WORKLIST_P (bb) = true; 3725 /* No need to insert the entry block, since it has an AVIN of 3726 null, and an AVOUT that has already been seeded in. */ 3727 if (bb != region->entry_block) 3728 *qin++ = bb; 3729 } 3730 3731 /* The entry block has been initialized with the local sets. */ 3732 BB_VISITED_P (region->entry_block) = true; 3733 3734 qin = worklist; 3735 qend = &worklist[qlen]; 3736 3737 /* Iterate until the worklist is empty. */ 3738 while (qlen) 3739 { 3740 /* Take the first entry off the worklist. */ 3741 bb = *qout++; 3742 qlen--; 3743 3744 if (qout >= qend) 3745 qout = worklist; 3746 3747 /* This block can be added to the worklist again if necessary. */ 3748 AVAIL_IN_WORKLIST_P (bb) = false; 3749 tm_memopt_compute_avin (bb); 3750 3751 /* Note: We do not add the LOCAL sets here because we already 3752 seeded the AVAIL_OUT sets with them. */ 3753 changed = bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_AVAIL_IN (bb)); 3754 changed |= bitmap_ior_into (READ_AVAIL_OUT (bb), READ_AVAIL_IN (bb)); 3755 if (changed 3756 && (region->exit_blocks == NULL 3757 || !bitmap_bit_p (region->exit_blocks, bb->index))) 3758 /* If the out state of this block changed, then we need to add 3759 its successors to the worklist if they are not already in. */ 3760 FOR_EACH_EDGE (e, ei, bb->succs) 3761 if (!AVAIL_IN_WORKLIST_P (e->dest) 3762 && e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun)) 3763 { 3764 *qin++ = e->dest; 3765 AVAIL_IN_WORKLIST_P (e->dest) = true; 3766 qlen++; 3767 3768 if (qin >= qend) 3769 qin = worklist; 3770 } 3771 } 3772 3773 free (worklist); 3774 3775 if (dump_file) 3776 dump_tm_memopt_sets (blocks); 3777} 3778 3779/* Compute ANTIC sets for every basic block in BLOCKS. 3780 3781 We compute STORE_ANTIC_OUT as follows: 3782 3783 STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb]) 3784 STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors]) 3785 3786 REGION is the TM region. 3787 BLOCKS are the basic blocks in the region. */ 3788 3789static void 3790tm_memopt_compute_antic (struct tm_region *region, 3791 vec<basic_block> blocks) 3792{ 3793 edge e; 3794 basic_block *worklist, *qin, *qout, *qend, bb; 3795 unsigned int qlen; 3796 int i; 3797 edge_iterator ei; 3798 3799 /* Allocate a worklist array/queue. Entries are only added to the 3800 list if they were not already on the list. So the size is 3801 bounded by the number of basic blocks in the region. */ 3802 qin = qout = worklist = XNEWVEC (basic_block, blocks.length ()); 3803 3804 for (qlen = 0, i = blocks.length () - 1; i >= 0; --i) 3805 { 3806 bb = blocks[i]; 3807 3808 /* Seed ANTIC_OUT with the LOCAL set. */ 3809 bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_LOCAL (bb)); 3810 3811 /* Put every block in the region on the worklist. */ 3812 AVAIL_IN_WORKLIST_P (bb) = true; 3813 /* No need to insert exit blocks, since their ANTIC_IN is NULL, 3814 and their ANTIC_OUT has already been seeded in. */ 3815 if (region->exit_blocks 3816 && !bitmap_bit_p (region->exit_blocks, bb->index)) 3817 { 3818 qlen++; 3819 *qin++ = bb; 3820 } 3821 } 3822 3823 /* The exit blocks have been initialized with the local sets. */ 3824 if (region->exit_blocks) 3825 { 3826 unsigned int i; 3827 bitmap_iterator bi; 3828 EXECUTE_IF_SET_IN_BITMAP (region->exit_blocks, 0, i, bi) 3829 BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun, i)) = true; 3830 } 3831 3832 qin = worklist; 3833 qend = &worklist[qlen]; 3834 3835 /* Iterate until the worklist is empty. */ 3836 while (qlen) 3837 { 3838 /* Take the first entry off the worklist. */ 3839 bb = *qout++; 3840 qlen--; 3841 3842 if (qout >= qend) 3843 qout = worklist; 3844 3845 /* This block can be added to the worklist again if necessary. */ 3846 AVAIL_IN_WORKLIST_P (bb) = false; 3847 tm_memopt_compute_antin (bb); 3848 3849 /* Note: We do not add the LOCAL sets here because we already 3850 seeded the ANTIC_OUT sets with them. */ 3851 if (bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_ANTIC_IN (bb)) 3852 && bb != region->entry_block) 3853 /* If the out state of this block changed, then we need to add 3854 its predecessors to the worklist if they are not already in. */ 3855 FOR_EACH_EDGE (e, ei, bb->preds) 3856 if (!AVAIL_IN_WORKLIST_P (e->src)) 3857 { 3858 *qin++ = e->src; 3859 AVAIL_IN_WORKLIST_P (e->src) = true; 3860 qlen++; 3861 3862 if (qin >= qend) 3863 qin = worklist; 3864 } 3865 } 3866 3867 free (worklist); 3868 3869 if (dump_file) 3870 dump_tm_memopt_sets (blocks); 3871} 3872 3873/* Offsets of load variants from TM_LOAD. For example, 3874 BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*. 3875 See gtm-builtins.def. */ 3876#define TRANSFORM_RAR 1 3877#define TRANSFORM_RAW 2 3878#define TRANSFORM_RFW 3 3879/* Offsets of store variants from TM_STORE. */ 3880#define TRANSFORM_WAR 1 3881#define TRANSFORM_WAW 2 3882 3883/* Inform about a load/store optimization. */ 3884 3885static void 3886dump_tm_memopt_transform (gimple *stmt) 3887{ 3888 if (dump_file) 3889 { 3890 fprintf (dump_file, "TM memopt: transforming: "); 3891 print_gimple_stmt (dump_file, stmt, 0); 3892 fprintf (dump_file, "\n"); 3893 } 3894} 3895 3896/* Perform a read/write optimization. Replaces the TM builtin in STMT 3897 by a builtin that is OFFSET entries down in the builtins table in 3898 gtm-builtins.def. */ 3899 3900static void 3901tm_memopt_transform_stmt (unsigned int offset, 3902 gcall *stmt, 3903 gimple_stmt_iterator *gsi) 3904{ 3905 tree fn = gimple_call_fn (stmt); 3906 gcc_assert (TREE_CODE (fn) == ADDR_EXPR); 3907 TREE_OPERAND (fn, 0) 3908 = builtin_decl_explicit ((enum built_in_function) 3909 (DECL_FUNCTION_CODE (TREE_OPERAND (fn, 0)) 3910 + offset)); 3911 gimple_call_set_fn (stmt, fn); 3912 gsi_replace (gsi, stmt, true); 3913 dump_tm_memopt_transform (stmt); 3914} 3915 3916/* Perform the actual TM memory optimization transformations in the 3917 basic blocks in BLOCKS. */ 3918 3919static void 3920tm_memopt_transform_blocks (vec<basic_block> blocks) 3921{ 3922 size_t i; 3923 basic_block bb; 3924 gimple_stmt_iterator gsi; 3925 3926 for (i = 0; blocks.iterate (i, &bb); ++i) 3927 { 3928 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 3929 { 3930 gimple *stmt = gsi_stmt (gsi); 3931 bitmap read_avail = READ_AVAIL_IN (bb); 3932 bitmap store_avail = STORE_AVAIL_IN (bb); 3933 bitmap store_antic = STORE_ANTIC_OUT (bb); 3934 unsigned int loc; 3935 3936 if (is_tm_simple_load (stmt)) 3937 { 3938 gcall *call_stmt = as_a <gcall *> (stmt); 3939 loc = tm_memopt_value_number (stmt, NO_INSERT); 3940 if (store_avail && bitmap_bit_p (store_avail, loc)) 3941 tm_memopt_transform_stmt (TRANSFORM_RAW, call_stmt, &gsi); 3942 else if (store_antic && bitmap_bit_p (store_antic, loc)) 3943 { 3944 tm_memopt_transform_stmt (TRANSFORM_RFW, call_stmt, &gsi); 3945 bitmap_set_bit (store_avail, loc); 3946 } 3947 else if (read_avail && bitmap_bit_p (read_avail, loc)) 3948 tm_memopt_transform_stmt (TRANSFORM_RAR, call_stmt, &gsi); 3949 else 3950 bitmap_set_bit (read_avail, loc); 3951 } 3952 else if (is_tm_simple_store (stmt)) 3953 { 3954 gcall *call_stmt = as_a <gcall *> (stmt); 3955 loc = tm_memopt_value_number (stmt, NO_INSERT); 3956 if (store_avail && bitmap_bit_p (store_avail, loc)) 3957 tm_memopt_transform_stmt (TRANSFORM_WAW, call_stmt, &gsi); 3958 else 3959 { 3960 if (read_avail && bitmap_bit_p (read_avail, loc)) 3961 tm_memopt_transform_stmt (TRANSFORM_WAR, call_stmt, &gsi); 3962 bitmap_set_bit (store_avail, loc); 3963 } 3964 } 3965 } 3966 } 3967} 3968 3969/* Return a new set of bitmaps for a BB. */ 3970 3971static struct tm_memopt_bitmaps * 3972tm_memopt_init_sets (void) 3973{ 3974 struct tm_memopt_bitmaps *b 3975 = XOBNEW (&tm_memopt_obstack.obstack, struct tm_memopt_bitmaps); 3976 b->store_avail_in = BITMAP_ALLOC (&tm_memopt_obstack); 3977 b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3978 b->store_antic_in = BITMAP_ALLOC (&tm_memopt_obstack); 3979 b->store_antic_out = BITMAP_ALLOC (&tm_memopt_obstack); 3980 b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3981 b->read_avail_in = BITMAP_ALLOC (&tm_memopt_obstack); 3982 b->read_avail_out = BITMAP_ALLOC (&tm_memopt_obstack); 3983 b->read_local = BITMAP_ALLOC (&tm_memopt_obstack); 3984 b->store_local = BITMAP_ALLOC (&tm_memopt_obstack); 3985 return b; 3986} 3987 3988/* Free sets computed for each BB. */ 3989 3990static void 3991tm_memopt_free_sets (vec<basic_block> blocks) 3992{ 3993 size_t i; 3994 basic_block bb; 3995 3996 for (i = 0; blocks.iterate (i, &bb); ++i) 3997 bb->aux = NULL; 3998} 3999 4000/* Clear the visited bit for every basic block in BLOCKS. */ 4001 4002static void 4003tm_memopt_clear_visited (vec<basic_block> blocks) 4004{ 4005 size_t i; 4006 basic_block bb; 4007 4008 for (i = 0; blocks.iterate (i, &bb); ++i) 4009 BB_VISITED_P (bb) = false; 4010} 4011 4012/* Replace TM load/stores with hints for the runtime. We handle 4013 things like read-after-write, write-after-read, read-after-read, 4014 read-for-write, etc. */ 4015 4016static unsigned int 4017execute_tm_memopt (void) 4018{ 4019 struct tm_region *region; 4020 vec<basic_block> bbs; 4021 4022 tm_memopt_value_id = 0; 4023 tm_memopt_value_numbers = new hash_table<tm_memop_hasher> (10); 4024 4025 for (region = all_tm_regions; region; region = region->next) 4026 { 4027 /* All the TM stores/loads in the current region. */ 4028 size_t i; 4029 basic_block bb; 4030 4031 bitmap_obstack_initialize (&tm_memopt_obstack); 4032 4033 /* Save all BBs for the current region. */ 4034 bbs = get_tm_region_blocks (region->entry_block, 4035 region->exit_blocks, 4036 region->irr_blocks, 4037 NULL, 4038 false); 4039 4040 /* Collect all the memory operations. */ 4041 for (i = 0; bbs.iterate (i, &bb); ++i) 4042 { 4043 bb->aux = tm_memopt_init_sets (); 4044 tm_memopt_accumulate_memops (bb); 4045 } 4046 4047 /* Solve data flow equations and transform each block accordingly. */ 4048 tm_memopt_clear_visited (bbs); 4049 tm_memopt_compute_available (region, bbs); 4050 tm_memopt_clear_visited (bbs); 4051 tm_memopt_compute_antic (region, bbs); 4052 tm_memopt_transform_blocks (bbs); 4053 4054 tm_memopt_free_sets (bbs); 4055 bbs.release (); 4056 bitmap_obstack_release (&tm_memopt_obstack); 4057 tm_memopt_value_numbers->empty (); 4058 } 4059 4060 delete tm_memopt_value_numbers; 4061 tm_memopt_value_numbers = NULL; 4062 return 0; 4063} 4064 4065namespace { 4066 4067const pass_data pass_data_tm_memopt = 4068{ 4069 GIMPLE_PASS, /* type */ 4070 "tmmemopt", /* name */ 4071 OPTGROUP_NONE, /* optinfo_flags */ 4072 TV_TRANS_MEM, /* tv_id */ 4073 ( PROP_ssa | PROP_cfg ), /* properties_required */ 4074 0, /* properties_provided */ 4075 0, /* properties_destroyed */ 4076 0, /* todo_flags_start */ 4077 0, /* todo_flags_finish */ 4078}; 4079 4080class pass_tm_memopt : public gimple_opt_pass 4081{ 4082public: 4083 pass_tm_memopt (gcc::context *ctxt) 4084 : gimple_opt_pass (pass_data_tm_memopt, ctxt) 4085 {} 4086 4087 /* opt_pass methods: */ 4088 virtual bool gate (function *) { return flag_tm && optimize > 0; } 4089 virtual unsigned int execute (function *) { return execute_tm_memopt (); } 4090 4091}; // class pass_tm_memopt 4092 4093} // anon namespace 4094 4095gimple_opt_pass * 4096make_pass_tm_memopt (gcc::context *ctxt) 4097{ 4098 return new pass_tm_memopt (ctxt); 4099} 4100 4101 4102/* Interprocedual analysis for the creation of transactional clones. 4103 The aim of this pass is to find which functions are referenced in 4104 a non-irrevocable transaction context, and for those over which 4105 we have control (or user directive), create a version of the 4106 function which uses only the transactional interface to reference 4107 protected memories. This analysis proceeds in several steps: 4108 4109 (1) Collect the set of all possible transactional clones: 4110 4111 (a) For all local public functions marked tm_callable, push 4112 it onto the tm_callee queue. 4113 4114 (b) For all local functions, scan for calls in transaction blocks. 4115 Push the caller and callee onto the tm_caller and tm_callee 4116 queues. Count the number of callers for each callee. 4117 4118 (c) For each local function on the callee list, assume we will 4119 create a transactional clone. Push *all* calls onto the 4120 callee queues; count the number of clone callers separately 4121 to the number of original callers. 4122 4123 (2) Propagate irrevocable status up the dominator tree: 4124 4125 (a) Any external function on the callee list that is not marked 4126 tm_callable is irrevocable. Push all callers of such onto 4127 a worklist. 4128 4129 (b) For each function on the worklist, mark each block that 4130 contains an irrevocable call. Use the AND operator to 4131 propagate that mark up the dominator tree. 4132 4133 (c) If we reach the entry block for a possible transactional 4134 clone, then the transactional clone is irrevocable, and 4135 we should not create the clone after all. Push all 4136 callers onto the worklist. 4137 4138 (d) Place tm_irrevocable calls at the beginning of the relevant 4139 blocks. Special case here is the entry block for the entire 4140 transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for 4141 the library to begin the region in serial mode. Decrement 4142 the call count for all callees in the irrevocable region. 4143 4144 (3) Create the transactional clones: 4145 4146 Any tm_callee that still has a non-zero call count is cloned. 4147*/ 4148 4149/* This structure is stored in the AUX field of each cgraph_node. */ 4150struct tm_ipa_cg_data 4151{ 4152 /* The clone of the function that got created. */ 4153 struct cgraph_node *clone; 4154 4155 /* The tm regions in the normal function. */ 4156 struct tm_region *all_tm_regions; 4157 4158 /* The blocks of the normal/clone functions that contain irrevocable 4159 calls, or blocks that are post-dominated by irrevocable calls. */ 4160 bitmap irrevocable_blocks_normal; 4161 bitmap irrevocable_blocks_clone; 4162 4163 /* The blocks of the normal function that are involved in transactions. */ 4164 bitmap transaction_blocks_normal; 4165 4166 /* The number of callers to the transactional clone of this function 4167 from normal and transactional clones respectively. */ 4168 unsigned tm_callers_normal; 4169 unsigned tm_callers_clone; 4170 4171 /* True if all calls to this function's transactional clone 4172 are irrevocable. Also automatically true if the function 4173 has no transactional clone. */ 4174 bool is_irrevocable; 4175 4176 /* Flags indicating the presence of this function in various queues. */ 4177 bool in_callee_queue; 4178 bool in_worklist; 4179 4180 /* Flags indicating the kind of scan desired while in the worklist. */ 4181 bool want_irr_scan_normal; 4182}; 4183 4184typedef vec<cgraph_node *> cgraph_node_queue; 4185 4186/* Return the ipa data associated with NODE, allocating zeroed memory 4187 if necessary. TRAVERSE_ALIASES is true if we must traverse aliases 4188 and set *NODE accordingly. */ 4189 4190static struct tm_ipa_cg_data * 4191get_cg_data (struct cgraph_node **node, bool traverse_aliases) 4192{ 4193 struct tm_ipa_cg_data *d; 4194 4195 if (traverse_aliases && (*node)->alias) 4196 *node = (*node)->get_alias_target (); 4197 4198 d = (struct tm_ipa_cg_data *) (*node)->aux; 4199 4200 if (d == NULL) 4201 { 4202 d = (struct tm_ipa_cg_data *) 4203 obstack_alloc (&tm_obstack.obstack, sizeof (*d)); 4204 (*node)->aux = (void *) d; 4205 memset (d, 0, sizeof (*d)); 4206 } 4207 4208 return d; 4209} 4210 4211/* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that 4212 it is already present. */ 4213 4214static void 4215maybe_push_queue (struct cgraph_node *node, 4216 cgraph_node_queue *queue_p, bool *in_queue_p) 4217{ 4218 if (!*in_queue_p) 4219 { 4220 *in_queue_p = true; 4221 queue_p->safe_push (node); 4222 } 4223} 4224 4225/* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone. 4226 Queue all callees within block BB. */ 4227 4228static void 4229ipa_tm_scan_calls_block (cgraph_node_queue *callees_p, 4230 basic_block bb, bool for_clone) 4231{ 4232 gimple_stmt_iterator gsi; 4233 4234 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4235 { 4236 gimple *stmt = gsi_stmt (gsi); 4237 if (is_gimple_call (stmt) && !is_tm_pure_call (stmt)) 4238 { 4239 tree fndecl = gimple_call_fndecl (stmt); 4240 if (fndecl) 4241 { 4242 struct tm_ipa_cg_data *d; 4243 unsigned *pcallers; 4244 struct cgraph_node *node; 4245 4246 if (is_tm_ending_fndecl (fndecl)) 4247 continue; 4248 if (find_tm_replacement_function (fndecl)) 4249 continue; 4250 4251 node = cgraph_node::get (fndecl); 4252 gcc_assert (node != NULL); 4253 d = get_cg_data (&node, true); 4254 4255 pcallers = (for_clone ? &d->tm_callers_clone 4256 : &d->tm_callers_normal); 4257 *pcallers += 1; 4258 4259 maybe_push_queue (node, callees_p, &d->in_callee_queue); 4260 } 4261 } 4262 } 4263} 4264 4265/* Scan all calls in NODE that are within a transaction region, 4266 and push the resulting nodes into the callee queue. */ 4267 4268static void 4269ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data *d, 4270 cgraph_node_queue *callees_p) 4271{ 4272 d->transaction_blocks_normal = BITMAP_ALLOC (&tm_obstack); 4273 d->all_tm_regions = all_tm_regions; 4274 4275 for (tm_region *r = all_tm_regions; r; r = r->next) 4276 { 4277 vec<basic_block> bbs; 4278 basic_block bb; 4279 unsigned i; 4280 4281 bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks, NULL, 4282 d->transaction_blocks_normal, false, false); 4283 4284 FOR_EACH_VEC_ELT (bbs, i, bb) 4285 ipa_tm_scan_calls_block (callees_p, bb, false); 4286 4287 bbs.release (); 4288 } 4289} 4290 4291/* Scan all calls in NODE as if this is the transactional clone, 4292 and push the destinations into the callee queue. */ 4293 4294static void 4295ipa_tm_scan_calls_clone (struct cgraph_node *node, 4296 cgraph_node_queue *callees_p) 4297{ 4298 struct function *fn = DECL_STRUCT_FUNCTION (node->decl); 4299 basic_block bb; 4300 4301 FOR_EACH_BB_FN (bb, fn) 4302 ipa_tm_scan_calls_block (callees_p, bb, true); 4303} 4304 4305/* The function NODE has been detected to be irrevocable. Push all 4306 of its callers onto WORKLIST for the purpose of re-scanning them. */ 4307 4308static void 4309ipa_tm_note_irrevocable (struct cgraph_node *node, 4310 cgraph_node_queue *worklist_p) 4311{ 4312 struct tm_ipa_cg_data *d = get_cg_data (&node, true); 4313 struct cgraph_edge *e; 4314 4315 d->is_irrevocable = true; 4316 4317 for (e = node->callers; e ; e = e->next_caller) 4318 { 4319 basic_block bb; 4320 struct cgraph_node *caller; 4321 4322 /* Don't examine recursive calls. */ 4323 if (e->caller == node) 4324 continue; 4325 /* Even if we think we can go irrevocable, believe the user 4326 above all. */ 4327 if (is_tm_safe_or_pure (e->caller->decl)) 4328 continue; 4329 4330 caller = e->caller; 4331 d = get_cg_data (&caller, true); 4332 4333 /* Check if the callee is in a transactional region. If so, 4334 schedule the function for normal re-scan as well. */ 4335 bb = gimple_bb (e->call_stmt); 4336 gcc_assert (bb != NULL); 4337 if (d->transaction_blocks_normal 4338 && bitmap_bit_p (d->transaction_blocks_normal, bb->index)) 4339 d->want_irr_scan_normal = true; 4340 4341 maybe_push_queue (caller, worklist_p, &d->in_worklist); 4342 } 4343} 4344 4345/* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement 4346 within the block is irrevocable. */ 4347 4348static bool 4349ipa_tm_scan_irr_block (basic_block bb) 4350{ 4351 gimple_stmt_iterator gsi; 4352 tree fn; 4353 4354 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4355 { 4356 gimple *stmt = gsi_stmt (gsi); 4357 switch (gimple_code (stmt)) 4358 { 4359 case GIMPLE_ASSIGN: 4360 if (gimple_assign_single_p (stmt)) 4361 { 4362 tree lhs = gimple_assign_lhs (stmt); 4363 tree rhs = gimple_assign_rhs1 (stmt); 4364 if (volatile_lvalue_p (lhs) || volatile_lvalue_p (rhs)) 4365 return true; 4366 } 4367 break; 4368 4369 case GIMPLE_CALL: 4370 { 4371 tree lhs = gimple_call_lhs (stmt); 4372 if (lhs && volatile_lvalue_p (lhs)) 4373 return true; 4374 4375 if (is_tm_pure_call (stmt)) 4376 break; 4377 4378 fn = gimple_call_fn (stmt); 4379 4380 /* Functions with the attribute are by definition irrevocable. */ 4381 if (is_tm_irrevocable (fn)) 4382 return true; 4383 4384 /* For direct function calls, go ahead and check for replacement 4385 functions, or transitive irrevocable functions. For indirect 4386 functions, we'll ask the runtime. */ 4387 if (TREE_CODE (fn) == ADDR_EXPR) 4388 { 4389 struct tm_ipa_cg_data *d; 4390 struct cgraph_node *node; 4391 4392 fn = TREE_OPERAND (fn, 0); 4393 if (is_tm_ending_fndecl (fn)) 4394 break; 4395 if (find_tm_replacement_function (fn)) 4396 break; 4397 4398 node = cgraph_node::get (fn); 4399 d = get_cg_data (&node, true); 4400 4401 /* Return true if irrevocable, but above all, believe 4402 the user. */ 4403 if (d->is_irrevocable 4404 && !is_tm_safe_or_pure (fn)) 4405 return true; 4406 } 4407 break; 4408 } 4409 4410 case GIMPLE_ASM: 4411 /* ??? The Approved Method of indicating that an inline 4412 assembly statement is not relevant to the transaction 4413 is to wrap it in a __tm_waiver block. This is not 4414 yet implemented, so we can't check for it. */ 4415 if (is_tm_safe (current_function_decl)) 4416 error_at (gimple_location (stmt), 4417 "%<asm%> not allowed in %<transaction_safe%> function"); 4418 return true; 4419 4420 default: 4421 break; 4422 } 4423 } 4424 4425 return false; 4426} 4427 4428/* For each of the blocks seeded witin PQUEUE, walk the CFG looking 4429 for new irrevocable blocks, marking them in NEW_IRR. Don't bother 4430 scanning past OLD_IRR or EXIT_BLOCKS. */ 4431 4432static bool 4433ipa_tm_scan_irr_blocks (vec<basic_block> *pqueue, bitmap new_irr, 4434 bitmap old_irr, bitmap exit_blocks) 4435{ 4436 bool any_new_irr = false; 4437 edge e; 4438 edge_iterator ei; 4439 bitmap visited_blocks = BITMAP_ALLOC (NULL); 4440 4441 do 4442 { 4443 basic_block bb = pqueue->pop (); 4444 4445 /* Don't re-scan blocks we know already are irrevocable. */ 4446 if (old_irr && bitmap_bit_p (old_irr, bb->index)) 4447 continue; 4448 4449 if (ipa_tm_scan_irr_block (bb)) 4450 { 4451 bitmap_set_bit (new_irr, bb->index); 4452 any_new_irr = true; 4453 } 4454 else if (exit_blocks == NULL || !bitmap_bit_p (exit_blocks, bb->index)) 4455 { 4456 FOR_EACH_EDGE (e, ei, bb->succs) 4457 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 4458 { 4459 bitmap_set_bit (visited_blocks, e->dest->index); 4460 pqueue->safe_push (e->dest); 4461 } 4462 } 4463 } 4464 while (!pqueue->is_empty ()); 4465 4466 BITMAP_FREE (visited_blocks); 4467 4468 return any_new_irr; 4469} 4470 4471/* Propagate the irrevocable property both up and down the dominator tree. 4472 BB is the current block being scanned; EXIT_BLOCKS are the edges of the 4473 TM regions; OLD_IRR are the results of a previous scan of the dominator 4474 tree which has been fully propagated; NEW_IRR is the set of new blocks 4475 which are gaining the irrevocable property during the current scan. */ 4476 4477static void 4478ipa_tm_propagate_irr (basic_block entry_block, bitmap new_irr, 4479 bitmap old_irr, bitmap exit_blocks) 4480{ 4481 vec<basic_block> bbs; 4482 bitmap all_region_blocks; 4483 4484 /* If this block is in the old set, no need to rescan. */ 4485 if (old_irr && bitmap_bit_p (old_irr, entry_block->index)) 4486 return; 4487 4488 all_region_blocks = BITMAP_ALLOC (&tm_obstack); 4489 bbs = get_tm_region_blocks (entry_block, exit_blocks, NULL, 4490 all_region_blocks, false); 4491 do 4492 { 4493 basic_block bb = bbs.pop (); 4494 bool this_irr = bitmap_bit_p (new_irr, bb->index); 4495 bool all_son_irr = false; 4496 edge_iterator ei; 4497 edge e; 4498 4499 /* Propagate up. If my children are, I am too, but we must have 4500 at least one child that is. */ 4501 if (!this_irr) 4502 { 4503 FOR_EACH_EDGE (e, ei, bb->succs) 4504 { 4505 if (!bitmap_bit_p (new_irr, e->dest->index)) 4506 { 4507 all_son_irr = false; 4508 break; 4509 } 4510 else 4511 all_son_irr = true; 4512 } 4513 if (all_son_irr) 4514 { 4515 /* Add block to new_irr if it hasn't already been processed. */ 4516 if (!old_irr || !bitmap_bit_p (old_irr, bb->index)) 4517 { 4518 bitmap_set_bit (new_irr, bb->index); 4519 this_irr = true; 4520 } 4521 } 4522 } 4523 4524 /* Propagate down to everyone we immediately dominate. */ 4525 if (this_irr) 4526 { 4527 basic_block son; 4528 for (son = first_dom_son (CDI_DOMINATORS, bb); 4529 son; 4530 son = next_dom_son (CDI_DOMINATORS, son)) 4531 { 4532 /* Make sure block is actually in a TM region, and it 4533 isn't already in old_irr. */ 4534 if ((!old_irr || !bitmap_bit_p (old_irr, son->index)) 4535 && bitmap_bit_p (all_region_blocks, son->index)) 4536 bitmap_set_bit (new_irr, son->index); 4537 } 4538 } 4539 } 4540 while (!bbs.is_empty ()); 4541 4542 BITMAP_FREE (all_region_blocks); 4543 bbs.release (); 4544} 4545 4546static void 4547ipa_tm_decrement_clone_counts (basic_block bb, bool for_clone) 4548{ 4549 gimple_stmt_iterator gsi; 4550 4551 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4552 { 4553 gimple *stmt = gsi_stmt (gsi); 4554 if (is_gimple_call (stmt) && !is_tm_pure_call (stmt)) 4555 { 4556 tree fndecl = gimple_call_fndecl (stmt); 4557 if (fndecl) 4558 { 4559 struct tm_ipa_cg_data *d; 4560 unsigned *pcallers; 4561 struct cgraph_node *tnode; 4562 4563 if (is_tm_ending_fndecl (fndecl)) 4564 continue; 4565 if (find_tm_replacement_function (fndecl)) 4566 continue; 4567 4568 tnode = cgraph_node::get (fndecl); 4569 d = get_cg_data (&tnode, true); 4570 4571 pcallers = (for_clone ? &d->tm_callers_clone 4572 : &d->tm_callers_normal); 4573 4574 gcc_assert (*pcallers > 0); 4575 *pcallers -= 1; 4576 } 4577 } 4578 } 4579} 4580 4581/* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions, 4582 as well as other irrevocable actions such as inline assembly. Mark all 4583 such blocks as irrevocable and decrement the number of calls to 4584 transactional clones. Return true if, for the transactional clone, the 4585 entire function is irrevocable. */ 4586 4587static bool 4588ipa_tm_scan_irr_function (struct cgraph_node *node, bool for_clone) 4589{ 4590 struct tm_ipa_cg_data *d; 4591 bitmap new_irr, old_irr; 4592 bool ret = false; 4593 4594 /* Builtin operators (operator new, and such). */ 4595 if (DECL_STRUCT_FUNCTION (node->decl) == NULL 4596 || DECL_STRUCT_FUNCTION (node->decl)->cfg == NULL) 4597 return false; 4598 4599 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 4600 calculate_dominance_info (CDI_DOMINATORS); 4601 4602 d = get_cg_data (&node, true); 4603 auto_vec<basic_block, 10> queue; 4604 new_irr = BITMAP_ALLOC (&tm_obstack); 4605 4606 /* Scan each tm region, propagating irrevocable status through the tree. */ 4607 if (for_clone) 4608 { 4609 old_irr = d->irrevocable_blocks_clone; 4610 queue.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))); 4611 if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr, NULL)) 4612 { 4613 ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 4614 new_irr, 4615 old_irr, NULL); 4616 ret = bitmap_bit_p (new_irr, 4617 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))->index); 4618 } 4619 } 4620 else 4621 { 4622 struct tm_region *region; 4623 4624 old_irr = d->irrevocable_blocks_normal; 4625 for (region = d->all_tm_regions; region; region = region->next) 4626 { 4627 queue.quick_push (region->entry_block); 4628 if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr, 4629 region->exit_blocks)) 4630 ipa_tm_propagate_irr (region->entry_block, new_irr, old_irr, 4631 region->exit_blocks); 4632 } 4633 } 4634 4635 /* If we found any new irrevocable blocks, reduce the call count for 4636 transactional clones within the irrevocable blocks. Save the new 4637 set of irrevocable blocks for next time. */ 4638 if (!bitmap_empty_p (new_irr)) 4639 { 4640 bitmap_iterator bmi; 4641 unsigned i; 4642 4643 EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi) 4644 ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun, i), 4645 for_clone); 4646 4647 if (old_irr) 4648 { 4649 bitmap_ior_into (old_irr, new_irr); 4650 BITMAP_FREE (new_irr); 4651 } 4652 else if (for_clone) 4653 d->irrevocable_blocks_clone = new_irr; 4654 else 4655 d->irrevocable_blocks_normal = new_irr; 4656 4657 if (dump_file && new_irr) 4658 { 4659 const char *dname; 4660 bitmap_iterator bmi; 4661 unsigned i; 4662 4663 dname = lang_hooks.decl_printable_name (current_function_decl, 2); 4664 EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi) 4665 fprintf (dump_file, "%s: bb %d goes irrevocable\n", dname, i); 4666 } 4667 } 4668 else 4669 BITMAP_FREE (new_irr); 4670 4671 pop_cfun (); 4672 4673 return ret; 4674} 4675 4676/* Return true if, for the transactional clone of NODE, any call 4677 may enter irrevocable mode. */ 4678 4679static bool 4680ipa_tm_mayenterirr_function (struct cgraph_node *node) 4681{ 4682 struct tm_ipa_cg_data *d; 4683 tree decl; 4684 unsigned flags; 4685 4686 d = get_cg_data (&node, true); 4687 decl = node->decl; 4688 flags = flags_from_decl_or_type (decl); 4689 4690 /* Handle some TM builtins. Ordinarily these aren't actually generated 4691 at this point, but handling these functions when written in by the 4692 user makes it easier to build unit tests. */ 4693 if (flags & ECF_TM_BUILTIN) 4694 return false; 4695 4696 /* Filter out all functions that are marked. */ 4697 if (flags & ECF_TM_PURE) 4698 return false; 4699 if (is_tm_safe (decl)) 4700 return false; 4701 if (is_tm_irrevocable (decl)) 4702 return true; 4703 if (is_tm_callable (decl)) 4704 return true; 4705 if (find_tm_replacement_function (decl)) 4706 return true; 4707 4708 /* If we aren't seeing the final version of the function we don't 4709 know what it will contain at runtime. */ 4710 if (node->get_availability () < AVAIL_AVAILABLE) 4711 return true; 4712 4713 /* If the function must go irrevocable, then of course true. */ 4714 if (d->is_irrevocable) 4715 return true; 4716 4717 /* If there are any blocks marked irrevocable, then the function 4718 as a whole may enter irrevocable. */ 4719 if (d->irrevocable_blocks_clone) 4720 return true; 4721 4722 /* We may have previously marked this function as tm_may_enter_irr; 4723 see pass_diagnose_tm_blocks. */ 4724 if (node->tm_may_enter_irr) 4725 return true; 4726 4727 /* Recurse on the main body for aliases. In general, this will 4728 result in one of the bits above being set so that we will not 4729 have to recurse next time. */ 4730 if (node->alias) 4731 return ipa_tm_mayenterirr_function 4732 (cgraph_node::get (thunk_info::get (node)->alias)); 4733 4734 /* What remains is unmarked local functions without items that force 4735 the function to go irrevocable. */ 4736 return false; 4737} 4738 4739/* Diagnose calls from transaction_safe functions to unmarked 4740 functions that are determined to not be safe. */ 4741 4742static void 4743ipa_tm_diagnose_tm_safe (struct cgraph_node *node) 4744{ 4745 struct cgraph_edge *e; 4746 4747 for (e = node->callees; e ; e = e->next_callee) 4748 if (!is_tm_callable (e->callee->decl) 4749 && e->callee->tm_may_enter_irr) 4750 error_at (gimple_location (e->call_stmt), 4751 "unsafe function call %qD within " 4752 "%<transaction_safe%> function", e->callee->decl); 4753} 4754 4755/* Diagnose call from atomic transactions to unmarked functions 4756 that are determined to not be safe. */ 4757 4758static void 4759ipa_tm_diagnose_transaction (struct cgraph_node *node, 4760 struct tm_region *all_tm_regions) 4761{ 4762 struct tm_region *r; 4763 4764 for (r = all_tm_regions; r ; r = r->next) 4765 if (gimple_transaction_subcode (r->get_transaction_stmt ()) 4766 & GTMA_IS_RELAXED) 4767 { 4768 /* Atomic transactions can be nested inside relaxed. */ 4769 if (r->inner) 4770 ipa_tm_diagnose_transaction (node, r->inner); 4771 } 4772 else 4773 { 4774 vec<basic_block> bbs; 4775 gimple_stmt_iterator gsi; 4776 basic_block bb; 4777 size_t i; 4778 4779 bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks, 4780 r->irr_blocks, NULL, false); 4781 4782 for (i = 0; bbs.iterate (i, &bb); ++i) 4783 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 4784 { 4785 gimple *stmt = gsi_stmt (gsi); 4786 tree fndecl; 4787 4788 if (gimple_code (stmt) == GIMPLE_ASM) 4789 { 4790 error_at (gimple_location (stmt), 4791 "%<asm%> not allowed in atomic transaction"); 4792 continue; 4793 } 4794 4795 if (!is_gimple_call (stmt)) 4796 continue; 4797 fndecl = gimple_call_fndecl (stmt); 4798 4799 /* Indirect function calls have been diagnosed already. */ 4800 if (!fndecl) 4801 continue; 4802 4803 /* Stop at the end of the transaction. */ 4804 if (is_tm_ending_fndecl (fndecl)) 4805 { 4806 if (bitmap_bit_p (r->exit_blocks, bb->index)) 4807 break; 4808 continue; 4809 } 4810 4811 /* Marked functions have been diagnosed already. */ 4812 if (is_tm_pure_call (stmt)) 4813 continue; 4814 if (is_tm_callable (fndecl)) 4815 continue; 4816 4817 if (cgraph_node::local_info_node (fndecl)->tm_may_enter_irr) 4818 error_at (gimple_location (stmt), 4819 "unsafe function call %qD within " 4820 "atomic transaction", fndecl); 4821 } 4822 4823 bbs.release (); 4824 } 4825} 4826 4827/* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in 4828 OLD_DECL. The returned value is a freshly malloced pointer that 4829 should be freed by the caller. */ 4830 4831static tree 4832tm_mangle (tree old_asm_id) 4833{ 4834 const char *old_asm_name; 4835 char *tm_name; 4836 void *alloc = NULL; 4837 struct demangle_component *dc; 4838 tree new_asm_id; 4839 4840 /* Determine if the symbol is already a valid C++ mangled name. Do this 4841 even for C, which might be interfacing with C++ code via appropriately 4842 ugly identifiers. */ 4843 /* ??? We could probably do just as well checking for "_Z" and be done. */ 4844 old_asm_name = IDENTIFIER_POINTER (old_asm_id); 4845 dc = cplus_demangle_v3_components (old_asm_name, DMGL_NO_OPTS, &alloc); 4846 4847 if (dc == NULL) 4848 { 4849 char length[12]; 4850 4851 do_unencoded: 4852 sprintf (length, "%u", IDENTIFIER_LENGTH (old_asm_id)); 4853 tm_name = concat ("_ZGTt", length, old_asm_name, NULL); 4854 } 4855 else 4856 { 4857 old_asm_name += 2; /* Skip _Z */ 4858 4859 switch (dc->type) 4860 { 4861 case DEMANGLE_COMPONENT_TRANSACTION_CLONE: 4862 case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE: 4863 /* Don't play silly games, you! */ 4864 goto do_unencoded; 4865 4866 case DEMANGLE_COMPONENT_HIDDEN_ALIAS: 4867 /* I'd really like to know if we can ever be passed one of 4868 these from the C++ front end. The Logical Thing would 4869 seem that hidden-alias should be outer-most, so that we 4870 get hidden-alias of a transaction-clone and not vice-versa. */ 4871 old_asm_name += 2; 4872 break; 4873 4874 default: 4875 break; 4876 } 4877 4878 tm_name = concat ("_ZGTt", old_asm_name, NULL); 4879 } 4880 free (alloc); 4881 4882 new_asm_id = get_identifier (tm_name); 4883 free (tm_name); 4884 4885 return new_asm_id; 4886} 4887 4888static inline void 4889ipa_tm_mark_force_output_node (struct cgraph_node *node) 4890{ 4891 node->mark_force_output (); 4892 node->analyzed = true; 4893} 4894 4895static inline void 4896ipa_tm_mark_forced_by_abi_node (struct cgraph_node *node) 4897{ 4898 node->forced_by_abi = true; 4899 node->analyzed = true; 4900} 4901 4902/* Callback data for ipa_tm_create_version_alias. */ 4903struct create_version_alias_info 4904{ 4905 struct cgraph_node *old_node; 4906 tree new_decl; 4907}; 4908 4909/* A subroutine of ipa_tm_create_version, called via 4910 cgraph_for_node_and_aliases. Create new tm clones for each of 4911 the existing aliases. */ 4912static bool 4913ipa_tm_create_version_alias (struct cgraph_node *node, void *data) 4914{ 4915 struct create_version_alias_info *info 4916 = (struct create_version_alias_info *)data; 4917 tree old_decl, new_decl, tm_name; 4918 struct cgraph_node *new_node; 4919 4920 if (!node->cpp_implicit_alias) 4921 return false; 4922 4923 old_decl = node->decl; 4924 tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl)); 4925 new_decl = build_decl (DECL_SOURCE_LOCATION (old_decl), 4926 TREE_CODE (old_decl), tm_name, 4927 TREE_TYPE (old_decl)); 4928 4929 SET_DECL_ASSEMBLER_NAME (new_decl, tm_name); 4930 SET_DECL_RTL (new_decl, NULL); 4931 4932 /* Based loosely on C++'s make_alias_for(). */ 4933 TREE_PUBLIC (new_decl) = TREE_PUBLIC (old_decl); 4934 DECL_CONTEXT (new_decl) = DECL_CONTEXT (old_decl); 4935 DECL_LANG_SPECIFIC (new_decl) = DECL_LANG_SPECIFIC (old_decl); 4936 TREE_READONLY (new_decl) = TREE_READONLY (old_decl); 4937 DECL_EXTERNAL (new_decl) = 0; 4938 DECL_ARTIFICIAL (new_decl) = 1; 4939 TREE_ADDRESSABLE (new_decl) = 1; 4940 TREE_USED (new_decl) = 1; 4941 TREE_SYMBOL_REFERENCED (tm_name) = 1; 4942 4943 /* Perform the same remapping to the comdat group. */ 4944 if (DECL_ONE_ONLY (new_decl)) 4945 varpool_node::get (new_decl)->set_comdat_group 4946 (tm_mangle (decl_comdat_group_id (old_decl))); 4947 4948 new_node = cgraph_node::create_same_body_alias (new_decl, info->new_decl); 4949 new_node->tm_clone = true; 4950 new_node->externally_visible = info->old_node->externally_visible; 4951 new_node->no_reorder = info->old_node->no_reorder; 4952 /* ?? Do not traverse aliases here. */ 4953 get_cg_data (&node, false)->clone = new_node; 4954 4955 record_tm_clone_pair (old_decl, new_decl); 4956 4957 if (info->old_node->force_output 4958 || info->old_node->ref_list.first_referring ()) 4959 ipa_tm_mark_force_output_node (new_node); 4960 if (info->old_node->forced_by_abi) 4961 ipa_tm_mark_forced_by_abi_node (new_node); 4962 return false; 4963} 4964 4965/* Create a copy of the function (possibly declaration only) of OLD_NODE, 4966 appropriate for the transactional clone. */ 4967 4968static void 4969ipa_tm_create_version (struct cgraph_node *old_node) 4970{ 4971 tree new_decl, old_decl, tm_name; 4972 struct cgraph_node *new_node; 4973 4974 old_decl = old_node->decl; 4975 new_decl = copy_node (old_decl); 4976 4977 /* DECL_ASSEMBLER_NAME needs to be set before we call 4978 cgraph_copy_node_for_versioning below, because cgraph_node will 4979 fill the assembler_name_hash. */ 4980 tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl)); 4981 SET_DECL_ASSEMBLER_NAME (new_decl, tm_name); 4982 SET_DECL_RTL (new_decl, NULL); 4983 TREE_SYMBOL_REFERENCED (tm_name) = 1; 4984 4985 /* Perform the same remapping to the comdat group. */ 4986 if (DECL_ONE_ONLY (new_decl)) 4987 varpool_node::get (new_decl)->set_comdat_group 4988 (tm_mangle (DECL_COMDAT_GROUP (old_decl))); 4989 4990 gcc_assert (!old_node->ipa_transforms_to_apply.exists ()); 4991 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 4992 new_node->local = false; 4993 new_node->externally_visible = old_node->externally_visible; 4994 new_node->lowered = true; 4995 new_node->tm_clone = 1; 4996 if (!old_node->implicit_section) 4997 new_node->set_section (*old_node); 4998 get_cg_data (&old_node, true)->clone = new_node; 4999 5000 if (old_node->get_availability () >= AVAIL_INTERPOSABLE) 5001 { 5002 /* Remap extern inline to static inline. */ 5003 /* ??? Is it worth trying to use make_decl_one_only? */ 5004 if (DECL_DECLARED_INLINE_P (new_decl) && DECL_EXTERNAL (new_decl)) 5005 { 5006 DECL_EXTERNAL (new_decl) = 0; 5007 TREE_PUBLIC (new_decl) = 0; 5008 DECL_WEAK (new_decl) = 0; 5009 } 5010 5011 tree_function_versioning (old_decl, new_decl, 5012 NULL, NULL, false, NULL, NULL); 5013 } 5014 5015 record_tm_clone_pair (old_decl, new_decl); 5016 5017 symtab->call_cgraph_insertion_hooks (new_node); 5018 if (old_node->force_output 5019 || old_node->ref_list.first_referring ()) 5020 ipa_tm_mark_force_output_node (new_node); 5021 if (old_node->forced_by_abi) 5022 ipa_tm_mark_forced_by_abi_node (new_node); 5023 5024 /* Do the same thing, but for any aliases of the original node. */ 5025 { 5026 struct create_version_alias_info data; 5027 data.old_node = old_node; 5028 data.new_decl = new_decl; 5029 old_node->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias, 5030 &data, true); 5031 } 5032} 5033 5034/* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */ 5035 5036static void 5037ipa_tm_insert_irr_call (struct cgraph_node *node, struct tm_region *region, 5038 basic_block bb) 5039{ 5040 gimple_stmt_iterator gsi; 5041 gcall *g; 5042 5043 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 5044 5045 g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE), 5046 1, build_int_cst (NULL_TREE, MODE_SERIALIRREVOCABLE)); 5047 5048 split_block_after_labels (bb); 5049 gsi = gsi_after_labels (bb); 5050 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 5051 5052 node->create_edge (cgraph_node::get_create 5053 (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE)), 5054 g, gimple_bb (g)->count); 5055} 5056 5057/* Construct a call to TM_GETTMCLONE and insert it before GSI. */ 5058 5059static bool 5060ipa_tm_insert_gettmclone_call (struct cgraph_node *node, 5061 struct tm_region *region, 5062 gimple_stmt_iterator *gsi, gcall *stmt) 5063{ 5064 tree gettm_fn, ret, old_fn, callfn; 5065 gcall *g; 5066 gassign *g2; 5067 bool safe; 5068 5069 old_fn = gimple_call_fn (stmt); 5070 5071 if (TREE_CODE (old_fn) == ADDR_EXPR) 5072 { 5073 tree fndecl = TREE_OPERAND (old_fn, 0); 5074 tree clone = get_tm_clone_pair (fndecl); 5075 5076 /* By transforming the call into a TM_GETTMCLONE, we are 5077 technically taking the address of the original function and 5078 its clone. Explain this so inlining will know this function 5079 is needed. */ 5080 cgraph_node::get (fndecl)->mark_address_taken () ; 5081 if (clone) 5082 cgraph_node::get (clone)->mark_address_taken (); 5083 } 5084 5085 safe = is_tm_safe (TREE_TYPE (old_fn)); 5086 gettm_fn = builtin_decl_explicit (safe ? BUILT_IN_TM_GETTMCLONE_SAFE 5087 : BUILT_IN_TM_GETTMCLONE_IRR); 5088 ret = create_tmp_var (ptr_type_node); 5089 5090 if (!safe) 5091 transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE); 5092 5093 /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */ 5094 if (TREE_CODE (old_fn) == OBJ_TYPE_REF) 5095 old_fn = OBJ_TYPE_REF_EXPR (old_fn); 5096 5097 g = gimple_build_call (gettm_fn, 1, old_fn); 5098 ret = make_ssa_name (ret, g); 5099 gimple_call_set_lhs (g, ret); 5100 5101 gsi_insert_before (gsi, g, GSI_SAME_STMT); 5102 5103 node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count); 5104 5105 /* Cast return value from tm_gettmclone* into appropriate function 5106 pointer. */ 5107 callfn = create_tmp_var (TREE_TYPE (old_fn)); 5108 g2 = gimple_build_assign (callfn, 5109 fold_build1 (NOP_EXPR, TREE_TYPE (callfn), ret)); 5110 callfn = make_ssa_name (callfn, g2); 5111 gimple_assign_set_lhs (g2, callfn); 5112 gsi_insert_before (gsi, g2, GSI_SAME_STMT); 5113 5114 /* ??? This is a hack to preserve the NOTHROW bit on the call, 5115 which we would have derived from the decl. Failure to save 5116 this bit means we might have to split the basic block. */ 5117 if (gimple_call_nothrow_p (stmt)) 5118 gimple_call_set_nothrow (stmt, true); 5119 5120 gimple_call_set_fn (stmt, callfn); 5121 5122 /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS 5123 for a call statement. Fix it. */ 5124 { 5125 tree lhs = gimple_call_lhs (stmt); 5126 tree rettype = TREE_TYPE (gimple_call_fntype (stmt)); 5127 if (lhs 5128 && !useless_type_conversion_p (TREE_TYPE (lhs), rettype)) 5129 { 5130 tree temp; 5131 5132 temp = create_tmp_reg (rettype); 5133 gimple_call_set_lhs (stmt, temp); 5134 5135 g2 = gimple_build_assign (lhs, 5136 fold_build1 (VIEW_CONVERT_EXPR, 5137 TREE_TYPE (lhs), temp)); 5138 gsi_insert_after (gsi, g2, GSI_SAME_STMT); 5139 } 5140 } 5141 5142 update_stmt (stmt); 5143 cgraph_edge *e = cgraph_node::get (current_function_decl)->get_edge (stmt); 5144 if (e && e->indirect_info) 5145 e->indirect_info->polymorphic = false; 5146 5147 return true; 5148} 5149 5150/* Helper function for ipa_tm_transform_calls*. Given a call 5151 statement in GSI which resides inside transaction REGION, redirect 5152 the call to either its wrapper function, or its clone. */ 5153 5154static void 5155ipa_tm_transform_calls_redirect (struct cgraph_node *node, 5156 struct tm_region *region, 5157 gimple_stmt_iterator *gsi, 5158 bool *need_ssa_rename_p) 5159{ 5160 gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi)); 5161 struct cgraph_node *new_node; 5162 struct cgraph_edge *e = node->get_edge (stmt); 5163 tree fndecl = gimple_call_fndecl (stmt); 5164 5165 /* For indirect calls, pass the address through the runtime. */ 5166 if (fndecl == NULL) 5167 { 5168 *need_ssa_rename_p |= 5169 ipa_tm_insert_gettmclone_call (node, region, gsi, stmt); 5170 return; 5171 } 5172 5173 /* Handle some TM builtins. Ordinarily these aren't actually generated 5174 at this point, but handling these functions when written in by the 5175 user makes it easier to build unit tests. */ 5176 if (flags_from_decl_or_type (fndecl) & ECF_TM_BUILTIN) 5177 return; 5178 5179 /* Fixup recursive calls inside clones. */ 5180 /* ??? Why did cgraph_copy_node_for_versioning update the call edges 5181 for recursion but not update the call statements themselves? */ 5182 if (e->caller == e->callee && decl_is_tm_clone (current_function_decl)) 5183 { 5184 gimple_call_set_fndecl (stmt, current_function_decl); 5185 return; 5186 } 5187 5188 /* If there is a replacement, use it. */ 5189 fndecl = find_tm_replacement_function (fndecl); 5190 if (fndecl) 5191 { 5192 new_node = cgraph_node::get_create (fndecl); 5193 5194 /* ??? Mark all transaction_wrap functions tm_may_enter_irr. 5195 5196 We can't do this earlier in record_tm_replacement because 5197 cgraph_remove_unreachable_nodes is called before we inject 5198 references to the node. Further, we can't do this in some 5199 nice central place in ipa_tm_execute because we don't have 5200 the exact list of wrapper functions that would be used. 5201 Marking more wrappers than necessary results in the creation 5202 of unnecessary cgraph_nodes, which can cause some of the 5203 other IPA passes to crash. 5204 5205 We do need to mark these nodes so that we get the proper 5206 result in expand_call_tm. */ 5207 /* ??? This seems broken. How is it that we're marking the 5208 CALLEE as may_enter_irr? Surely we should be marking the 5209 CALLER. Also note that find_tm_replacement_function also 5210 contains mappings into the TM runtime, e.g. memcpy. These 5211 we know won't go irrevocable. */ 5212 new_node->tm_may_enter_irr = 1; 5213 } 5214 else 5215 { 5216 struct tm_ipa_cg_data *d; 5217 struct cgraph_node *tnode = e->callee; 5218 5219 d = get_cg_data (&tnode, true); 5220 new_node = d->clone; 5221 5222 /* As we've already skipped pure calls and appropriate builtins, 5223 and we've already marked irrevocable blocks, if we can't come 5224 up with a static replacement, then ask the runtime. */ 5225 if (new_node == NULL) 5226 { 5227 *need_ssa_rename_p |= 5228 ipa_tm_insert_gettmclone_call (node, region, gsi, stmt); 5229 return; 5230 } 5231 5232 fndecl = new_node->decl; 5233 } 5234 5235 e->redirect_callee (new_node); 5236 gimple_call_set_fndecl (stmt, fndecl); 5237} 5238 5239/* Helper function for ipa_tm_transform_calls. For a given BB, 5240 install calls to tm_irrevocable when IRR_BLOCKS are reached, 5241 redirect other calls to the generated transactional clone. */ 5242 5243static bool 5244ipa_tm_transform_calls_1 (struct cgraph_node *node, struct tm_region *region, 5245 basic_block bb, bitmap irr_blocks) 5246{ 5247 gimple_stmt_iterator gsi; 5248 bool need_ssa_rename = false; 5249 5250 if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index)) 5251 { 5252 ipa_tm_insert_irr_call (node, region, bb); 5253 return true; 5254 } 5255 5256 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 5257 { 5258 gimple *stmt = gsi_stmt (gsi); 5259 5260 if (!is_gimple_call (stmt)) 5261 continue; 5262 if (is_tm_pure_call (stmt)) 5263 continue; 5264 5265 /* Redirect edges to the appropriate replacement or clone. */ 5266 ipa_tm_transform_calls_redirect (node, region, &gsi, &need_ssa_rename); 5267 } 5268 5269 return need_ssa_rename; 5270} 5271 5272/* Walk the CFG for REGION, beginning at BB. Install calls to 5273 tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to 5274 the generated transactional clone. */ 5275 5276static bool 5277ipa_tm_transform_calls (struct cgraph_node *node, struct tm_region *region, 5278 basic_block bb, bitmap irr_blocks) 5279{ 5280 bool need_ssa_rename = false; 5281 edge e; 5282 edge_iterator ei; 5283 auto_vec<basic_block> queue; 5284 bitmap visited_blocks = BITMAP_ALLOC (NULL); 5285 5286 queue.safe_push (bb); 5287 do 5288 { 5289 bb = queue.pop (); 5290 5291 need_ssa_rename |= 5292 ipa_tm_transform_calls_1 (node, region, bb, irr_blocks); 5293 5294 if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index)) 5295 continue; 5296 5297 if (region && bitmap_bit_p (region->exit_blocks, bb->index)) 5298 continue; 5299 5300 FOR_EACH_EDGE (e, ei, bb->succs) 5301 if (!bitmap_bit_p (visited_blocks, e->dest->index)) 5302 { 5303 bitmap_set_bit (visited_blocks, e->dest->index); 5304 queue.safe_push (e->dest); 5305 } 5306 } 5307 while (!queue.is_empty ()); 5308 5309 BITMAP_FREE (visited_blocks); 5310 5311 return need_ssa_rename; 5312} 5313 5314/* Transform the calls within the TM regions within NODE. */ 5315 5316static void 5317ipa_tm_transform_transaction (struct cgraph_node *node) 5318{ 5319 struct tm_ipa_cg_data *d; 5320 struct tm_region *region; 5321 bool need_ssa_rename = false; 5322 5323 d = get_cg_data (&node, true); 5324 5325 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 5326 calculate_dominance_info (CDI_DOMINATORS); 5327 5328 for (region = d->all_tm_regions; region; region = region->next) 5329 { 5330 /* If we're sure to go irrevocable, don't transform anything. */ 5331 if (d->irrevocable_blocks_normal 5332 && bitmap_bit_p (d->irrevocable_blocks_normal, 5333 region->entry_block->index)) 5334 { 5335 transaction_subcode_ior (region, GTMA_DOES_GO_IRREVOCABLE 5336 | GTMA_MAY_ENTER_IRREVOCABLE 5337 | GTMA_HAS_NO_INSTRUMENTATION); 5338 continue; 5339 } 5340 5341 need_ssa_rename |= 5342 ipa_tm_transform_calls (node, region, region->entry_block, 5343 d->irrevocable_blocks_normal); 5344 } 5345 5346 if (need_ssa_rename) 5347 update_ssa (TODO_update_ssa_only_virtuals); 5348 5349 pop_cfun (); 5350} 5351 5352/* Transform the calls within the transactional clone of NODE. */ 5353 5354static void 5355ipa_tm_transform_clone (struct cgraph_node *node) 5356{ 5357 struct tm_ipa_cg_data *d; 5358 bool need_ssa_rename; 5359 5360 d = get_cg_data (&node, true); 5361 5362 /* If this function makes no calls and has no irrevocable blocks, 5363 then there's nothing to do. */ 5364 /* ??? Remove non-aborting top-level transactions. */ 5365 if (!node->callees && !node->indirect_calls && !d->irrevocable_blocks_clone) 5366 return; 5367 5368 push_cfun (DECL_STRUCT_FUNCTION (d->clone->decl)); 5369 calculate_dominance_info (CDI_DOMINATORS); 5370 5371 need_ssa_rename = 5372 ipa_tm_transform_calls (d->clone, NULL, 5373 single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)), 5374 d->irrevocable_blocks_clone); 5375 5376 if (need_ssa_rename) 5377 update_ssa (TODO_update_ssa_only_virtuals); 5378 5379 pop_cfun (); 5380} 5381 5382/* Main entry point for the transactional memory IPA pass. */ 5383 5384static unsigned int 5385ipa_tm_execute (void) 5386{ 5387 cgraph_node_queue tm_callees = cgraph_node_queue (); 5388 /* List of functions that will go irrevocable. */ 5389 cgraph_node_queue irr_worklist = cgraph_node_queue (); 5390 5391 struct cgraph_node *node; 5392 struct tm_ipa_cg_data *d; 5393 enum availability a; 5394 unsigned int i; 5395 5396 cgraph_node::checking_verify_cgraph_nodes (); 5397 5398 bitmap_obstack_initialize (&tm_obstack); 5399 initialize_original_copy_tables (); 5400 5401 /* For all local functions marked tm_callable, queue them. */ 5402 FOR_EACH_DEFINED_FUNCTION (node) 5403 if (is_tm_callable (node->decl) 5404 && node->get_availability () >= AVAIL_INTERPOSABLE) 5405 { 5406 d = get_cg_data (&node, true); 5407 maybe_push_queue (node, &tm_callees, &d->in_callee_queue); 5408 } 5409 5410 /* For all local reachable functions... */ 5411 FOR_EACH_DEFINED_FUNCTION (node) 5412 if (node->lowered 5413 && node->get_availability () >= AVAIL_INTERPOSABLE) 5414 { 5415 /* ... marked tm_pure, record that fact for the runtime by 5416 indicating that the pure function is its own tm_callable. 5417 No need to do this if the function's address can't be taken. */ 5418 if (is_tm_pure (node->decl)) 5419 { 5420 if (!node->local) 5421 record_tm_clone_pair (node->decl, node->decl); 5422 continue; 5423 } 5424 5425 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 5426 calculate_dominance_info (CDI_DOMINATORS); 5427 5428 tm_region_init (NULL); 5429 if (all_tm_regions) 5430 { 5431 d = get_cg_data (&node, true); 5432 5433 /* Scan for calls that are in each transaction, and 5434 generate the uninstrumented code path. */ 5435 ipa_tm_scan_calls_transaction (d, &tm_callees); 5436 5437 /* Put it in the worklist so we can scan the function 5438 later (ipa_tm_scan_irr_function) and mark the 5439 irrevocable blocks. */ 5440 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5441 d->want_irr_scan_normal = true; 5442 } 5443 5444 pop_cfun (); 5445 } 5446 5447 /* For every local function on the callee list, scan as if we will be 5448 creating a transactional clone, queueing all new functions we find 5449 along the way. */ 5450 for (i = 0; i < tm_callees.length (); ++i) 5451 { 5452 node = tm_callees[i]; 5453 a = node->get_availability (); 5454 d = get_cg_data (&node, true); 5455 5456 /* Put it in the worklist so we can scan the function later 5457 (ipa_tm_scan_irr_function) and mark the irrevocable 5458 blocks. */ 5459 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5460 5461 /* Some callees cannot be arbitrarily cloned. These will always be 5462 irrevocable. Mark these now, so that we need not scan them. */ 5463 if (is_tm_irrevocable (node->decl)) 5464 ipa_tm_note_irrevocable (node, &irr_worklist); 5465 else if (a <= AVAIL_NOT_AVAILABLE 5466 && !is_tm_safe_or_pure (node->decl)) 5467 ipa_tm_note_irrevocable (node, &irr_worklist); 5468 else if (a >= AVAIL_INTERPOSABLE) 5469 { 5470 if (!tree_versionable_function_p (node->decl)) 5471 ipa_tm_note_irrevocable (node, &irr_worklist); 5472 else if (!d->is_irrevocable) 5473 { 5474 /* If this is an alias, make sure its base is queued as well. 5475 we need not scan the callees now, as the base will do. */ 5476 if (node->alias) 5477 { 5478 node = cgraph_node::get (thunk_info::get (node)->alias); 5479 d = get_cg_data (&node, true); 5480 maybe_push_queue (node, &tm_callees, &d->in_callee_queue); 5481 continue; 5482 } 5483 5484 /* Add all nodes called by this function into 5485 tm_callees as well. */ 5486 ipa_tm_scan_calls_clone (node, &tm_callees); 5487 } 5488 } 5489 } 5490 5491 /* Iterate scans until no more work to be done. Prefer not to use 5492 vec::pop because the worklist tends to follow a breadth-first 5493 search of the callgraph, which should allow convergance with a 5494 minimum number of scans. But we also don't want the worklist 5495 array to grow without bound, so we shift the array up periodically. */ 5496 for (i = 0; i < irr_worklist.length (); ++i) 5497 { 5498 if (i > 256 && i == irr_worklist.length () / 8) 5499 { 5500 irr_worklist.block_remove (0, i); 5501 i = 0; 5502 } 5503 5504 node = irr_worklist[i]; 5505 d = get_cg_data (&node, true); 5506 d->in_worklist = false; 5507 5508 if (d->want_irr_scan_normal) 5509 { 5510 d->want_irr_scan_normal = false; 5511 ipa_tm_scan_irr_function (node, false); 5512 } 5513 if (d->in_callee_queue && ipa_tm_scan_irr_function (node, true)) 5514 ipa_tm_note_irrevocable (node, &irr_worklist); 5515 } 5516 5517 /* For every function on the callee list, collect the tm_may_enter_irr 5518 bit on the node. */ 5519 irr_worklist.truncate (0); 5520 for (i = 0; i < tm_callees.length (); ++i) 5521 { 5522 node = tm_callees[i]; 5523 if (ipa_tm_mayenterirr_function (node)) 5524 { 5525 d = get_cg_data (&node, true); 5526 gcc_assert (d->in_worklist == false); 5527 maybe_push_queue (node, &irr_worklist, &d->in_worklist); 5528 } 5529 } 5530 5531 /* Propagate the tm_may_enter_irr bit to callers until stable. */ 5532 for (i = 0; i < irr_worklist.length (); ++i) 5533 { 5534 struct cgraph_node *caller; 5535 struct cgraph_edge *e; 5536 struct ipa_ref *ref; 5537 5538 if (i > 256 && i == irr_worklist.length () / 8) 5539 { 5540 irr_worklist.block_remove (0, i); 5541 i = 0; 5542 } 5543 5544 node = irr_worklist[i]; 5545 d = get_cg_data (&node, true); 5546 d->in_worklist = false; 5547 node->tm_may_enter_irr = true; 5548 5549 /* Propagate back to normal callers. */ 5550 for (e = node->callers; e ; e = e->next_caller) 5551 { 5552 caller = e->caller; 5553 if (!is_tm_safe_or_pure (caller->decl) 5554 && !caller->tm_may_enter_irr) 5555 { 5556 d = get_cg_data (&caller, true); 5557 maybe_push_queue (caller, &irr_worklist, &d->in_worklist); 5558 } 5559 } 5560 5561 /* Propagate back to referring aliases as well. */ 5562 FOR_EACH_ALIAS (node, ref) 5563 { 5564 caller = dyn_cast<cgraph_node *> (ref->referring); 5565 if (!caller->tm_may_enter_irr) 5566 { 5567 /* ?? Do not traverse aliases here. */ 5568 d = get_cg_data (&caller, false); 5569 maybe_push_queue (caller, &irr_worklist, &d->in_worklist); 5570 } 5571 } 5572 } 5573 5574 /* Now validate all tm_safe functions, and all atomic regions in 5575 other functions. */ 5576 FOR_EACH_DEFINED_FUNCTION (node) 5577 if (node->lowered 5578 && node->get_availability () >= AVAIL_INTERPOSABLE) 5579 { 5580 d = get_cg_data (&node, true); 5581 if (is_tm_safe (node->decl)) 5582 ipa_tm_diagnose_tm_safe (node); 5583 else if (d->all_tm_regions) 5584 ipa_tm_diagnose_transaction (node, d->all_tm_regions); 5585 } 5586 5587 /* Create clones. Do those that are not irrevocable and have a 5588 positive call count. Do those publicly visible functions that 5589 the user directed us to clone. */ 5590 for (i = 0; i < tm_callees.length (); ++i) 5591 { 5592 bool doit = false; 5593 5594 node = tm_callees[i]; 5595 if (node->cpp_implicit_alias) 5596 continue; 5597 5598 a = node->get_availability (); 5599 d = get_cg_data (&node, true); 5600 5601 if (a <= AVAIL_NOT_AVAILABLE) 5602 doit = is_tm_callable (node->decl); 5603 else if (a <= AVAIL_AVAILABLE && is_tm_callable (node->decl)) 5604 doit = true; 5605 else if (!d->is_irrevocable 5606 && d->tm_callers_normal + d->tm_callers_clone > 0) 5607 doit = true; 5608 5609 if (doit) 5610 ipa_tm_create_version (node); 5611 } 5612 5613 /* Redirect calls to the new clones, and insert irrevocable marks. */ 5614 for (i = 0; i < tm_callees.length (); ++i) 5615 { 5616 node = tm_callees[i]; 5617 if (node->analyzed) 5618 { 5619 d = get_cg_data (&node, true); 5620 if (d->clone) 5621 ipa_tm_transform_clone (node); 5622 } 5623 } 5624 FOR_EACH_DEFINED_FUNCTION (node) 5625 if (node->lowered 5626 && node->get_availability () >= AVAIL_INTERPOSABLE) 5627 { 5628 d = get_cg_data (&node, true); 5629 if (d->all_tm_regions) 5630 ipa_tm_transform_transaction (node); 5631 } 5632 5633 /* Free and clear all data structures. */ 5634 tm_callees.release (); 5635 irr_worklist.release (); 5636 bitmap_obstack_release (&tm_obstack); 5637 free_original_copy_tables (); 5638 5639 FOR_EACH_FUNCTION (node) 5640 node->aux = NULL; 5641 5642 cgraph_node::checking_verify_cgraph_nodes (); 5643 5644 return 0; 5645} 5646 5647namespace { 5648 5649const pass_data pass_data_ipa_tm = 5650{ 5651 SIMPLE_IPA_PASS, /* type */ 5652 "tmipa", /* name */ 5653 OPTGROUP_NONE, /* optinfo_flags */ 5654 TV_TRANS_MEM, /* tv_id */ 5655 ( PROP_ssa | PROP_cfg ), /* properties_required */ 5656 0, /* properties_provided */ 5657 0, /* properties_destroyed */ 5658 0, /* todo_flags_start */ 5659 0, /* todo_flags_finish */ 5660}; 5661 5662class pass_ipa_tm : public simple_ipa_opt_pass 5663{ 5664public: 5665 pass_ipa_tm (gcc::context *ctxt) 5666 : simple_ipa_opt_pass (pass_data_ipa_tm, ctxt) 5667 {} 5668 5669 /* opt_pass methods: */ 5670 virtual bool gate (function *) { return flag_tm; } 5671 virtual unsigned int execute (function *) { return ipa_tm_execute (); } 5672 5673}; // class pass_ipa_tm 5674 5675} // anon namespace 5676 5677simple_ipa_opt_pass * 5678make_pass_ipa_tm (gcc::context *ctxt) 5679{ 5680 return new pass_ipa_tm (ctxt); 5681} 5682 5683#include "gt-trans-mem.h" 5684