1/* Callgraph clones 2 Copyright (C) 2003-2020 Free Software Foundation, Inc. 3 Contributed by Jan Hubicka 4 5This file is part of GCC. 6 7GCC is free software; you can redistribute it and/or modify it under 8the terms of the GNU General Public License as published by the Free 9Software Foundation; either version 3, or (at your option) any later 10version. 11 12GCC is distributed in the hope that it will be useful, but WITHOUT ANY 13WARRANTY; without even the implied warranty of MERCHANTABILITY or 14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 15for more details. 16 17You should have received a copy of the GNU General Public License 18along with GCC; see the file COPYING3. If not see 19<http://www.gnu.org/licenses/>. */ 20 21/* This module provide facilities for cloning functions. I.e. creating 22 new functions based on existing functions with simple modifications, 23 such as replacement of parameters. 24 25 To allow whole program optimization without actual presence of function 26 bodies, an additional infrastructure is provided for so-called virtual 27 clones 28 29 A virtual clone in the callgraph is a function that has no 30 associated body, just a description of how to create its body based 31 on a different function (which itself may be a virtual clone). 32 33 The description of function modifications includes adjustments to 34 the function's signature (which allows, for example, removing or 35 adding function arguments), substitutions to perform on the 36 function body, and, for inlined functions, a pointer to the 37 function that it will be inlined into. 38 39 It is also possible to redirect any edge of the callgraph from a 40 function to its virtual clone. This implies updating of the call 41 site to adjust for the new function signature. 42 43 Most of the transformations performed by inter-procedural 44 optimizations can be represented via virtual clones. For 45 instance, a constant propagation pass can produce a virtual clone 46 of the function which replaces one of its arguments by a 47 constant. The inliner can represent its decisions by producing a 48 clone of a function whose body will be later integrated into 49 a given function. 50 51 Using virtual clones, the program can be easily updated 52 during the Execute stage, solving most of pass interactions 53 problems that would otherwise occur during Transform. 54 55 Virtual clones are later materialized in the LTRANS stage and 56 turned into real functions. Passes executed after the virtual 57 clone were introduced also perform their Transform stage 58 on new functions, so for a pass there is no significant 59 difference between operating on a real function or a virtual 60 clone introduced before its Execute stage. 61 62 Optimization passes then work on virtual clones introduced before 63 their Execute stage as if they were real functions. The 64 only difference is that clones are not visible during the 65 Generate Summary stage. */ 66 67#include "config.h" 68#include "system.h" 69#include "coretypes.h" 70#include "backend.h" 71#include "target.h" 72#include "rtl.h" 73#include "tree.h" 74#include "gimple.h" 75#include "stringpool.h" 76#include "cgraph.h" 77#include "lto-streamer.h" 78#include "tree-eh.h" 79#include "tree-cfg.h" 80#include "tree-inline.h" 81#include "dumpfile.h" 82#include "gimple-pretty-print.h" 83#include "alloc-pool.h" 84#include "symbol-summary.h" 85#include "tree-vrp.h" 86#include "ipa-prop.h" 87#include "ipa-fnsummary.h" 88 89/* Create clone of edge in the node N represented by CALL_EXPR 90 the callgraph. */ 91 92cgraph_edge * 93cgraph_edge::clone (cgraph_node *n, gcall *call_stmt, unsigned stmt_uid, 94 profile_count num, profile_count den, 95 bool update_original) 96{ 97 cgraph_edge *new_edge; 98 profile_count::adjust_for_ipa_scaling (&num, &den); 99 profile_count prof_count = count.apply_scale (num, den); 100 101 if (indirect_unknown_callee) 102 { 103 tree decl; 104 105 if (call_stmt && (decl = gimple_call_fndecl (call_stmt)) 106 /* When the call is speculative, we need to resolve it 107 via cgraph_resolve_speculation and not here. */ 108 && !speculative) 109 { 110 cgraph_node *callee = cgraph_node::get (decl); 111 gcc_checking_assert (callee); 112 new_edge = n->create_edge (callee, call_stmt, prof_count, true); 113 } 114 else 115 { 116 new_edge = n->create_indirect_edge (call_stmt, 117 indirect_info->ecf_flags, 118 prof_count, true); 119 *new_edge->indirect_info = *indirect_info; 120 } 121 } 122 else 123 { 124 new_edge = n->create_edge (callee, call_stmt, prof_count, true); 125 if (indirect_info) 126 { 127 new_edge->indirect_info 128 = ggc_cleared_alloc<cgraph_indirect_call_info> (); 129 *new_edge->indirect_info = *indirect_info; 130 } 131 } 132 133 new_edge->inline_failed = inline_failed; 134 new_edge->indirect_inlining_edge = indirect_inlining_edge; 135 if (!call_stmt) 136 new_edge->lto_stmt_uid = stmt_uid; 137 new_edge->speculative_id = speculative_id; 138 /* Clone flags that depend on call_stmt availability manually. */ 139 new_edge->can_throw_external = can_throw_external; 140 new_edge->call_stmt_cannot_inline_p = call_stmt_cannot_inline_p; 141 new_edge->speculative = speculative; 142 new_edge->in_polymorphic_cdtor = in_polymorphic_cdtor; 143 144 /* Update IPA profile. Local profiles need no updating in original. */ 145 if (update_original) 146 count = count.combine_with_ipa_count_within (count.ipa () 147 - new_edge->count.ipa (), 148 caller->count); 149 symtab->call_edge_duplication_hooks (this, new_edge); 150 return new_edge; 151} 152 153/* Set flags of NEW_NODE and its decl. NEW_NODE is a newly created private 154 clone or its thunk. */ 155 156static void 157set_new_clone_decl_and_node_flags (cgraph_node *new_node) 158{ 159 DECL_EXTERNAL (new_node->decl) = 0; 160 TREE_PUBLIC (new_node->decl) = 0; 161 DECL_COMDAT (new_node->decl) = 0; 162 DECL_WEAK (new_node->decl) = 0; 163 DECL_VIRTUAL_P (new_node->decl) = 0; 164 DECL_STATIC_CONSTRUCTOR (new_node->decl) = 0; 165 DECL_STATIC_DESTRUCTOR (new_node->decl) = 0; 166 DECL_SET_IS_OPERATOR_NEW (new_node->decl, 0); 167 DECL_SET_IS_OPERATOR_DELETE (new_node->decl, 0); 168 DECL_IS_REPLACEABLE_OPERATOR (new_node->decl) = 0; 169 170 new_node->externally_visible = 0; 171 new_node->local = 1; 172 new_node->lowered = true; 173} 174 175/* Duplicate thunk THUNK if necessary but make it to refer to NODE. 176 ARGS_TO_SKIP, if non-NULL, determines which parameters should be omitted. 177 Function can return NODE if no thunk is necessary, which can happen when 178 thunk is this_adjusting but we are removing this parameter. */ 179 180static cgraph_node * 181duplicate_thunk_for_node (cgraph_node *thunk, cgraph_node *node) 182{ 183 cgraph_node *new_thunk, *thunk_of; 184 thunk_of = thunk->callees->callee->ultimate_alias_target (); 185 186 if (thunk_of->thunk.thunk_p) 187 node = duplicate_thunk_for_node (thunk_of, node); 188 189 if (!DECL_ARGUMENTS (thunk->decl)) 190 thunk->get_untransformed_body (); 191 192 cgraph_edge *cs; 193 for (cs = node->callers; cs; cs = cs->next_caller) 194 if (cs->caller->thunk.thunk_p 195 && cs->caller->thunk.fixed_offset == thunk->thunk.fixed_offset 196 && cs->caller->thunk.virtual_value == thunk->thunk.virtual_value 197 && cs->caller->thunk.indirect_offset == thunk->thunk.indirect_offset 198 && cs->caller->thunk.this_adjusting == thunk->thunk.this_adjusting 199 && cs->caller->thunk.virtual_offset_p == thunk->thunk.virtual_offset_p) 200 return cs->caller; 201 202 tree new_decl; 203 if (node->clone.param_adjustments) 204 { 205 /* We do not need to duplicate this_adjusting thunks if we have removed 206 this. */ 207 if (thunk->thunk.this_adjusting 208 && !node->clone.param_adjustments->first_param_intact_p ()) 209 return node; 210 211 new_decl = copy_node (thunk->decl); 212 ipa_param_body_adjustments body_adj (node->clone.param_adjustments, 213 new_decl); 214 body_adj.modify_formal_parameters (); 215 } 216 else 217 { 218 new_decl = copy_node (thunk->decl); 219 for (tree *arg = &DECL_ARGUMENTS (new_decl); 220 *arg; arg = &DECL_CHAIN (*arg)) 221 { 222 tree next = DECL_CHAIN (*arg); 223 *arg = copy_node (*arg); 224 DECL_CONTEXT (*arg) = new_decl; 225 DECL_CHAIN (*arg) = next; 226 } 227 } 228 229 gcc_checking_assert (!DECL_STRUCT_FUNCTION (new_decl)); 230 gcc_checking_assert (!DECL_INITIAL (new_decl)); 231 gcc_checking_assert (!DECL_RESULT (new_decl)); 232 gcc_checking_assert (!DECL_RTL_SET_P (new_decl)); 233 234 DECL_NAME (new_decl) = clone_function_name_numbered (thunk->decl, 235 "artificial_thunk"); 236 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 237 238 /* We need to force DECL_IGNORED_P because the new thunk is created after 239 early debug was run. */ 240 DECL_IGNORED_P (new_decl) = 1; 241 242 new_thunk = cgraph_node::create (new_decl); 243 set_new_clone_decl_and_node_flags (new_thunk); 244 new_thunk->definition = true; 245 new_thunk->can_change_signature = node->can_change_signature; 246 new_thunk->thunk = thunk->thunk; 247 new_thunk->unique_name = in_lto_p; 248 new_thunk->former_clone_of = thunk->decl; 249 new_thunk->clone.param_adjustments = node->clone.param_adjustments; 250 new_thunk->unit_id = thunk->unit_id; 251 new_thunk->merged_comdat = thunk->merged_comdat; 252 new_thunk->merged_extern_inline = thunk->merged_extern_inline; 253 254 cgraph_edge *e = new_thunk->create_edge (node, NULL, new_thunk->count); 255 symtab->call_edge_duplication_hooks (thunk->callees, e); 256 symtab->call_cgraph_duplication_hooks (thunk, new_thunk); 257 return new_thunk; 258} 259 260/* If E does not lead to a thunk, simply redirect it to N. Otherwise create 261 one or more equivalent thunks for N and redirect E to the first in the 262 chain. Note that it is then necessary to call 263 n->expand_all_artificial_thunks once all callers are redirected. */ 264 265void 266cgraph_edge::redirect_callee_duplicating_thunks (cgraph_node *n) 267{ 268 cgraph_node *orig_to = callee->ultimate_alias_target (); 269 if (orig_to->thunk.thunk_p) 270 n = duplicate_thunk_for_node (orig_to, n); 271 272 redirect_callee (n); 273} 274 275/* Call expand_thunk on all callers that are thunks and if analyze those nodes 276 that were expanded. */ 277 278void 279cgraph_node::expand_all_artificial_thunks () 280{ 281 cgraph_edge *e; 282 for (e = callers; e;) 283 if (e->caller->thunk.thunk_p) 284 { 285 cgraph_node *thunk = e->caller; 286 287 e = e->next_caller; 288 if (thunk->expand_thunk (false, false)) 289 { 290 thunk->thunk.thunk_p = false; 291 thunk->analyze (); 292 ipa_analyze_node (thunk); 293 inline_analyze_function (thunk); 294 } 295 thunk->expand_all_artificial_thunks (); 296 } 297 else 298 e = e->next_caller; 299} 300 301void 302dump_callgraph_transformation (const cgraph_node *original, 303 const cgraph_node *clone, 304 const char *suffix) 305{ 306 if (symtab->ipa_clones_dump_file) 307 { 308 fprintf (symtab->ipa_clones_dump_file, 309 "Callgraph clone;%s;%d;%s;%d;%d;%s;%d;%s;%d;%d;%s\n", 310 original->asm_name (), original->order, 311 DECL_SOURCE_FILE (original->decl), 312 DECL_SOURCE_LINE (original->decl), 313 DECL_SOURCE_COLUMN (original->decl), clone->asm_name (), 314 clone->order, DECL_SOURCE_FILE (clone->decl), 315 DECL_SOURCE_LINE (clone->decl), DECL_SOURCE_COLUMN (clone->decl), 316 suffix); 317 318 symtab->cloned_nodes.add (original); 319 symtab->cloned_nodes.add (clone); 320 } 321} 322 323/* Turn profile of N to local profile. */ 324 325static void 326localize_profile (cgraph_node *n) 327{ 328 n->count = n->count.guessed_local (); 329 for (cgraph_edge *e = n->callees; e; e=e->next_callee) 330 { 331 e->count = e->count.guessed_local (); 332 if (!e->inline_failed) 333 localize_profile (e->callee); 334 } 335 for (cgraph_edge *e = n->indirect_calls; e; e=e->next_callee) 336 e->count = e->count.guessed_local (); 337} 338 339/* Create node representing clone of N executed COUNT times. Decrease 340 the execution counts from original node too. 341 The new clone will have decl set to DECL that may or may not be the same 342 as decl of N. 343 344 When UPDATE_ORIGINAL is true, the counts are subtracted from the original 345 function's profile to reflect the fact that part of execution is handled 346 by node. 347 When CALL_DUPLICATION_HOOK is true, the ipa passes are acknowledged about 348 the new clone. Otherwise the caller is responsible for doing so later. 349 350 If the new node is being inlined into another one, NEW_INLINED_TO should be 351 the outline function the new one is (even indirectly) inlined to. All hooks 352 will see this in node's inlined_to, when invoked. Can be NULL if the 353 node is not inlined. 354 355 If PARAM_ADJUSTMENTS is non-NULL, the parameter manipulation information 356 will be overwritten by the new structure. Otherwise the new node will 357 share parameter manipulation information with the original node. */ 358 359cgraph_node * 360cgraph_node::create_clone (tree new_decl, profile_count prof_count, 361 bool update_original, 362 vec<cgraph_edge *> redirect_callers, 363 bool call_duplication_hook, 364 cgraph_node *new_inlined_to, 365 ipa_param_adjustments *param_adjustments, 366 const char *suffix) 367{ 368 cgraph_node *new_node = symtab->create_empty (); 369 cgraph_edge *e; 370 unsigned i; 371 profile_count old_count = count; 372 bool nonzero = count.ipa ().nonzero_p (); 373 374 if (new_inlined_to) 375 dump_callgraph_transformation (this, new_inlined_to, "inlining to"); 376 377 /* When inlining we scale precisely to prof_count, when cloning we can 378 preserve local profile. */ 379 if (!new_inlined_to) 380 prof_count = count.combine_with_ipa_count (prof_count); 381 new_node->count = prof_count; 382 383 /* Update IPA profile. Local profiles need no updating in original. */ 384 if (update_original) 385 { 386 if (inlined_to) 387 count = count.combine_with_ipa_count_within (count.ipa () 388 - prof_count.ipa (), 389 inlined_to->count); 390 else 391 count = count.combine_with_ipa_count (count.ipa () - prof_count.ipa ()); 392 } 393 new_node->decl = new_decl; 394 new_node->register_symbol (); 395 new_node->origin = origin; 396 new_node->lto_file_data = lto_file_data; 397 if (new_node->origin) 398 { 399 new_node->next_nested = new_node->origin->nested; 400 new_node->origin->nested = new_node; 401 } 402 new_node->analyzed = analyzed; 403 new_node->definition = definition; 404 new_node->versionable = versionable; 405 new_node->can_change_signature = can_change_signature; 406 new_node->redefined_extern_inline = redefined_extern_inline; 407 new_node->tm_may_enter_irr = tm_may_enter_irr; 408 new_node->externally_visible = false; 409 new_node->no_reorder = no_reorder; 410 new_node->local = true; 411 new_node->inlined_to = new_inlined_to; 412 new_node->rtl = rtl; 413 new_node->frequency = frequency; 414 new_node->tp_first_run = tp_first_run; 415 new_node->tm_clone = tm_clone; 416 new_node->icf_merged = icf_merged; 417 new_node->thunk = thunk; 418 new_node->unit_id = unit_id; 419 new_node->merged_comdat = merged_comdat; 420 new_node->merged_extern_inline = merged_extern_inline; 421 422 if (param_adjustments) 423 new_node->clone.param_adjustments = param_adjustments; 424 else 425 new_node->clone.param_adjustments = clone.param_adjustments; 426 new_node->clone.tree_map = NULL; 427 new_node->clone.performed_splits = vec_safe_copy (clone.performed_splits); 428 new_node->split_part = split_part; 429 430 FOR_EACH_VEC_ELT (redirect_callers, i, e) 431 { 432 /* Redirect calls to the old version node to point to its new 433 version. The only exception is when the edge was proved to 434 be unreachable during the cloning procedure. */ 435 if (!e->callee 436 || !fndecl_built_in_p (e->callee->decl, BUILT_IN_UNREACHABLE)) 437 e->redirect_callee_duplicating_thunks (new_node); 438 } 439 new_node->expand_all_artificial_thunks (); 440 441 for (e = callees;e; e=e->next_callee) 442 e->clone (new_node, e->call_stmt, e->lto_stmt_uid, new_node->count, old_count, 443 update_original); 444 445 for (e = indirect_calls; e; e = e->next_callee) 446 e->clone (new_node, e->call_stmt, e->lto_stmt_uid, 447 new_node->count, old_count, update_original); 448 new_node->clone_references (this); 449 450 new_node->next_sibling_clone = clones; 451 if (clones) 452 clones->prev_sibling_clone = new_node; 453 clones = new_node; 454 new_node->clone_of = this; 455 456 if (call_duplication_hook) 457 symtab->call_cgraph_duplication_hooks (this, new_node); 458 /* With partial train run we do not want to assume that original's 459 count is zero whenever we redurect all executed edges to clone. 460 Simply drop profile to local one in this case. */ 461 if (update_original 462 && opt_for_fn (decl, flag_profile_partial_training) 463 && nonzero 464 && count.ipa_p () 465 && !count.ipa ().nonzero_p () 466 && !inlined_to) 467 localize_profile (this); 468 469 if (!new_inlined_to) 470 dump_callgraph_transformation (this, new_node, suffix); 471 472 return new_node; 473} 474 475static GTY(()) hash_map<const char *, unsigned> *clone_fn_ids; 476 477/* Return a new assembler name for a clone of decl named NAME. Apart 478 from the string SUFFIX, the new name will end with a unique (for 479 each NAME) unspecified number. If clone numbering is not needed 480 then the two argument clone_function_name should be used instead. 481 Should not be called directly except for by 482 lto-partition.c:privatize_symbol_name_1. */ 483 484tree 485clone_function_name_numbered (const char *name, const char *suffix) 486{ 487 /* Initialize the function->counter mapping the first time it's 488 needed. */ 489 if (!clone_fn_ids) 490 clone_fn_ids = hash_map<const char *, unsigned int>::create_ggc (64); 491 unsigned int &suffix_counter = clone_fn_ids->get_or_insert ( 492 IDENTIFIER_POINTER (get_identifier (name))); 493 return clone_function_name (name, suffix, suffix_counter++); 494} 495 496/* Return a new assembler name for a clone of DECL. Apart from string 497 SUFFIX, the new name will end with a unique (for each DECL 498 assembler name) unspecified number. If clone numbering is not 499 needed then the two argument clone_function_name should be used 500 instead. */ 501 502tree 503clone_function_name_numbered (tree decl, const char *suffix) 504{ 505 tree name = DECL_ASSEMBLER_NAME (decl); 506 return clone_function_name_numbered (IDENTIFIER_POINTER (name), 507 suffix); 508} 509 510/* Return a new assembler name for a clone of decl named NAME. Apart 511 from the string SUFFIX, the new name will end with the specified 512 NUMBER. If clone numbering is not needed then the two argument 513 clone_function_name should be used instead. */ 514 515tree 516clone_function_name (const char *name, const char *suffix, 517 unsigned long number) 518{ 519 size_t len = strlen (name); 520 char *tmp_name, *prefix; 521 522 prefix = XALLOCAVEC (char, len + strlen (suffix) + 2); 523 memcpy (prefix, name, len); 524 strcpy (prefix + len + 1, suffix); 525 prefix[len] = symbol_table::symbol_suffix_separator (); 526 ASM_FORMAT_PRIVATE_NAME (tmp_name, prefix, number); 527 return get_identifier (tmp_name); 528} 529 530/* Return a new assembler name for a clone of DECL. Apart from the 531 string SUFFIX, the new name will end with the specified NUMBER. If 532 clone numbering is not needed then the two argument 533 clone_function_name should be used instead. */ 534 535tree 536clone_function_name (tree decl, const char *suffix, 537 unsigned long number) 538{ 539 return clone_function_name ( 540 IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (decl)), suffix, number); 541} 542 543/* Return a new assembler name ending with the string SUFFIX for a 544 clone of DECL. */ 545 546tree 547clone_function_name (tree decl, const char *suffix) 548{ 549 tree identifier = DECL_ASSEMBLER_NAME (decl); 550 /* For consistency this needs to behave the same way as 551 ASM_FORMAT_PRIVATE_NAME does, but without the final number 552 suffix. */ 553 char *separator = XALLOCAVEC (char, 2); 554 separator[0] = symbol_table::symbol_suffix_separator (); 555 separator[1] = 0; 556#if defined (NO_DOT_IN_LABEL) && defined (NO_DOLLAR_IN_LABEL) 557 const char *prefix = "__"; 558#else 559 const char *prefix = ""; 560#endif 561 char *result = ACONCAT ((prefix, 562 IDENTIFIER_POINTER (identifier), 563 separator, 564 suffix, 565 (char*)0)); 566 return get_identifier (result); 567} 568 569 570/* Create callgraph node clone with new declaration. The actual body will be 571 copied later at compilation stage. The name of the new clone will be 572 constructed from the name of the original node, SUFFIX and NUM_SUFFIX. 573 574 TODO: after merging in ipa-sra use function call notes instead of args_to_skip 575 bitmap interface. 576 */ 577cgraph_node * 578cgraph_node::create_virtual_clone (vec<cgraph_edge *> redirect_callers, 579 vec<ipa_replace_map *, va_gc> *tree_map, 580 ipa_param_adjustments *param_adjustments, 581 const char * suffix, unsigned num_suffix) 582{ 583 tree old_decl = decl; 584 cgraph_node *new_node = NULL; 585 tree new_decl; 586 size_t len, i; 587 ipa_replace_map *map; 588 char *name; 589 590 gcc_checking_assert (versionable); 591 /* TODO: It would be nice if we could recognize that param_adjustments do not 592 actually perform any changes, but at the moment let's require it simply 593 does not exist. */ 594 gcc_assert (can_change_signature || !param_adjustments); 595 596 /* Make a new FUNCTION_DECL tree node */ 597 if (!param_adjustments) 598 new_decl = copy_node (old_decl); 599 else 600 new_decl = param_adjustments->adjust_decl (old_decl); 601 602 /* These pointers represent function body and will be populated only when clone 603 is materialized. */ 604 gcc_assert (new_decl != old_decl); 605 DECL_STRUCT_FUNCTION (new_decl) = NULL; 606 DECL_ARGUMENTS (new_decl) = NULL; 607 DECL_INITIAL (new_decl) = NULL; 608 DECL_RESULT (new_decl) = NULL; 609 /* We cannot do DECL_RESULT (new_decl) = NULL; here because of LTO partitioning 610 sometimes storing only clone decl instead of original. */ 611 612 /* Generate a new name for the new version. */ 613 len = IDENTIFIER_LENGTH (DECL_NAME (old_decl)); 614 name = XALLOCAVEC (char, len + strlen (suffix) + 2); 615 memcpy (name, IDENTIFIER_POINTER (DECL_NAME (old_decl)), len); 616 strcpy (name + len + 1, suffix); 617 name[len] = '.'; 618 DECL_NAME (new_decl) = get_identifier (name); 619 SET_DECL_ASSEMBLER_NAME (new_decl, 620 clone_function_name (old_decl, suffix, num_suffix)); 621 SET_DECL_RTL (new_decl, NULL); 622 623 new_node = create_clone (new_decl, count, false, 624 redirect_callers, false, NULL, param_adjustments, 625 suffix); 626 627 /* Update the properties. 628 Make clone visible only within this translation unit. Make sure 629 that is not weak also. 630 ??? We cannot use COMDAT linkage because there is no 631 ABI support for this. */ 632 set_new_clone_decl_and_node_flags (new_node); 633 new_node->ipcp_clone = ipcp_clone; 634 new_node->clone.tree_map = tree_map; 635 if (!implicit_section) 636 new_node->set_section (get_section ()); 637 638 /* Clones of global symbols or symbols with unique names are unique. */ 639 if ((TREE_PUBLIC (old_decl) 640 && !DECL_EXTERNAL (old_decl) 641 && !DECL_WEAK (old_decl) 642 && !DECL_COMDAT (old_decl)) 643 || in_lto_p) 644 new_node->unique_name = true; 645 FOR_EACH_VEC_SAFE_ELT (tree_map, i, map) 646 new_node->maybe_create_reference (map->new_tree, NULL); 647 648 if (ipa_transforms_to_apply.exists ()) 649 new_node->ipa_transforms_to_apply 650 = ipa_transforms_to_apply.copy (); 651 652 symtab->call_cgraph_duplication_hooks (this, new_node); 653 654 return new_node; 655} 656 657/* callgraph node being removed from symbol table; see if its entry can be 658 replaced by other inline clone. */ 659cgraph_node * 660cgraph_node::find_replacement (void) 661{ 662 cgraph_node *next_inline_clone, *replacement; 663 664 for (next_inline_clone = clones; 665 next_inline_clone 666 && next_inline_clone->decl != decl; 667 next_inline_clone = next_inline_clone->next_sibling_clone) 668 ; 669 670 /* If there is inline clone of the node being removed, we need 671 to put it into the position of removed node and reorganize all 672 other clones to be based on it. */ 673 if (next_inline_clone) 674 { 675 cgraph_node *n; 676 cgraph_node *new_clones; 677 678 replacement = next_inline_clone; 679 680 /* Unlink inline clone from the list of clones of removed node. */ 681 if (next_inline_clone->next_sibling_clone) 682 next_inline_clone->next_sibling_clone->prev_sibling_clone 683 = next_inline_clone->prev_sibling_clone; 684 if (next_inline_clone->prev_sibling_clone) 685 { 686 gcc_assert (clones != next_inline_clone); 687 next_inline_clone->prev_sibling_clone->next_sibling_clone 688 = next_inline_clone->next_sibling_clone; 689 } 690 else 691 { 692 gcc_assert (clones == next_inline_clone); 693 clones = next_inline_clone->next_sibling_clone; 694 } 695 696 new_clones = clones; 697 clones = NULL; 698 699 /* Copy clone info. */ 700 next_inline_clone->clone = clone; 701 702 /* Now place it into clone tree at same level at NODE. */ 703 next_inline_clone->clone_of = clone_of; 704 next_inline_clone->prev_sibling_clone = NULL; 705 next_inline_clone->next_sibling_clone = NULL; 706 if (clone_of) 707 { 708 if (clone_of->clones) 709 clone_of->clones->prev_sibling_clone = next_inline_clone; 710 next_inline_clone->next_sibling_clone = clone_of->clones; 711 clone_of->clones = next_inline_clone; 712 } 713 714 /* Merge the clone list. */ 715 if (new_clones) 716 { 717 if (!next_inline_clone->clones) 718 next_inline_clone->clones = new_clones; 719 else 720 { 721 n = next_inline_clone->clones; 722 while (n->next_sibling_clone) 723 n = n->next_sibling_clone; 724 n->next_sibling_clone = new_clones; 725 new_clones->prev_sibling_clone = n; 726 } 727 } 728 729 /* Update clone_of pointers. */ 730 n = new_clones; 731 while (n) 732 { 733 n->clone_of = next_inline_clone; 734 n = n->next_sibling_clone; 735 } 736 737 /* Update order in order to be able to find a LTO section 738 with function body. */ 739 replacement->order = order; 740 741 return replacement; 742 } 743 else 744 return NULL; 745} 746 747/* Like cgraph_set_call_stmt but walk the clone tree and update all 748 clones sharing the same function body. 749 When WHOLE_SPECULATIVE_EDGES is true, all three components of 750 speculative edge gets updated. Otherwise we update only direct 751 call. */ 752 753void 754cgraph_node::set_call_stmt_including_clones (gimple *old_stmt, 755 gcall *new_stmt, 756 bool update_speculative) 757{ 758 cgraph_node *node; 759 cgraph_edge *master_edge = get_edge (old_stmt); 760 761 if (master_edge) 762 cgraph_edge::set_call_stmt (master_edge, new_stmt, update_speculative); 763 764 node = clones; 765 if (node) 766 while (node != this) 767 { 768 cgraph_edge *edge = node->get_edge (old_stmt); 769 if (edge) 770 { 771 edge = cgraph_edge::set_call_stmt (edge, new_stmt, 772 update_speculative); 773 /* If UPDATE_SPECULATIVE is false, it means that we are turning 774 speculative call into a real code sequence. Update the 775 callgraph edges. */ 776 if (edge->speculative && !update_speculative) 777 { 778 cgraph_edge *indirect = edge->speculative_call_indirect_edge (); 779 780 for (cgraph_edge *next, *direct 781 = edge->first_speculative_call_target (); 782 direct; 783 direct = next) 784 { 785 next = direct->next_speculative_call_target (); 786 direct->speculative_call_target_ref ()->speculative = false; 787 direct->speculative = false; 788 } 789 indirect->speculative = false; 790 } 791 } 792 if (node->clones) 793 node = node->clones; 794 else if (node->next_sibling_clone) 795 node = node->next_sibling_clone; 796 else 797 { 798 while (node != this && !node->next_sibling_clone) 799 node = node->clone_of; 800 if (node != this) 801 node = node->next_sibling_clone; 802 } 803 } 804} 805 806/* Like cgraph_create_edge walk the clone tree and update all clones sharing 807 same function body. If clones already have edge for OLD_STMT; only 808 update the edge same way as cgraph_set_call_stmt_including_clones does. 809 810 TODO: COUNT and LOOP_DEPTH should be properly distributed based on relative 811 frequencies of the clones. */ 812 813void 814cgraph_node::create_edge_including_clones (cgraph_node *callee, 815 gimple *old_stmt, gcall *stmt, 816 profile_count count, 817 cgraph_inline_failed_t reason) 818{ 819 cgraph_node *node; 820 821 if (!get_edge (stmt)) 822 { 823 cgraph_edge *edge = create_edge (callee, stmt, count); 824 edge->inline_failed = reason; 825 } 826 827 node = clones; 828 if (node) 829 while (node != this) 830 /* Thunk clones do not get updated while copying inline function body. */ 831 if (!node->thunk.thunk_p) 832 { 833 cgraph_edge *edge = node->get_edge (old_stmt); 834 835 /* It is possible that clones already contain the edge while 836 master didn't. Either we promoted indirect call into direct 837 call in the clone or we are processing clones of unreachable 838 master where edges has been removed. */ 839 if (edge) 840 edge = cgraph_edge::set_call_stmt (edge, stmt); 841 else if (! node->get_edge (stmt)) 842 { 843 edge = node->create_edge (callee, stmt, count); 844 edge->inline_failed = reason; 845 } 846 847 if (node->clones) 848 node = node->clones; 849 else if (node->next_sibling_clone) 850 node = node->next_sibling_clone; 851 else 852 { 853 while (node != this && !node->next_sibling_clone) 854 node = node->clone_of; 855 if (node != this) 856 node = node->next_sibling_clone; 857 } 858 } 859} 860 861/* Remove the node from cgraph and all inline clones inlined into it. 862 Skip however removal of FORBIDDEN_NODE and return true if it needs to be 863 removed. This allows to call the function from outer loop walking clone 864 tree. */ 865 866bool 867cgraph_node::remove_symbol_and_inline_clones (cgraph_node *forbidden_node) 868{ 869 cgraph_edge *e, *next; 870 bool found = false; 871 872 if (this == forbidden_node) 873 { 874 cgraph_edge::remove (callers); 875 return true; 876 } 877 for (e = callees; e; e = next) 878 { 879 next = e->next_callee; 880 if (!e->inline_failed) 881 found |= e->callee->remove_symbol_and_inline_clones (forbidden_node); 882 } 883 remove (); 884 return found; 885} 886 887/* The edges representing the callers of the NEW_VERSION node were 888 fixed by cgraph_function_versioning (), now the call_expr in their 889 respective tree code should be updated to call the NEW_VERSION. */ 890 891static void 892update_call_expr (cgraph_node *new_version) 893{ 894 cgraph_edge *e; 895 896 gcc_assert (new_version); 897 898 /* Update the call expr on the edges to call the new version. */ 899 for (e = new_version->callers; e; e = e->next_caller) 900 { 901 function *inner_function = DECL_STRUCT_FUNCTION (e->caller->decl); 902 gimple_call_set_fndecl (e->call_stmt, new_version->decl); 903 maybe_clean_eh_stmt_fn (inner_function, e->call_stmt); 904 } 905} 906 907 908/* Create a new cgraph node which is the new version of 909 callgraph node. REDIRECT_CALLERS holds the callers 910 edges which should be redirected to point to 911 NEW_VERSION. ALL the callees edges of the node 912 are cloned to the new version node. Return the new 913 version node. 914 915 If non-NULL BLOCK_TO_COPY determine what basic blocks 916 was copied to prevent duplications of calls that are dead 917 in the clone. */ 918 919cgraph_node * 920cgraph_node::create_version_clone (tree new_decl, 921 vec<cgraph_edge *> redirect_callers, 922 bitmap bbs_to_copy, 923 const char *suffix) 924 { 925 cgraph_node *new_version; 926 cgraph_edge *e; 927 unsigned i; 928 929 new_version = cgraph_node::create (new_decl); 930 931 new_version->analyzed = analyzed; 932 new_version->definition = definition; 933 new_version->local = local; 934 new_version->externally_visible = false; 935 new_version->no_reorder = no_reorder; 936 new_version->local = new_version->definition; 937 new_version->inlined_to = inlined_to; 938 new_version->rtl = rtl; 939 new_version->count = count; 940 new_version->unit_id = unit_id; 941 new_version->merged_comdat = merged_comdat; 942 new_version->merged_extern_inline = merged_extern_inline; 943 944 for (e = callees; e; e=e->next_callee) 945 if (!bbs_to_copy 946 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) 947 e->clone (new_version, e->call_stmt, 948 e->lto_stmt_uid, count, count, 949 true); 950 for (e = indirect_calls; e; e=e->next_callee) 951 if (!bbs_to_copy 952 || bitmap_bit_p (bbs_to_copy, gimple_bb (e->call_stmt)->index)) 953 e->clone (new_version, e->call_stmt, 954 e->lto_stmt_uid, count, count, 955 true); 956 FOR_EACH_VEC_ELT (redirect_callers, i, e) 957 { 958 /* Redirect calls to the old version node to point to its new 959 version. */ 960 e->redirect_callee (new_version); 961 } 962 963 dump_callgraph_transformation (this, new_version, suffix); 964 965 return new_version; 966 } 967 968/* Perform function versioning. 969 Function versioning includes copying of the tree and 970 a callgraph update (creating a new cgraph node and updating 971 its callees and callers). 972 973 REDIRECT_CALLERS varray includes the edges to be redirected 974 to the new version. 975 976 TREE_MAP is a mapping of tree nodes we want to replace with 977 new ones (according to results of prior analysis). 978 979 If non-NULL ARGS_TO_SKIP determine function parameters to remove 980 from new version. 981 If SKIP_RETURN is true, the new version will return void. 982 If non-NULL BLOCK_TO_COPY determine what basic blocks to copy. 983 If non_NULL NEW_ENTRY determine new entry BB of the clone. 984 985 If TARGET_ATTRIBUTES is non-null, when creating a new declaration, 986 add the attributes to DECL_ATTRIBUTES. And call valid_attribute_p 987 that will promote value of the attribute DECL_FUNCTION_SPECIFIC_TARGET 988 of the declaration. 989 990 Return the new version's cgraph node. */ 991 992cgraph_node * 993cgraph_node::create_version_clone_with_body 994 (vec<cgraph_edge *> redirect_callers, 995 vec<ipa_replace_map *, va_gc> *tree_map, 996 ipa_param_adjustments *param_adjustments, 997 bitmap bbs_to_copy, basic_block new_entry_block, const char *suffix, 998 tree target_attributes) 999{ 1000 tree old_decl = decl; 1001 cgraph_node *new_version_node = NULL; 1002 tree new_decl; 1003 1004 if (!tree_versionable_function_p (old_decl)) 1005 return NULL; 1006 1007 /* TODO: Restore an assert that we do not change signature if 1008 can_change_signature is false. We cannot just check that 1009 param_adjustments is NULL because unfortunately ipa-split removes return 1010 values from such functions. */ 1011 1012 /* Make a new FUNCTION_DECL tree node for the new version. */ 1013 if (param_adjustments) 1014 new_decl = param_adjustments->adjust_decl (old_decl); 1015 else 1016 new_decl = copy_node (old_decl); 1017 1018 /* Generate a new name for the new version. */ 1019 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl, suffix); 1020 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 1021 SET_DECL_RTL (new_decl, NULL); 1022 1023 DECL_VIRTUAL_P (new_decl) = 0; 1024 1025 if (target_attributes) 1026 { 1027 DECL_ATTRIBUTES (new_decl) = target_attributes; 1028 1029 location_t saved_loc = input_location; 1030 tree v = TREE_VALUE (target_attributes); 1031 input_location = DECL_SOURCE_LOCATION (new_decl); 1032 bool r = targetm.target_option.valid_attribute_p (new_decl, NULL, v, 1); 1033 input_location = saved_loc; 1034 if (!r) 1035 return NULL; 1036 } 1037 1038 /* When the old decl was a con-/destructor make sure the clone isn't. */ 1039 DECL_STATIC_CONSTRUCTOR (new_decl) = 0; 1040 DECL_STATIC_DESTRUCTOR (new_decl) = 0; 1041 DECL_SET_IS_OPERATOR_NEW (new_decl, 0); 1042 DECL_SET_IS_OPERATOR_DELETE (new_decl, 0); 1043 DECL_IS_REPLACEABLE_OPERATOR (new_decl) = 0; 1044 1045 /* Create the new version's call-graph node. 1046 and update the edges of the new node. */ 1047 new_version_node = create_version_clone (new_decl, redirect_callers, 1048 bbs_to_copy, suffix); 1049 1050 if (ipa_transforms_to_apply.exists ()) 1051 new_version_node->ipa_transforms_to_apply 1052 = ipa_transforms_to_apply.copy (); 1053 /* Copy the OLD_VERSION_NODE function tree to the new version. */ 1054 tree_function_versioning (old_decl, new_decl, tree_map, param_adjustments, 1055 false, bbs_to_copy, new_entry_block); 1056 1057 /* Update the new version's properties. 1058 Make The new version visible only within this translation unit. Make sure 1059 that is not weak also. 1060 ??? We cannot use COMDAT linkage because there is no 1061 ABI support for this. */ 1062 new_version_node->make_decl_local (); 1063 DECL_VIRTUAL_P (new_version_node->decl) = 0; 1064 new_version_node->externally_visible = 0; 1065 new_version_node->local = 1; 1066 new_version_node->lowered = true; 1067 if (!implicit_section) 1068 new_version_node->set_section (get_section ()); 1069 /* Clones of global symbols or symbols with unique names are unique. */ 1070 if ((TREE_PUBLIC (old_decl) 1071 && !DECL_EXTERNAL (old_decl) 1072 && !DECL_WEAK (old_decl) 1073 && !DECL_COMDAT (old_decl)) 1074 || in_lto_p) 1075 new_version_node->unique_name = true; 1076 1077 /* Update the call_expr on the edges to call the new version node. */ 1078 update_call_expr (new_version_node); 1079 1080 symtab->call_cgraph_insertion_hooks (new_version_node); 1081 return new_version_node; 1082} 1083 1084/* Remove the node from the tree of virtual and inline clones and make it a 1085 standalone node - not a clone any more. */ 1086 1087void cgraph_node::remove_from_clone_tree () 1088{ 1089 if (next_sibling_clone) 1090 next_sibling_clone->prev_sibling_clone = prev_sibling_clone; 1091 if (prev_sibling_clone) 1092 prev_sibling_clone->next_sibling_clone = next_sibling_clone; 1093 else 1094 clone_of->clones = next_sibling_clone; 1095 next_sibling_clone = NULL; 1096 prev_sibling_clone = NULL; 1097 clone_of = NULL; 1098} 1099 1100/* Given virtual clone, turn it into actual clone. */ 1101 1102static void 1103cgraph_materialize_clone (cgraph_node *node) 1104{ 1105 bitmap_obstack_initialize (NULL); 1106 node->former_clone_of = node->clone_of->decl; 1107 if (node->clone_of->former_clone_of) 1108 node->former_clone_of = node->clone_of->former_clone_of; 1109 /* Copy the OLD_VERSION_NODE function tree to the new version. */ 1110 tree_function_versioning (node->clone_of->decl, node->decl, 1111 node->clone.tree_map, node->clone.param_adjustments, 1112 true, NULL, NULL); 1113 if (symtab->dump_file) 1114 { 1115 dump_function_to_file (node->clone_of->decl, symtab->dump_file, 1116 dump_flags); 1117 dump_function_to_file (node->decl, symtab->dump_file, dump_flags); 1118 } 1119 1120 cgraph_node *clone_of = node->clone_of; 1121 /* Function is no longer clone. */ 1122 node->remove_from_clone_tree (); 1123 if (!clone_of->analyzed && !clone_of->clones) 1124 { 1125 clone_of->release_body (); 1126 clone_of->remove_callees (); 1127 clone_of->remove_all_references (); 1128 } 1129 bitmap_obstack_release (NULL); 1130} 1131 1132/* Once all functions from compilation unit are in memory, produce all clones 1133 and update all calls. We might also do this on demand if we don't want to 1134 bring all functions to memory prior compilation, but current WHOPR 1135 implementation does that and it is a bit easier to keep everything right in 1136 this order. */ 1137 1138void 1139symbol_table::materialize_all_clones (void) 1140{ 1141 cgraph_node *node; 1142 bool stabilized = false; 1143 1144 1145 if (symtab->dump_file) 1146 fprintf (symtab->dump_file, "Materializing clones\n"); 1147 1148 cgraph_node::checking_verify_cgraph_nodes (); 1149 1150 /* We can also do topological order, but number of iterations should be 1151 bounded by number of IPA passes since single IPA pass is probably not 1152 going to create clones of clones it created itself. */ 1153 while (!stabilized) 1154 { 1155 stabilized = true; 1156 FOR_EACH_FUNCTION (node) 1157 { 1158 if (node->clone_of && node->decl != node->clone_of->decl 1159 && !gimple_has_body_p (node->decl)) 1160 { 1161 if (!node->clone_of->clone_of) 1162 node->clone_of->get_untransformed_body (); 1163 if (gimple_has_body_p (node->clone_of->decl)) 1164 { 1165 if (symtab->dump_file) 1166 { 1167 fprintf (symtab->dump_file, "cloning %s to %s\n", 1168 node->clone_of->dump_name (), 1169 node->dump_name ()); 1170 if (node->clone.tree_map) 1171 { 1172 unsigned int i; 1173 fprintf (symtab->dump_file, " replace map: "); 1174 for (i = 0; 1175 i < vec_safe_length (node->clone.tree_map); 1176 i++) 1177 { 1178 ipa_replace_map *replace_info; 1179 replace_info = (*node->clone.tree_map)[i]; 1180 fprintf (symtab->dump_file, "%i -> ", 1181 (*node->clone.tree_map)[i]->parm_num); 1182 print_generic_expr (symtab->dump_file, 1183 replace_info->new_tree); 1184 } 1185 fprintf (symtab->dump_file, "\n"); 1186 } 1187 if (node->clone.param_adjustments) 1188 node->clone.param_adjustments->dump (symtab->dump_file); 1189 } 1190 cgraph_materialize_clone (node); 1191 stabilized = false; 1192 } 1193 } 1194 } 1195 } 1196 FOR_EACH_FUNCTION (node) 1197 if (!node->analyzed && node->callees) 1198 { 1199 node->remove_callees (); 1200 node->remove_all_references (); 1201 } 1202 else 1203 node->clear_stmts_in_references (); 1204 if (symtab->dump_file) 1205 fprintf (symtab->dump_file, "Materialization Call site updates done.\n"); 1206 1207 cgraph_node::checking_verify_cgraph_nodes (); 1208 1209 symtab->remove_unreachable_nodes (symtab->dump_file); 1210} 1211 1212#include "gt-cgraphclones.h" 1213