1/* Instruction scheduling pass. Selective scheduler and pipeliner. 2 Copyright (C) 2006-2022 Free Software Foundation, Inc. 3 4This file is part of GCC. 5 6GCC is free software; you can redistribute it and/or modify it under 7the terms of the GNU General Public License as published by the Free 8Software Foundation; either version 3, or (at your option) any later 9version. 10 11GCC is distributed in the hope that it will be useful, but WITHOUT ANY 12WARRANTY; without even the implied warranty of MERCHANTABILITY or 13FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 14for more details. 15 16You should have received a copy of the GNU General Public License 17along with GCC; see the file COPYING3. If not see 18<http://www.gnu.org/licenses/>. */ 19 20#include "config.h" 21#include "system.h" 22#include "coretypes.h" 23#include "backend.h" 24#include "tree.h" 25#include "rtl.h" 26#include "df.h" 27#include "memmodel.h" 28#include "tm_p.h" 29#include "regs.h" 30#include "cfgbuild.h" 31#include "cfgcleanup.h" 32#include "insn-config.h" 33#include "insn-attr.h" 34#include "target.h" 35#include "sched-int.h" 36#include "rtlhooks-def.h" 37#include "ira.h" 38#include "ira-int.h" 39#include "rtl-iter.h" 40 41#ifdef INSN_SCHEDULING 42#include "regset.h" 43#include "cfgloop.h" 44#include "sel-sched-ir.h" 45#include "sel-sched-dump.h" 46#include "sel-sched.h" 47#include "dbgcnt.h" 48#include "function-abi.h" 49 50/* Implementation of selective scheduling approach. 51 The below implementation follows the original approach with the following 52 changes: 53 54 o the scheduler works after register allocation (but can be also tuned 55 to work before RA); 56 o some instructions are not copied or register renamed; 57 o conditional jumps are not moved with code duplication; 58 o several jumps in one parallel group are not supported; 59 o when pipelining outer loops, code motion through inner loops 60 is not supported; 61 o control and data speculation are supported; 62 o some improvements for better compile time/performance were made. 63 64 Terminology 65 =========== 66 67 A vinsn, or virtual insn, is an insn with additional data characterizing 68 insn pattern, such as LHS, RHS, register sets used/set/clobbered, etc. 69 Vinsns also act as smart pointers to save memory by reusing them in 70 different expressions. A vinsn is described by vinsn_t type. 71 72 An expression is a vinsn with additional data characterizing its properties 73 at some point in the control flow graph. The data may be its usefulness, 74 priority, speculative status, whether it was renamed/subsituted, etc. 75 An expression is described by expr_t type. 76 77 Availability set (av_set) is a set of expressions at a given control flow 78 point. It is represented as av_set_t. The expressions in av sets are kept 79 sorted in the terms of expr_greater_p function. It allows to truncate 80 the set while leaving the best expressions. 81 82 A fence is a point through which code motion is prohibited. On each step, 83 we gather a parallel group of insns at a fence. It is possible to have 84 multiple fences. A fence is represented via fence_t. 85 86 A boundary is the border between the fence group and the rest of the code. 87 Currently, we never have more than one boundary per fence, as we finalize 88 the fence group when a jump is scheduled. A boundary is represented 89 via bnd_t. 90 91 High-level overview 92 =================== 93 94 The scheduler finds regions to schedule, schedules each one, and finalizes. 95 The regions are formed starting from innermost loops, so that when the inner 96 loop is pipelined, its prologue can be scheduled together with yet unprocessed 97 outer loop. The rest of acyclic regions are found using extend_rgns: 98 the blocks that are not yet allocated to any regions are traversed in top-down 99 order, and a block is added to a region to which all its predecessors belong; 100 otherwise, the block starts its own region. 101 102 The main scheduling loop (sel_sched_region_2) consists of just 103 scheduling on each fence and updating fences. For each fence, 104 we fill a parallel group of insns (fill_insns) until some insns can be added. 105 First, we compute available exprs (av-set) at the boundary of the current 106 group. Second, we choose the best expression from it. If the stall is 107 required to schedule any of the expressions, we advance the current cycle 108 appropriately. So, the final group does not exactly correspond to a VLIW 109 word. Third, we move the chosen expression to the boundary (move_op) 110 and update the intermediate av sets and liveness sets. We quit fill_insns 111 when either no insns left for scheduling or we have scheduled enough insns 112 so we feel like advancing a scheduling point. 113 114 Computing available expressions 115 =============================== 116 117 The computation (compute_av_set) is a bottom-up traversal. At each insn, 118 we're moving the union of its successors' sets through it via 119 moveup_expr_set. The dependent expressions are removed. Local 120 transformations (substitution, speculation) are applied to move more 121 exprs. Then the expr corresponding to the current insn is added. 122 The result is saved on each basic block header. 123 124 When traversing the CFG, we're moving down for no more than max_ws insns. 125 Also, we do not move down to ineligible successors (is_ineligible_successor), 126 which include moving along a back-edge, moving to already scheduled code, 127 and moving to another fence. The first two restrictions are lifted during 128 pipelining, which allows us to move insns along a back-edge. We always have 129 an acyclic region for scheduling because we forbid motion through fences. 130 131 Choosing the best expression 132 ============================ 133 134 We sort the final availability set via sel_rank_for_schedule, then we remove 135 expressions which are not yet ready (tick_check_p) or which dest registers 136 cannot be used. For some of them, we choose another register via 137 find_best_reg. To do this, we run find_used_regs to calculate the set of 138 registers which cannot be used. The find_used_regs function performs 139 a traversal of code motion paths for an expr. We consider for renaming 140 only registers which are from the same regclass as the original one and 141 using which does not interfere with any live ranges. Finally, we convert 142 the resulting set to the ready list format and use max_issue and reorder* 143 hooks similarly to the Haifa scheduler. 144 145 Scheduling the best expression 146 ============================== 147 148 We run the move_op routine to perform the same type of code motion paths 149 traversal as in find_used_regs. (These are working via the same driver, 150 code_motion_path_driver.) When moving down the CFG, we look for original 151 instruction that gave birth to a chosen expression. We undo 152 the transformations performed on an expression via the history saved in it. 153 When found, we remove the instruction or leave a reg-reg copy/speculation 154 check if needed. On a way up, we insert bookkeeping copies at each join 155 point. If a copy is not needed, it will be removed later during this 156 traversal. We update the saved av sets and liveness sets on the way up, too. 157 158 Finalizing the schedule 159 ======================= 160 161 When pipelining, we reschedule the blocks from which insns were pipelined 162 to get a tighter schedule. On Itanium, we also perform bundling via 163 the same routine from ia64.cc. 164 165 Dependence analysis changes 166 =========================== 167 168 We augmented the sched-deps.cc with hooks that get called when a particular 169 dependence is found in a particular part of an insn. Using these hooks, we 170 can do several actions such as: determine whether an insn can be moved through 171 another (has_dependence_p, moveup_expr); find out whether an insn can be 172 scheduled on the current cycle (tick_check_p); find out registers that 173 are set/used/clobbered by an insn and find out all the strange stuff that 174 restrict its movement, like SCHED_GROUP_P or CANT_MOVE (done in 175 init_global_and_expr_for_insn). 176 177 Initialization changes 178 ====================== 179 180 There are parts of haifa-sched.cc, sched-deps.cc, and sched-rgn.cc that are 181 reused in all of the schedulers. We have split up the initialization of data 182 of such parts into different functions prefixed with scheduler type and 183 postfixed with the type of data initialized: {,sel_,haifa_}sched_{init,finish}, 184 sched_rgn_init/finish, sched_deps_init/finish, sched_init_{luids/bbs}, etc. 185 The same splitting is done with current_sched_info structure: 186 dependence-related parts are in sched_deps_info, common part is in 187 common_sched_info, and haifa/sel/etc part is in current_sched_info. 188 189 Target contexts 190 =============== 191 192 As we now have multiple-point scheduling, this would not work with backends 193 which save some of the scheduler state to use it in the target hooks. 194 For this purpose, we introduce a concept of target contexts, which 195 encapsulate such information. The backend should implement simple routines 196 of allocating/freeing/setting such a context. The scheduler calls these 197 as target hooks and handles the target context as an opaque pointer (similar 198 to the DFA state type, state_t). 199 200 Various speedups 201 ================ 202 203 As the correct data dependence graph is not supported during scheduling (which 204 is to be changed in mid-term), we cache as much of the dependence analysis 205 results as possible to avoid reanalyzing. This includes: bitmap caches on 206 each insn in stream of the region saying yes/no for a query with a pair of 207 UIDs; hashtables with the previously done transformations on each insn in 208 stream; a vector keeping a history of transformations on each expr. 209 210 Also, we try to minimize the dependence context used on each fence to check 211 whether the given expression is ready for scheduling by removing from it 212 insns that are definitely completed the execution. The results of 213 tick_check_p checks are also cached in a vector on each fence. 214 215 We keep a valid liveness set on each insn in a region to avoid the high 216 cost of recomputation on large basic blocks. 217 218 Finally, we try to minimize the number of needed updates to the availability 219 sets. The updates happen in two cases: when fill_insns terminates, 220 we advance all fences and increase the stage number to show that the region 221 has changed and the sets are to be recomputed; and when the next iteration 222 of a loop in fill_insns happens (but this one reuses the saved av sets 223 on bb headers.) Thus, we try to break the fill_insns loop only when 224 "significant" number of insns from the current scheduling window was 225 scheduled. This should be made a target param. 226 227 228 TODO: correctly support the data dependence graph at all stages and get rid 229 of all caches. This should speed up the scheduler. 230 TODO: implement moving cond jumps with bookkeeping copies on both targets. 231 TODO: tune the scheduler before RA so it does not create too much pseudos. 232 233 234 References: 235 S.-M. Moon and K. Ebcioglu. Parallelizing nonnumerical code with 236 selective scheduling and software pipelining. 237 ACM TOPLAS, Vol 19, No. 6, pages 853--898, Nov. 1997. 238 239 Andrey Belevantsev, Maxim Kuvyrkov, Vladimir Makarov, Dmitry Melnik, 240 and Dmitry Zhurikhin. An interblock VLIW-targeted instruction scheduler 241 for GCC. In Proceedings of GCC Developers' Summit 2006. 242 243 Arutyun Avetisyan, Andrey Belevantsev, and Dmitry Melnik. GCC Instruction 244 Scheduler and Software Pipeliner on the Itanium Platform. EPIC-7 Workshop. 245 http://rogue.colorado.edu/EPIC7/. 246 247*/ 248 249/* True when pipelining is enabled. */ 250bool pipelining_p; 251 252/* True if bookkeeping is enabled. */ 253bool bookkeeping_p; 254 255/* Maximum number of insns that are eligible for renaming. */ 256int max_insns_to_rename; 257 258 259/* Definitions of local types and macros. */ 260 261/* Represents possible outcomes of moving an expression through an insn. */ 262enum MOVEUP_EXPR_CODE 263 { 264 /* The expression is not changed. */ 265 MOVEUP_EXPR_SAME, 266 267 /* Not changed, but requires a new destination register. */ 268 MOVEUP_EXPR_AS_RHS, 269 270 /* Cannot be moved. */ 271 MOVEUP_EXPR_NULL, 272 273 /* Changed (substituted or speculated). */ 274 MOVEUP_EXPR_CHANGED 275 }; 276 277/* The container to be passed into rtx search & replace functions. */ 278struct rtx_search_arg 279{ 280 /* What we are searching for. */ 281 rtx x; 282 283 /* The occurrence counter. */ 284 int n; 285}; 286 287typedef struct rtx_search_arg *rtx_search_arg_p; 288 289/* This struct contains precomputed hard reg sets that are needed when 290 computing registers available for renaming. */ 291struct hard_regs_data 292{ 293 /* For every mode, this stores registers available for use with 294 that mode. */ 295 HARD_REG_SET regs_for_mode[NUM_MACHINE_MODES]; 296 297 /* True when regs_for_mode[mode] is initialized. */ 298 bool regs_for_mode_ok[NUM_MACHINE_MODES]; 299 300 /* For every register, it has regs that are ok to rename into it. 301 The register in question is always set. If not, this means 302 that the whole set is not computed yet. */ 303 HARD_REG_SET regs_for_rename[FIRST_PSEUDO_REGISTER]; 304 305 /* All registers that are used or call used. */ 306 HARD_REG_SET regs_ever_used; 307 308#ifdef STACK_REGS 309 /* Stack registers. */ 310 HARD_REG_SET stack_regs; 311#endif 312}; 313 314/* Holds the results of computation of available for renaming and 315 unavailable hard registers. */ 316struct reg_rename 317{ 318 /* These are unavailable due to calls crossing, globalness, etc. */ 319 HARD_REG_SET unavailable_hard_regs; 320 321 /* These are *available* for renaming. */ 322 HARD_REG_SET available_for_renaming; 323 324 /* The set of ABIs used by calls that the code motion path crosses. */ 325 unsigned int crossed_call_abis : NUM_ABI_IDS; 326}; 327 328/* A global structure that contains the needed information about harg 329 regs. */ 330static struct hard_regs_data sel_hrd; 331 332 333/* This structure holds local data used in code_motion_path_driver hooks on 334 the same or adjacent levels of recursion. Here we keep those parameters 335 that are not used in code_motion_path_driver routine itself, but only in 336 its hooks. Moreover, all parameters that can be modified in hooks are 337 in this structure, so all other parameters passed explicitly to hooks are 338 read-only. */ 339struct cmpd_local_params 340{ 341 /* Local params used in move_op_* functions. */ 342 343 /* Edges for bookkeeping generation. */ 344 edge e1, e2; 345 346 /* C_EXPR merged from all successors and locally allocated temporary C_EXPR. */ 347 expr_t c_expr_merged, c_expr_local; 348 349 /* Local params used in fur_* functions. */ 350 /* Copy of the ORIGINAL_INSN list, stores the original insns already 351 found before entering the current level of code_motion_path_driver. */ 352 def_list_t old_original_insns; 353 354 /* Local params used in move_op_* functions. */ 355 /* True when we have removed last insn in the block which was 356 also a boundary. Do not update anything or create bookkeeping copies. */ 357 BOOL_BITFIELD removed_last_insn : 1; 358}; 359 360/* Stores the static parameters for move_op_* calls. */ 361struct moveop_static_params 362{ 363 /* Destination register. */ 364 rtx dest; 365 366 /* Current C_EXPR. */ 367 expr_t c_expr; 368 369 /* An UID of expr_vliw which is to be moved up. If we find other exprs, 370 they are to be removed. */ 371 int uid; 372 373 /* This is initialized to the insn on which the driver stopped its traversal. */ 374 insn_t failed_insn; 375 376 /* True if we scheduled an insn with different register. */ 377 bool was_renamed; 378}; 379 380/* Stores the static parameters for fur_* calls. */ 381struct fur_static_params 382{ 383 /* Set of registers unavailable on the code motion path. */ 384 regset used_regs; 385 386 /* Pointer to the list of original insns definitions. */ 387 def_list_t *original_insns; 388 389 /* The set of ABIs used by calls that the code motion path crosses. */ 390 unsigned int crossed_call_abis : NUM_ABI_IDS; 391}; 392 393typedef struct fur_static_params *fur_static_params_p; 394typedef struct cmpd_local_params *cmpd_local_params_p; 395typedef struct moveop_static_params *moveop_static_params_p; 396 397/* Set of hooks and parameters that determine behavior specific to 398 move_op or find_used_regs functions. */ 399struct code_motion_path_driver_info_def 400{ 401 /* Called on enter to the basic block. */ 402 int (*on_enter) (insn_t, cmpd_local_params_p, void *, bool); 403 404 /* Called when original expr is found. */ 405 void (*orig_expr_found) (insn_t, expr_t, cmpd_local_params_p, void *); 406 407 /* Called while descending current basic block if current insn is not 408 the original EXPR we're searching for. */ 409 bool (*orig_expr_not_found) (insn_t, av_set_t, void *); 410 411 /* Function to merge C_EXPRes from different successors. */ 412 void (*merge_succs) (insn_t, insn_t, int, cmpd_local_params_p, void *); 413 414 /* Function to finalize merge from different successors and possibly 415 deallocate temporary data structures used for merging. */ 416 void (*after_merge_succs) (cmpd_local_params_p, void *); 417 418 /* Called on the backward stage of recursion to do moveup_expr. 419 Used only with move_op_*. */ 420 void (*ascend) (insn_t, void *); 421 422 /* Called on the ascending pass, before returning from the current basic 423 block or from the whole traversal. */ 424 void (*at_first_insn) (insn_t, cmpd_local_params_p, void *); 425 426 /* When processing successors in move_op we need only descend into 427 SUCCS_NORMAL successors, while in find_used_regs we need SUCCS_ALL. */ 428 int succ_flags; 429 430 /* The routine name to print in dumps ("move_op" of "find_used_regs"). */ 431 const char *routine_name; 432}; 433 434/* Global pointer to current hooks, either points to MOVE_OP_HOOKS or 435 FUR_HOOKS. */ 436struct code_motion_path_driver_info_def *code_motion_path_driver_info; 437 438/* Set of hooks for performing move_op and find_used_regs routines with 439 code_motion_path_driver. */ 440extern struct code_motion_path_driver_info_def move_op_hooks, fur_hooks; 441 442/* True if/when we want to emulate Haifa scheduler in the common code. 443 This is used in sched_rgn_local_init and in various places in 444 sched-deps.cc. */ 445int sched_emulate_haifa_p; 446 447/* GLOBAL_LEVEL is used to discard information stored in basic block headers 448 av_sets. Av_set of bb header is valid if its (bb header's) level is equal 449 to GLOBAL_LEVEL. And invalid if lesser. This is primarily used to advance 450 scheduling window. */ 451int global_level; 452 453/* Current fences. */ 454flist_t fences; 455 456/* True when separable insns should be scheduled as RHSes. */ 457static bool enable_schedule_as_rhs_p; 458 459/* Used in verify_target_availability to assert that target reg is reported 460 unavailabile by both TARGET_UNAVAILABLE and find_used_regs only if 461 we haven't scheduled anything on the previous fence. 462 if scheduled_something_on_previous_fence is true, TARGET_UNAVAILABLE can 463 have more conservative value than the one returned by the 464 find_used_regs, thus we shouldn't assert that these values are equal. */ 465static bool scheduled_something_on_previous_fence; 466 467/* All newly emitted insns will have their uids greater than this value. */ 468static int first_emitted_uid; 469 470/* Set of basic blocks that are forced to start new ebbs. This is a subset 471 of all the ebb heads. */ 472bitmap forced_ebb_heads; 473 474/* Blocks that need to be rescheduled after pipelining. */ 475bitmap blocks_to_reschedule = NULL; 476 477/* True when the first lv set should be ignored when updating liveness. */ 478static bool ignore_first = false; 479 480/* Number of insns max_issue has initialized data structures for. */ 481static int max_issue_size = 0; 482 483/* Whether we can issue more instructions. */ 484static int can_issue_more; 485 486/* Maximum software lookahead window size, reduced when rescheduling after 487 pipelining. */ 488static int max_ws; 489 490/* Number of insns scheduled in current region. */ 491static int num_insns_scheduled; 492 493/* A vector of expressions is used to be able to sort them. */ 494static vec<expr_t> vec_av_set; 495 496/* A vector of vinsns is used to hold temporary lists of vinsns. */ 497typedef vec<vinsn_t> vinsn_vec_t; 498 499/* This vector has the exprs which may still present in av_sets, but actually 500 can't be moved up due to bookkeeping created during code motion to another 501 fence. See comment near the call to update_and_record_unavailable_insns 502 for the detailed explanations. */ 503static vinsn_vec_t vec_bookkeeping_blocked_vinsns = vinsn_vec_t (); 504 505/* This vector has vinsns which are scheduled with renaming on the first fence 506 and then seen on the second. For expressions with such vinsns, target 507 availability information may be wrong. */ 508static vinsn_vec_t vec_target_unavailable_vinsns = vinsn_vec_t (); 509 510/* Vector to store temporary nops inserted in move_op to prevent removal 511 of empty bbs. */ 512static vec<insn_t> vec_temp_moveop_nops; 513 514/* These bitmaps record original instructions scheduled on the current 515 iteration and bookkeeping copies created by them. */ 516static bitmap current_originators = NULL; 517static bitmap current_copies = NULL; 518 519/* This bitmap marks the blocks visited by code_motion_path_driver so we don't 520 visit them afterwards. */ 521static bitmap code_motion_visited_blocks = NULL; 522 523/* Variables to accumulate different statistics. */ 524 525/* The number of bookkeeping copies created. */ 526static int stat_bookkeeping_copies; 527 528/* The number of insns that required bookkeeiping for their scheduling. */ 529static int stat_insns_needed_bookkeeping; 530 531/* The number of insns that got renamed. */ 532static int stat_renamed_scheduled; 533 534/* The number of substitutions made during scheduling. */ 535static int stat_substitutions_total; 536 537 538/* Forward declarations of static functions. */ 539static bool rtx_ok_for_substitution_p (rtx, rtx); 540static int sel_rank_for_schedule (const void *, const void *); 541static av_set_t find_sequential_best_exprs (bnd_t, expr_t, bool); 542static basic_block find_block_for_bookkeeping (edge e1, edge e2, bool lax); 543 544static rtx get_dest_from_orig_ops (av_set_t); 545static basic_block generate_bookkeeping_insn (expr_t, edge, edge); 546static bool find_used_regs (insn_t, av_set_t, regset, struct reg_rename *, 547 def_list_t *); 548static bool move_op (insn_t, av_set_t, expr_t, rtx, expr_t, bool*); 549static int code_motion_path_driver (insn_t, av_set_t, ilist_t, 550 cmpd_local_params_p, void *); 551static void sel_sched_region_1 (void); 552static void sel_sched_region_2 (int); 553static av_set_t compute_av_set_inside_bb (insn_t, ilist_t, int, bool); 554 555static void debug_state (state_t); 556 557 558/* Functions that work with fences. */ 559 560/* Advance one cycle on FENCE. */ 561static void 562advance_one_cycle (fence_t fence) 563{ 564 unsigned i; 565 int cycle; 566 rtx_insn *insn; 567 568 advance_state (FENCE_STATE (fence)); 569 cycle = ++FENCE_CYCLE (fence); 570 FENCE_ISSUED_INSNS (fence) = 0; 571 FENCE_STARTS_CYCLE_P (fence) = 1; 572 can_issue_more = issue_rate; 573 FENCE_ISSUE_MORE (fence) = can_issue_more; 574 575 for (i = 0; vec_safe_iterate (FENCE_EXECUTING_INSNS (fence), i, &insn); ) 576 { 577 if (INSN_READY_CYCLE (insn) < cycle) 578 { 579 remove_from_deps (FENCE_DC (fence), insn); 580 FENCE_EXECUTING_INSNS (fence)->unordered_remove (i); 581 continue; 582 } 583 i++; 584 } 585 if (sched_verbose >= 2) 586 { 587 sel_print ("Finished a cycle. Current cycle = %d\n", FENCE_CYCLE (fence)); 588 debug_state (FENCE_STATE (fence)); 589 } 590} 591 592/* Returns true when SUCC in a fallthru bb of INSN, possibly 593 skipping empty basic blocks. */ 594static bool 595in_fallthru_bb_p (rtx_insn *insn, rtx succ) 596{ 597 basic_block bb = BLOCK_FOR_INSN (insn); 598 edge e; 599 600 if (bb == BLOCK_FOR_INSN (succ)) 601 return true; 602 603 e = find_fallthru_edge_from (bb); 604 if (e) 605 bb = e->dest; 606 else 607 return false; 608 609 while (sel_bb_empty_p (bb)) 610 bb = bb->next_bb; 611 612 return bb == BLOCK_FOR_INSN (succ); 613} 614 615/* Construct successor fences from OLD_FENCEs and put them in NEW_FENCES. 616 When a successor will continue a ebb, transfer all parameters of a fence 617 to the new fence. ORIG_MAX_SEQNO is the maximal seqno before this round 618 of scheduling helping to distinguish between the old and the new code. */ 619static void 620extract_new_fences_from (flist_t old_fences, flist_tail_t new_fences, 621 int orig_max_seqno) 622{ 623 bool was_here_p = false; 624 insn_t insn = NULL; 625 insn_t succ; 626 succ_iterator si; 627 ilist_iterator ii; 628 fence_t fence = FLIST_FENCE (old_fences); 629 basic_block bb; 630 631 /* Get the only element of FENCE_BNDS (fence). */ 632 FOR_EACH_INSN (insn, ii, FENCE_BNDS (fence)) 633 { 634 gcc_assert (!was_here_p); 635 was_here_p = true; 636 } 637 gcc_assert (was_here_p && insn != NULL_RTX); 638 639 /* When in the "middle" of the block, just move this fence 640 to the new list. */ 641 bb = BLOCK_FOR_INSN (insn); 642 if (! sel_bb_end_p (insn) 643 || (single_succ_p (bb) 644 && single_pred_p (single_succ (bb)))) 645 { 646 insn_t succ; 647 648 succ = (sel_bb_end_p (insn) 649 ? sel_bb_head (single_succ (bb)) 650 : NEXT_INSN (insn)); 651 652 if (INSN_SEQNO (succ) > 0 653 && INSN_SEQNO (succ) <= orig_max_seqno 654 && INSN_SCHED_TIMES (succ) <= 0) 655 { 656 FENCE_INSN (fence) = succ; 657 move_fence_to_fences (old_fences, new_fences); 658 659 if (sched_verbose >= 1) 660 sel_print ("Fence %d continues as %d[%d] (state continue)\n", 661 INSN_UID (insn), INSN_UID (succ), BLOCK_NUM (succ)); 662 } 663 return; 664 } 665 666 /* Otherwise copy fence's structures to (possibly) multiple successors. */ 667 FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS) 668 { 669 int seqno = INSN_SEQNO (succ); 670 671 if (seqno > 0 && seqno <= orig_max_seqno 672 && (pipelining_p || INSN_SCHED_TIMES (succ) <= 0)) 673 { 674 bool b = (in_same_ebb_p (insn, succ) 675 || in_fallthru_bb_p (insn, succ)); 676 677 if (sched_verbose >= 1) 678 sel_print ("Fence %d continues as %d[%d] (state %s)\n", 679 INSN_UID (insn), INSN_UID (succ), 680 BLOCK_NUM (succ), b ? "continue" : "reset"); 681 682 if (b) 683 add_dirty_fence_to_fences (new_fences, succ, fence); 684 else 685 { 686 /* Mark block of the SUCC as head of the new ebb. */ 687 bitmap_set_bit (forced_ebb_heads, BLOCK_NUM (succ)); 688 add_clean_fence_to_fences (new_fences, succ, fence); 689 } 690 } 691 } 692} 693 694 695/* Functions to support substitution. */ 696 697/* Returns whether INSN with dependence status DS is eligible for 698 substitution, i.e. it's a copy operation x := y, and RHS that is 699 moved up through this insn should be substituted. */ 700static bool 701can_substitute_through_p (insn_t insn, ds_t ds) 702{ 703 /* We can substitute only true dependencies. */ 704 if ((ds & DEP_OUTPUT) 705 || (ds & DEP_ANTI) 706 || ! INSN_RHS (insn) 707 || ! INSN_LHS (insn)) 708 return false; 709 710 /* Now we just need to make sure the INSN_RHS consists of only one 711 simple REG rtx. */ 712 if (REG_P (INSN_LHS (insn)) 713 && REG_P (INSN_RHS (insn))) 714 return true; 715 return false; 716} 717 718/* Substitute all occurrences of INSN's destination in EXPR' vinsn with INSN's 719 source (if INSN is eligible for substitution). Returns TRUE if 720 substitution was actually performed, FALSE otherwise. Substitution might 721 be not performed because it's either EXPR' vinsn doesn't contain INSN's 722 destination or the resulting insn is invalid for the target machine. 723 When UNDO is true, perform unsubstitution instead (the difference is in 724 the part of rtx on which validate_replace_rtx is called). */ 725static bool 726substitute_reg_in_expr (expr_t expr, insn_t insn, bool undo) 727{ 728 rtx *where; 729 bool new_insn_valid; 730 vinsn_t *vi = &EXPR_VINSN (expr); 731 bool has_rhs = VINSN_RHS (*vi) != NULL; 732 rtx old, new_rtx; 733 734 /* Do not try to replace in SET_DEST. Although we'll choose new 735 register for the RHS, we don't want to change RHS' original reg. 736 If the insn is not SET, we may still be able to substitute something 737 in it, and if we're here (don't have deps), it doesn't write INSN's 738 dest. */ 739 where = (has_rhs 740 ? &VINSN_RHS (*vi) 741 : &PATTERN (VINSN_INSN_RTX (*vi))); 742 old = undo ? INSN_RHS (insn) : INSN_LHS (insn); 743 744 /* Substitute if INSN has a form of x:=y and LHS(INSN) occurs in *VI. */ 745 if (rtx_ok_for_substitution_p (old, *where)) 746 { 747 rtx_insn *new_insn; 748 rtx *where_replace; 749 750 /* We should copy these rtxes before substitution. */ 751 new_rtx = copy_rtx (undo ? INSN_LHS (insn) : INSN_RHS (insn)); 752 new_insn = create_copy_of_insn_rtx (VINSN_INSN_RTX (*vi)); 753 754 /* Where we'll replace. 755 WHERE_REPLACE should point inside NEW_INSN, so INSN_RHS couldn't be 756 used instead of SET_SRC. */ 757 where_replace = (has_rhs 758 ? &SET_SRC (PATTERN (new_insn)) 759 : &PATTERN (new_insn)); 760 761 new_insn_valid 762 = validate_replace_rtx_part_nosimplify (old, new_rtx, where_replace, 763 new_insn); 764 765 /* ??? Actually, constrain_operands result depends upon choice of 766 destination register. E.g. if we allow single register to be an rhs, 767 and if we try to move dx=ax(as rhs) through ax=dx, we'll result 768 in invalid insn dx=dx, so we'll loose this rhs here. 769 Just can't come up with significant testcase for this, so just 770 leaving it for now. */ 771 if (new_insn_valid) 772 { 773 change_vinsn_in_expr (expr, 774 create_vinsn_from_insn_rtx (new_insn, false)); 775 776 /* Do not allow clobbering the address register of speculative 777 insns. */ 778 if ((EXPR_SPEC_DONE_DS (expr) & SPECULATIVE) 779 && register_unavailable_p (VINSN_REG_USES (EXPR_VINSN (expr)), 780 expr_dest_reg (expr))) 781 EXPR_TARGET_AVAILABLE (expr) = false; 782 783 return true; 784 } 785 else 786 return false; 787 } 788 else 789 return false; 790} 791 792/* Return the number of places WHAT appears within WHERE. 793 Bail out when we found a reference occupying several hard registers. */ 794static int 795count_occurrences_equiv (const_rtx what, const_rtx where) 796{ 797 int count = 0; 798 subrtx_iterator::array_type array; 799 FOR_EACH_SUBRTX (iter, array, where, NONCONST) 800 { 801 const_rtx x = *iter; 802 if (REG_P (x) && REGNO (x) == REGNO (what)) 803 { 804 /* Bail out if mode is different or more than one register is 805 used. */ 806 if (GET_MODE (x) != GET_MODE (what) || REG_NREGS (x) > 1) 807 return 0; 808 count += 1; 809 } 810 else if (GET_CODE (x) == SUBREG 811 && (!REG_P (SUBREG_REG (x)) 812 || REGNO (SUBREG_REG (x)) == REGNO (what))) 813 /* ??? Do not support substituting regs inside subregs. In that case, 814 simplify_subreg will be called by validate_replace_rtx, and 815 unsubstitution will fail later. */ 816 return 0; 817 } 818 return count; 819} 820 821/* Returns TRUE if WHAT is found in WHERE rtx tree. */ 822static bool 823rtx_ok_for_substitution_p (rtx what, rtx where) 824{ 825 return (count_occurrences_equiv (what, where) > 0); 826} 827 828 829/* Functions to support register renaming. */ 830 831/* Substitute VI's set source with REGNO. Returns newly created pattern 832 that has REGNO as its source. */ 833static rtx_insn * 834create_insn_rtx_with_rhs (vinsn_t vi, rtx rhs_rtx) 835{ 836 rtx lhs_rtx; 837 rtx pattern; 838 rtx_insn *insn_rtx; 839 840 lhs_rtx = copy_rtx (VINSN_LHS (vi)); 841 842 pattern = gen_rtx_SET (lhs_rtx, rhs_rtx); 843 insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX); 844 845 return insn_rtx; 846} 847 848/* Returns whether INSN's src can be replaced with register number 849 NEW_SRC_REG. E.g. the following insn is valid for i386: 850 851 (insn:HI 2205 6585 2207 727 ../../gcc/libiberty/regex.c:3337 852 (set (mem/s:QI (plus:SI (plus:SI (reg/f:SI 7 sp) 853 (reg:SI 0 ax [orig:770 c1 ] [770])) 854 (const_int 288 [0x120])) [0 str S1 A8]) 855 (const_int 0 [0x0])) 43 {*movqi_1} (nil) 856 (nil)) 857 858 But if we change (const_int 0 [0x0]) to (reg:QI 4 si), it will be invalid 859 because of operand constraints: 860 861 (define_insn "*movqi_1" 862 [(set (match_operand:QI 0 "nonimmediate_operand" "=q,q ,q ,r,r ,?r,m") 863 (match_operand:QI 1 "general_operand" " q,qn,qm,q,rn,qm,qn") 864 )] 865 866 So do constrain_operands here, before choosing NEW_SRC_REG as best 867 reg for rhs. */ 868 869static bool 870replace_src_with_reg_ok_p (insn_t insn, rtx new_src_reg) 871{ 872 vinsn_t vi = INSN_VINSN (insn); 873 machine_mode mode; 874 rtx dst_loc; 875 bool res; 876 877 gcc_assert (VINSN_SEPARABLE_P (vi)); 878 879 get_dest_and_mode (insn, &dst_loc, &mode); 880 gcc_assert (mode == GET_MODE (new_src_reg)); 881 882 if (REG_P (dst_loc) && REGNO (new_src_reg) == REGNO (dst_loc)) 883 return true; 884 885 /* See whether SET_SRC can be replaced with this register. */ 886 validate_change (insn, &SET_SRC (PATTERN (insn)), new_src_reg, 1); 887 res = verify_changes (0); 888 cancel_changes (0); 889 890 return res; 891} 892 893/* Returns whether INSN still be valid after replacing it's DEST with 894 register NEW_REG. */ 895static bool 896replace_dest_with_reg_ok_p (insn_t insn, rtx new_reg) 897{ 898 vinsn_t vi = INSN_VINSN (insn); 899 bool res; 900 901 /* We should deal here only with separable insns. */ 902 gcc_assert (VINSN_SEPARABLE_P (vi)); 903 gcc_assert (GET_MODE (VINSN_LHS (vi)) == GET_MODE (new_reg)); 904 905 /* See whether SET_DEST can be replaced with this register. */ 906 validate_change (insn, &SET_DEST (PATTERN (insn)), new_reg, 1); 907 res = verify_changes (0); 908 cancel_changes (0); 909 910 return res; 911} 912 913/* Create a pattern with rhs of VI and lhs of LHS_RTX. */ 914static rtx_insn * 915create_insn_rtx_with_lhs (vinsn_t vi, rtx lhs_rtx) 916{ 917 rtx rhs_rtx; 918 rtx pattern; 919 rtx_insn *insn_rtx; 920 921 rhs_rtx = copy_rtx (VINSN_RHS (vi)); 922 923 pattern = gen_rtx_SET (lhs_rtx, rhs_rtx); 924 insn_rtx = create_insn_rtx_from_pattern (pattern, NULL_RTX); 925 926 return insn_rtx; 927} 928 929/* Substitute lhs in the given expression EXPR for the register with number 930 NEW_REGNO. SET_DEST may be arbitrary rtx, not only register. */ 931static void 932replace_dest_with_reg_in_expr (expr_t expr, rtx new_reg) 933{ 934 rtx_insn *insn_rtx; 935 vinsn_t vinsn; 936 937 insn_rtx = create_insn_rtx_with_lhs (EXPR_VINSN (expr), new_reg); 938 vinsn = create_vinsn_from_insn_rtx (insn_rtx, false); 939 940 change_vinsn_in_expr (expr, vinsn); 941 EXPR_WAS_RENAMED (expr) = 1; 942 EXPR_TARGET_AVAILABLE (expr) = 1; 943} 944 945/* Returns whether VI writes either one of the USED_REGS registers or, 946 if a register is a hard one, one of the UNAVAILABLE_HARD_REGS registers. */ 947static bool 948vinsn_writes_one_of_regs_p (vinsn_t vi, regset used_regs, 949 HARD_REG_SET unavailable_hard_regs) 950{ 951 unsigned regno; 952 reg_set_iterator rsi; 953 954 EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_SETS (vi), 0, regno, rsi) 955 { 956 if (REGNO_REG_SET_P (used_regs, regno)) 957 return true; 958 if (HARD_REGISTER_NUM_P (regno) 959 && TEST_HARD_REG_BIT (unavailable_hard_regs, regno)) 960 return true; 961 } 962 963 EXECUTE_IF_SET_IN_REG_SET (VINSN_REG_CLOBBERS (vi), 0, regno, rsi) 964 { 965 if (REGNO_REG_SET_P (used_regs, regno)) 966 return true; 967 if (HARD_REGISTER_NUM_P (regno) 968 && TEST_HARD_REG_BIT (unavailable_hard_regs, regno)) 969 return true; 970 } 971 972 return false; 973} 974 975/* Returns register class of the output register in INSN. 976 Returns NO_REGS for call insns because some targets have constraints on 977 destination register of a call insn. 978 979 Code adopted from regrename.cc::build_def_use. */ 980static enum reg_class 981get_reg_class (rtx_insn *insn) 982{ 983 int i, n_ops; 984 985 extract_constrain_insn (insn); 986 preprocess_constraints (insn); 987 n_ops = recog_data.n_operands; 988 989 const operand_alternative *op_alt = which_op_alt (); 990 if (asm_noperands (PATTERN (insn)) > 0) 991 { 992 for (i = 0; i < n_ops; i++) 993 if (recog_data.operand_type[i] == OP_OUT) 994 { 995 rtx *loc = recog_data.operand_loc[i]; 996 rtx op = *loc; 997 enum reg_class cl = alternative_class (op_alt, i); 998 999 if (REG_P (op) 1000 && REGNO (op) == ORIGINAL_REGNO (op)) 1001 continue; 1002 1003 return cl; 1004 } 1005 } 1006 else if (!CALL_P (insn)) 1007 { 1008 for (i = 0; i < n_ops + recog_data.n_dups; i++) 1009 { 1010 int opn = i < n_ops ? i : recog_data.dup_num[i - n_ops]; 1011 enum reg_class cl = alternative_class (op_alt, opn); 1012 1013 if (recog_data.operand_type[opn] == OP_OUT || 1014 recog_data.operand_type[opn] == OP_INOUT) 1015 return cl; 1016 } 1017 } 1018 1019/* Insns like 1020 (insn (set (reg:CCZ 17 flags) (compare:CCZ ...))) 1021 may result in returning NO_REGS, cause flags is written implicitly through 1022 CMP insn, which has no OP_OUT | OP_INOUT operands. */ 1023 return NO_REGS; 1024} 1025 1026/* Calculate HARD_REGNO_RENAME_OK data for REGNO. */ 1027static void 1028init_hard_regno_rename (int regno) 1029{ 1030 int cur_reg; 1031 1032 SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], regno); 1033 1034 for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++) 1035 { 1036 /* We are not interested in renaming in other regs. */ 1037 if (!TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg)) 1038 continue; 1039 1040 if (HARD_REGNO_RENAME_OK (regno, cur_reg)) 1041 SET_HARD_REG_BIT (sel_hrd.regs_for_rename[regno], cur_reg); 1042 } 1043} 1044 1045/* A wrapper around HARD_REGNO_RENAME_OK that will look into the hard regs 1046 data first. */ 1047static inline bool 1048sel_hard_regno_rename_ok (int from ATTRIBUTE_UNUSED, int to ATTRIBUTE_UNUSED) 1049{ 1050 /* Check whether this is all calculated. */ 1051 if (TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], from)) 1052 return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to); 1053 1054 init_hard_regno_rename (from); 1055 1056 return TEST_HARD_REG_BIT (sel_hrd.regs_for_rename[from], to); 1057} 1058 1059/* Calculate set of registers that are capable of holding MODE. */ 1060static void 1061init_regs_for_mode (machine_mode mode) 1062{ 1063 int cur_reg; 1064 1065 CLEAR_HARD_REG_SET (sel_hrd.regs_for_mode[mode]); 1066 1067 for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++) 1068 { 1069 int nregs; 1070 int i; 1071 1072 /* See whether it accepts all modes that occur in 1073 original insns. */ 1074 if (!targetm.hard_regno_mode_ok (cur_reg, mode)) 1075 continue; 1076 1077 nregs = hard_regno_nregs (cur_reg, mode); 1078 1079 for (i = nregs - 1; i >= 0; --i) 1080 if (fixed_regs[cur_reg + i] 1081 || global_regs[cur_reg + i] 1082 /* Can't use regs which aren't saved by 1083 the prologue. */ 1084 || !TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg + i) 1085 /* Can't use regs with non-null REG_BASE_VALUE, because adjusting 1086 it affects aliasing globally and invalidates all AV sets. */ 1087 || get_reg_base_value (cur_reg + i) 1088#ifdef LEAF_REGISTERS 1089 /* We can't use a non-leaf register if we're in a 1090 leaf function. */ 1091 || (crtl->is_leaf 1092 && !LEAF_REGISTERS[cur_reg + i]) 1093#endif 1094 ) 1095 break; 1096 1097 if (i >= 0) 1098 continue; 1099 1100 /* If the CUR_REG passed all the checks above, 1101 then it's ok. */ 1102 SET_HARD_REG_BIT (sel_hrd.regs_for_mode[mode], cur_reg); 1103 } 1104 1105 sel_hrd.regs_for_mode_ok[mode] = true; 1106} 1107 1108/* Init all register sets gathered in HRD. */ 1109static void 1110init_hard_regs_data (void) 1111{ 1112 int cur_reg = 0; 1113 int cur_mode = 0; 1114 1115 CLEAR_HARD_REG_SET (sel_hrd.regs_ever_used); 1116 for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++) 1117 if (df_regs_ever_live_p (cur_reg) 1118 || crtl->abi->clobbers_full_reg_p (cur_reg)) 1119 SET_HARD_REG_BIT (sel_hrd.regs_ever_used, cur_reg); 1120 1121 /* Initialize registers that are valid based on mode when this is 1122 really needed. */ 1123 for (cur_mode = 0; cur_mode < NUM_MACHINE_MODES; cur_mode++) 1124 sel_hrd.regs_for_mode_ok[cur_mode] = false; 1125 1126 /* Mark that all HARD_REGNO_RENAME_OK is not calculated. */ 1127 for (cur_reg = 0; cur_reg < FIRST_PSEUDO_REGISTER; cur_reg++) 1128 CLEAR_HARD_REG_SET (sel_hrd.regs_for_rename[cur_reg]); 1129 1130#ifdef STACK_REGS 1131 CLEAR_HARD_REG_SET (sel_hrd.stack_regs); 1132 1133 for (cur_reg = FIRST_STACK_REG; cur_reg <= LAST_STACK_REG; cur_reg++) 1134 SET_HARD_REG_BIT (sel_hrd.stack_regs, cur_reg); 1135#endif 1136} 1137 1138/* Mark hardware regs in REG_RENAME_P that are not suitable 1139 for renaming rhs in INSN due to hardware restrictions (register class, 1140 modes compatibility etc). This doesn't affect original insn's dest reg, 1141 if it isn't in USED_REGS. DEF is a definition insn of rhs for which the 1142 destination register is sought. LHS (DEF->ORIG_INSN) may be REG or MEM. 1143 Registers that are in used_regs are always marked in 1144 unavailable_hard_regs as well. */ 1145 1146static void 1147mark_unavailable_hard_regs (def_t def, struct reg_rename *reg_rename_p, 1148 regset used_regs ATTRIBUTE_UNUSED) 1149{ 1150 machine_mode mode; 1151 enum reg_class cl = NO_REGS; 1152 rtx orig_dest; 1153 unsigned cur_reg, regno; 1154 hard_reg_set_iterator hrsi; 1155 1156 gcc_assert (GET_CODE (PATTERN (def->orig_insn)) == SET); 1157 gcc_assert (reg_rename_p); 1158 1159 orig_dest = SET_DEST (PATTERN (def->orig_insn)); 1160 1161 /* We have decided not to rename 'mem = something;' insns, as 'something' 1162 is usually a register. */ 1163 if (!REG_P (orig_dest)) 1164 return; 1165 1166 regno = REGNO (orig_dest); 1167 1168 /* If before reload, don't try to work with pseudos. */ 1169 if (!reload_completed && !HARD_REGISTER_NUM_P (regno)) 1170 return; 1171 1172 if (reload_completed) 1173 cl = get_reg_class (def->orig_insn); 1174 1175 /* Stop if the original register is one of the fixed_regs, global_regs or 1176 frame pointer, or we could not discover its class. */ 1177 if (fixed_regs[regno] 1178 || global_regs[regno] 1179 || (!HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed 1180 && regno == HARD_FRAME_POINTER_REGNUM) 1181 || (HARD_FRAME_POINTER_IS_FRAME_POINTER && frame_pointer_needed 1182 && regno == FRAME_POINTER_REGNUM) 1183 || (reload_completed && cl == NO_REGS)) 1184 { 1185 SET_HARD_REG_SET (reg_rename_p->unavailable_hard_regs); 1186 1187 /* Give a chance for original register, if it isn't in used_regs. */ 1188 if (!def->crossed_call_abis) 1189 CLEAR_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno); 1190 1191 return; 1192 } 1193 1194 /* If something allocated on stack in this function, mark frame pointer 1195 register unavailable, considering also modes. 1196 FIXME: it is enough to do this once per all original defs. */ 1197 if (frame_pointer_needed) 1198 { 1199 add_to_hard_reg_set (®_rename_p->unavailable_hard_regs, 1200 Pmode, FRAME_POINTER_REGNUM); 1201 1202 if (!HARD_FRAME_POINTER_IS_FRAME_POINTER) 1203 add_to_hard_reg_set (®_rename_p->unavailable_hard_regs, 1204 Pmode, HARD_FRAME_POINTER_REGNUM); 1205 } 1206 1207#ifdef STACK_REGS 1208 /* For the stack registers the presence of FIRST_STACK_REG in USED_REGS 1209 is equivalent to as if all stack regs were in this set. 1210 I.e. no stack register can be renamed, and even if it's an original 1211 register here we make sure it won't be lifted over it's previous def 1212 (it's previous def will appear as if it's a FIRST_STACK_REG def. 1213 The HARD_REGNO_RENAME_OK covers other cases in condition below. */ 1214 if (IN_RANGE (REGNO (orig_dest), FIRST_STACK_REG, LAST_STACK_REG) 1215 && REGNO_REG_SET_P (used_regs, FIRST_STACK_REG)) 1216 reg_rename_p->unavailable_hard_regs |= sel_hrd.stack_regs; 1217#endif 1218 1219 mode = GET_MODE (orig_dest); 1220 1221 /* If there's a call on this path, make regs from full_reg_clobbers 1222 unavailable. 1223 1224 ??? It would be better to track the set of clobbered registers 1225 directly, but that would be quite expensive in a def_t. */ 1226 if (def->crossed_call_abis) 1227 reg_rename_p->unavailable_hard_regs 1228 |= call_clobbers_in_region (def->crossed_call_abis, 1229 reg_class_contents[ALL_REGS], mode); 1230 1231 /* Stop here before reload: we need FRAME_REGS, STACK_REGS, and 1232 crossed_call_abis, but not register classes. */ 1233 if (!reload_completed) 1234 return; 1235 1236 /* Leave regs as 'available' only from the current 1237 register class. */ 1238 reg_rename_p->available_for_renaming = reg_class_contents[cl]; 1239 1240 /* Leave only registers available for this mode. */ 1241 if (!sel_hrd.regs_for_mode_ok[mode]) 1242 init_regs_for_mode (mode); 1243 reg_rename_p->available_for_renaming &= sel_hrd.regs_for_mode[mode]; 1244 1245 /* Leave only those that are ok to rename. */ 1246 EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming, 1247 0, cur_reg, hrsi) 1248 { 1249 int nregs; 1250 int i; 1251 1252 nregs = hard_regno_nregs (cur_reg, mode); 1253 gcc_assert (nregs > 0); 1254 1255 for (i = nregs - 1; i >= 0; --i) 1256 if (! sel_hard_regno_rename_ok (regno + i, cur_reg + i)) 1257 break; 1258 1259 if (i >= 0) 1260 CLEAR_HARD_REG_BIT (reg_rename_p->available_for_renaming, 1261 cur_reg); 1262 } 1263 1264 reg_rename_p->available_for_renaming &= ~reg_rename_p->unavailable_hard_regs; 1265 1266 /* Regno is always ok from the renaming part of view, but it really 1267 could be in *unavailable_hard_regs already, so set it here instead 1268 of there. */ 1269 SET_HARD_REG_BIT (reg_rename_p->available_for_renaming, regno); 1270} 1271 1272/* reg_rename_tick[REG1] > reg_rename_tick[REG2] if REG1 was chosen as the 1273 best register more recently than REG2. */ 1274static int reg_rename_tick[FIRST_PSEUDO_REGISTER]; 1275 1276/* Indicates the number of times renaming happened before the current one. */ 1277static int reg_rename_this_tick; 1278 1279/* Choose the register among free, that is suitable for storing 1280 the rhs value. 1281 1282 ORIGINAL_INSNS is the list of insns where the operation (rhs) 1283 originally appears. There could be multiple original operations 1284 for single rhs since we moving it up and merging along different 1285 paths. 1286 1287 Some code is adapted from regrename.cc (regrename_optimize). 1288 If original register is available, function returns it. 1289 Otherwise it performs the checks, so the new register should 1290 comply with the following: 1291 - it should not violate any live ranges (such registers are in 1292 REG_RENAME_P->available_for_renaming set); 1293 - it should not be in the HARD_REGS_USED regset; 1294 - it should be in the class compatible with original uses; 1295 - it should not be clobbered through reference with different mode; 1296 - if we're in the leaf function, then the new register should 1297 not be in the LEAF_REGISTERS; 1298 - etc. 1299 1300 If several registers meet the conditions, the register with smallest 1301 tick is returned to achieve more even register allocation. 1302 1303 If original register seems to be ok, we set *IS_ORIG_REG_P_PTR to true. 1304 1305 If no register satisfies the above conditions, NULL_RTX is returned. */ 1306static rtx 1307choose_best_reg_1 (HARD_REG_SET hard_regs_used, 1308 struct reg_rename *reg_rename_p, 1309 def_list_t original_insns, bool *is_orig_reg_p_ptr) 1310{ 1311 int best_new_reg; 1312 unsigned cur_reg; 1313 machine_mode mode = VOIDmode; 1314 unsigned regno, i, n; 1315 hard_reg_set_iterator hrsi; 1316 def_list_iterator di; 1317 def_t def; 1318 1319 /* If original register is available, return it. */ 1320 *is_orig_reg_p_ptr = true; 1321 1322 FOR_EACH_DEF (def, di, original_insns) 1323 { 1324 rtx orig_dest = SET_DEST (PATTERN (def->orig_insn)); 1325 1326 gcc_assert (REG_P (orig_dest)); 1327 1328 /* Check that all original operations have the same mode. 1329 This is done for the next loop; if we'd return from this 1330 loop, we'd check only part of them, but in this case 1331 it doesn't matter. */ 1332 if (mode == VOIDmode) 1333 mode = GET_MODE (orig_dest); 1334 gcc_assert (mode == GET_MODE (orig_dest)); 1335 1336 regno = REGNO (orig_dest); 1337 for (i = 0, n = REG_NREGS (orig_dest); i < n; i++) 1338 if (TEST_HARD_REG_BIT (hard_regs_used, regno + i)) 1339 break; 1340 1341 /* All hard registers are available. */ 1342 if (i == n) 1343 { 1344 gcc_assert (mode != VOIDmode); 1345 1346 /* Hard registers should not be shared. */ 1347 return gen_rtx_REG (mode, regno); 1348 } 1349 } 1350 1351 *is_orig_reg_p_ptr = false; 1352 best_new_reg = -1; 1353 1354 /* Among all available regs choose the register that was 1355 allocated earliest. */ 1356 EXECUTE_IF_SET_IN_HARD_REG_SET (reg_rename_p->available_for_renaming, 1357 0, cur_reg, hrsi) 1358 if (! TEST_HARD_REG_BIT (hard_regs_used, cur_reg)) 1359 { 1360 /* Check that all hard regs for mode are available. */ 1361 for (i = 1, n = hard_regno_nregs (cur_reg, mode); i < n; i++) 1362 if (TEST_HARD_REG_BIT (hard_regs_used, cur_reg + i) 1363 || !TEST_HARD_REG_BIT (reg_rename_p->available_for_renaming, 1364 cur_reg + i)) 1365 break; 1366 1367 if (i < n) 1368 continue; 1369 1370 /* All hard registers are available. */ 1371 if (best_new_reg < 0 1372 || reg_rename_tick[cur_reg] < reg_rename_tick[best_new_reg]) 1373 { 1374 best_new_reg = cur_reg; 1375 1376 /* Return immediately when we know there's no better reg. */ 1377 if (! reg_rename_tick[best_new_reg]) 1378 break; 1379 } 1380 } 1381 1382 if (best_new_reg >= 0) 1383 { 1384 /* Use the check from the above loop. */ 1385 gcc_assert (mode != VOIDmode); 1386 return gen_rtx_REG (mode, best_new_reg); 1387 } 1388 1389 return NULL_RTX; 1390} 1391 1392/* A wrapper around choose_best_reg_1 () to verify that we make correct 1393 assumptions about available registers in the function. */ 1394static rtx 1395choose_best_reg (HARD_REG_SET hard_regs_used, struct reg_rename *reg_rename_p, 1396 def_list_t original_insns, bool *is_orig_reg_p_ptr) 1397{ 1398 rtx best_reg = choose_best_reg_1 (hard_regs_used, reg_rename_p, 1399 original_insns, is_orig_reg_p_ptr); 1400 1401 /* FIXME loop over hard_regno_nregs here. */ 1402 gcc_assert (best_reg == NULL_RTX 1403 || TEST_HARD_REG_BIT (sel_hrd.regs_ever_used, REGNO (best_reg))); 1404 1405 return best_reg; 1406} 1407 1408/* Choose the pseudo register for storing rhs value. As this is supposed 1409 to work before reload, we return either the original register or make 1410 the new one. The parameters are the same that in choose_nest_reg_1 1411 functions, except that USED_REGS may contain pseudos. 1412 If we work with hard regs, check also REG_RENAME_P->UNAVAILABLE_HARD_REGS. 1413 1414 TODO: take into account register pressure while doing this. Up to this 1415 moment, this function would never return NULL for pseudos, but we should 1416 not rely on this. */ 1417static rtx 1418choose_best_pseudo_reg (regset used_regs, 1419 struct reg_rename *reg_rename_p, 1420 def_list_t original_insns, bool *is_orig_reg_p_ptr) 1421{ 1422 def_list_iterator i; 1423 def_t def; 1424 machine_mode mode = VOIDmode; 1425 bool bad_hard_regs = false; 1426 1427 /* We should not use this after reload. */ 1428 gcc_assert (!reload_completed); 1429 1430 /* If original register is available, return it. */ 1431 *is_orig_reg_p_ptr = true; 1432 1433 FOR_EACH_DEF (def, i, original_insns) 1434 { 1435 rtx dest = SET_DEST (PATTERN (def->orig_insn)); 1436 int orig_regno; 1437 1438 gcc_assert (REG_P (dest)); 1439 1440 /* Check that all original operations have the same mode. */ 1441 if (mode == VOIDmode) 1442 mode = GET_MODE (dest); 1443 else 1444 gcc_assert (mode == GET_MODE (dest)); 1445 orig_regno = REGNO (dest); 1446 1447 /* Check that nothing in used_regs intersects with orig_regno. When 1448 we have a hard reg here, still loop over hard_regno_nregs. */ 1449 if (HARD_REGISTER_NUM_P (orig_regno)) 1450 { 1451 int j, n; 1452 for (j = 0, n = REG_NREGS (dest); j < n; j++) 1453 if (REGNO_REG_SET_P (used_regs, orig_regno + j)) 1454 break; 1455 if (j < n) 1456 continue; 1457 } 1458 else 1459 { 1460 if (REGNO_REG_SET_P (used_regs, orig_regno)) 1461 continue; 1462 } 1463 if (HARD_REGISTER_NUM_P (orig_regno)) 1464 { 1465 gcc_assert (df_regs_ever_live_p (orig_regno)); 1466 1467 /* For hard registers, we have to check hardware imposed 1468 limitations (frame/stack registers, calls crossed). */ 1469 if (!TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, 1470 orig_regno)) 1471 { 1472 /* Don't let register cross a call if it doesn't already 1473 cross one. This condition is written in accordance with 1474 that in sched-deps.cc sched_analyze_reg(). */ 1475 if (!reg_rename_p->crossed_call_abis 1476 || REG_N_CALLS_CROSSED (orig_regno) > 0) 1477 return gen_rtx_REG (mode, orig_regno); 1478 } 1479 1480 bad_hard_regs = true; 1481 } 1482 else 1483 return dest; 1484 } 1485 1486 *is_orig_reg_p_ptr = false; 1487 1488 /* We had some original hard registers that couldn't be used. 1489 Those were likely special. Don't try to create a pseudo. */ 1490 if (bad_hard_regs) 1491 return NULL_RTX; 1492 1493 /* We haven't found a register from original operations. Get a new one. 1494 FIXME: control register pressure somehow. */ 1495 { 1496 rtx new_reg = gen_reg_rtx (mode); 1497 1498 gcc_assert (mode != VOIDmode); 1499 1500 max_regno = max_reg_num (); 1501 maybe_extend_reg_info_p (); 1502 REG_N_CALLS_CROSSED (REGNO (new_reg)) 1503 = reg_rename_p->crossed_call_abis ? 1 : 0; 1504 1505 return new_reg; 1506 } 1507} 1508 1509/* True when target of EXPR is available due to EXPR_TARGET_AVAILABLE, 1510 USED_REGS and REG_RENAME_P->UNAVAILABLE_HARD_REGS. */ 1511static void 1512verify_target_availability (expr_t expr, regset used_regs, 1513 struct reg_rename *reg_rename_p) 1514{ 1515 unsigned n, i, regno; 1516 machine_mode mode; 1517 bool target_available, live_available, hard_available; 1518 1519 if (!REG_P (EXPR_LHS (expr)) || EXPR_TARGET_AVAILABLE (expr) < 0) 1520 return; 1521 1522 regno = expr_dest_regno (expr); 1523 mode = GET_MODE (EXPR_LHS (expr)); 1524 target_available = EXPR_TARGET_AVAILABLE (expr) == 1; 1525 n = HARD_REGISTER_NUM_P (regno) ? hard_regno_nregs (regno, mode) : 1; 1526 1527 live_available = hard_available = true; 1528 for (i = 0; i < n; i++) 1529 { 1530 if (bitmap_bit_p (used_regs, regno + i)) 1531 live_available = false; 1532 if (TEST_HARD_REG_BIT (reg_rename_p->unavailable_hard_regs, regno + i)) 1533 hard_available = false; 1534 } 1535 1536 /* When target is not available, it may be due to hard register 1537 restrictions, e.g. crosses calls, so we check hard_available too. */ 1538 if (target_available) 1539 gcc_assert (live_available); 1540 else 1541 /* Check only if we haven't scheduled something on the previous fence, 1542 cause due to MAX_SOFTWARE_LOOKAHEAD_WINDOW_SIZE issues 1543 and having more than one fence, we may end having targ_un in a block 1544 in which successors target register is actually available. 1545 1546 The last condition handles the case when a dependence from a call insn 1547 was created in sched-deps.cc for insns with destination registers that 1548 never crossed a call before, but do cross one after our code motion. 1549 1550 FIXME: in the latter case, we just uselessly called find_used_regs, 1551 because we can't move this expression with any other register 1552 as well. */ 1553 gcc_assert (scheduled_something_on_previous_fence || !live_available 1554 || !hard_available 1555 || (!reload_completed 1556 && reg_rename_p->crossed_call_abis 1557 && REG_N_CALLS_CROSSED (regno) == 0)); 1558} 1559 1560/* Collect unavailable registers due to liveness for EXPR from BNDS 1561 into USED_REGS. Save additional information about available 1562 registers and unavailable due to hardware restriction registers 1563 into REG_RENAME_P structure. Save original insns into ORIGINAL_INSNS 1564 list. */ 1565static void 1566collect_unavailable_regs_from_bnds (expr_t expr, blist_t bnds, regset used_regs, 1567 struct reg_rename *reg_rename_p, 1568 def_list_t *original_insns) 1569{ 1570 for (; bnds; bnds = BLIST_NEXT (bnds)) 1571 { 1572 bool res; 1573 av_set_t orig_ops = NULL; 1574 bnd_t bnd = BLIST_BND (bnds); 1575 1576 /* If the chosen best expr doesn't belong to current boundary, 1577 skip it. */ 1578 if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr))) 1579 continue; 1580 1581 /* Put in ORIG_OPS all exprs from this boundary that became 1582 RES on top. */ 1583 orig_ops = find_sequential_best_exprs (bnd, expr, false); 1584 1585 /* Compute used regs and OR it into the USED_REGS. */ 1586 res = find_used_regs (BND_TO (bnd), orig_ops, used_regs, 1587 reg_rename_p, original_insns); 1588 1589 /* FIXME: the assert is true until we'd have several boundaries. */ 1590 gcc_assert (res); 1591 av_set_clear (&orig_ops); 1592 } 1593} 1594 1595/* Return TRUE if it is possible to replace LHSes of ORIG_INSNS with BEST_REG. 1596 If BEST_REG is valid, replace LHS of EXPR with it. */ 1597static bool 1598try_replace_dest_reg (ilist_t orig_insns, rtx best_reg, expr_t expr) 1599{ 1600 /* Try whether we'll be able to generate the insn 1601 'dest := best_reg' at the place of the original operation. */ 1602 for (; orig_insns; orig_insns = ILIST_NEXT (orig_insns)) 1603 { 1604 insn_t orig_insn = DEF_LIST_DEF (orig_insns)->orig_insn; 1605 1606 gcc_assert (EXPR_SEPARABLE_P (INSN_EXPR (orig_insn))); 1607 1608 if (REGNO (best_reg) != REGNO (INSN_LHS (orig_insn)) 1609 && (! replace_src_with_reg_ok_p (orig_insn, best_reg) 1610 || ! replace_dest_with_reg_ok_p (orig_insn, best_reg))) 1611 return false; 1612 } 1613 1614 /* Make sure that EXPR has the right destination 1615 register. */ 1616 if (expr_dest_regno (expr) != REGNO (best_reg)) 1617 replace_dest_with_reg_in_expr (expr, best_reg); 1618 else 1619 EXPR_TARGET_AVAILABLE (expr) = 1; 1620 1621 return true; 1622} 1623 1624/* Select and assign best register to EXPR searching from BNDS. 1625 Set *IS_ORIG_REG_P to TRUE if original register was selected. 1626 Return FALSE if no register can be chosen, which could happen when: 1627 * EXPR_SEPARABLE_P is true but we were unable to find suitable register; 1628 * EXPR_SEPARABLE_P is false but the insn sets/clobbers one of the registers 1629 that are used on the moving path. */ 1630static bool 1631find_best_reg_for_expr (expr_t expr, blist_t bnds, bool *is_orig_reg_p) 1632{ 1633 static struct reg_rename reg_rename_data; 1634 1635 regset used_regs; 1636 def_list_t original_insns = NULL; 1637 bool reg_ok; 1638 1639 *is_orig_reg_p = false; 1640 1641 /* Don't bother to do anything if this insn doesn't set any registers. */ 1642 if (bitmap_empty_p (VINSN_REG_SETS (EXPR_VINSN (expr))) 1643 && bitmap_empty_p (VINSN_REG_CLOBBERS (EXPR_VINSN (expr)))) 1644 return true; 1645 1646 used_regs = get_clear_regset_from_pool (); 1647 CLEAR_HARD_REG_SET (reg_rename_data.unavailable_hard_regs); 1648 1649 collect_unavailable_regs_from_bnds (expr, bnds, used_regs, ®_rename_data, 1650 &original_insns); 1651 1652 /* If after reload, make sure we're working with hard regs here. */ 1653 if (flag_checking && reload_completed) 1654 { 1655 reg_set_iterator rsi; 1656 unsigned i; 1657 1658 EXECUTE_IF_SET_IN_REG_SET (used_regs, FIRST_PSEUDO_REGISTER, i, rsi) 1659 gcc_unreachable (); 1660 } 1661 1662 if (EXPR_SEPARABLE_P (expr)) 1663 { 1664 rtx best_reg = NULL_RTX; 1665 /* Check that we have computed availability of a target register 1666 correctly. */ 1667 verify_target_availability (expr, used_regs, ®_rename_data); 1668 1669 /* Turn everything in hard regs after reload. */ 1670 if (reload_completed) 1671 { 1672 HARD_REG_SET hard_regs_used; 1673 REG_SET_TO_HARD_REG_SET (hard_regs_used, used_regs); 1674 1675 /* Join hard registers unavailable due to register class 1676 restrictions and live range intersection. */ 1677 hard_regs_used |= reg_rename_data.unavailable_hard_regs; 1678 1679 best_reg = choose_best_reg (hard_regs_used, ®_rename_data, 1680 original_insns, is_orig_reg_p); 1681 } 1682 else 1683 best_reg = choose_best_pseudo_reg (used_regs, ®_rename_data, 1684 original_insns, is_orig_reg_p); 1685 1686 if (!best_reg) 1687 reg_ok = false; 1688 else if (*is_orig_reg_p) 1689 { 1690 /* In case of unification BEST_REG may be different from EXPR's LHS 1691 when EXPR's LHS is unavailable, and there is another LHS among 1692 ORIGINAL_INSNS. */ 1693 reg_ok = try_replace_dest_reg (original_insns, best_reg, expr); 1694 } 1695 else 1696 { 1697 /* Forbid renaming of low-cost insns. */ 1698 if (sel_vinsn_cost (EXPR_VINSN (expr)) < 2) 1699 reg_ok = false; 1700 else 1701 reg_ok = try_replace_dest_reg (original_insns, best_reg, expr); 1702 } 1703 } 1704 else 1705 { 1706 /* If !EXPR_SCHEDULE_AS_RHS (EXPR), just make sure INSN doesn't set 1707 any of the HARD_REGS_USED set. */ 1708 if (vinsn_writes_one_of_regs_p (EXPR_VINSN (expr), used_regs, 1709 reg_rename_data.unavailable_hard_regs)) 1710 { 1711 reg_ok = false; 1712 gcc_assert (EXPR_TARGET_AVAILABLE (expr) <= 0); 1713 } 1714 else 1715 { 1716 reg_ok = true; 1717 gcc_assert (EXPR_TARGET_AVAILABLE (expr) != 0); 1718 } 1719 } 1720 1721 ilist_clear (&original_insns); 1722 return_regset_to_pool (used_regs); 1723 1724 return reg_ok; 1725} 1726 1727 1728/* Return true if dependence described by DS can be overcomed. */ 1729static bool 1730can_speculate_dep_p (ds_t ds) 1731{ 1732 if (spec_info == NULL) 1733 return false; 1734 1735 /* Leave only speculative data. */ 1736 ds &= SPECULATIVE; 1737 1738 if (ds == 0) 1739 return false; 1740 1741 { 1742 /* FIXME: make sched-deps.cc produce only those non-hard dependencies, 1743 that we can overcome. */ 1744 ds_t spec_mask = spec_info->mask; 1745 1746 if ((ds & spec_mask) != ds) 1747 return false; 1748 } 1749 1750 if (ds_weak (ds) < spec_info->data_weakness_cutoff) 1751 return false; 1752 1753 return true; 1754} 1755 1756/* Get a speculation check instruction. 1757 C_EXPR is a speculative expression, 1758 CHECK_DS describes speculations that should be checked, 1759 ORIG_INSN is the original non-speculative insn in the stream. */ 1760static insn_t 1761create_speculation_check (expr_t c_expr, ds_t check_ds, insn_t orig_insn) 1762{ 1763 rtx check_pattern; 1764 rtx_insn *insn_rtx; 1765 insn_t insn; 1766 basic_block recovery_block; 1767 rtx_insn *label; 1768 1769 /* Create a recovery block if target is going to emit branchy check, or if 1770 ORIG_INSN was speculative already. */ 1771 if (targetm.sched.needs_block_p (check_ds) 1772 || EXPR_SPEC_DONE_DS (INSN_EXPR (orig_insn)) != 0) 1773 { 1774 recovery_block = sel_create_recovery_block (orig_insn); 1775 label = BB_HEAD (recovery_block); 1776 } 1777 else 1778 { 1779 recovery_block = NULL; 1780 label = NULL; 1781 } 1782 1783 /* Get pattern of the check. */ 1784 check_pattern = targetm.sched.gen_spec_check (EXPR_INSN_RTX (c_expr), label, 1785 check_ds); 1786 1787 gcc_assert (check_pattern != NULL); 1788 1789 /* Emit check. */ 1790 insn_rtx = create_insn_rtx_from_pattern (check_pattern, label); 1791 1792 insn = sel_gen_insn_from_rtx_after (insn_rtx, INSN_EXPR (orig_insn), 1793 INSN_SEQNO (orig_insn), orig_insn); 1794 1795 /* Make check to be non-speculative. */ 1796 EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0; 1797 INSN_SPEC_CHECKED_DS (insn) = check_ds; 1798 1799 /* Decrease priority of check by difference of load/check instruction 1800 latencies. */ 1801 EXPR_PRIORITY (INSN_EXPR (insn)) -= (sel_vinsn_cost (INSN_VINSN (orig_insn)) 1802 - sel_vinsn_cost (INSN_VINSN (insn))); 1803 1804 /* Emit copy of original insn (though with replaced target register, 1805 if needed) to the recovery block. */ 1806 if (recovery_block != NULL) 1807 { 1808 rtx twin_rtx; 1809 1810 twin_rtx = copy_rtx (PATTERN (EXPR_INSN_RTX (c_expr))); 1811 twin_rtx = create_insn_rtx_from_pattern (twin_rtx, NULL_RTX); 1812 sel_gen_recovery_insn_from_rtx_after (twin_rtx, 1813 INSN_EXPR (orig_insn), 1814 INSN_SEQNO (insn), 1815 bb_note (recovery_block)); 1816 } 1817 1818 /* If we've generated a data speculation check, make sure 1819 that all the bookkeeping instruction we'll create during 1820 this move_op () will allocate an ALAT entry so that the 1821 check won't fail. 1822 In case of control speculation we must convert C_EXPR to control 1823 speculative mode, because failing to do so will bring us an exception 1824 thrown by the non-control-speculative load. */ 1825 check_ds = ds_get_max_dep_weak (check_ds); 1826 speculate_expr (c_expr, check_ds); 1827 1828 return insn; 1829} 1830 1831/* True when INSN is a "regN = regN" copy. */ 1832static bool 1833identical_copy_p (rtx_insn *insn) 1834{ 1835 rtx lhs, rhs, pat; 1836 1837 pat = PATTERN (insn); 1838 1839 if (GET_CODE (pat) != SET) 1840 return false; 1841 1842 lhs = SET_DEST (pat); 1843 if (!REG_P (lhs)) 1844 return false; 1845 1846 rhs = SET_SRC (pat); 1847 if (!REG_P (rhs)) 1848 return false; 1849 1850 return REGNO (lhs) == REGNO (rhs); 1851} 1852 1853/* Undo all transformations on *AV_PTR that were done when 1854 moving through INSN. */ 1855static void 1856undo_transformations (av_set_t *av_ptr, rtx_insn *insn) 1857{ 1858 av_set_iterator av_iter; 1859 expr_t expr; 1860 av_set_t new_set = NULL; 1861 1862 /* First, kill any EXPR that uses registers set by an insn. This is 1863 required for correctness. */ 1864 FOR_EACH_EXPR_1 (expr, av_iter, av_ptr) 1865 if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (expr)) 1866 && bitmap_intersect_p (INSN_REG_SETS (insn), 1867 VINSN_REG_USES (EXPR_VINSN (expr))) 1868 /* When an insn looks like 'r1 = r1', we could substitute through 1869 it, but the above condition will still hold. This happened with 1870 gcc.c-torture/execute/961125-1.c. */ 1871 && !identical_copy_p (insn)) 1872 { 1873 if (sched_verbose >= 6) 1874 sel_print ("Expr %d removed due to use/set conflict\n", 1875 INSN_UID (EXPR_INSN_RTX (expr))); 1876 av_set_iter_remove (&av_iter); 1877 } 1878 1879 /* Undo transformations looking at the history vector. */ 1880 FOR_EACH_EXPR (expr, av_iter, *av_ptr) 1881 { 1882 int index = find_in_history_vect (EXPR_HISTORY_OF_CHANGES (expr), 1883 insn, EXPR_VINSN (expr), true); 1884 1885 if (index >= 0) 1886 { 1887 expr_history_def *phist; 1888 1889 phist = &EXPR_HISTORY_OF_CHANGES (expr)[index]; 1890 1891 switch (phist->type) 1892 { 1893 case TRANS_SPECULATION: 1894 { 1895 ds_t old_ds, new_ds; 1896 1897 /* Compute the difference between old and new speculative 1898 statuses: that's what we need to check. 1899 Earlier we used to assert that the status will really 1900 change. This no longer works because only the probability 1901 bits in the status may have changed during compute_av_set, 1902 and in the case of merging different probabilities of the 1903 same speculative status along different paths we do not 1904 record this in the history vector. */ 1905 old_ds = phist->spec_ds; 1906 new_ds = EXPR_SPEC_DONE_DS (expr); 1907 1908 old_ds &= SPECULATIVE; 1909 new_ds &= SPECULATIVE; 1910 new_ds &= ~old_ds; 1911 1912 EXPR_SPEC_TO_CHECK_DS (expr) |= new_ds; 1913 break; 1914 } 1915 case TRANS_SUBSTITUTION: 1916 { 1917 expr_def _tmp_expr, *tmp_expr = &_tmp_expr; 1918 vinsn_t new_vi; 1919 bool add = true; 1920 1921 new_vi = phist->old_expr_vinsn; 1922 1923 gcc_assert (VINSN_SEPARABLE_P (new_vi) 1924 == EXPR_SEPARABLE_P (expr)); 1925 copy_expr (tmp_expr, expr); 1926 1927 if (vinsn_equal_p (phist->new_expr_vinsn, 1928 EXPR_VINSN (tmp_expr))) 1929 change_vinsn_in_expr (tmp_expr, new_vi); 1930 else 1931 /* This happens when we're unsubstituting on a bookkeeping 1932 copy, which was in turn substituted. The history is wrong 1933 in this case. Do it the hard way. */ 1934 add = substitute_reg_in_expr (tmp_expr, insn, true); 1935 if (add) 1936 av_set_add (&new_set, tmp_expr); 1937 clear_expr (tmp_expr); 1938 break; 1939 } 1940 default: 1941 gcc_unreachable (); 1942 } 1943 } 1944 1945 } 1946 1947 av_set_union_and_clear (av_ptr, &new_set, NULL); 1948} 1949 1950 1951/* Moveup_* helpers for code motion and computing av sets. */ 1952 1953/* Propagates EXPR inside an insn group through THROUGH_INSN. 1954 The difference from the below function is that only substitution is 1955 performed. */ 1956static enum MOVEUP_EXPR_CODE 1957moveup_expr_inside_insn_group (expr_t expr, insn_t through_insn) 1958{ 1959 vinsn_t vi = EXPR_VINSN (expr); 1960 ds_t *has_dep_p; 1961 ds_t full_ds; 1962 1963 /* Do this only inside insn group. */ 1964 gcc_assert (INSN_SCHED_CYCLE (through_insn) > 0); 1965 1966 full_ds = has_dependence_p (expr, through_insn, &has_dep_p); 1967 if (full_ds == 0) 1968 return MOVEUP_EXPR_SAME; 1969 1970 /* Substitution is the possible choice in this case. */ 1971 if (has_dep_p[DEPS_IN_RHS]) 1972 { 1973 /* Can't substitute UNIQUE VINSNs. */ 1974 gcc_assert (!VINSN_UNIQUE_P (vi)); 1975 1976 if (can_substitute_through_p (through_insn, 1977 has_dep_p[DEPS_IN_RHS]) 1978 && substitute_reg_in_expr (expr, through_insn, false)) 1979 { 1980 EXPR_WAS_SUBSTITUTED (expr) = true; 1981 return MOVEUP_EXPR_CHANGED; 1982 } 1983 1984 /* Don't care about this, as even true dependencies may be allowed 1985 in an insn group. */ 1986 return MOVEUP_EXPR_SAME; 1987 } 1988 1989 /* This can catch output dependencies in COND_EXECs. */ 1990 if (has_dep_p[DEPS_IN_INSN]) 1991 return MOVEUP_EXPR_NULL; 1992 1993 /* This is either an output or an anti dependence, which usually have 1994 a zero latency. Allow this here, if we'd be wrong, tick_check_p 1995 will fix this. */ 1996 gcc_assert (has_dep_p[DEPS_IN_LHS]); 1997 return MOVEUP_EXPR_AS_RHS; 1998} 1999 2000/* True when a trapping EXPR cannot be moved through THROUGH_INSN. */ 2001#define CANT_MOVE_TRAPPING(expr, through_insn) \ 2002 (VINSN_MAY_TRAP_P (EXPR_VINSN (expr)) \ 2003 && !sel_insn_has_single_succ_p ((through_insn), SUCCS_ALL) \ 2004 && !sel_insn_is_speculation_check (through_insn)) 2005 2006/* True when a conflict on a target register was found during moveup_expr. */ 2007static bool was_target_conflict = false; 2008 2009/* Return true when moving a debug INSN across THROUGH_INSN will 2010 create a bookkeeping block. We don't want to create such blocks, 2011 for they would cause codegen differences between compilations with 2012 and without debug info. */ 2013 2014static bool 2015moving_insn_creates_bookkeeping_block_p (insn_t insn, 2016 insn_t through_insn) 2017{ 2018 basic_block bbi, bbt; 2019 edge e1, e2; 2020 edge_iterator ei1, ei2; 2021 2022 if (!bookkeeping_can_be_created_if_moved_through_p (through_insn)) 2023 { 2024 if (sched_verbose >= 9) 2025 sel_print ("no bookkeeping required: "); 2026 return FALSE; 2027 } 2028 2029 bbi = BLOCK_FOR_INSN (insn); 2030 2031 if (EDGE_COUNT (bbi->preds) == 1) 2032 { 2033 if (sched_verbose >= 9) 2034 sel_print ("only one pred edge: "); 2035 return TRUE; 2036 } 2037 2038 bbt = BLOCK_FOR_INSN (through_insn); 2039 2040 FOR_EACH_EDGE (e1, ei1, bbt->succs) 2041 { 2042 FOR_EACH_EDGE (e2, ei2, bbi->preds) 2043 { 2044 if (find_block_for_bookkeeping (e1, e2, TRUE)) 2045 { 2046 if (sched_verbose >= 9) 2047 sel_print ("found existing block: "); 2048 return FALSE; 2049 } 2050 } 2051 } 2052 2053 if (sched_verbose >= 9) 2054 sel_print ("would create bookkeeping block: "); 2055 2056 return TRUE; 2057} 2058 2059/* Return true when the conflict with newly created implicit clobbers 2060 between EXPR and THROUGH_INSN is found because of renaming. */ 2061static bool 2062implicit_clobber_conflict_p (insn_t through_insn, expr_t expr) 2063{ 2064 HARD_REG_SET temp; 2065 rtx_insn *insn; 2066 rtx reg, rhs, pat; 2067 hard_reg_set_iterator hrsi; 2068 unsigned regno; 2069 bool valid; 2070 2071 /* Make a new pseudo register. */ 2072 reg = gen_reg_rtx (GET_MODE (EXPR_LHS (expr))); 2073 max_regno = max_reg_num (); 2074 maybe_extend_reg_info_p (); 2075 2076 /* Validate a change and bail out early. */ 2077 insn = EXPR_INSN_RTX (expr); 2078 validate_change (insn, &SET_DEST (PATTERN (insn)), reg, true); 2079 valid = verify_changes (0); 2080 cancel_changes (0); 2081 if (!valid) 2082 { 2083 if (sched_verbose >= 6) 2084 sel_print ("implicit clobbers failed validation, "); 2085 return true; 2086 } 2087 2088 /* Make a new insn with it. */ 2089 rhs = copy_rtx (VINSN_RHS (EXPR_VINSN (expr))); 2090 pat = gen_rtx_SET (reg, rhs); 2091 start_sequence (); 2092 insn = emit_insn (pat); 2093 end_sequence (); 2094 2095 /* Calculate implicit clobbers. */ 2096 extract_insn (insn); 2097 preprocess_constraints (insn); 2098 alternative_mask prefrred = get_preferred_alternatives (insn); 2099 ira_implicitly_set_insn_hard_regs (&temp, prefrred); 2100 temp &= ~ira_no_alloc_regs; 2101 2102 /* If any implicit clobber registers intersect with regular ones in 2103 through_insn, we have a dependency and thus bail out. */ 2104 EXECUTE_IF_SET_IN_HARD_REG_SET (temp, 0, regno, hrsi) 2105 { 2106 vinsn_t vi = INSN_VINSN (through_insn); 2107 if (bitmap_bit_p (VINSN_REG_SETS (vi), regno) 2108 || bitmap_bit_p (VINSN_REG_CLOBBERS (vi), regno) 2109 || bitmap_bit_p (VINSN_REG_USES (vi), regno)) 2110 return true; 2111 } 2112 2113 return false; 2114} 2115 2116/* Modifies EXPR so it can be moved through the THROUGH_INSN, 2117 performing necessary transformations. Record the type of transformation 2118 made in PTRANS_TYPE, when it is not NULL. When INSIDE_INSN_GROUP, 2119 permit all dependencies except true ones, and try to remove those 2120 too via forward substitution. All cases when a non-eliminable 2121 non-zero cost dependency exists inside an insn group will be fixed 2122 in tick_check_p instead. */ 2123static enum MOVEUP_EXPR_CODE 2124moveup_expr (expr_t expr, insn_t through_insn, bool inside_insn_group, 2125 enum local_trans_type *ptrans_type) 2126{ 2127 vinsn_t vi = EXPR_VINSN (expr); 2128 insn_t insn = VINSN_INSN_RTX (vi); 2129 bool was_changed = false; 2130 bool as_rhs = false; 2131 ds_t *has_dep_p; 2132 ds_t full_ds; 2133 2134 /* ??? We use dependencies of non-debug insns on debug insns to 2135 indicate that the debug insns need to be reset if the non-debug 2136 insn is pulled ahead of it. It's hard to figure out how to 2137 introduce such a notion in sel-sched, but it already fails to 2138 support debug insns in other ways, so we just go ahead and 2139 let the deug insns go corrupt for now. */ 2140 if (DEBUG_INSN_P (through_insn) && !DEBUG_INSN_P (insn)) 2141 return MOVEUP_EXPR_SAME; 2142 2143 /* When inside_insn_group, delegate to the helper. */ 2144 if (inside_insn_group) 2145 return moveup_expr_inside_insn_group (expr, through_insn); 2146 2147 /* Deal with unique insns and control dependencies. */ 2148 if (VINSN_UNIQUE_P (vi)) 2149 { 2150 /* We can move jumps without side-effects or jumps that are 2151 mutually exclusive with instruction THROUGH_INSN (all in cases 2152 dependencies allow to do so and jump is not speculative). */ 2153 if (control_flow_insn_p (insn)) 2154 { 2155 basic_block fallthru_bb; 2156 2157 /* Do not move checks and do not move jumps through other 2158 jumps. */ 2159 if (control_flow_insn_p (through_insn) 2160 || sel_insn_is_speculation_check (insn)) 2161 return MOVEUP_EXPR_NULL; 2162 2163 /* Don't move jumps through CFG joins. */ 2164 if (bookkeeping_can_be_created_if_moved_through_p (through_insn)) 2165 return MOVEUP_EXPR_NULL; 2166 2167 /* The jump should have a clear fallthru block, and 2168 this block should be in the current region. */ 2169 if ((fallthru_bb = fallthru_bb_of_jump (insn)) == NULL 2170 || ! in_current_region_p (fallthru_bb)) 2171 return MOVEUP_EXPR_NULL; 2172 2173 /* And it should be mutually exclusive with through_insn. */ 2174 if (! sched_insns_conditions_mutex_p (insn, through_insn) 2175 && ! DEBUG_INSN_P (through_insn)) 2176 return MOVEUP_EXPR_NULL; 2177 } 2178 2179 /* Don't move what we can't move. */ 2180 if (EXPR_CANT_MOVE (expr) 2181 && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn)) 2182 return MOVEUP_EXPR_NULL; 2183 2184 /* Don't move SCHED_GROUP instruction through anything. 2185 If we don't force this, then it will be possible to start 2186 scheduling a sched_group before all its dependencies are 2187 resolved. 2188 ??? Haifa deals with this issue by delaying the SCHED_GROUP 2189 as late as possible through rank_for_schedule. */ 2190 if (SCHED_GROUP_P (insn)) 2191 return MOVEUP_EXPR_NULL; 2192 } 2193 else 2194 gcc_assert (!control_flow_insn_p (insn)); 2195 2196 /* Don't move debug insns if this would require bookkeeping. */ 2197 if (DEBUG_INSN_P (insn) 2198 && BLOCK_FOR_INSN (through_insn) != BLOCK_FOR_INSN (insn) 2199 && moving_insn_creates_bookkeeping_block_p (insn, through_insn)) 2200 return MOVEUP_EXPR_NULL; 2201 2202 /* Deal with data dependencies. */ 2203 was_target_conflict = false; 2204 full_ds = has_dependence_p (expr, through_insn, &has_dep_p); 2205 if (full_ds == 0) 2206 { 2207 if (!CANT_MOVE_TRAPPING (expr, through_insn)) 2208 return MOVEUP_EXPR_SAME; 2209 } 2210 else 2211 { 2212 /* We can move UNIQUE insn up only as a whole and unchanged, 2213 so it shouldn't have any dependencies. */ 2214 if (VINSN_UNIQUE_P (vi)) 2215 return MOVEUP_EXPR_NULL; 2216 } 2217 2218 if (full_ds != 0 && can_speculate_dep_p (full_ds)) 2219 { 2220 int res; 2221 2222 res = speculate_expr (expr, full_ds); 2223 if (res >= 0) 2224 { 2225 /* Speculation was successful. */ 2226 full_ds = 0; 2227 was_changed = (res > 0); 2228 if (res == 2) 2229 was_target_conflict = true; 2230 if (ptrans_type) 2231 *ptrans_type = TRANS_SPECULATION; 2232 sel_clear_has_dependence (); 2233 } 2234 } 2235 2236 if (has_dep_p[DEPS_IN_INSN]) 2237 /* We have some dependency that cannot be discarded. */ 2238 return MOVEUP_EXPR_NULL; 2239 2240 if (has_dep_p[DEPS_IN_LHS]) 2241 { 2242 /* Only separable insns can be moved up with the new register. 2243 Anyways, we should mark that the original register is 2244 unavailable. */ 2245 if (!enable_schedule_as_rhs_p || !EXPR_SEPARABLE_P (expr)) 2246 return MOVEUP_EXPR_NULL; 2247 2248 /* When renaming a hard register to a pseudo before reload, extra 2249 dependencies can occur from the implicit clobbers of the insn. 2250 Filter out such cases here. */ 2251 if (!reload_completed && REG_P (EXPR_LHS (expr)) 2252 && HARD_REGISTER_P (EXPR_LHS (expr)) 2253 && implicit_clobber_conflict_p (through_insn, expr)) 2254 { 2255 if (sched_verbose >= 6) 2256 sel_print ("implicit clobbers conflict detected, "); 2257 return MOVEUP_EXPR_NULL; 2258 } 2259 EXPR_TARGET_AVAILABLE (expr) = false; 2260 was_target_conflict = true; 2261 as_rhs = true; 2262 } 2263 2264 /* At this point we have either separable insns, that will be lifted 2265 up only as RHSes, or non-separable insns with no dependency in lhs. 2266 If dependency is in RHS, then try to perform substitution and move up 2267 substituted RHS: 2268 2269 Ex. 1: Ex.2 2270 y = x; y = x; 2271 z = y*2; y = y*2; 2272 2273 In Ex.1 y*2 can be substituted for x*2 and the whole operation can be 2274 moved above y=x assignment as z=x*2. 2275 2276 In Ex.2 y*2 also can be substituted for x*2, but only the right hand 2277 side can be moved because of the output dependency. The operation was 2278 cropped to its rhs above. */ 2279 if (has_dep_p[DEPS_IN_RHS]) 2280 { 2281 ds_t *rhs_dsp = &has_dep_p[DEPS_IN_RHS]; 2282 2283 /* Can't substitute UNIQUE VINSNs. */ 2284 gcc_assert (!VINSN_UNIQUE_P (vi)); 2285 2286 if (can_speculate_dep_p (*rhs_dsp)) 2287 { 2288 int res; 2289 2290 res = speculate_expr (expr, *rhs_dsp); 2291 if (res >= 0) 2292 { 2293 /* Speculation was successful. */ 2294 *rhs_dsp = 0; 2295 was_changed = (res > 0); 2296 if (res == 2) 2297 was_target_conflict = true; 2298 if (ptrans_type) 2299 *ptrans_type = TRANS_SPECULATION; 2300 } 2301 else 2302 return MOVEUP_EXPR_NULL; 2303 } 2304 else if (can_substitute_through_p (through_insn, 2305 *rhs_dsp) 2306 && substitute_reg_in_expr (expr, through_insn, false)) 2307 { 2308 /* ??? We cannot perform substitution AND speculation on the same 2309 insn. */ 2310 gcc_assert (!was_changed); 2311 was_changed = true; 2312 if (ptrans_type) 2313 *ptrans_type = TRANS_SUBSTITUTION; 2314 EXPR_WAS_SUBSTITUTED (expr) = true; 2315 } 2316 else 2317 return MOVEUP_EXPR_NULL; 2318 } 2319 2320 /* Don't move trapping insns through jumps. 2321 This check should be at the end to give a chance to control speculation 2322 to perform its duties. */ 2323 if (CANT_MOVE_TRAPPING (expr, through_insn)) 2324 return MOVEUP_EXPR_NULL; 2325 2326 return (was_changed 2327 ? MOVEUP_EXPR_CHANGED 2328 : (as_rhs 2329 ? MOVEUP_EXPR_AS_RHS 2330 : MOVEUP_EXPR_SAME)); 2331} 2332 2333/* Try to look at bitmap caches for EXPR and INSN pair, return true 2334 if successful. When INSIDE_INSN_GROUP, also try ignore dependencies 2335 that can exist within a parallel group. Write to RES the resulting 2336 code for moveup_expr. */ 2337static bool 2338try_bitmap_cache (expr_t expr, insn_t insn, 2339 bool inside_insn_group, 2340 enum MOVEUP_EXPR_CODE *res) 2341{ 2342 int expr_uid = INSN_UID (EXPR_INSN_RTX (expr)); 2343 2344 /* First check whether we've analyzed this situation already. */ 2345 if (bitmap_bit_p (INSN_ANALYZED_DEPS (insn), expr_uid)) 2346 { 2347 if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid)) 2348 { 2349 if (sched_verbose >= 6) 2350 sel_print ("removed (cached)\n"); 2351 *res = MOVEUP_EXPR_NULL; 2352 return true; 2353 } 2354 else 2355 { 2356 if (sched_verbose >= 6) 2357 sel_print ("unchanged (cached)\n"); 2358 *res = MOVEUP_EXPR_SAME; 2359 return true; 2360 } 2361 } 2362 else if (bitmap_bit_p (INSN_FOUND_DEPS (insn), expr_uid)) 2363 { 2364 if (inside_insn_group) 2365 { 2366 if (sched_verbose >= 6) 2367 sel_print ("unchanged (as RHS, cached, inside insn group)\n"); 2368 *res = MOVEUP_EXPR_SAME; 2369 return true; 2370 2371 } 2372 else 2373 EXPR_TARGET_AVAILABLE (expr) = false; 2374 2375 /* This is the only case when propagation result can change over time, 2376 as we can dynamically switch off scheduling as RHS. In this case, 2377 just check the flag to reach the correct decision. */ 2378 if (enable_schedule_as_rhs_p) 2379 { 2380 if (sched_verbose >= 6) 2381 sel_print ("unchanged (as RHS, cached)\n"); 2382 *res = MOVEUP_EXPR_AS_RHS; 2383 return true; 2384 } 2385 else 2386 { 2387 if (sched_verbose >= 6) 2388 sel_print ("removed (cached as RHS, but renaming" 2389 " is now disabled)\n"); 2390 *res = MOVEUP_EXPR_NULL; 2391 return true; 2392 } 2393 } 2394 2395 return false; 2396} 2397 2398/* Try to look at bitmap caches for EXPR and INSN pair, return true 2399 if successful. Write to RES the resulting code for moveup_expr. */ 2400static bool 2401try_transformation_cache (expr_t expr, insn_t insn, 2402 enum MOVEUP_EXPR_CODE *res) 2403{ 2404 struct transformed_insns *pti 2405 = (struct transformed_insns *) 2406 htab_find_with_hash (INSN_TRANSFORMED_INSNS (insn), 2407 &EXPR_VINSN (expr), 2408 VINSN_HASH_RTX (EXPR_VINSN (expr))); 2409 if (pti) 2410 { 2411 /* This EXPR was already moved through this insn and was 2412 changed as a result. Fetch the proper data from 2413 the hashtable. */ 2414 insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr), 2415 INSN_UID (insn), pti->type, 2416 pti->vinsn_old, pti->vinsn_new, 2417 EXPR_SPEC_DONE_DS (expr)); 2418 2419 if (INSN_IN_STREAM_P (VINSN_INSN_RTX (pti->vinsn_new))) 2420 pti->vinsn_new = vinsn_copy (pti->vinsn_new, true); 2421 change_vinsn_in_expr (expr, pti->vinsn_new); 2422 if (pti->was_target_conflict) 2423 EXPR_TARGET_AVAILABLE (expr) = false; 2424 if (pti->type == TRANS_SPECULATION) 2425 { 2426 EXPR_SPEC_DONE_DS (expr) = pti->ds; 2427 EXPR_NEEDS_SPEC_CHECK_P (expr) |= pti->needs_check; 2428 } 2429 2430 if (sched_verbose >= 6) 2431 { 2432 sel_print ("changed (cached): "); 2433 dump_expr (expr); 2434 sel_print ("\n"); 2435 } 2436 2437 *res = MOVEUP_EXPR_CHANGED; 2438 return true; 2439 } 2440 2441 return false; 2442} 2443 2444/* Update bitmap caches on INSN with result RES of propagating EXPR. */ 2445static void 2446update_bitmap_cache (expr_t expr, insn_t insn, bool inside_insn_group, 2447 enum MOVEUP_EXPR_CODE res) 2448{ 2449 int expr_uid = INSN_UID (EXPR_INSN_RTX (expr)); 2450 2451 /* Do not cache result of propagating jumps through an insn group, 2452 as it is always true, which is not useful outside the group. */ 2453 if (inside_insn_group) 2454 return; 2455 2456 if (res == MOVEUP_EXPR_NULL) 2457 { 2458 bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid); 2459 bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid); 2460 } 2461 else if (res == MOVEUP_EXPR_SAME) 2462 { 2463 bitmap_set_bit (INSN_ANALYZED_DEPS (insn), expr_uid); 2464 bitmap_clear_bit (INSN_FOUND_DEPS (insn), expr_uid); 2465 } 2466 else if (res == MOVEUP_EXPR_AS_RHS) 2467 { 2468 bitmap_clear_bit (INSN_ANALYZED_DEPS (insn), expr_uid); 2469 bitmap_set_bit (INSN_FOUND_DEPS (insn), expr_uid); 2470 } 2471 else 2472 gcc_unreachable (); 2473} 2474 2475/* Update hashtable on INSN with changed EXPR, old EXPR_OLD_VINSN 2476 and transformation type TRANS_TYPE. */ 2477static void 2478update_transformation_cache (expr_t expr, insn_t insn, 2479 bool inside_insn_group, 2480 enum local_trans_type trans_type, 2481 vinsn_t expr_old_vinsn) 2482{ 2483 struct transformed_insns *pti; 2484 2485 if (inside_insn_group) 2486 return; 2487 2488 pti = XNEW (struct transformed_insns); 2489 pti->vinsn_old = expr_old_vinsn; 2490 pti->vinsn_new = EXPR_VINSN (expr); 2491 pti->type = trans_type; 2492 pti->was_target_conflict = was_target_conflict; 2493 pti->ds = EXPR_SPEC_DONE_DS (expr); 2494 pti->needs_check = EXPR_NEEDS_SPEC_CHECK_P (expr); 2495 vinsn_attach (pti->vinsn_old); 2496 vinsn_attach (pti->vinsn_new); 2497 *((struct transformed_insns **) 2498 htab_find_slot_with_hash (INSN_TRANSFORMED_INSNS (insn), 2499 pti, VINSN_HASH_RTX (expr_old_vinsn), 2500 INSERT)) = pti; 2501} 2502 2503/* Same as moveup_expr, but first looks up the result of 2504 transformation in caches. */ 2505static enum MOVEUP_EXPR_CODE 2506moveup_expr_cached (expr_t expr, insn_t insn, bool inside_insn_group) 2507{ 2508 enum MOVEUP_EXPR_CODE res; 2509 bool got_answer = false; 2510 2511 if (sched_verbose >= 6) 2512 { 2513 sel_print ("Moving "); 2514 dump_expr (expr); 2515 sel_print (" through %d: ", INSN_UID (insn)); 2516 } 2517 2518 if (DEBUG_INSN_P (EXPR_INSN_RTX (expr)) 2519 && BLOCK_FOR_INSN (EXPR_INSN_RTX (expr)) 2520 && (sel_bb_head (BLOCK_FOR_INSN (EXPR_INSN_RTX (expr))) 2521 == EXPR_INSN_RTX (expr))) 2522 /* Don't use cached information for debug insns that are heads of 2523 basic blocks. */; 2524 else if (try_bitmap_cache (expr, insn, inside_insn_group, &res)) 2525 /* When inside insn group, we do not want remove stores conflicting 2526 with previosly issued loads. */ 2527 got_answer = ! inside_insn_group || res != MOVEUP_EXPR_NULL; 2528 else if (try_transformation_cache (expr, insn, &res)) 2529 got_answer = true; 2530 2531 if (! got_answer) 2532 { 2533 /* Invoke moveup_expr and record the results. */ 2534 vinsn_t expr_old_vinsn = EXPR_VINSN (expr); 2535 ds_t expr_old_spec_ds = EXPR_SPEC_DONE_DS (expr); 2536 int expr_uid = INSN_UID (VINSN_INSN_RTX (expr_old_vinsn)); 2537 bool unique_p = VINSN_UNIQUE_P (expr_old_vinsn); 2538 enum local_trans_type trans_type = TRANS_SUBSTITUTION; 2539 2540 /* ??? Invent something better than this. We can't allow old_vinsn 2541 to go, we need it for the history vector. */ 2542 vinsn_attach (expr_old_vinsn); 2543 2544 res = moveup_expr (expr, insn, inside_insn_group, 2545 &trans_type); 2546 switch (res) 2547 { 2548 case MOVEUP_EXPR_NULL: 2549 update_bitmap_cache (expr, insn, inside_insn_group, res); 2550 if (sched_verbose >= 6) 2551 sel_print ("removed\n"); 2552 break; 2553 2554 case MOVEUP_EXPR_SAME: 2555 update_bitmap_cache (expr, insn, inside_insn_group, res); 2556 if (sched_verbose >= 6) 2557 sel_print ("unchanged\n"); 2558 break; 2559 2560 case MOVEUP_EXPR_AS_RHS: 2561 gcc_assert (!unique_p || inside_insn_group); 2562 update_bitmap_cache (expr, insn, inside_insn_group, res); 2563 if (sched_verbose >= 6) 2564 sel_print ("unchanged (as RHS)\n"); 2565 break; 2566 2567 case MOVEUP_EXPR_CHANGED: 2568 gcc_assert (INSN_UID (EXPR_INSN_RTX (expr)) != expr_uid 2569 || EXPR_SPEC_DONE_DS (expr) != expr_old_spec_ds); 2570 insert_in_history_vect (&EXPR_HISTORY_OF_CHANGES (expr), 2571 INSN_UID (insn), trans_type, 2572 expr_old_vinsn, EXPR_VINSN (expr), 2573 expr_old_spec_ds); 2574 update_transformation_cache (expr, insn, inside_insn_group, 2575 trans_type, expr_old_vinsn); 2576 if (sched_verbose >= 6) 2577 { 2578 sel_print ("changed: "); 2579 dump_expr (expr); 2580 sel_print ("\n"); 2581 } 2582 break; 2583 default: 2584 gcc_unreachable (); 2585 } 2586 2587 vinsn_detach (expr_old_vinsn); 2588 } 2589 2590 return res; 2591} 2592 2593/* Moves an av set AVP up through INSN, performing necessary 2594 transformations. */ 2595static void 2596moveup_set_expr (av_set_t *avp, insn_t insn, bool inside_insn_group) 2597{ 2598 av_set_iterator i; 2599 expr_t expr; 2600 2601 FOR_EACH_EXPR_1 (expr, i, avp) 2602 { 2603 2604 switch (moveup_expr_cached (expr, insn, inside_insn_group)) 2605 { 2606 case MOVEUP_EXPR_SAME: 2607 case MOVEUP_EXPR_AS_RHS: 2608 break; 2609 2610 case MOVEUP_EXPR_NULL: 2611 av_set_iter_remove (&i); 2612 break; 2613 2614 case MOVEUP_EXPR_CHANGED: 2615 expr = merge_with_other_exprs (avp, &i, expr); 2616 break; 2617 2618 default: 2619 gcc_unreachable (); 2620 } 2621 } 2622} 2623 2624/* Moves AVP set along PATH. */ 2625static void 2626moveup_set_inside_insn_group (av_set_t *avp, ilist_t path) 2627{ 2628 int last_cycle; 2629 2630 if (sched_verbose >= 6) 2631 sel_print ("Moving expressions up in the insn group...\n"); 2632 if (! path) 2633 return; 2634 last_cycle = INSN_SCHED_CYCLE (ILIST_INSN (path)); 2635 while (path 2636 && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle) 2637 { 2638 moveup_set_expr (avp, ILIST_INSN (path), true); 2639 path = ILIST_NEXT (path); 2640 } 2641} 2642 2643/* Returns true if after moving EXPR along PATH it equals to EXPR_VLIW. */ 2644static bool 2645equal_after_moveup_path_p (expr_t expr, ilist_t path, expr_t expr_vliw) 2646{ 2647 expr_def _tmp, *tmp = &_tmp; 2648 int last_cycle; 2649 bool res = true; 2650 2651 copy_expr_onside (tmp, expr); 2652 last_cycle = path ? INSN_SCHED_CYCLE (ILIST_INSN (path)) : 0; 2653 while (path 2654 && res 2655 && INSN_SCHED_CYCLE (ILIST_INSN (path)) == last_cycle) 2656 { 2657 res = (moveup_expr_cached (tmp, ILIST_INSN (path), true) 2658 != MOVEUP_EXPR_NULL); 2659 path = ILIST_NEXT (path); 2660 } 2661 2662 if (res) 2663 { 2664 vinsn_t tmp_vinsn = EXPR_VINSN (tmp); 2665 vinsn_t expr_vliw_vinsn = EXPR_VINSN (expr_vliw); 2666 2667 if (tmp_vinsn != expr_vliw_vinsn) 2668 res = vinsn_equal_p (tmp_vinsn, expr_vliw_vinsn); 2669 } 2670 2671 clear_expr (tmp); 2672 return res; 2673} 2674 2675 2676/* Functions that compute av and lv sets. */ 2677 2678/* Returns true if INSN is not a downward continuation of the given path P in 2679 the current stage. */ 2680static bool 2681is_ineligible_successor (insn_t insn, ilist_t p) 2682{ 2683 insn_t prev_insn; 2684 2685 /* Check if insn is not deleted. */ 2686 if (PREV_INSN (insn) && NEXT_INSN (PREV_INSN (insn)) != insn) 2687 gcc_unreachable (); 2688 else if (NEXT_INSN (insn) && PREV_INSN (NEXT_INSN (insn)) != insn) 2689 gcc_unreachable (); 2690 2691 /* If it's the first insn visited, then the successor is ok. */ 2692 if (!p) 2693 return false; 2694 2695 prev_insn = ILIST_INSN (p); 2696 2697 if (/* a backward edge. */ 2698 INSN_SEQNO (insn) < INSN_SEQNO (prev_insn) 2699 /* is already visited. */ 2700 || (INSN_SEQNO (insn) == INSN_SEQNO (prev_insn) 2701 && (ilist_is_in_p (p, insn) 2702 /* We can reach another fence here and still seqno of insn 2703 would be equal to seqno of prev_insn. This is possible 2704 when prev_insn is a previously created bookkeeping copy. 2705 In that case it'd get a seqno of insn. Thus, check here 2706 whether insn is in current fence too. */ 2707 || IN_CURRENT_FENCE_P (insn))) 2708 /* Was already scheduled on this round. */ 2709 || (INSN_SEQNO (insn) > INSN_SEQNO (prev_insn) 2710 && IN_CURRENT_FENCE_P (insn)) 2711 /* An insn from another fence could also be 2712 scheduled earlier even if this insn is not in 2713 a fence list right now. Check INSN_SCHED_CYCLE instead. */ 2714 || (!pipelining_p 2715 && INSN_SCHED_TIMES (insn) > 0)) 2716 return true; 2717 else 2718 return false; 2719} 2720 2721/* Computes the av_set below the last bb insn INSN, doing all the 'dirty work' 2722 of handling multiple successors and properly merging its av_sets. P is 2723 the current path traversed. WS is the size of lookahead window. 2724 Return the av set computed. */ 2725static av_set_t 2726compute_av_set_at_bb_end (insn_t insn, ilist_t p, int ws) 2727{ 2728 struct succs_info *sinfo; 2729 av_set_t expr_in_all_succ_branches = NULL; 2730 int is; 2731 insn_t succ, zero_succ = NULL; 2732 av_set_t av1 = NULL; 2733 2734 gcc_assert (sel_bb_end_p (insn)); 2735 2736 /* Find different kind of successors needed for correct computing of 2737 SPEC and TARGET_AVAILABLE attributes. */ 2738 sinfo = compute_succs_info (insn, SUCCS_NORMAL); 2739 2740 /* Debug output. */ 2741 if (sched_verbose >= 6) 2742 { 2743 sel_print ("successors of bb end (%d): ", INSN_UID (insn)); 2744 dump_insn_vector (sinfo->succs_ok); 2745 sel_print ("\n"); 2746 if (sinfo->succs_ok_n != sinfo->all_succs_n) 2747 sel_print ("real successors num: %d\n", sinfo->all_succs_n); 2748 } 2749 2750 /* Add insn to the tail of current path. */ 2751 ilist_add (&p, insn); 2752 2753 FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ) 2754 { 2755 av_set_t succ_set; 2756 2757 /* We will edit SUCC_SET and EXPR_SPEC field of its elements. */ 2758 succ_set = compute_av_set_inside_bb (succ, p, ws, true); 2759 2760 av_set_split_usefulness (succ_set, 2761 sinfo->probs_ok[is], 2762 sinfo->all_prob); 2763 2764 if (sinfo->all_succs_n > 1) 2765 { 2766 /* Find EXPR'es that came from *all* successors and save them 2767 into expr_in_all_succ_branches. This set will be used later 2768 for calculating speculation attributes of EXPR'es. */ 2769 if (is == 0) 2770 { 2771 expr_in_all_succ_branches = av_set_copy (succ_set); 2772 2773 /* Remember the first successor for later. */ 2774 zero_succ = succ; 2775 } 2776 else 2777 { 2778 av_set_iterator i; 2779 expr_t expr; 2780 2781 FOR_EACH_EXPR_1 (expr, i, &expr_in_all_succ_branches) 2782 if (!av_set_is_in_p (succ_set, EXPR_VINSN (expr))) 2783 av_set_iter_remove (&i); 2784 } 2785 } 2786 2787 /* Union the av_sets. Check liveness restrictions on target registers 2788 in special case of two successors. */ 2789 if (sinfo->succs_ok_n == 2 && is == 1) 2790 { 2791 basic_block bb0 = BLOCK_FOR_INSN (zero_succ); 2792 basic_block bb1 = BLOCK_FOR_INSN (succ); 2793 2794 gcc_assert (BB_LV_SET_VALID_P (bb0) && BB_LV_SET_VALID_P (bb1)); 2795 av_set_union_and_live (&av1, &succ_set, 2796 BB_LV_SET (bb0), 2797 BB_LV_SET (bb1), 2798 insn); 2799 } 2800 else 2801 av_set_union_and_clear (&av1, &succ_set, insn); 2802 } 2803 2804 /* Check liveness restrictions via hard way when there are more than 2805 two successors. */ 2806 if (sinfo->succs_ok_n > 2) 2807 FOR_EACH_VEC_ELT (sinfo->succs_ok, is, succ) 2808 { 2809 basic_block succ_bb = BLOCK_FOR_INSN (succ); 2810 av_set_t av_succ = (is_ineligible_successor (succ, p) 2811 ? NULL 2812 : BB_AV_SET (succ_bb)); 2813 2814 gcc_assert (BB_LV_SET_VALID_P (succ_bb)); 2815 mark_unavailable_targets (av1, av_succ, BB_LV_SET (succ_bb)); 2816 } 2817 2818 /* Finally, check liveness restrictions on paths leaving the region. */ 2819 if (sinfo->all_succs_n > sinfo->succs_ok_n) 2820 FOR_EACH_VEC_ELT (sinfo->succs_other, is, succ) 2821 mark_unavailable_targets 2822 (av1, NULL, BB_LV_SET (BLOCK_FOR_INSN (succ))); 2823 2824 if (sinfo->all_succs_n > 1) 2825 { 2826 av_set_iterator i; 2827 expr_t expr; 2828 2829 /* Increase the spec attribute of all EXPR'es that didn't come 2830 from all successors. */ 2831 FOR_EACH_EXPR (expr, i, av1) 2832 if (!av_set_is_in_p (expr_in_all_succ_branches, EXPR_VINSN (expr))) 2833 EXPR_SPEC (expr)++; 2834 2835 av_set_clear (&expr_in_all_succ_branches); 2836 2837 /* Do not move conditional branches through other 2838 conditional branches. So, remove all conditional 2839 branches from av_set if current operator is a conditional 2840 branch. */ 2841 av_set_substract_cond_branches (&av1); 2842 } 2843 2844 ilist_remove (&p); 2845 free_succs_info (sinfo); 2846 2847 if (sched_verbose >= 6) 2848 { 2849 sel_print ("av_succs (%d): ", INSN_UID (insn)); 2850 dump_av_set (av1); 2851 sel_print ("\n"); 2852 } 2853 2854 return av1; 2855} 2856 2857/* This function computes av_set for the FIRST_INSN by dragging valid 2858 av_set through all basic block insns either from the end of basic block 2859 (computed using compute_av_set_at_bb_end) or from the insn on which 2860 MAX_WS was exceeded. It uses compute_av_set_at_bb_end to compute av_set 2861 below the basic block and handling conditional branches. 2862 FIRST_INSN - the basic block head, P - path consisting of the insns 2863 traversed on the way to the FIRST_INSN (the path is sparse, only bb heads 2864 and bb ends are added to the path), WS - current window size, 2865 NEED_COPY_P - true if we'll make a copy of av_set before returning it. */ 2866static av_set_t 2867compute_av_set_inside_bb (insn_t first_insn, ilist_t p, int ws, 2868 bool need_copy_p) 2869{ 2870 insn_t cur_insn; 2871 int end_ws = ws; 2872 insn_t bb_end = sel_bb_end (BLOCK_FOR_INSN (first_insn)); 2873 insn_t after_bb_end = NEXT_INSN (bb_end); 2874 insn_t last_insn; 2875 av_set_t av = NULL; 2876 basic_block cur_bb = BLOCK_FOR_INSN (first_insn); 2877 2878 /* Return NULL if insn is not on the legitimate downward path. */ 2879 if (is_ineligible_successor (first_insn, p)) 2880 { 2881 if (sched_verbose >= 6) 2882 sel_print ("Insn %d is ineligible_successor\n", INSN_UID (first_insn)); 2883 2884 return NULL; 2885 } 2886 2887 /* If insn already has valid av(insn) computed, just return it. */ 2888 if (AV_SET_VALID_P (first_insn)) 2889 { 2890 av_set_t av_set; 2891 2892 if (sel_bb_head_p (first_insn)) 2893 av_set = BB_AV_SET (BLOCK_FOR_INSN (first_insn)); 2894 else 2895 av_set = NULL; 2896 2897 if (sched_verbose >= 6) 2898 { 2899 sel_print ("Insn %d has a valid av set: ", INSN_UID (first_insn)); 2900 dump_av_set (av_set); 2901 sel_print ("\n"); 2902 } 2903 2904 return need_copy_p ? av_set_copy (av_set) : av_set; 2905 } 2906 2907 ilist_add (&p, first_insn); 2908 2909 /* As the result after this loop have completed, in LAST_INSN we'll 2910 have the insn which has valid av_set to start backward computation 2911 from: it either will be NULL because on it the window size was exceeded 2912 or other valid av_set as returned by compute_av_set for the last insn 2913 of the basic block. */ 2914 for (last_insn = first_insn; last_insn != after_bb_end; 2915 last_insn = NEXT_INSN (last_insn)) 2916 { 2917 /* We may encounter valid av_set not only on bb_head, but also on 2918 those insns on which previously MAX_WS was exceeded. */ 2919 if (AV_SET_VALID_P (last_insn)) 2920 { 2921 if (sched_verbose >= 6) 2922 sel_print ("Insn %d has a valid empty av set\n", INSN_UID (last_insn)); 2923 break; 2924 } 2925 2926 /* The special case: the last insn of the BB may be an 2927 ineligible_successor due to its SEQ_NO that was set on 2928 it as a bookkeeping. */ 2929 if (last_insn != first_insn 2930 && is_ineligible_successor (last_insn, p)) 2931 { 2932 if (sched_verbose >= 6) 2933 sel_print ("Insn %d is ineligible_successor\n", INSN_UID (last_insn)); 2934 break; 2935 } 2936 2937 if (DEBUG_INSN_P (last_insn)) 2938 continue; 2939 2940 if (end_ws > max_ws) 2941 { 2942 /* We can reach max lookahead size at bb_header, so clean av_set 2943 first. */ 2944 INSN_WS_LEVEL (last_insn) = global_level; 2945 2946 if (sched_verbose >= 6) 2947 sel_print ("Insn %d is beyond the software lookahead window size\n", 2948 INSN_UID (last_insn)); 2949 break; 2950 } 2951 2952 end_ws++; 2953 } 2954 2955 /* Get the valid av_set into AV above the LAST_INSN to start backward 2956 computation from. It either will be empty av_set or av_set computed from 2957 the successors on the last insn of the current bb. */ 2958 if (last_insn != after_bb_end) 2959 { 2960 av = NULL; 2961 2962 /* This is needed only to obtain av_sets that are identical to 2963 those computed by the old compute_av_set version. */ 2964 if (last_insn == first_insn && !INSN_NOP_P (last_insn)) 2965 av_set_add (&av, INSN_EXPR (last_insn)); 2966 } 2967 else 2968 /* END_WS is always already increased by 1 if LAST_INSN == AFTER_BB_END. */ 2969 av = compute_av_set_at_bb_end (bb_end, p, end_ws); 2970 2971 /* Compute av_set in AV starting from below the LAST_INSN up to 2972 location above the FIRST_INSN. */ 2973 for (cur_insn = PREV_INSN (last_insn); cur_insn != PREV_INSN (first_insn); 2974 cur_insn = PREV_INSN (cur_insn)) 2975 if (!INSN_NOP_P (cur_insn)) 2976 { 2977 expr_t expr; 2978 2979 moveup_set_expr (&av, cur_insn, false); 2980 2981 /* If the expression for CUR_INSN is already in the set, 2982 replace it by the new one. */ 2983 expr = av_set_lookup (av, INSN_VINSN (cur_insn)); 2984 if (expr != NULL) 2985 { 2986 clear_expr (expr); 2987 copy_expr (expr, INSN_EXPR (cur_insn)); 2988 } 2989 else 2990 av_set_add (&av, INSN_EXPR (cur_insn)); 2991 } 2992 2993 /* Clear stale bb_av_set. */ 2994 if (sel_bb_head_p (first_insn)) 2995 { 2996 av_set_clear (&BB_AV_SET (cur_bb)); 2997 BB_AV_SET (cur_bb) = need_copy_p ? av_set_copy (av) : av; 2998 BB_AV_LEVEL (cur_bb) = global_level; 2999 } 3000 3001 if (sched_verbose >= 6) 3002 { 3003 sel_print ("Computed av set for insn %d: ", INSN_UID (first_insn)); 3004 dump_av_set (av); 3005 sel_print ("\n"); 3006 } 3007 3008 ilist_remove (&p); 3009 return av; 3010} 3011 3012/* Compute av set before INSN. 3013 INSN - the current operation (actual rtx INSN) 3014 P - the current path, which is list of insns visited so far 3015 WS - software lookahead window size. 3016 UNIQUE_P - TRUE, if returned av_set will be changed, hence 3017 if we want to save computed av_set in s_i_d, we should make a copy of it. 3018 3019 In the resulting set we will have only expressions that don't have delay 3020 stalls and nonsubstitutable dependences. */ 3021static av_set_t 3022compute_av_set (insn_t insn, ilist_t p, int ws, bool unique_p) 3023{ 3024 return compute_av_set_inside_bb (insn, p, ws, unique_p); 3025} 3026 3027/* Propagate a liveness set LV through INSN. */ 3028static void 3029propagate_lv_set (regset lv, insn_t insn) 3030{ 3031 gcc_assert (INSN_P (insn)); 3032 3033 if (INSN_NOP_P (insn)) 3034 return; 3035 3036 df_simulate_one_insn_backwards (BLOCK_FOR_INSN (insn), insn, lv); 3037} 3038 3039/* Return livness set at the end of BB. */ 3040static regset 3041compute_live_after_bb (basic_block bb) 3042{ 3043 edge e; 3044 edge_iterator ei; 3045 regset lv = get_clear_regset_from_pool (); 3046 3047 gcc_assert (!ignore_first); 3048 3049 FOR_EACH_EDGE (e, ei, bb->succs) 3050 if (sel_bb_empty_p (e->dest)) 3051 { 3052 if (! BB_LV_SET_VALID_P (e->dest)) 3053 { 3054 gcc_unreachable (); 3055 gcc_assert (BB_LV_SET (e->dest) == NULL); 3056 BB_LV_SET (e->dest) = compute_live_after_bb (e->dest); 3057 BB_LV_SET_VALID_P (e->dest) = true; 3058 } 3059 IOR_REG_SET (lv, BB_LV_SET (e->dest)); 3060 } 3061 else 3062 IOR_REG_SET (lv, compute_live (sel_bb_head (e->dest))); 3063 3064 return lv; 3065} 3066 3067/* Compute the set of all live registers at the point before INSN and save 3068 it at INSN if INSN is bb header. */ 3069regset 3070compute_live (insn_t insn) 3071{ 3072 basic_block bb = BLOCK_FOR_INSN (insn); 3073 insn_t final, temp; 3074 regset lv; 3075 3076 /* Return the valid set if we're already on it. */ 3077 if (!ignore_first) 3078 { 3079 regset src = NULL; 3080 3081 if (sel_bb_head_p (insn) && BB_LV_SET_VALID_P (bb)) 3082 src = BB_LV_SET (bb); 3083 else 3084 { 3085 gcc_assert (in_current_region_p (bb)); 3086 if (INSN_LIVE_VALID_P (insn)) 3087 src = INSN_LIVE (insn); 3088 } 3089 3090 if (src) 3091 { 3092 lv = get_regset_from_pool (); 3093 COPY_REG_SET (lv, src); 3094 3095 if (sel_bb_head_p (insn) && ! BB_LV_SET_VALID_P (bb)) 3096 { 3097 COPY_REG_SET (BB_LV_SET (bb), lv); 3098 BB_LV_SET_VALID_P (bb) = true; 3099 } 3100 3101 return_regset_to_pool (lv); 3102 return lv; 3103 } 3104 } 3105 3106 /* We've skipped the wrong lv_set. Don't skip the right one. */ 3107 ignore_first = false; 3108 gcc_assert (in_current_region_p (bb)); 3109 3110 /* Find a valid LV set in this block or below, if needed. 3111 Start searching from the next insn: either ignore_first is true, or 3112 INSN doesn't have a correct live set. */ 3113 temp = NEXT_INSN (insn); 3114 final = NEXT_INSN (BB_END (bb)); 3115 while (temp != final && ! INSN_LIVE_VALID_P (temp)) 3116 temp = NEXT_INSN (temp); 3117 if (temp == final) 3118 { 3119 lv = compute_live_after_bb (bb); 3120 temp = PREV_INSN (temp); 3121 } 3122 else 3123 { 3124 lv = get_regset_from_pool (); 3125 COPY_REG_SET (lv, INSN_LIVE (temp)); 3126 } 3127 3128 /* Put correct lv sets on the insns which have bad sets. */ 3129 final = PREV_INSN (insn); 3130 while (temp != final) 3131 { 3132 propagate_lv_set (lv, temp); 3133 COPY_REG_SET (INSN_LIVE (temp), lv); 3134 INSN_LIVE_VALID_P (temp) = true; 3135 temp = PREV_INSN (temp); 3136 } 3137 3138 /* Also put it in a BB. */ 3139 if (sel_bb_head_p (insn)) 3140 { 3141 basic_block bb = BLOCK_FOR_INSN (insn); 3142 3143 COPY_REG_SET (BB_LV_SET (bb), lv); 3144 BB_LV_SET_VALID_P (bb) = true; 3145 } 3146 3147 /* We return LV to the pool, but will not clear it there. Thus we can 3148 legimatelly use LV till the next use of regset_pool_get (). */ 3149 return_regset_to_pool (lv); 3150 return lv; 3151} 3152 3153/* Update liveness sets for INSN. */ 3154static inline void 3155update_liveness_on_insn (rtx_insn *insn) 3156{ 3157 ignore_first = true; 3158 compute_live (insn); 3159} 3160 3161/* Compute liveness below INSN and write it into REGS. */ 3162static inline void 3163compute_live_below_insn (rtx_insn *insn, regset regs) 3164{ 3165 rtx_insn *succ; 3166 succ_iterator si; 3167 3168 FOR_EACH_SUCC_1 (succ, si, insn, SUCCS_ALL) 3169 IOR_REG_SET (regs, compute_live (succ)); 3170} 3171 3172/* Update the data gathered in av and lv sets starting from INSN. */ 3173static void 3174update_data_sets (rtx_insn *insn) 3175{ 3176 update_liveness_on_insn (insn); 3177 if (sel_bb_head_p (insn)) 3178 { 3179 gcc_assert (AV_LEVEL (insn) != 0); 3180 BB_AV_LEVEL (BLOCK_FOR_INSN (insn)) = -1; 3181 compute_av_set (insn, NULL, 0, 0); 3182 } 3183} 3184 3185 3186/* Helper for move_op () and find_used_regs (). 3187 Return speculation type for which a check should be created on the place 3188 of INSN. EXPR is one of the original ops we are searching for. */ 3189static ds_t 3190get_spec_check_type_for_insn (insn_t insn, expr_t expr) 3191{ 3192 ds_t to_check_ds; 3193 ds_t already_checked_ds = EXPR_SPEC_DONE_DS (INSN_EXPR (insn)); 3194 3195 to_check_ds = EXPR_SPEC_TO_CHECK_DS (expr); 3196 3197 if (targetm.sched.get_insn_checked_ds) 3198 already_checked_ds |= targetm.sched.get_insn_checked_ds (insn); 3199 3200 if (spec_info != NULL 3201 && (spec_info->flags & SEL_SCHED_SPEC_DONT_CHECK_CONTROL)) 3202 already_checked_ds |= BEGIN_CONTROL; 3203 3204 already_checked_ds = ds_get_speculation_types (already_checked_ds); 3205 3206 to_check_ds &= ~already_checked_ds; 3207 3208 return to_check_ds; 3209} 3210 3211/* Find the set of registers that are unavailable for storing expres 3212 while moving ORIG_OPS up on the path starting from INSN due to 3213 liveness (USED_REGS) or hardware restrictions (REG_RENAME_P). 3214 3215 All the original operations found during the traversal are saved in the 3216 ORIGINAL_INSNS list. 3217 3218 REG_RENAME_P denotes the set of hardware registers that 3219 cannot be used with renaming due to the register class restrictions, 3220 mode restrictions and other (the register we'll choose should be 3221 compatible class with the original uses, shouldn't be in call_used_regs, 3222 should be HARD_REGNO_RENAME_OK etc). 3223 3224 Returns TRUE if we've found all original insns, FALSE otherwise. 3225 3226 This function utilizes code_motion_path_driver (formerly find_used_regs_1) 3227 to traverse the code motion paths. This helper function finds registers 3228 that are not available for storing expres while moving ORIG_OPS up on the 3229 path starting from INSN. A register considered as used on the moving path, 3230 if one of the following conditions is not satisfied: 3231 3232 (1) a register not set or read on any path from xi to an instance of 3233 the original operation, 3234 (2) not among the live registers of the point immediately following the 3235 first original operation on a given downward path, except for the 3236 original target register of the operation, 3237 (3) not live on the other path of any conditional branch that is passed 3238 by the operation, in case original operations are not present on 3239 both paths of the conditional branch. 3240 3241 All the original operations found during the traversal are saved in the 3242 ORIGINAL_INSNS list. 3243 3244 REG_RENAME_P->CROSSED_CALL_ABIS is true, if there is a call insn on the path 3245 from INSN to original insn. In this case CALL_USED_REG_SET will be added 3246 to unavailable hard regs at the point original operation is found. */ 3247 3248static bool 3249find_used_regs (insn_t insn, av_set_t orig_ops, regset used_regs, 3250 struct reg_rename *reg_rename_p, def_list_t *original_insns) 3251{ 3252 def_list_iterator i; 3253 def_t def; 3254 int res; 3255 bool needs_spec_check_p = false; 3256 expr_t expr; 3257 av_set_iterator expr_iter; 3258 struct fur_static_params sparams; 3259 struct cmpd_local_params lparams; 3260 3261 /* We haven't visited any blocks yet. */ 3262 bitmap_clear (code_motion_visited_blocks); 3263 3264 /* Init parameters for code_motion_path_driver. */ 3265 sparams.crossed_call_abis = 0; 3266 sparams.original_insns = original_insns; 3267 sparams.used_regs = used_regs; 3268 3269 /* Set the appropriate hooks and data. */ 3270 code_motion_path_driver_info = &fur_hooks; 3271 3272 res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams); 3273 3274 reg_rename_p->crossed_call_abis |= sparams.crossed_call_abis; 3275 3276 gcc_assert (res == 1); 3277 gcc_assert (original_insns && *original_insns); 3278 3279 /* ??? We calculate whether an expression needs a check when computing 3280 av sets. This information is not as precise as it could be due to 3281 merging this bit in merge_expr. We can do better in find_used_regs, 3282 but we want to avoid multiple traversals of the same code motion 3283 paths. */ 3284 FOR_EACH_EXPR (expr, expr_iter, orig_ops) 3285 needs_spec_check_p |= EXPR_NEEDS_SPEC_CHECK_P (expr); 3286 3287 /* Mark hardware regs in REG_RENAME_P that are not suitable 3288 for renaming expr in INSN due to hardware restrictions (register class, 3289 modes compatibility etc). */ 3290 FOR_EACH_DEF (def, i, *original_insns) 3291 { 3292 vinsn_t vinsn = INSN_VINSN (def->orig_insn); 3293 3294 if (VINSN_SEPARABLE_P (vinsn)) 3295 mark_unavailable_hard_regs (def, reg_rename_p, used_regs); 3296 3297 /* Do not allow clobbering of ld.[sa] address in case some of the 3298 original operations need a check. */ 3299 if (needs_spec_check_p) 3300 IOR_REG_SET (used_regs, VINSN_REG_USES (vinsn)); 3301 } 3302 3303 return true; 3304} 3305 3306 3307/* Functions to choose the best insn from available ones. */ 3308 3309/* Adjusts the priority for EXPR using the backend *_adjust_priority hook. */ 3310static int 3311sel_target_adjust_priority (expr_t expr) 3312{ 3313 int priority = EXPR_PRIORITY (expr); 3314 int new_priority; 3315 3316 if (targetm.sched.adjust_priority) 3317 new_priority = targetm.sched.adjust_priority (EXPR_INSN_RTX (expr), priority); 3318 else 3319 new_priority = priority; 3320 3321 /* If the priority has changed, adjust EXPR_PRIORITY_ADJ accordingly. */ 3322 EXPR_PRIORITY_ADJ (expr) = new_priority - EXPR_PRIORITY (expr); 3323 3324 if (sched_verbose >= 4) 3325 sel_print ("sel_target_adjust_priority: insn %d, %d+%d = %d.\n", 3326 INSN_UID (EXPR_INSN_RTX (expr)), EXPR_PRIORITY (expr), 3327 EXPR_PRIORITY_ADJ (expr), new_priority); 3328 3329 return new_priority; 3330} 3331 3332/* Rank two available exprs for schedule. Never return 0 here. */ 3333static int 3334sel_rank_for_schedule (const void *x, const void *y) 3335{ 3336 expr_t tmp = *(const expr_t *) y; 3337 expr_t tmp2 = *(const expr_t *) x; 3338 insn_t tmp_insn, tmp2_insn; 3339 vinsn_t tmp_vinsn, tmp2_vinsn; 3340 int val; 3341 3342 tmp_vinsn = EXPR_VINSN (tmp); 3343 tmp2_vinsn = EXPR_VINSN (tmp2); 3344 tmp_insn = EXPR_INSN_RTX (tmp); 3345 tmp2_insn = EXPR_INSN_RTX (tmp2); 3346 3347 /* Schedule debug insns as early as possible. */ 3348 if (DEBUG_INSN_P (tmp_insn) && !DEBUG_INSN_P (tmp2_insn)) 3349 return -1; 3350 else if (DEBUG_INSN_P (tmp2_insn)) 3351 return 1; 3352 3353 /* Prefer SCHED_GROUP_P insns to any others. */ 3354 if (SCHED_GROUP_P (tmp_insn) != SCHED_GROUP_P (tmp2_insn)) 3355 { 3356 if (VINSN_UNIQUE_P (tmp_vinsn) && VINSN_UNIQUE_P (tmp2_vinsn)) 3357 return SCHED_GROUP_P (tmp2_insn) ? 1 : -1; 3358 3359 /* Now uniqueness means SCHED_GROUP_P is set, because schedule groups 3360 cannot be cloned. */ 3361 if (VINSN_UNIQUE_P (tmp2_vinsn)) 3362 return 1; 3363 return -1; 3364 } 3365 3366 /* Discourage scheduling of speculative checks. */ 3367 val = (sel_insn_is_speculation_check (tmp_insn) 3368 - sel_insn_is_speculation_check (tmp2_insn)); 3369 if (val) 3370 return val; 3371 3372 /* Prefer not scheduled insn over scheduled one. */ 3373 if (EXPR_SCHED_TIMES (tmp) > 0 || EXPR_SCHED_TIMES (tmp2) > 0) 3374 { 3375 val = EXPR_SCHED_TIMES (tmp) - EXPR_SCHED_TIMES (tmp2); 3376 if (val) 3377 return val; 3378 } 3379 3380 /* Prefer jump over non-jump instruction. */ 3381 if (control_flow_insn_p (tmp_insn) && !control_flow_insn_p (tmp2_insn)) 3382 return -1; 3383 else if (control_flow_insn_p (tmp2_insn) && !control_flow_insn_p (tmp_insn)) 3384 return 1; 3385 3386 /* Prefer an expr with non-zero usefulness. */ 3387 int u1 = EXPR_USEFULNESS (tmp), u2 = EXPR_USEFULNESS (tmp2); 3388 3389 if (u1 == 0) 3390 { 3391 if (u2 == 0) 3392 u1 = u2 = 1; 3393 else 3394 return 1; 3395 } 3396 else if (u2 == 0) 3397 return -1; 3398 3399 /* Prefer an expr with greater priority. */ 3400 val = (u2 * (EXPR_PRIORITY (tmp2) + EXPR_PRIORITY_ADJ (tmp2)) 3401 - u1 * (EXPR_PRIORITY (tmp) + EXPR_PRIORITY_ADJ (tmp))); 3402 if (val) 3403 return val; 3404 3405 if (spec_info != NULL && spec_info->mask != 0) 3406 /* This code was taken from haifa-sched.cc: rank_for_schedule (). */ 3407 { 3408 ds_t ds1, ds2; 3409 dw_t dw1, dw2; 3410 int dw; 3411 3412 ds1 = EXPR_SPEC_DONE_DS (tmp); 3413 if (ds1) 3414 dw1 = ds_weak (ds1); 3415 else 3416 dw1 = NO_DEP_WEAK; 3417 3418 ds2 = EXPR_SPEC_DONE_DS (tmp2); 3419 if (ds2) 3420 dw2 = ds_weak (ds2); 3421 else 3422 dw2 = NO_DEP_WEAK; 3423 3424 dw = dw2 - dw1; 3425 if (dw > (NO_DEP_WEAK / 8) || dw < -(NO_DEP_WEAK / 8)) 3426 return dw; 3427 } 3428 3429 /* Prefer an old insn to a bookkeeping insn. */ 3430 if (INSN_UID (tmp_insn) < first_emitted_uid 3431 && INSN_UID (tmp2_insn) >= first_emitted_uid) 3432 return -1; 3433 if (INSN_UID (tmp_insn) >= first_emitted_uid 3434 && INSN_UID (tmp2_insn) < first_emitted_uid) 3435 return 1; 3436 3437 /* Prefer an insn with smaller UID, as a last resort. 3438 We can't safely use INSN_LUID as it is defined only for those insns 3439 that are in the stream. */ 3440 return INSN_UID (tmp_insn) - INSN_UID (tmp2_insn); 3441} 3442 3443/* Filter out expressions from av set pointed to by AV_PTR 3444 that are pipelined too many times. */ 3445static void 3446process_pipelined_exprs (av_set_t *av_ptr) 3447{ 3448 expr_t expr; 3449 av_set_iterator si; 3450 3451 /* Don't pipeline already pipelined code as that would increase 3452 number of unnecessary register moves. */ 3453 FOR_EACH_EXPR_1 (expr, si, av_ptr) 3454 { 3455 if (EXPR_SCHED_TIMES (expr) 3456 >= param_selsched_max_sched_times) 3457 av_set_iter_remove (&si); 3458 } 3459} 3460 3461/* Filter speculative insns from AV_PTR if we don't want them. */ 3462static void 3463process_spec_exprs (av_set_t *av_ptr) 3464{ 3465 expr_t expr; 3466 av_set_iterator si; 3467 3468 if (spec_info == NULL) 3469 return; 3470 3471 /* Scan *AV_PTR to find out if we want to consider speculative 3472 instructions for scheduling. */ 3473 FOR_EACH_EXPR_1 (expr, si, av_ptr) 3474 { 3475 ds_t ds; 3476 3477 ds = EXPR_SPEC_DONE_DS (expr); 3478 3479 /* The probability of a success is too low - don't speculate. */ 3480 if ((ds & SPECULATIVE) 3481 && (ds_weak (ds) < spec_info->data_weakness_cutoff 3482 || EXPR_USEFULNESS (expr) < spec_info->control_weakness_cutoff 3483 || (pipelining_p && false 3484 && (ds & DATA_SPEC) 3485 && (ds & CONTROL_SPEC)))) 3486 { 3487 av_set_iter_remove (&si); 3488 continue; 3489 } 3490 } 3491} 3492 3493/* Search for any use-like insns in AV_PTR and decide on scheduling 3494 them. Return one when found, and NULL otherwise. 3495 Note that we check here whether a USE could be scheduled to avoid 3496 an infinite loop later. */ 3497static expr_t 3498process_use_exprs (av_set_t *av_ptr) 3499{ 3500 expr_t expr; 3501 av_set_iterator si; 3502 bool uses_present_p = false; 3503 bool try_uses_p = true; 3504 3505 FOR_EACH_EXPR_1 (expr, si, av_ptr) 3506 { 3507 /* This will also initialize INSN_CODE for later use. */ 3508 if (recog_memoized (EXPR_INSN_RTX (expr)) < 0) 3509 { 3510 /* If we have a USE in *AV_PTR that was not scheduled yet, 3511 do so because it will do good only. */ 3512 if (EXPR_SCHED_TIMES (expr) <= 0) 3513 { 3514 if (EXPR_TARGET_AVAILABLE (expr) == 1) 3515 return expr; 3516 3517 av_set_iter_remove (&si); 3518 } 3519 else 3520 { 3521 gcc_assert (pipelining_p); 3522 3523 uses_present_p = true; 3524 } 3525 } 3526 else 3527 try_uses_p = false; 3528 } 3529 3530 if (uses_present_p) 3531 { 3532 /* If we don't want to schedule any USEs right now and we have some 3533 in *AV_PTR, remove them, else just return the first one found. */ 3534 if (!try_uses_p) 3535 { 3536 FOR_EACH_EXPR_1 (expr, si, av_ptr) 3537 if (INSN_CODE (EXPR_INSN_RTX (expr)) < 0) 3538 av_set_iter_remove (&si); 3539 } 3540 else 3541 { 3542 FOR_EACH_EXPR_1 (expr, si, av_ptr) 3543 { 3544 gcc_assert (INSN_CODE (EXPR_INSN_RTX (expr)) < 0); 3545 3546 if (EXPR_TARGET_AVAILABLE (expr) == 1) 3547 return expr; 3548 3549 av_set_iter_remove (&si); 3550 } 3551 } 3552 } 3553 3554 return NULL; 3555} 3556 3557/* Lookup EXPR in VINSN_VEC and return TRUE if found. Also check patterns from 3558 EXPR's history of changes. */ 3559static bool 3560vinsn_vec_has_expr_p (vinsn_vec_t vinsn_vec, expr_t expr) 3561{ 3562 vinsn_t vinsn, expr_vinsn; 3563 int n; 3564 unsigned i; 3565 3566 /* Start with checking expr itself and then proceed with all the old forms 3567 of expr taken from its history vector. */ 3568 for (i = 0, expr_vinsn = EXPR_VINSN (expr); 3569 expr_vinsn; 3570 expr_vinsn = (i < EXPR_HISTORY_OF_CHANGES (expr).length () 3571 ? EXPR_HISTORY_OF_CHANGES (expr)[i++].old_expr_vinsn 3572 : NULL)) 3573 FOR_EACH_VEC_ELT (vinsn_vec, n, vinsn) 3574 if (VINSN_SEPARABLE_P (vinsn)) 3575 { 3576 if (vinsn_equal_p (vinsn, expr_vinsn)) 3577 return true; 3578 } 3579 else 3580 { 3581 /* For non-separable instructions, the blocking insn can have 3582 another pattern due to substitution, and we can't choose 3583 different register as in the above case. Check all registers 3584 being written instead. */ 3585 if (bitmap_intersect_p (VINSN_REG_SETS (vinsn), 3586 VINSN_REG_SETS (expr_vinsn))) 3587 return true; 3588 } 3589 3590 return false; 3591} 3592 3593/* Return true if either of expressions from ORIG_OPS can be blocked 3594 by previously created bookkeeping code. STATIC_PARAMS points to static 3595 parameters of move_op. */ 3596static bool 3597av_set_could_be_blocked_by_bookkeeping_p (av_set_t orig_ops, void *static_params) 3598{ 3599 expr_t expr; 3600 av_set_iterator iter; 3601 moveop_static_params_p sparams; 3602 3603 /* This checks that expressions in ORIG_OPS are not blocked by bookkeeping 3604 created while scheduling on another fence. */ 3605 FOR_EACH_EXPR (expr, iter, orig_ops) 3606 if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr)) 3607 return true; 3608 3609 gcc_assert (code_motion_path_driver_info == &move_op_hooks); 3610 sparams = (moveop_static_params_p) static_params; 3611 3612 /* Expressions can be also blocked by bookkeeping created during current 3613 move_op. */ 3614 if (bitmap_bit_p (current_copies, INSN_UID (sparams->failed_insn))) 3615 FOR_EACH_EXPR (expr, iter, orig_ops) 3616 if (moveup_expr_cached (expr, sparams->failed_insn, false) != MOVEUP_EXPR_NULL) 3617 return true; 3618 3619 /* Expressions in ORIG_OPS may have wrong destination register due to 3620 renaming. Check with the right register instead. */ 3621 if (sparams->dest && REG_P (sparams->dest)) 3622 { 3623 rtx reg = sparams->dest; 3624 vinsn_t failed_vinsn = INSN_VINSN (sparams->failed_insn); 3625 3626 if (register_unavailable_p (VINSN_REG_SETS (failed_vinsn), reg) 3627 || register_unavailable_p (VINSN_REG_USES (failed_vinsn), reg) 3628 || register_unavailable_p (VINSN_REG_CLOBBERS (failed_vinsn), reg)) 3629 return true; 3630 } 3631 3632 return false; 3633} 3634 3635/* Clear VINSN_VEC and detach vinsns. */ 3636static void 3637vinsn_vec_clear (vinsn_vec_t *vinsn_vec) 3638{ 3639 unsigned len = vinsn_vec->length (); 3640 if (len > 0) 3641 { 3642 vinsn_t vinsn; 3643 int n; 3644 3645 FOR_EACH_VEC_ELT (*vinsn_vec, n, vinsn) 3646 vinsn_detach (vinsn); 3647 vinsn_vec->block_remove (0, len); 3648 } 3649} 3650 3651/* Add the vinsn of EXPR to the VINSN_VEC. */ 3652static void 3653vinsn_vec_add (vinsn_vec_t *vinsn_vec, expr_t expr) 3654{ 3655 vinsn_attach (EXPR_VINSN (expr)); 3656 vinsn_vec->safe_push (EXPR_VINSN (expr)); 3657} 3658 3659/* Free the vector representing blocked expressions. */ 3660static void 3661vinsn_vec_free (vinsn_vec_t &vinsn_vec) 3662{ 3663 vinsn_vec.release (); 3664} 3665 3666/* Increase EXPR_PRIORITY_ADJ for INSN by AMOUNT. */ 3667 3668void sel_add_to_insn_priority (rtx insn, int amount) 3669{ 3670 EXPR_PRIORITY_ADJ (INSN_EXPR (insn)) += amount; 3671 3672 if (sched_verbose >= 2) 3673 sel_print ("sel_add_to_insn_priority: insn %d, by %d (now %d+%d).\n", 3674 INSN_UID (insn), amount, EXPR_PRIORITY (INSN_EXPR (insn)), 3675 EXPR_PRIORITY_ADJ (INSN_EXPR (insn))); 3676} 3677 3678/* Turn AV into a vector, filter inappropriate insns and sort it. Return 3679 true if there is something to schedule. BNDS and FENCE are current 3680 boundaries and fence, respectively. If we need to stall for some cycles 3681 before an expr from AV would become available, write this number to 3682 *PNEED_STALL. */ 3683static bool 3684fill_vec_av_set (av_set_t av, blist_t bnds, fence_t fence, 3685 int *pneed_stall) 3686{ 3687 av_set_iterator si; 3688 expr_t expr; 3689 int sched_next_worked = 0, stalled, n; 3690 static int av_max_prio, est_ticks_till_branch; 3691 int min_need_stall = -1; 3692 deps_t dc = BND_DC (BLIST_BND (bnds)); 3693 3694 /* Bail out early when the ready list contained only USEs/CLOBBERs that are 3695 already scheduled. */ 3696 if (av == NULL) 3697 return false; 3698 3699 /* Empty vector from the previous stuff. */ 3700 if (vec_av_set.length () > 0) 3701 vec_av_set.block_remove (0, vec_av_set.length ()); 3702 3703 /* Turn the set into a vector for sorting and call sel_target_adjust_priority 3704 for each insn. */ 3705 gcc_assert (vec_av_set.is_empty ()); 3706 FOR_EACH_EXPR (expr, si, av) 3707 { 3708 vec_av_set.safe_push (expr); 3709 3710 gcc_assert (EXPR_PRIORITY_ADJ (expr) == 0 || *pneed_stall); 3711 3712 /* Adjust priority using target backend hook. */ 3713 sel_target_adjust_priority (expr); 3714 } 3715 3716 /* Sort the vector. */ 3717 vec_av_set.qsort (sel_rank_for_schedule); 3718 3719 /* We record maximal priority of insns in av set for current instruction 3720 group. */ 3721 if (FENCE_STARTS_CYCLE_P (fence)) 3722 av_max_prio = est_ticks_till_branch = INT_MIN; 3723 3724 /* Filter out inappropriate expressions. Loop's direction is reversed to 3725 visit "best" instructions first. We assume that vec::unordered_remove 3726 moves last element in place of one being deleted. */ 3727 for (n = vec_av_set.length () - 1, stalled = 0; n >= 0; n--) 3728 { 3729 expr_t expr = vec_av_set[n]; 3730 insn_t insn = EXPR_INSN_RTX (expr); 3731 signed char target_available; 3732 bool is_orig_reg_p = true; 3733 int need_cycles, new_prio; 3734 bool fence_insn_p = INSN_UID (insn) == INSN_UID (FENCE_INSN (fence)); 3735 3736 /* Don't allow any insns other than from SCHED_GROUP if we have one. */ 3737 if (FENCE_SCHED_NEXT (fence) && insn != FENCE_SCHED_NEXT (fence)) 3738 { 3739 vec_av_set.unordered_remove (n); 3740 continue; 3741 } 3742 3743 /* Set number of sched_next insns (just in case there 3744 could be several). */ 3745 if (FENCE_SCHED_NEXT (fence)) 3746 sched_next_worked++; 3747 3748 /* Check all liveness requirements and try renaming. 3749 FIXME: try to minimize calls to this. */ 3750 target_available = EXPR_TARGET_AVAILABLE (expr); 3751 3752 /* If insn was already scheduled on the current fence, 3753 set TARGET_AVAILABLE to -1 no matter what expr's attribute says. */ 3754 if (vinsn_vec_has_expr_p (vec_target_unavailable_vinsns, expr) 3755 && !fence_insn_p) 3756 target_available = -1; 3757 3758 /* If the availability of the EXPR is invalidated by the insertion of 3759 bookkeeping earlier, make sure that we won't choose this expr for 3760 scheduling if it's not separable, and if it is separable, then 3761 we have to recompute the set of available registers for it. */ 3762 if (vinsn_vec_has_expr_p (vec_bookkeeping_blocked_vinsns, expr)) 3763 { 3764 vec_av_set.unordered_remove (n); 3765 if (sched_verbose >= 4) 3766 sel_print ("Expr %d is blocked by bookkeeping inserted earlier\n", 3767 INSN_UID (insn)); 3768 continue; 3769 } 3770 3771 if (target_available == true) 3772 { 3773 /* Do nothing -- we can use an existing register. */ 3774 is_orig_reg_p = EXPR_SEPARABLE_P (expr); 3775 } 3776 else if (/* Non-separable instruction will never 3777 get another register. */ 3778 (target_available == false 3779 && !EXPR_SEPARABLE_P (expr)) 3780 /* Don't try to find a register for low-priority expression. */ 3781 || (int) vec_av_set.length () - 1 - n >= max_insns_to_rename 3782 /* ??? FIXME: Don't try to rename data speculation. */ 3783 || (EXPR_SPEC_DONE_DS (expr) & BEGIN_DATA) 3784 || ! find_best_reg_for_expr (expr, bnds, &is_orig_reg_p)) 3785 { 3786 vec_av_set.unordered_remove (n); 3787 if (sched_verbose >= 4) 3788 sel_print ("Expr %d has no suitable target register\n", 3789 INSN_UID (insn)); 3790 3791 /* A fence insn should not get here. */ 3792 gcc_assert (!fence_insn_p); 3793 continue; 3794 } 3795 3796 /* At this point a fence insn should always be available. */ 3797 gcc_assert (!fence_insn_p 3798 || INSN_UID (FENCE_INSN (fence)) == INSN_UID (EXPR_INSN_RTX (expr))); 3799 3800 /* Filter expressions that need to be renamed or speculated when 3801 pipelining, because compensating register copies or speculation 3802 checks are likely to be placed near the beginning of the loop, 3803 causing a stall. */ 3804 if (pipelining_p && EXPR_ORIG_SCHED_CYCLE (expr) > 0 3805 && (!is_orig_reg_p || EXPR_SPEC_DONE_DS (expr) != 0)) 3806 { 3807 /* Estimation of number of cycles until loop branch for 3808 renaming/speculation to be successful. */ 3809 int need_n_ticks_till_branch = sel_vinsn_cost (EXPR_VINSN (expr)); 3810 3811 if ((int) current_loop_nest->ninsns < 9) 3812 { 3813 vec_av_set.unordered_remove (n); 3814 if (sched_verbose >= 4) 3815 sel_print ("Pipelining expr %d will likely cause stall\n", 3816 INSN_UID (insn)); 3817 continue; 3818 } 3819 3820 if ((int) current_loop_nest->ninsns - num_insns_scheduled 3821 < need_n_ticks_till_branch * issue_rate / 2 3822 && est_ticks_till_branch < need_n_ticks_till_branch) 3823 { 3824 vec_av_set.unordered_remove (n); 3825 if (sched_verbose >= 4) 3826 sel_print ("Pipelining expr %d will likely cause stall\n", 3827 INSN_UID (insn)); 3828 continue; 3829 } 3830 } 3831 3832 /* We want to schedule speculation checks as late as possible. Discard 3833 them from av set if there are instructions with higher priority. */ 3834 if (sel_insn_is_speculation_check (insn) 3835 && EXPR_PRIORITY (expr) < av_max_prio) 3836 { 3837 stalled++; 3838 min_need_stall = min_need_stall < 0 ? 1 : MIN (min_need_stall, 1); 3839 vec_av_set.unordered_remove (n); 3840 if (sched_verbose >= 4) 3841 sel_print ("Delaying speculation check %d until its first use\n", 3842 INSN_UID (insn)); 3843 continue; 3844 } 3845 3846 /* Ignore EXPRs available from pipelining to update AV_MAX_PRIO. */ 3847 if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0) 3848 av_max_prio = MAX (av_max_prio, EXPR_PRIORITY (expr)); 3849 3850 /* Don't allow any insns whose data is not yet ready. 3851 Check first whether we've already tried them and failed. */ 3852 if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence)) 3853 { 3854 need_cycles = (FENCE_READY_TICKS (fence)[INSN_UID (insn)] 3855 - FENCE_CYCLE (fence)); 3856 if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0) 3857 est_ticks_till_branch = MAX (est_ticks_till_branch, 3858 EXPR_PRIORITY (expr) + need_cycles); 3859 3860 if (need_cycles > 0) 3861 { 3862 stalled++; 3863 min_need_stall = (min_need_stall < 0 3864 ? need_cycles 3865 : MIN (min_need_stall, need_cycles)); 3866 vec_av_set.unordered_remove (n); 3867 3868 if (sched_verbose >= 4) 3869 sel_print ("Expr %d is not ready until cycle %d (cached)\n", 3870 INSN_UID (insn), 3871 FENCE_READY_TICKS (fence)[INSN_UID (insn)]); 3872 continue; 3873 } 3874 } 3875 3876 /* Now resort to dependence analysis to find whether EXPR might be 3877 stalled due to dependencies from FENCE's context. */ 3878 need_cycles = tick_check_p (expr, dc, fence); 3879 new_prio = EXPR_PRIORITY (expr) + EXPR_PRIORITY_ADJ (expr) + need_cycles; 3880 3881 if (EXPR_ORIG_SCHED_CYCLE (expr) <= 0) 3882 est_ticks_till_branch = MAX (est_ticks_till_branch, 3883 new_prio); 3884 3885 if (need_cycles > 0) 3886 { 3887 if (INSN_UID (insn) >= FENCE_READY_TICKS_SIZE (fence)) 3888 { 3889 int new_size = INSN_UID (insn) * 3 / 2; 3890 3891 FENCE_READY_TICKS (fence) 3892 = (int *) xrecalloc (FENCE_READY_TICKS (fence), 3893 new_size, FENCE_READY_TICKS_SIZE (fence), 3894 sizeof (int)); 3895 } 3896 FENCE_READY_TICKS (fence)[INSN_UID (insn)] 3897 = FENCE_CYCLE (fence) + need_cycles; 3898 3899 stalled++; 3900 min_need_stall = (min_need_stall < 0 3901 ? need_cycles 3902 : MIN (min_need_stall, need_cycles)); 3903 3904 vec_av_set.unordered_remove (n); 3905 3906 if (sched_verbose >= 4) 3907 sel_print ("Expr %d is not ready yet until cycle %d\n", 3908 INSN_UID (insn), 3909 FENCE_READY_TICKS (fence)[INSN_UID (insn)]); 3910 continue; 3911 } 3912 3913 if (sched_verbose >= 4) 3914 sel_print ("Expr %d is ok\n", INSN_UID (insn)); 3915 min_need_stall = 0; 3916 } 3917 3918 /* Clear SCHED_NEXT. */ 3919 if (FENCE_SCHED_NEXT (fence)) 3920 { 3921 gcc_assert (sched_next_worked == 1); 3922 FENCE_SCHED_NEXT (fence) = NULL; 3923 } 3924 3925 /* No need to stall if this variable was not initialized. */ 3926 if (min_need_stall < 0) 3927 min_need_stall = 0; 3928 3929 if (vec_av_set.is_empty ()) 3930 { 3931 /* We need to set *pneed_stall here, because later we skip this code 3932 when ready list is empty. */ 3933 *pneed_stall = min_need_stall; 3934 return false; 3935 } 3936 else 3937 gcc_assert (min_need_stall == 0); 3938 3939 /* Sort the vector. */ 3940 vec_av_set.qsort (sel_rank_for_schedule); 3941 3942 if (sched_verbose >= 4) 3943 { 3944 sel_print ("Total ready exprs: %d, stalled: %d\n", 3945 vec_av_set.length (), stalled); 3946 sel_print ("Sorted av set (%d): ", vec_av_set.length ()); 3947 FOR_EACH_VEC_ELT (vec_av_set, n, expr) 3948 dump_expr (expr); 3949 sel_print ("\n"); 3950 } 3951 3952 *pneed_stall = 0; 3953 return true; 3954} 3955 3956/* Convert a vectored and sorted av set to the ready list that 3957 the rest of the backend wants to see. */ 3958static void 3959convert_vec_av_set_to_ready (void) 3960{ 3961 int n; 3962 expr_t expr; 3963 3964 /* Allocate and fill the ready list from the sorted vector. */ 3965 ready.n_ready = vec_av_set.length (); 3966 ready.first = ready.n_ready - 1; 3967 3968 gcc_assert (ready.n_ready > 0); 3969 3970 if (ready.n_ready > max_issue_size) 3971 { 3972 max_issue_size = ready.n_ready; 3973 sched_extend_ready_list (ready.n_ready); 3974 } 3975 3976 FOR_EACH_VEC_ELT (vec_av_set, n, expr) 3977 { 3978 vinsn_t vi = EXPR_VINSN (expr); 3979 insn_t insn = VINSN_INSN_RTX (vi); 3980 3981 ready_try[n] = 0; 3982 ready.vec[n] = insn; 3983 } 3984} 3985 3986/* Initialize ready list from *AV_PTR for the max_issue () call. 3987 If any unrecognizable insn found in *AV_PTR, return it (and skip 3988 max_issue). BND and FENCE are current boundary and fence, 3989 respectively. If we need to stall for some cycles before an expr 3990 from *AV_PTR would become available, write this number to *PNEED_STALL. */ 3991static expr_t 3992fill_ready_list (av_set_t *av_ptr, blist_t bnds, fence_t fence, 3993 int *pneed_stall) 3994{ 3995 expr_t expr; 3996 3997 /* We do not support multiple boundaries per fence. */ 3998 gcc_assert (BLIST_NEXT (bnds) == NULL); 3999 4000 /* Process expressions required special handling, i.e. pipelined, 4001 speculative and recog() < 0 expressions first. */ 4002 process_pipelined_exprs (av_ptr); 4003 process_spec_exprs (av_ptr); 4004 4005 /* A USE could be scheduled immediately. */ 4006 expr = process_use_exprs (av_ptr); 4007 if (expr) 4008 { 4009 *pneed_stall = 0; 4010 return expr; 4011 } 4012 4013 /* Turn the av set to a vector for sorting. */ 4014 if (! fill_vec_av_set (*av_ptr, bnds, fence, pneed_stall)) 4015 { 4016 ready.n_ready = 0; 4017 return NULL; 4018 } 4019 4020 /* Build the final ready list. */ 4021 convert_vec_av_set_to_ready (); 4022 return NULL; 4023} 4024 4025/* Wrapper for dfa_new_cycle (). Returns TRUE if cycle was advanced. */ 4026static bool 4027sel_dfa_new_cycle (insn_t insn, fence_t fence) 4028{ 4029 int last_scheduled_cycle = FENCE_LAST_SCHEDULED_INSN (fence) 4030 ? INSN_SCHED_CYCLE (FENCE_LAST_SCHEDULED_INSN (fence)) 4031 : FENCE_CYCLE (fence) - 1; 4032 bool res = false; 4033 int sort_p = 0; 4034 4035 if (!targetm.sched.dfa_new_cycle) 4036 return false; 4037 4038 memcpy (curr_state, FENCE_STATE (fence), dfa_state_size); 4039 4040 while (!sort_p && targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, 4041 insn, last_scheduled_cycle, 4042 FENCE_CYCLE (fence), &sort_p)) 4043 { 4044 memcpy (FENCE_STATE (fence), curr_state, dfa_state_size); 4045 advance_one_cycle (fence); 4046 memcpy (curr_state, FENCE_STATE (fence), dfa_state_size); 4047 res = true; 4048 } 4049 4050 return res; 4051} 4052 4053/* Invoke reorder* target hooks on the ready list. Return the number of insns 4054 we can issue. FENCE is the current fence. */ 4055static int 4056invoke_reorder_hooks (fence_t fence) 4057{ 4058 int issue_more; 4059 bool ran_hook = false; 4060 4061 /* Call the reorder hook at the beginning of the cycle, and call 4062 the reorder2 hook in the middle of the cycle. */ 4063 if (FENCE_ISSUED_INSNS (fence) == 0) 4064 { 4065 if (targetm.sched.reorder 4066 && !SCHED_GROUP_P (ready_element (&ready, 0)) 4067 && ready.n_ready > 1) 4068 { 4069 /* Don't give reorder the most prioritized insn as it can break 4070 pipelining. */ 4071 if (pipelining_p) 4072 --ready.n_ready; 4073 4074 issue_more 4075 = targetm.sched.reorder (sched_dump, sched_verbose, 4076 ready_lastpos (&ready), 4077 &ready.n_ready, FENCE_CYCLE (fence)); 4078 4079 if (pipelining_p) 4080 ++ready.n_ready; 4081 4082 ran_hook = true; 4083 } 4084 else 4085 /* Initialize can_issue_more for variable_issue. */ 4086 issue_more = issue_rate; 4087 } 4088 else if (targetm.sched.reorder2 4089 && !SCHED_GROUP_P (ready_element (&ready, 0))) 4090 { 4091 if (ready.n_ready == 1) 4092 issue_more = 4093 targetm.sched.reorder2 (sched_dump, sched_verbose, 4094 ready_lastpos (&ready), 4095 &ready.n_ready, FENCE_CYCLE (fence)); 4096 else 4097 { 4098 if (pipelining_p) 4099 --ready.n_ready; 4100 4101 issue_more = 4102 targetm.sched.reorder2 (sched_dump, sched_verbose, 4103 ready.n_ready 4104 ? ready_lastpos (&ready) : NULL, 4105 &ready.n_ready, FENCE_CYCLE (fence)); 4106 4107 if (pipelining_p) 4108 ++ready.n_ready; 4109 } 4110 4111 ran_hook = true; 4112 } 4113 else 4114 issue_more = FENCE_ISSUE_MORE (fence); 4115 4116 /* Ensure that ready list and vec_av_set are in line with each other, 4117 i.e. vec_av_set[i] == ready_element (&ready, i). */ 4118 if (issue_more && ran_hook) 4119 { 4120 int i, j, n; 4121 rtx_insn **arr = ready.vec; 4122 expr_t *vec = vec_av_set.address (); 4123 4124 for (i = 0, n = ready.n_ready; i < n; i++) 4125 if (EXPR_INSN_RTX (vec[i]) != arr[i]) 4126 { 4127 for (j = i; j < n; j++) 4128 if (EXPR_INSN_RTX (vec[j]) == arr[i]) 4129 break; 4130 gcc_assert (j < n); 4131 4132 std::swap (vec[i], vec[j]); 4133 } 4134 } 4135 4136 return issue_more; 4137} 4138 4139/* Return an EXPR corresponding to INDEX element of ready list, if 4140 FOLLOW_READY_ELEMENT is true (i.e., an expr of 4141 ready_element (&ready, INDEX) will be returned), and to INDEX element of 4142 ready.vec otherwise. */ 4143static inline expr_t 4144find_expr_for_ready (int index, bool follow_ready_element) 4145{ 4146 expr_t expr; 4147 int real_index; 4148 4149 real_index = follow_ready_element ? ready.first - index : index; 4150 4151 expr = vec_av_set[real_index]; 4152 gcc_assert (ready.vec[real_index] == EXPR_INSN_RTX (expr)); 4153 4154 return expr; 4155} 4156 4157/* Calculate insns worth trying via lookahead_guard hook. Return a number 4158 of such insns found. */ 4159static int 4160invoke_dfa_lookahead_guard (void) 4161{ 4162 int i, n; 4163 bool have_hook 4164 = targetm.sched.first_cycle_multipass_dfa_lookahead_guard != NULL; 4165 4166 if (sched_verbose >= 2) 4167 sel_print ("ready after reorder: "); 4168 4169 for (i = 0, n = 0; i < ready.n_ready; i++) 4170 { 4171 expr_t expr; 4172 insn_t insn; 4173 int r; 4174 4175 /* In this loop insn is Ith element of the ready list given by 4176 ready_element, not Ith element of ready.vec. */ 4177 insn = ready_element (&ready, i); 4178 4179 if (! have_hook || i == 0) 4180 r = 0; 4181 else 4182 r = targetm.sched.first_cycle_multipass_dfa_lookahead_guard (insn, i); 4183 4184 gcc_assert (INSN_CODE (insn) >= 0); 4185 4186 /* Only insns with ready_try = 0 can get here 4187 from fill_ready_list. */ 4188 gcc_assert (ready_try [i] == 0); 4189 ready_try[i] = r; 4190 if (!r) 4191 n++; 4192 4193 expr = find_expr_for_ready (i, true); 4194 4195 if (sched_verbose >= 2) 4196 { 4197 dump_vinsn (EXPR_VINSN (expr)); 4198 sel_print (":%d; ", ready_try[i]); 4199 } 4200 } 4201 4202 if (sched_verbose >= 2) 4203 sel_print ("\n"); 4204 return n; 4205} 4206 4207/* Calculate the number of privileged insns and return it. */ 4208static int 4209calculate_privileged_insns (void) 4210{ 4211 expr_t cur_expr, min_spec_expr = NULL; 4212 int privileged_n = 0, i; 4213 4214 for (i = 0; i < ready.n_ready; i++) 4215 { 4216 if (ready_try[i]) 4217 continue; 4218 4219 if (! min_spec_expr) 4220 min_spec_expr = find_expr_for_ready (i, true); 4221 4222 cur_expr = find_expr_for_ready (i, true); 4223 4224 if (EXPR_SPEC (cur_expr) > EXPR_SPEC (min_spec_expr)) 4225 break; 4226 4227 ++privileged_n; 4228 } 4229 4230 if (i == ready.n_ready) 4231 privileged_n = 0; 4232 4233 if (sched_verbose >= 2) 4234 sel_print ("privileged_n: %d insns with SPEC %d\n", 4235 privileged_n, privileged_n ? EXPR_SPEC (min_spec_expr) : -1); 4236 return privileged_n; 4237} 4238 4239/* Call the rest of the hooks after the choice was made. Return 4240 the number of insns that still can be issued given that the current 4241 number is ISSUE_MORE. FENCE and BEST_INSN are the current fence 4242 and the insn chosen for scheduling, respectively. */ 4243static int 4244invoke_aftermath_hooks (fence_t fence, rtx_insn *best_insn, int issue_more) 4245{ 4246 gcc_assert (INSN_P (best_insn)); 4247 4248 /* First, call dfa_new_cycle, and then variable_issue, if available. */ 4249 sel_dfa_new_cycle (best_insn, fence); 4250 4251 if (targetm.sched.variable_issue) 4252 { 4253 memcpy (curr_state, FENCE_STATE (fence), dfa_state_size); 4254 issue_more = 4255 targetm.sched.variable_issue (sched_dump, sched_verbose, best_insn, 4256 issue_more); 4257 memcpy (FENCE_STATE (fence), curr_state, dfa_state_size); 4258 } 4259 else if (!DEBUG_INSN_P (best_insn) 4260 && GET_CODE (PATTERN (best_insn)) != USE 4261 && GET_CODE (PATTERN (best_insn)) != CLOBBER) 4262 issue_more--; 4263 4264 return issue_more; 4265} 4266 4267/* Estimate the cost of issuing INSN on DFA state STATE. */ 4268static int 4269estimate_insn_cost (rtx_insn *insn, state_t state) 4270{ 4271 static state_t temp = NULL; 4272 int cost; 4273 4274 if (!temp) 4275 temp = xmalloc (dfa_state_size); 4276 4277 memcpy (temp, state, dfa_state_size); 4278 cost = state_transition (temp, insn); 4279 4280 if (cost < 0) 4281 return 0; 4282 else if (cost == 0) 4283 return 1; 4284 return cost; 4285} 4286 4287/* Return the cost of issuing EXPR on the FENCE as estimated by DFA. 4288 This function properly handles ASMs, USEs etc. */ 4289static int 4290get_expr_cost (expr_t expr, fence_t fence) 4291{ 4292 rtx_insn *insn = EXPR_INSN_RTX (expr); 4293 4294 if (recog_memoized (insn) < 0) 4295 { 4296 if (!FENCE_STARTS_CYCLE_P (fence) 4297 && INSN_ASM_P (insn)) 4298 /* This is asm insn which is tryed to be issued on the 4299 cycle not first. Issue it on the next cycle. */ 4300 return 1; 4301 else 4302 /* A USE insn, or something else we don't need to 4303 understand. We can't pass these directly to 4304 state_transition because it will trigger a 4305 fatal error for unrecognizable insns. */ 4306 return 0; 4307 } 4308 else 4309 return estimate_insn_cost (insn, FENCE_STATE (fence)); 4310} 4311 4312/* Find the best insn for scheduling, either via max_issue or just take 4313 the most prioritized available. */ 4314static int 4315choose_best_insn (fence_t fence, int privileged_n, int *index) 4316{ 4317 int can_issue = 0; 4318 4319 if (dfa_lookahead > 0) 4320 { 4321 cycle_issued_insns = FENCE_ISSUED_INSNS (fence); 4322 /* TODO: pass equivalent of first_cycle_insn_p to max_issue (). */ 4323 can_issue = max_issue (&ready, privileged_n, 4324 FENCE_STATE (fence), true, index); 4325 if (sched_verbose >= 2) 4326 sel_print ("max_issue: we can issue %d insns, already did %d insns\n", 4327 can_issue, FENCE_ISSUED_INSNS (fence)); 4328 } 4329 else 4330 { 4331 /* We can't use max_issue; just return the first available element. */ 4332 int i; 4333 4334 for (i = 0; i < ready.n_ready; i++) 4335 { 4336 expr_t expr = find_expr_for_ready (i, true); 4337 4338 if (get_expr_cost (expr, fence) < 1) 4339 { 4340 can_issue = can_issue_more; 4341 *index = i; 4342 4343 if (sched_verbose >= 2) 4344 sel_print ("using %dth insn from the ready list\n", i + 1); 4345 4346 break; 4347 } 4348 } 4349 4350 if (i == ready.n_ready) 4351 { 4352 can_issue = 0; 4353 *index = -1; 4354 } 4355 } 4356 4357 return can_issue; 4358} 4359 4360/* Choose the best expr from *AV_VLIW_PTR and a suitable register for it. 4361 BNDS and FENCE are current boundaries and scheduling fence respectively. 4362 Return the expr found and NULL if nothing can be issued atm. 4363 Write to PNEED_STALL the number of cycles to stall if no expr was found. */ 4364static expr_t 4365find_best_expr (av_set_t *av_vliw_ptr, blist_t bnds, fence_t fence, 4366 int *pneed_stall) 4367{ 4368 expr_t best; 4369 4370 /* Choose the best insn for scheduling via: 4371 1) sorting the ready list based on priority; 4372 2) calling the reorder hook; 4373 3) calling max_issue. */ 4374 best = fill_ready_list (av_vliw_ptr, bnds, fence, pneed_stall); 4375 if (best == NULL && ready.n_ready > 0) 4376 { 4377 int privileged_n, index; 4378 4379 can_issue_more = invoke_reorder_hooks (fence); 4380 if (can_issue_more > 0) 4381 { 4382 /* Try choosing the best insn until we find one that is could be 4383 scheduled due to liveness restrictions on its destination register. 4384 In the future, we'd like to choose once and then just probe insns 4385 in the order of their priority. */ 4386 invoke_dfa_lookahead_guard (); 4387 privileged_n = calculate_privileged_insns (); 4388 can_issue_more = choose_best_insn (fence, privileged_n, &index); 4389 if (can_issue_more) 4390 best = find_expr_for_ready (index, true); 4391 } 4392 /* We had some available insns, so if we can't issue them, 4393 we have a stall. */ 4394 if (can_issue_more == 0) 4395 { 4396 best = NULL; 4397 *pneed_stall = 1; 4398 } 4399 } 4400 4401 if (best != NULL) 4402 { 4403 can_issue_more = invoke_aftermath_hooks (fence, EXPR_INSN_RTX (best), 4404 can_issue_more); 4405 if (targetm.sched.variable_issue 4406 && can_issue_more == 0) 4407 *pneed_stall = 1; 4408 } 4409 4410 if (sched_verbose >= 2) 4411 { 4412 if (best != NULL) 4413 { 4414 sel_print ("Best expression (vliw form): "); 4415 dump_expr (best); 4416 sel_print ("; cycle %d\n", FENCE_CYCLE (fence)); 4417 } 4418 else 4419 sel_print ("No best expr found!\n"); 4420 } 4421 4422 return best; 4423} 4424 4425 4426/* Functions that implement the core of the scheduler. */ 4427 4428 4429/* Emit an instruction from EXPR with SEQNO and VINSN after 4430 PLACE_TO_INSERT. */ 4431static insn_t 4432emit_insn_from_expr_after (expr_t expr, vinsn_t vinsn, int seqno, 4433 insn_t place_to_insert) 4434{ 4435 /* This assert fails when we have identical instructions 4436 one of which dominates the other. In this case move_op () 4437 finds the first instruction and doesn't search for second one. 4438 The solution would be to compute av_set after the first found 4439 insn and, if insn present in that set, continue searching. 4440 For now we workaround this issue in move_op. */ 4441 gcc_assert (!INSN_IN_STREAM_P (EXPR_INSN_RTX (expr))); 4442 4443 if (EXPR_WAS_RENAMED (expr)) 4444 { 4445 unsigned regno = expr_dest_regno (expr); 4446 4447 if (HARD_REGISTER_NUM_P (regno)) 4448 { 4449 df_set_regs_ever_live (regno, true); 4450 reg_rename_tick[regno] = ++reg_rename_this_tick; 4451 } 4452 } 4453 4454 return sel_gen_insn_from_expr_after (expr, vinsn, seqno, 4455 place_to_insert); 4456} 4457 4458/* Return TRUE if BB can hold bookkeeping code. */ 4459static bool 4460block_valid_for_bookkeeping_p (basic_block bb) 4461{ 4462 insn_t bb_end = BB_END (bb); 4463 4464 if (!in_current_region_p (bb) || EDGE_COUNT (bb->succs) > 1) 4465 return false; 4466 4467 if (INSN_P (bb_end)) 4468 { 4469 if (INSN_SCHED_TIMES (bb_end) > 0) 4470 return false; 4471 } 4472 else 4473 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (bb_end)); 4474 4475 return true; 4476} 4477 4478/* Attempt to find a block that can hold bookkeeping code for path(s) incoming 4479 into E2->dest, except from E1->src (there may be a sequence of empty basic 4480 blocks between E1->src and E2->dest). Return found block, or NULL if new 4481 one must be created. If LAX holds, don't assume there is a simple path 4482 from E1->src to E2->dest. */ 4483static basic_block 4484find_block_for_bookkeeping (edge e1, edge e2, bool lax) 4485{ 4486 basic_block candidate_block = NULL; 4487 edge e; 4488 4489 /* Loop over edges from E1 to E2, inclusive. */ 4490 for (e = e1; !lax || e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun); e = 4491 EDGE_SUCC (e->dest, 0)) 4492 { 4493 if (EDGE_COUNT (e->dest->preds) == 2) 4494 { 4495 if (candidate_block == NULL) 4496 candidate_block = (EDGE_PRED (e->dest, 0) == e 4497 ? EDGE_PRED (e->dest, 1)->src 4498 : EDGE_PRED (e->dest, 0)->src); 4499 else 4500 /* Found additional edge leading to path from e1 to e2 4501 from aside. */ 4502 return NULL; 4503 } 4504 else if (EDGE_COUNT (e->dest->preds) > 2) 4505 /* Several edges leading to path from e1 to e2 from aside. */ 4506 return NULL; 4507 4508 if (e == e2) 4509 return ((!lax || candidate_block) 4510 && block_valid_for_bookkeeping_p (candidate_block) 4511 ? candidate_block 4512 : NULL); 4513 4514 if (lax && EDGE_COUNT (e->dest->succs) != 1) 4515 return NULL; 4516 } 4517 4518 if (lax) 4519 return NULL; 4520 4521 gcc_unreachable (); 4522} 4523 4524/* Create new basic block for bookkeeping code for path(s) incoming into 4525 E2->dest, except from E1->src. Return created block. */ 4526static basic_block 4527create_block_for_bookkeeping (edge e1, edge e2) 4528{ 4529 basic_block new_bb, bb = e2->dest; 4530 4531 /* Check that we don't spoil the loop structure. */ 4532 if (current_loop_nest) 4533 { 4534 basic_block latch = current_loop_nest->latch; 4535 4536 /* We do not split header. */ 4537 gcc_assert (e2->dest != current_loop_nest->header); 4538 4539 /* We do not redirect the only edge to the latch block. */ 4540 gcc_assert (e1->dest != latch 4541 || !single_pred_p (latch) 4542 || e1 != single_pred_edge (latch)); 4543 } 4544 4545 /* Split BB to insert BOOK_INSN there. */ 4546 new_bb = sched_split_block (bb, NULL); 4547 4548 /* Move note_list from the upper bb. */ 4549 gcc_assert (BB_NOTE_LIST (new_bb) == NULL_RTX); 4550 BB_NOTE_LIST (new_bb) = BB_NOTE_LIST (bb); 4551 BB_NOTE_LIST (bb) = NULL; 4552 4553 gcc_assert (e2->dest == bb); 4554 4555 /* Skip block for bookkeeping copy when leaving E1->src. */ 4556 if (e1->flags & EDGE_FALLTHRU) 4557 sel_redirect_edge_and_branch_force (e1, new_bb); 4558 else 4559 sel_redirect_edge_and_branch (e1, new_bb); 4560 4561 gcc_assert (e1->dest == new_bb); 4562 gcc_assert (sel_bb_empty_p (bb)); 4563 4564 /* To keep basic block numbers in sync between debug and non-debug 4565 compilations, we have to rotate blocks here. Consider that we 4566 started from (a,b)->d, (c,d)->e, and d contained only debug 4567 insns. It would have been removed before if the debug insns 4568 weren't there, so we'd have split e rather than d. So what we do 4569 now is to swap the block numbers of new_bb and 4570 single_succ(new_bb) == e, so that the insns that were in e before 4571 get the new block number. */ 4572 4573 if (MAY_HAVE_DEBUG_INSNS) 4574 { 4575 basic_block succ; 4576 insn_t insn = sel_bb_head (new_bb); 4577 insn_t last; 4578 4579 if (DEBUG_INSN_P (insn) 4580 && single_succ_p (new_bb) 4581 && (succ = single_succ (new_bb)) 4582 && succ != EXIT_BLOCK_PTR_FOR_FN (cfun) 4583 && DEBUG_INSN_P ((last = sel_bb_end (new_bb)))) 4584 { 4585 while (insn != last && (DEBUG_INSN_P (insn) || NOTE_P (insn))) 4586 insn = NEXT_INSN (insn); 4587 4588 if (insn == last) 4589 { 4590 sel_global_bb_info_def gbi; 4591 sel_region_bb_info_def rbi; 4592 4593 if (sched_verbose >= 2) 4594 sel_print ("Swapping block ids %i and %i\n", 4595 new_bb->index, succ->index); 4596 4597 std::swap (new_bb->index, succ->index); 4598 4599 SET_BASIC_BLOCK_FOR_FN (cfun, new_bb->index, new_bb); 4600 SET_BASIC_BLOCK_FOR_FN (cfun, succ->index, succ); 4601 4602 memcpy (&gbi, SEL_GLOBAL_BB_INFO (new_bb), sizeof (gbi)); 4603 memcpy (SEL_GLOBAL_BB_INFO (new_bb), SEL_GLOBAL_BB_INFO (succ), 4604 sizeof (gbi)); 4605 memcpy (SEL_GLOBAL_BB_INFO (succ), &gbi, sizeof (gbi)); 4606 4607 memcpy (&rbi, SEL_REGION_BB_INFO (new_bb), sizeof (rbi)); 4608 memcpy (SEL_REGION_BB_INFO (new_bb), SEL_REGION_BB_INFO (succ), 4609 sizeof (rbi)); 4610 memcpy (SEL_REGION_BB_INFO (succ), &rbi, sizeof (rbi)); 4611 4612 std::swap (BLOCK_TO_BB (new_bb->index), 4613 BLOCK_TO_BB (succ->index)); 4614 4615 std::swap (CONTAINING_RGN (new_bb->index), 4616 CONTAINING_RGN (succ->index)); 4617 4618 for (int i = 0; i < current_nr_blocks; i++) 4619 if (BB_TO_BLOCK (i) == succ->index) 4620 BB_TO_BLOCK (i) = new_bb->index; 4621 else if (BB_TO_BLOCK (i) == new_bb->index) 4622 BB_TO_BLOCK (i) = succ->index; 4623 4624 FOR_BB_INSNS (new_bb, insn) 4625 if (INSN_P (insn)) 4626 EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = new_bb->index; 4627 4628 FOR_BB_INSNS (succ, insn) 4629 if (INSN_P (insn)) 4630 EXPR_ORIG_BB_INDEX (INSN_EXPR (insn)) = succ->index; 4631 4632 if (bitmap_clear_bit (code_motion_visited_blocks, new_bb->index)) 4633 bitmap_set_bit (code_motion_visited_blocks, succ->index); 4634 4635 gcc_assert (LABEL_P (BB_HEAD (new_bb)) 4636 && LABEL_P (BB_HEAD (succ))); 4637 4638 if (sched_verbose >= 4) 4639 sel_print ("Swapping code labels %i and %i\n", 4640 CODE_LABEL_NUMBER (BB_HEAD (new_bb)), 4641 CODE_LABEL_NUMBER (BB_HEAD (succ))); 4642 4643 std::swap (CODE_LABEL_NUMBER (BB_HEAD (new_bb)), 4644 CODE_LABEL_NUMBER (BB_HEAD (succ))); 4645 } 4646 } 4647 } 4648 4649 return bb; 4650} 4651 4652/* Return insn after which we must insert bookkeeping code for path(s) incoming 4653 into E2->dest, except from E1->src. If the returned insn immediately 4654 precedes a fence, assign that fence to *FENCE_TO_REWIND. */ 4655static insn_t 4656find_place_for_bookkeeping (edge e1, edge e2, fence_t *fence_to_rewind) 4657{ 4658 insn_t place_to_insert; 4659 /* Find a basic block that can hold bookkeeping. If it can be found, do not 4660 create new basic block, but insert bookkeeping there. */ 4661 basic_block book_block = find_block_for_bookkeeping (e1, e2, FALSE); 4662 4663 if (book_block) 4664 { 4665 place_to_insert = BB_END (book_block); 4666 4667 /* Don't use a block containing only debug insns for 4668 bookkeeping, this causes scheduling differences between debug 4669 and non-debug compilations, for the block would have been 4670 removed already. */ 4671 if (DEBUG_INSN_P (place_to_insert)) 4672 { 4673 rtx_insn *insn = sel_bb_head (book_block); 4674 4675 while (insn != place_to_insert && 4676 (DEBUG_INSN_P (insn) || NOTE_P (insn))) 4677 insn = NEXT_INSN (insn); 4678 4679 if (insn == place_to_insert) 4680 book_block = NULL; 4681 } 4682 } 4683 4684 if (!book_block) 4685 { 4686 book_block = create_block_for_bookkeeping (e1, e2); 4687 place_to_insert = BB_END (book_block); 4688 if (sched_verbose >= 9) 4689 sel_print ("New block is %i, split from bookkeeping block %i\n", 4690 EDGE_SUCC (book_block, 0)->dest->index, book_block->index); 4691 } 4692 else 4693 { 4694 if (sched_verbose >= 9) 4695 sel_print ("Pre-existing bookkeeping block is %i\n", book_block->index); 4696 } 4697 4698 *fence_to_rewind = NULL; 4699 /* If basic block ends with a jump, insert bookkeeping code right before it. 4700 Notice if we are crossing a fence when taking PREV_INSN. */ 4701 if (INSN_P (place_to_insert) && control_flow_insn_p (place_to_insert)) 4702 { 4703 *fence_to_rewind = flist_lookup (fences, place_to_insert); 4704 place_to_insert = PREV_INSN (place_to_insert); 4705 } 4706 4707 return place_to_insert; 4708} 4709 4710/* Find a proper seqno for bookkeeing insn inserted at PLACE_TO_INSERT 4711 for JOIN_POINT. */ 4712static int 4713find_seqno_for_bookkeeping (insn_t place_to_insert, insn_t join_point) 4714{ 4715 int seqno; 4716 4717 /* Check if we are about to insert bookkeeping copy before a jump, and use 4718 jump's seqno for the copy; otherwise, use JOIN_POINT's seqno. */ 4719 rtx_insn *next = NEXT_INSN (place_to_insert); 4720 if (INSN_P (next) 4721 && JUMP_P (next) 4722 && BLOCK_FOR_INSN (next) == BLOCK_FOR_INSN (place_to_insert)) 4723 { 4724 gcc_assert (INSN_SCHED_TIMES (next) == 0); 4725 seqno = INSN_SEQNO (next); 4726 } 4727 else if (INSN_SEQNO (join_point) > 0) 4728 seqno = INSN_SEQNO (join_point); 4729 else 4730 { 4731 seqno = get_seqno_by_preds (place_to_insert); 4732 4733 /* Sometimes the fences can move in such a way that there will be 4734 no instructions with positive seqno around this bookkeeping. 4735 This means that there will be no way to get to it by a regular 4736 fence movement. Never mind because we pick up such pieces for 4737 rescheduling anyways, so any positive value will do for now. */ 4738 if (seqno < 0) 4739 { 4740 gcc_assert (pipelining_p); 4741 seqno = 1; 4742 } 4743 } 4744 4745 gcc_assert (seqno > 0); 4746 return seqno; 4747} 4748 4749/* Insert bookkeeping copy of C_EXPS's insn after PLACE_TO_INSERT, assigning 4750 NEW_SEQNO to it. Return created insn. */ 4751static insn_t 4752emit_bookkeeping_insn (insn_t place_to_insert, expr_t c_expr, int new_seqno) 4753{ 4754 rtx_insn *new_insn_rtx = create_copy_of_insn_rtx (EXPR_INSN_RTX (c_expr)); 4755 4756 vinsn_t new_vinsn 4757 = create_vinsn_from_insn_rtx (new_insn_rtx, 4758 VINSN_UNIQUE_P (EXPR_VINSN (c_expr))); 4759 4760 insn_t new_insn = emit_insn_from_expr_after (c_expr, new_vinsn, new_seqno, 4761 place_to_insert); 4762 4763 INSN_SCHED_TIMES (new_insn) = 0; 4764 bitmap_set_bit (current_copies, INSN_UID (new_insn)); 4765 4766 return new_insn; 4767} 4768 4769/* Generate a bookkeeping copy of C_EXPR's insn for path(s) incoming into to 4770 E2->dest, except from E1->src (there may be a sequence of empty blocks 4771 between E1->src and E2->dest). Return block containing the copy. 4772 All scheduler data is initialized for the newly created insn. */ 4773static basic_block 4774generate_bookkeeping_insn (expr_t c_expr, edge e1, edge e2) 4775{ 4776 insn_t join_point, place_to_insert, new_insn; 4777 int new_seqno; 4778 bool need_to_exchange_data_sets; 4779 fence_t fence_to_rewind; 4780 4781 if (sched_verbose >= 4) 4782 sel_print ("Generating bookkeeping insn (%d->%d)\n", e1->src->index, 4783 e2->dest->index); 4784 4785 join_point = sel_bb_head (e2->dest); 4786 place_to_insert = find_place_for_bookkeeping (e1, e2, &fence_to_rewind); 4787 new_seqno = find_seqno_for_bookkeeping (place_to_insert, join_point); 4788 need_to_exchange_data_sets 4789 = sel_bb_empty_p (BLOCK_FOR_INSN (place_to_insert)); 4790 4791 new_insn = emit_bookkeeping_insn (place_to_insert, c_expr, new_seqno); 4792 4793 if (fence_to_rewind) 4794 FENCE_INSN (fence_to_rewind) = new_insn; 4795 4796 /* When inserting bookkeeping insn in new block, av sets should be 4797 following: old basic block (that now holds bookkeeping) data sets are 4798 the same as was before generation of bookkeeping, and new basic block 4799 (that now hold all other insns of old basic block) data sets are 4800 invalid. So exchange data sets for these basic blocks as sel_split_block 4801 mistakenly exchanges them in this case. Cannot do it earlier because 4802 when single instruction is added to new basic block it should hold NULL 4803 lv_set. */ 4804 if (need_to_exchange_data_sets) 4805 exchange_data_sets (BLOCK_FOR_INSN (new_insn), 4806 BLOCK_FOR_INSN (join_point)); 4807 4808 stat_bookkeeping_copies++; 4809 return BLOCK_FOR_INSN (new_insn); 4810} 4811 4812/* Remove from AV_PTR all insns that may need bookkeeping when scheduling 4813 on FENCE, but we are unable to copy them. */ 4814static void 4815remove_insns_that_need_bookkeeping (fence_t fence, av_set_t *av_ptr) 4816{ 4817 expr_t expr; 4818 av_set_iterator i; 4819 4820 /* An expression does not need bookkeeping if it is available on all paths 4821 from current block to original block and current block dominates 4822 original block. We check availability on all paths by examining 4823 EXPR_SPEC; this is not equivalent, because it may be positive even 4824 if expr is available on all paths (but if expr is not available on 4825 any path, EXPR_SPEC will be positive). */ 4826 4827 FOR_EACH_EXPR_1 (expr, i, av_ptr) 4828 { 4829 if (!control_flow_insn_p (EXPR_INSN_RTX (expr)) 4830 && (!bookkeeping_p || VINSN_UNIQUE_P (EXPR_VINSN (expr))) 4831 && (EXPR_SPEC (expr) 4832 || !EXPR_ORIG_BB_INDEX (expr) 4833 || !dominated_by_p (CDI_DOMINATORS, 4834 BASIC_BLOCK_FOR_FN (cfun, 4835 EXPR_ORIG_BB_INDEX (expr)), 4836 BLOCK_FOR_INSN (FENCE_INSN (fence))))) 4837 { 4838 if (sched_verbose >= 4) 4839 sel_print ("Expr %d removed because it would need bookkeeping, which " 4840 "cannot be created\n", INSN_UID (EXPR_INSN_RTX (expr))); 4841 av_set_iter_remove (&i); 4842 } 4843 } 4844} 4845 4846/* Moving conditional jump through some instructions. 4847 4848 Consider example: 4849 4850 ... <- current scheduling point 4851 NOTE BASIC BLOCK: <- bb header 4852 (p8) add r14=r14+0x9;; 4853 (p8) mov [r14]=r23 4854 (!p8) jump L1;; 4855 NOTE BASIC BLOCK: 4856 ... 4857 4858 We can schedule jump one cycle earlier, than mov, because they cannot be 4859 executed together as their predicates are mutually exclusive. 4860 4861 This is done in this way: first, new fallthrough basic block is created 4862 after jump (it is always can be done, because there already should be a 4863 fallthrough block, where control flow goes in case of predicate being true - 4864 in our example; otherwise there should be a dependence between those 4865 instructions and jump and we cannot schedule jump right now); 4866 next, all instructions between jump and current scheduling point are moved 4867 to this new block. And the result is this: 4868 4869 NOTE BASIC BLOCK: 4870 (!p8) jump L1 <- current scheduling point 4871 NOTE BASIC BLOCK: <- bb header 4872 (p8) add r14=r14+0x9;; 4873 (p8) mov [r14]=r23 4874 NOTE BASIC BLOCK: 4875 ... 4876*/ 4877static void 4878move_cond_jump (rtx_insn *insn, bnd_t bnd) 4879{ 4880 edge ft_edge; 4881 basic_block block_from, block_next, block_new, block_bnd, bb; 4882 rtx_insn *next, *prev, *link, *head; 4883 4884 block_from = BLOCK_FOR_INSN (insn); 4885 block_bnd = BLOCK_FOR_INSN (BND_TO (bnd)); 4886 prev = BND_TO (bnd); 4887 4888 /* Moving of jump should not cross any other jumps or beginnings of new 4889 basic blocks. The only exception is when we move a jump through 4890 mutually exclusive insns along fallthru edges. */ 4891 if (flag_checking && block_from != block_bnd) 4892 { 4893 bb = block_from; 4894 for (link = PREV_INSN (insn); link != PREV_INSN (prev); 4895 link = PREV_INSN (link)) 4896 { 4897 if (INSN_P (link)) 4898 gcc_assert (sched_insns_conditions_mutex_p (insn, link)); 4899 if (BLOCK_FOR_INSN (link) && BLOCK_FOR_INSN (link) != bb) 4900 { 4901 gcc_assert (single_pred (bb) == BLOCK_FOR_INSN (link)); 4902 bb = BLOCK_FOR_INSN (link); 4903 } 4904 } 4905 } 4906 4907 /* Jump is moved to the boundary. */ 4908 next = PREV_INSN (insn); 4909 BND_TO (bnd) = insn; 4910 4911 ft_edge = find_fallthru_edge_from (block_from); 4912 block_next = ft_edge->dest; 4913 /* There must be a fallthrough block (or where should go 4914 control flow in case of false jump predicate otherwise?). */ 4915 gcc_assert (block_next); 4916 4917 /* Create new empty basic block after source block. */ 4918 block_new = sel_split_edge (ft_edge); 4919 gcc_assert (block_new->next_bb == block_next 4920 && block_from->next_bb == block_new); 4921 4922 /* Move all instructions except INSN to BLOCK_NEW. */ 4923 bb = block_bnd; 4924 head = BB_HEAD (block_new); 4925 while (bb != block_from->next_bb) 4926 { 4927 rtx_insn *from, *to; 4928 from = bb == block_bnd ? prev : sel_bb_head (bb); 4929 to = bb == block_from ? next : sel_bb_end (bb); 4930 4931 /* The jump being moved can be the first insn in the block. 4932 In this case we don't have to move anything in this block. */ 4933 if (NEXT_INSN (to) != from) 4934 { 4935 reorder_insns (from, to, head); 4936 4937 for (link = to; link != head; link = PREV_INSN (link)) 4938 EXPR_ORIG_BB_INDEX (INSN_EXPR (link)) = block_new->index; 4939 head = to; 4940 } 4941 4942 /* Cleanup possibly empty blocks left. */ 4943 block_next = bb->next_bb; 4944 if (bb != block_from) 4945 tidy_control_flow (bb, false); 4946 bb = block_next; 4947 } 4948 4949 /* Assert there is no jump to BLOCK_NEW, only fallthrough edge. */ 4950 gcc_assert (NOTE_INSN_BASIC_BLOCK_P (BB_HEAD (block_new))); 4951 4952 gcc_assert (!sel_bb_empty_p (block_from) 4953 && !sel_bb_empty_p (block_new)); 4954 4955 /* Update data sets for BLOCK_NEW to represent that INSN and 4956 instructions from the other branch of INSN is no longer 4957 available at BLOCK_NEW. */ 4958 BB_AV_LEVEL (block_new) = global_level; 4959 gcc_assert (BB_LV_SET (block_new) == NULL); 4960 BB_LV_SET (block_new) = get_clear_regset_from_pool (); 4961 update_data_sets (sel_bb_head (block_new)); 4962 4963 /* INSN is a new basic block header - so prepare its data 4964 structures and update availability and liveness sets. */ 4965 update_data_sets (insn); 4966 4967 if (sched_verbose >= 4) 4968 sel_print ("Moving jump %d\n", INSN_UID (insn)); 4969} 4970 4971/* Remove nops generated during move_op for preventing removal of empty 4972 basic blocks. */ 4973static void 4974remove_temp_moveop_nops (bool full_tidying) 4975{ 4976 int i; 4977 insn_t insn; 4978 4979 FOR_EACH_VEC_ELT (vec_temp_moveop_nops, i, insn) 4980 { 4981 gcc_assert (INSN_NOP_P (insn)); 4982 return_nop_to_pool (insn, full_tidying); 4983 } 4984 4985 /* Empty the vector. */ 4986 if (vec_temp_moveop_nops.length () > 0) 4987 vec_temp_moveop_nops.block_remove (0, vec_temp_moveop_nops.length ()); 4988} 4989 4990/* Records the maximal UID before moving up an instruction. Used for 4991 distinguishing between bookkeeping copies and original insns. */ 4992static int max_uid_before_move_op = 0; 4993 4994/* When true, we're always scheduling next insn on the already scheduled code 4995 to get the right insn data for the following bundling or other passes. */ 4996static int force_next_insn = 0; 4997 4998/* Remove from AV_VLIW_P all instructions but next when debug counter 4999 tells us so. Next instruction is fetched from BNDS. */ 5000static void 5001remove_insns_for_debug (blist_t bnds, av_set_t *av_vliw_p) 5002{ 5003 if (! dbg_cnt (sel_sched_insn_cnt) || force_next_insn) 5004 /* Leave only the next insn in av_vliw. */ 5005 { 5006 av_set_iterator av_it; 5007 expr_t expr; 5008 bnd_t bnd = BLIST_BND (bnds); 5009 insn_t next = BND_TO (bnd); 5010 5011 gcc_assert (BLIST_NEXT (bnds) == NULL); 5012 5013 FOR_EACH_EXPR_1 (expr, av_it, av_vliw_p) 5014 if (EXPR_INSN_RTX (expr) != next) 5015 av_set_iter_remove (&av_it); 5016 } 5017} 5018 5019/* Compute available instructions on BNDS. FENCE is the current fence. Write 5020 the computed set to *AV_VLIW_P. */ 5021static void 5022compute_av_set_on_boundaries (fence_t fence, blist_t bnds, av_set_t *av_vliw_p) 5023{ 5024 if (sched_verbose >= 2) 5025 { 5026 sel_print ("Boundaries: "); 5027 dump_blist (bnds); 5028 sel_print ("\n"); 5029 } 5030 5031 for (; bnds; bnds = BLIST_NEXT (bnds)) 5032 { 5033 bnd_t bnd = BLIST_BND (bnds); 5034 av_set_t av1_copy; 5035 insn_t bnd_to = BND_TO (bnd); 5036 5037 /* Rewind BND->TO to the basic block header in case some bookkeeping 5038 instructions were inserted before BND->TO and it needs to be 5039 adjusted. */ 5040 if (sel_bb_head_p (bnd_to)) 5041 gcc_assert (INSN_SCHED_TIMES (bnd_to) == 0); 5042 else 5043 while (INSN_SCHED_TIMES (PREV_INSN (bnd_to)) == 0) 5044 { 5045 bnd_to = PREV_INSN (bnd_to); 5046 if (sel_bb_head_p (bnd_to)) 5047 break; 5048 } 5049 5050 if (BND_TO (bnd) != bnd_to) 5051 { 5052 gcc_assert (FENCE_INSN (fence) == BND_TO (bnd)); 5053 FENCE_INSN (fence) = bnd_to; 5054 BND_TO (bnd) = bnd_to; 5055 } 5056 5057 av_set_clear (&BND_AV (bnd)); 5058 BND_AV (bnd) = compute_av_set (BND_TO (bnd), NULL, 0, true); 5059 5060 av_set_clear (&BND_AV1 (bnd)); 5061 BND_AV1 (bnd) = av_set_copy (BND_AV (bnd)); 5062 5063 moveup_set_inside_insn_group (&BND_AV1 (bnd), NULL); 5064 5065 av1_copy = av_set_copy (BND_AV1 (bnd)); 5066 av_set_union_and_clear (av_vliw_p, &av1_copy, NULL); 5067 } 5068 5069 if (sched_verbose >= 2) 5070 { 5071 sel_print ("Available exprs (vliw form): "); 5072 dump_av_set (*av_vliw_p); 5073 sel_print ("\n"); 5074 } 5075} 5076 5077/* Calculate the sequential av set on BND corresponding to the EXPR_VLIW 5078 expression. When FOR_MOVEOP is true, also replace the register of 5079 expressions found with the register from EXPR_VLIW. */ 5080static av_set_t 5081find_sequential_best_exprs (bnd_t bnd, expr_t expr_vliw, bool for_moveop) 5082{ 5083 av_set_t expr_seq = NULL; 5084 expr_t expr; 5085 av_set_iterator i; 5086 5087 FOR_EACH_EXPR (expr, i, BND_AV (bnd)) 5088 { 5089 if (equal_after_moveup_path_p (expr, NULL, expr_vliw)) 5090 { 5091 if (for_moveop) 5092 { 5093 /* The sequential expression has the right form to pass 5094 to move_op except when renaming happened. Put the 5095 correct register in EXPR then. */ 5096 if (EXPR_SEPARABLE_P (expr) && REG_P (EXPR_LHS (expr))) 5097 { 5098 if (expr_dest_regno (expr) != expr_dest_regno (expr_vliw)) 5099 { 5100 replace_dest_with_reg_in_expr (expr, EXPR_LHS (expr_vliw)); 5101 stat_renamed_scheduled++; 5102 } 5103 /* Also put the correct TARGET_AVAILABLE bit on the expr. 5104 This is needed when renaming came up with original 5105 register. */ 5106 else if (EXPR_TARGET_AVAILABLE (expr) 5107 != EXPR_TARGET_AVAILABLE (expr_vliw)) 5108 { 5109 gcc_assert (EXPR_TARGET_AVAILABLE (expr_vliw) == 1); 5110 EXPR_TARGET_AVAILABLE (expr) = 1; 5111 } 5112 } 5113 if (EXPR_WAS_SUBSTITUTED (expr)) 5114 stat_substitutions_total++; 5115 } 5116 5117 av_set_add (&expr_seq, expr); 5118 5119 /* With substitution inside insn group, it is possible 5120 that more than one expression in expr_seq will correspond 5121 to expr_vliw. In this case, choose one as the attempt to 5122 move both leads to miscompiles. */ 5123 break; 5124 } 5125 } 5126 5127 if (for_moveop && sched_verbose >= 2) 5128 { 5129 sel_print ("Best expression(s) (sequential form): "); 5130 dump_av_set (expr_seq); 5131 sel_print ("\n"); 5132 } 5133 5134 return expr_seq; 5135} 5136 5137 5138/* Move nop to previous block. */ 5139static void ATTRIBUTE_UNUSED 5140move_nop_to_previous_block (insn_t nop, basic_block prev_bb) 5141{ 5142 insn_t prev_insn, next_insn; 5143 5144 gcc_assert (sel_bb_head_p (nop) 5145 && prev_bb == BLOCK_FOR_INSN (nop)->prev_bb); 5146 rtx_note *note = bb_note (BLOCK_FOR_INSN (nop)); 5147 prev_insn = sel_bb_end (prev_bb); 5148 next_insn = NEXT_INSN (nop); 5149 gcc_assert (prev_insn != NULL_RTX 5150 && PREV_INSN (note) == prev_insn); 5151 5152 SET_NEXT_INSN (prev_insn) = nop; 5153 SET_PREV_INSN (nop) = prev_insn; 5154 5155 SET_PREV_INSN (note) = nop; 5156 SET_NEXT_INSN (note) = next_insn; 5157 5158 SET_NEXT_INSN (nop) = note; 5159 SET_PREV_INSN (next_insn) = note; 5160 5161 BB_END (prev_bb) = nop; 5162 BLOCK_FOR_INSN (nop) = prev_bb; 5163} 5164 5165/* Prepare a place to insert the chosen expression on BND. */ 5166static insn_t 5167prepare_place_to_insert (bnd_t bnd) 5168{ 5169 insn_t place_to_insert; 5170 5171 /* Init place_to_insert before calling move_op, as the later 5172 can possibly remove BND_TO (bnd). */ 5173 if (/* If this is not the first insn scheduled. */ 5174 BND_PTR (bnd)) 5175 { 5176 /* Add it after last scheduled. */ 5177 place_to_insert = ILIST_INSN (BND_PTR (bnd)); 5178 if (DEBUG_INSN_P (place_to_insert)) 5179 { 5180 ilist_t l = BND_PTR (bnd); 5181 while ((l = ILIST_NEXT (l)) && 5182 DEBUG_INSN_P (ILIST_INSN (l))) 5183 ; 5184 if (!l) 5185 place_to_insert = NULL; 5186 } 5187 } 5188 else 5189 place_to_insert = NULL; 5190 5191 if (!place_to_insert) 5192 { 5193 /* Add it before BND_TO. The difference is in the 5194 basic block, where INSN will be added. */ 5195 place_to_insert = get_nop_from_pool (BND_TO (bnd)); 5196 gcc_assert (BLOCK_FOR_INSN (place_to_insert) 5197 == BLOCK_FOR_INSN (BND_TO (bnd))); 5198 } 5199 5200 return place_to_insert; 5201} 5202 5203/* Find original instructions for EXPR_SEQ and move it to BND boundary. 5204 Return the expression to emit in C_EXPR. */ 5205static bool 5206move_exprs_to_boundary (bnd_t bnd, expr_t expr_vliw, 5207 av_set_t expr_seq, expr_t c_expr) 5208{ 5209 bool b, should_move; 5210 unsigned book_uid; 5211 bitmap_iterator bi; 5212 int n_bookkeeping_copies_before_moveop; 5213 5214 /* Make a move. This call will remove the original operation, 5215 insert all necessary bookkeeping instructions and update the 5216 data sets. After that all we have to do is add the operation 5217 at before BND_TO (BND). */ 5218 n_bookkeeping_copies_before_moveop = stat_bookkeeping_copies; 5219 max_uid_before_move_op = get_max_uid (); 5220 bitmap_clear (current_copies); 5221 bitmap_clear (current_originators); 5222 5223 b = move_op (BND_TO (bnd), expr_seq, expr_vliw, 5224 get_dest_from_orig_ops (expr_seq), c_expr, &should_move); 5225 5226 /* We should be able to find the expression we've chosen for 5227 scheduling. */ 5228 gcc_assert (b); 5229 5230 if (stat_bookkeeping_copies > n_bookkeeping_copies_before_moveop) 5231 stat_insns_needed_bookkeeping++; 5232 5233 EXECUTE_IF_SET_IN_BITMAP (current_copies, 0, book_uid, bi) 5234 { 5235 unsigned uid; 5236 bitmap_iterator bi; 5237 5238 /* We allocate these bitmaps lazily. */ 5239 if (! INSN_ORIGINATORS_BY_UID (book_uid)) 5240 INSN_ORIGINATORS_BY_UID (book_uid) = BITMAP_ALLOC (NULL); 5241 5242 bitmap_copy (INSN_ORIGINATORS_BY_UID (book_uid), 5243 current_originators); 5244 5245 /* Transitively add all originators' originators. */ 5246 EXECUTE_IF_SET_IN_BITMAP (current_originators, 0, uid, bi) 5247 if (INSN_ORIGINATORS_BY_UID (uid)) 5248 bitmap_ior_into (INSN_ORIGINATORS_BY_UID (book_uid), 5249 INSN_ORIGINATORS_BY_UID (uid)); 5250 } 5251 5252 return should_move; 5253} 5254 5255 5256/* Debug a DFA state as an array of bytes. */ 5257static void 5258debug_state (state_t state) 5259{ 5260 unsigned char *p; 5261 unsigned int i, size = dfa_state_size; 5262 5263 sel_print ("state (%u):", size); 5264 for (i = 0, p = (unsigned char *) state; i < size; i++) 5265 sel_print (" %d", p[i]); 5266 sel_print ("\n"); 5267} 5268 5269/* Advance state on FENCE with INSN. Return true if INSN is 5270 an ASM, and we should advance state once more. */ 5271static bool 5272advance_state_on_fence (fence_t fence, insn_t insn) 5273{ 5274 bool asm_p; 5275 5276 if (recog_memoized (insn) >= 0) 5277 { 5278 int res; 5279 state_t temp_state = alloca (dfa_state_size); 5280 5281 gcc_assert (!INSN_ASM_P (insn)); 5282 asm_p = false; 5283 5284 memcpy (temp_state, FENCE_STATE (fence), dfa_state_size); 5285 res = state_transition (FENCE_STATE (fence), insn); 5286 gcc_assert (res < 0); 5287 5288 if (memcmp (temp_state, FENCE_STATE (fence), dfa_state_size)) 5289 { 5290 FENCE_ISSUED_INSNS (fence)++; 5291 5292 /* We should never issue more than issue_rate insns. */ 5293 if (FENCE_ISSUED_INSNS (fence) > issue_rate) 5294 gcc_unreachable (); 5295 } 5296 } 5297 else 5298 { 5299 /* This could be an ASM insn which we'd like to schedule 5300 on the next cycle. */ 5301 asm_p = INSN_ASM_P (insn); 5302 if (!FENCE_STARTS_CYCLE_P (fence) && asm_p) 5303 advance_one_cycle (fence); 5304 } 5305 5306 if (sched_verbose >= 2) 5307 debug_state (FENCE_STATE (fence)); 5308 if (!DEBUG_INSN_P (insn)) 5309 FENCE_STARTS_CYCLE_P (fence) = 0; 5310 FENCE_ISSUE_MORE (fence) = can_issue_more; 5311 return asm_p; 5312} 5313 5314/* Update FENCE on which INSN was scheduled and this INSN, too. NEED_STALL 5315 is nonzero if we need to stall after issuing INSN. */ 5316static void 5317update_fence_and_insn (fence_t fence, insn_t insn, int need_stall) 5318{ 5319 bool asm_p; 5320 5321 /* First, reflect that something is scheduled on this fence. */ 5322 asm_p = advance_state_on_fence (fence, insn); 5323 FENCE_LAST_SCHEDULED_INSN (fence) = insn; 5324 vec_safe_push (FENCE_EXECUTING_INSNS (fence), insn); 5325 if (SCHED_GROUP_P (insn)) 5326 { 5327 FENCE_SCHED_NEXT (fence) = INSN_SCHED_NEXT (insn); 5328 SCHED_GROUP_P (insn) = 0; 5329 } 5330 else 5331 FENCE_SCHED_NEXT (fence) = NULL; 5332 if (INSN_UID (insn) < FENCE_READY_TICKS_SIZE (fence)) 5333 FENCE_READY_TICKS (fence) [INSN_UID (insn)] = 0; 5334 5335 /* Set instruction scheduling info. This will be used in bundling, 5336 pipelining, tick computations etc. */ 5337 ++INSN_SCHED_TIMES (insn); 5338 EXPR_TARGET_AVAILABLE (INSN_EXPR (insn)) = true; 5339 EXPR_ORIG_SCHED_CYCLE (INSN_EXPR (insn)) = FENCE_CYCLE (fence); 5340 INSN_AFTER_STALL_P (insn) = FENCE_AFTER_STALL_P (fence); 5341 INSN_SCHED_CYCLE (insn) = FENCE_CYCLE (fence); 5342 5343 /* This does not account for adjust_cost hooks, just add the biggest 5344 constant the hook may add to the latency. TODO: make this 5345 a target dependent constant. */ 5346 INSN_READY_CYCLE (insn) 5347 = INSN_SCHED_CYCLE (insn) + (INSN_CODE (insn) < 0 5348 ? 1 5349 : maximal_insn_latency (insn) + 1); 5350 5351 /* Change these fields last, as they're used above. */ 5352 FENCE_AFTER_STALL_P (fence) = 0; 5353 if (asm_p || need_stall) 5354 advance_one_cycle (fence); 5355 5356 /* Indicate that we've scheduled something on this fence. */ 5357 FENCE_SCHEDULED_P (fence) = true; 5358 scheduled_something_on_previous_fence = true; 5359 5360 /* Print debug information when insn's fields are updated. */ 5361 if (sched_verbose >= 2) 5362 { 5363 sel_print ("Scheduling insn: "); 5364 dump_insn_1 (insn, 1); 5365 sel_print ("\n"); 5366 } 5367} 5368 5369/* Update boundary BND (and, if needed, FENCE) with INSN, remove the 5370 old boundary from BNDSP, add new boundaries to BNDS_TAIL_P and 5371 return it. */ 5372static blist_t * 5373update_boundaries (fence_t fence, bnd_t bnd, insn_t insn, blist_t *bndsp, 5374 blist_t *bnds_tailp) 5375{ 5376 succ_iterator si; 5377 insn_t succ; 5378 5379 advance_deps_context (BND_DC (bnd), insn); 5380 FOR_EACH_SUCC_1 (succ, si, insn, 5381 SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS) 5382 { 5383 ilist_t ptr = ilist_copy (BND_PTR (bnd)); 5384 5385 ilist_add (&ptr, insn); 5386 5387 if (DEBUG_INSN_P (insn) && sel_bb_end_p (insn) 5388 && is_ineligible_successor (succ, ptr)) 5389 { 5390 ilist_clear (&ptr); 5391 continue; 5392 } 5393 5394 if (FENCE_INSN (fence) == insn && !sel_bb_end_p (insn)) 5395 { 5396 if (sched_verbose >= 9) 5397 sel_print ("Updating fence insn from %i to %i\n", 5398 INSN_UID (insn), INSN_UID (succ)); 5399 FENCE_INSN (fence) = succ; 5400 } 5401 blist_add (bnds_tailp, succ, ptr, BND_DC (bnd)); 5402 bnds_tailp = &BLIST_NEXT (*bnds_tailp); 5403 } 5404 5405 blist_remove (bndsp); 5406 return bnds_tailp; 5407} 5408 5409/* Schedule EXPR_VLIW on BND. Return the insn emitted. */ 5410static insn_t 5411schedule_expr_on_boundary (bnd_t bnd, expr_t expr_vliw, int seqno) 5412{ 5413 av_set_t expr_seq; 5414 expr_t c_expr = XALLOCA (expr_def); 5415 insn_t place_to_insert; 5416 insn_t insn; 5417 bool should_move; 5418 5419 expr_seq = find_sequential_best_exprs (bnd, expr_vliw, true); 5420 5421 /* In case of scheduling a jump skipping some other instructions, 5422 prepare CFG. After this, jump is at the boundary and can be 5423 scheduled as usual insn by MOVE_OP. */ 5424 if (vinsn_cond_branch_p (EXPR_VINSN (expr_vliw))) 5425 { 5426 insn = EXPR_INSN_RTX (expr_vliw); 5427 5428 /* Speculative jumps are not handled. */ 5429 if (insn != BND_TO (bnd) 5430 && !sel_insn_is_speculation_check (insn)) 5431 move_cond_jump (insn, bnd); 5432 } 5433 5434 /* Find a place for C_EXPR to schedule. */ 5435 place_to_insert = prepare_place_to_insert (bnd); 5436 should_move = move_exprs_to_boundary (bnd, expr_vliw, expr_seq, c_expr); 5437 clear_expr (c_expr); 5438 5439 /* Add the instruction. The corner case to care about is when 5440 the expr_seq set has more than one expr, and we chose the one that 5441 is not equal to expr_vliw. Then expr_vliw may be insn in stream, and 5442 we can't use it. Generate the new vinsn. */ 5443 if (INSN_IN_STREAM_P (EXPR_INSN_RTX (expr_vliw))) 5444 { 5445 vinsn_t vinsn_new; 5446 5447 vinsn_new = vinsn_copy (EXPR_VINSN (expr_vliw), false); 5448 change_vinsn_in_expr (expr_vliw, vinsn_new); 5449 should_move = false; 5450 } 5451 if (should_move) 5452 insn = sel_move_insn (expr_vliw, seqno, place_to_insert); 5453 else 5454 insn = emit_insn_from_expr_after (expr_vliw, NULL, seqno, 5455 place_to_insert); 5456 5457 /* Return the nops generated for preserving of data sets back 5458 into pool. */ 5459 if (INSN_NOP_P (place_to_insert)) 5460 return_nop_to_pool (place_to_insert, !DEBUG_INSN_P (insn)); 5461 remove_temp_moveop_nops (!DEBUG_INSN_P (insn)); 5462 5463 av_set_clear (&expr_seq); 5464 5465 /* Save the expression scheduled so to reset target availability if we'll 5466 meet it later on the same fence. */ 5467 if (EXPR_WAS_RENAMED (expr_vliw)) 5468 vinsn_vec_add (&vec_target_unavailable_vinsns, INSN_EXPR (insn)); 5469 5470 /* Check that the recent movement didn't destroyed loop 5471 structure. */ 5472 gcc_assert (!pipelining_p 5473 || current_loop_nest == NULL 5474 || loop_latch_edge (current_loop_nest)); 5475 return insn; 5476} 5477 5478/* Stall for N cycles on FENCE. */ 5479static void 5480stall_for_cycles (fence_t fence, int n) 5481{ 5482 int could_more; 5483 5484 could_more = n > 1 || FENCE_ISSUED_INSNS (fence) < issue_rate; 5485 while (n--) 5486 advance_one_cycle (fence); 5487 if (could_more) 5488 FENCE_AFTER_STALL_P (fence) = 1; 5489} 5490 5491/* Gather a parallel group of insns at FENCE and assign their seqno 5492 to SEQNO. All scheduled insns are gathered in SCHEDULED_INSNS_TAILPP 5493 list for later recalculation of seqnos. */ 5494static void 5495fill_insns (fence_t fence, int seqno, ilist_t **scheduled_insns_tailpp) 5496{ 5497 blist_t bnds = NULL, *bnds_tailp; 5498 av_set_t av_vliw = NULL; 5499 insn_t insn = FENCE_INSN (fence); 5500 5501 if (sched_verbose >= 2) 5502 sel_print ("Starting fill_insns for insn %d, cycle %d\n", 5503 INSN_UID (insn), FENCE_CYCLE (fence)); 5504 5505 blist_add (&bnds, insn, NULL, FENCE_DC (fence)); 5506 bnds_tailp = &BLIST_NEXT (bnds); 5507 set_target_context (FENCE_TC (fence)); 5508 can_issue_more = FENCE_ISSUE_MORE (fence); 5509 target_bb = INSN_BB (insn); 5510 5511 /* Do while we can add any operation to the current group. */ 5512 do 5513 { 5514 blist_t *bnds_tailp1, *bndsp; 5515 expr_t expr_vliw; 5516 int need_stall = false; 5517 int was_stall = 0, scheduled_insns = 0; 5518 int max_insns = pipelining_p ? issue_rate : 2 * issue_rate; 5519 int max_stall = pipelining_p ? 1 : 3; 5520 bool last_insn_was_debug = false; 5521 bool was_debug_bb_end_p = false; 5522 5523 compute_av_set_on_boundaries (fence, bnds, &av_vliw); 5524 remove_insns_that_need_bookkeeping (fence, &av_vliw); 5525 remove_insns_for_debug (bnds, &av_vliw); 5526 5527 /* Return early if we have nothing to schedule. */ 5528 if (av_vliw == NULL) 5529 break; 5530 5531 /* Choose the best expression and, if needed, destination register 5532 for it. */ 5533 do 5534 { 5535 expr_vliw = find_best_expr (&av_vliw, bnds, fence, &need_stall); 5536 if (! expr_vliw && need_stall) 5537 { 5538 /* All expressions required a stall. Do not recompute av sets 5539 as we'll get the same answer (modulo the insns between 5540 the fence and its boundary, which will not be available for 5541 pipelining). 5542 If we are going to stall for too long, break to recompute av 5543 sets and bring more insns for pipelining. */ 5544 was_stall++; 5545 if (need_stall <= 3) 5546 stall_for_cycles (fence, need_stall); 5547 else 5548 { 5549 stall_for_cycles (fence, 1); 5550 break; 5551 } 5552 } 5553 } 5554 while (! expr_vliw && need_stall); 5555 5556 /* Now either we've selected expr_vliw or we have nothing to schedule. */ 5557 if (!expr_vliw) 5558 { 5559 av_set_clear (&av_vliw); 5560 break; 5561 } 5562 5563 bndsp = &bnds; 5564 bnds_tailp1 = bnds_tailp; 5565 5566 do 5567 /* This code will be executed only once until we'd have several 5568 boundaries per fence. */ 5569 { 5570 bnd_t bnd = BLIST_BND (*bndsp); 5571 5572 if (!av_set_is_in_p (BND_AV1 (bnd), EXPR_VINSN (expr_vliw))) 5573 { 5574 bndsp = &BLIST_NEXT (*bndsp); 5575 continue; 5576 } 5577 5578 insn = schedule_expr_on_boundary (bnd, expr_vliw, seqno); 5579 last_insn_was_debug = DEBUG_INSN_P (insn); 5580 if (last_insn_was_debug) 5581 was_debug_bb_end_p = (insn == BND_TO (bnd) && sel_bb_end_p (insn)); 5582 update_fence_and_insn (fence, insn, need_stall); 5583 bnds_tailp = update_boundaries (fence, bnd, insn, bndsp, bnds_tailp); 5584 5585 /* Add insn to the list of scheduled on this cycle instructions. */ 5586 ilist_add (*scheduled_insns_tailpp, insn); 5587 *scheduled_insns_tailpp = &ILIST_NEXT (**scheduled_insns_tailpp); 5588 } 5589 while (*bndsp != *bnds_tailp1); 5590 5591 av_set_clear (&av_vliw); 5592 if (!last_insn_was_debug) 5593 scheduled_insns++; 5594 5595 /* We currently support information about candidate blocks only for 5596 one 'target_bb' block. Hence we can't schedule after jump insn, 5597 as this will bring two boundaries and, hence, necessity to handle 5598 information for two or more blocks concurrently. */ 5599 if ((last_insn_was_debug ? was_debug_bb_end_p : sel_bb_end_p (insn)) 5600 || (was_stall 5601 && (was_stall >= max_stall 5602 || scheduled_insns >= max_insns))) 5603 break; 5604 } 5605 while (bnds); 5606 5607 gcc_assert (!FENCE_BNDS (fence)); 5608 5609 /* Update boundaries of the FENCE. */ 5610 while (bnds) 5611 { 5612 ilist_t ptr = BND_PTR (BLIST_BND (bnds)); 5613 5614 if (ptr) 5615 { 5616 insn = ILIST_INSN (ptr); 5617 5618 if (!ilist_is_in_p (FENCE_BNDS (fence), insn)) 5619 ilist_add (&FENCE_BNDS (fence), insn); 5620 } 5621 5622 blist_remove (&bnds); 5623 } 5624 5625 /* Update target context on the fence. */ 5626 reset_target_context (FENCE_TC (fence), false); 5627} 5628 5629/* All exprs in ORIG_OPS must have the same destination register or memory. 5630 Return that destination. */ 5631static rtx 5632get_dest_from_orig_ops (av_set_t orig_ops) 5633{ 5634 rtx dest = NULL_RTX; 5635 av_set_iterator av_it; 5636 expr_t expr; 5637 bool first_p = true; 5638 5639 FOR_EACH_EXPR (expr, av_it, orig_ops) 5640 { 5641 rtx x = EXPR_LHS (expr); 5642 5643 if (first_p) 5644 { 5645 first_p = false; 5646 dest = x; 5647 } 5648 else 5649 gcc_assert (dest == x 5650 || (dest != NULL_RTX && x != NULL_RTX 5651 && rtx_equal_p (dest, x))); 5652 } 5653 5654 return dest; 5655} 5656 5657/* Update data sets for the bookkeeping block and record those expressions 5658 which become no longer available after inserting this bookkeeping. */ 5659static void 5660update_and_record_unavailable_insns (basic_block book_block) 5661{ 5662 av_set_iterator i; 5663 av_set_t old_av_set = NULL; 5664 expr_t cur_expr; 5665 rtx_insn *bb_end = sel_bb_end (book_block); 5666 5667 /* First, get correct liveness in the bookkeeping block. The problem is 5668 the range between the bookeeping insn and the end of block. */ 5669 update_liveness_on_insn (bb_end); 5670 if (control_flow_insn_p (bb_end)) 5671 update_liveness_on_insn (PREV_INSN (bb_end)); 5672 5673 /* If there's valid av_set on BOOK_BLOCK, then there might exist another 5674 fence above, where we may choose to schedule an insn which is 5675 actually blocked from moving up with the bookkeeping we create here. */ 5676 if (AV_SET_VALID_P (sel_bb_head (book_block))) 5677 { 5678 old_av_set = av_set_copy (BB_AV_SET (book_block)); 5679 update_data_sets (sel_bb_head (book_block)); 5680 5681 /* Traverse all the expressions in the old av_set and check whether 5682 CUR_EXPR is in new AV_SET. */ 5683 FOR_EACH_EXPR (cur_expr, i, old_av_set) 5684 { 5685 expr_t new_expr = av_set_lookup (BB_AV_SET (book_block), 5686 EXPR_VINSN (cur_expr)); 5687 5688 if (! new_expr 5689 /* In this case, we can just turn off the E_T_A bit, but we can't 5690 represent this information with the current vector. */ 5691 || EXPR_TARGET_AVAILABLE (new_expr) 5692 != EXPR_TARGET_AVAILABLE (cur_expr)) 5693 /* Unfortunately, the below code could be also fired up on 5694 separable insns, e.g. when moving insns through the new 5695 speculation check as in PR 53701. */ 5696 vinsn_vec_add (&vec_bookkeeping_blocked_vinsns, cur_expr); 5697 } 5698 5699 av_set_clear (&old_av_set); 5700 } 5701} 5702 5703/* The main effect of this function is that sparams->c_expr is merged 5704 with (or copied to) lparams->c_expr_merged. If there's only one successor, 5705 we avoid merging anything by copying sparams->c_expr to lparams->c_expr_merged. 5706 lparams->c_expr_merged is copied back to sparams->c_expr after all 5707 successors has been traversed. lparams->c_expr_local is an expr allocated 5708 on stack in the caller function, and is used if there is more than one 5709 successor. 5710 5711 SUCC is one of the SUCCS_NORMAL successors of INSN, 5712 MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ, 5713 LPARAMS and STATIC_PARAMS contain the parameters described above. */ 5714static void 5715move_op_merge_succs (insn_t insn ATTRIBUTE_UNUSED, 5716 insn_t succ ATTRIBUTE_UNUSED, 5717 int moveop_drv_call_res, 5718 cmpd_local_params_p lparams, void *static_params) 5719{ 5720 moveop_static_params_p sparams = (moveop_static_params_p) static_params; 5721 5722 /* Nothing to do, if original expr wasn't found below. */ 5723 if (moveop_drv_call_res != 1) 5724 return; 5725 5726 /* If this is a first successor. */ 5727 if (!lparams->c_expr_merged) 5728 { 5729 lparams->c_expr_merged = sparams->c_expr; 5730 sparams->c_expr = lparams->c_expr_local; 5731 } 5732 else 5733 { 5734 /* We must merge all found expressions to get reasonable 5735 EXPR_SPEC_DONE_DS for the resulting insn. If we don't 5736 do so then we can first find the expr with epsilon 5737 speculation success probability and only then with the 5738 good probability. As a result the insn will get epsilon 5739 probability and will never be scheduled because of 5740 weakness_cutoff in find_best_expr. 5741 5742 We call merge_expr_data here instead of merge_expr 5743 because due to speculation C_EXPR and X may have the 5744 same insns with different speculation types. And as of 5745 now such insns are considered non-equal. 5746 5747 However, EXPR_SCHED_TIMES is different -- we must get 5748 SCHED_TIMES from a real insn, not a bookkeeping copy. 5749 We force this here. Instead, we may consider merging 5750 SCHED_TIMES to the maximum instead of minimum in the 5751 below function. */ 5752 int old_times = EXPR_SCHED_TIMES (lparams->c_expr_merged); 5753 5754 merge_expr_data (lparams->c_expr_merged, sparams->c_expr, NULL); 5755 if (EXPR_SCHED_TIMES (sparams->c_expr) == 0) 5756 EXPR_SCHED_TIMES (lparams->c_expr_merged) = old_times; 5757 5758 clear_expr (sparams->c_expr); 5759 } 5760} 5761 5762/* Add used regs for the successor SUCC into SPARAMS->USED_REGS. 5763 5764 SUCC is one of the SUCCS_NORMAL successors of INSN, 5765 MOVEOP_DRV_CALL_RES is the result of call code_motion_path_driver on succ or 0, 5766 if SUCC is one of SUCCS_BACK or SUCCS_OUT. 5767 STATIC_PARAMS contain USED_REGS set. */ 5768static void 5769fur_merge_succs (insn_t insn ATTRIBUTE_UNUSED, insn_t succ, 5770 int moveop_drv_call_res, 5771 cmpd_local_params_p lparams ATTRIBUTE_UNUSED, 5772 void *static_params) 5773{ 5774 regset succ_live; 5775 fur_static_params_p sparams = (fur_static_params_p) static_params; 5776 5777 /* Here we compute live regsets only for branches that do not lie 5778 on the code motion paths. These branches correspond to value 5779 MOVEOP_DRV_CALL_RES==0 and include SUCCS_BACK and SUCCS_OUT, though 5780 for such branches code_motion_path_driver is not called. */ 5781 if (moveop_drv_call_res != 0) 5782 return; 5783 5784 /* Mark all registers that do not meet the following condition: 5785 (3) not live on the other path of any conditional branch 5786 that is passed by the operation, in case original 5787 operations are not present on both paths of the 5788 conditional branch. */ 5789 succ_live = compute_live (succ); 5790 IOR_REG_SET (sparams->used_regs, succ_live); 5791} 5792 5793/* This function is called after the last successor. Copies LP->C_EXPR_MERGED 5794 into SP->CEXPR. */ 5795static void 5796move_op_after_merge_succs (cmpd_local_params_p lp, void *sparams) 5797{ 5798 moveop_static_params_p sp = (moveop_static_params_p) sparams; 5799 5800 sp->c_expr = lp->c_expr_merged; 5801} 5802 5803/* Track bookkeeping copies created, insns scheduled, and blocks for 5804 rescheduling when INSN is found by move_op. */ 5805static void 5806track_scheduled_insns_and_blocks (rtx_insn *insn) 5807{ 5808 /* Even if this insn can be a copy that will be removed during current move_op, 5809 we still need to count it as an originator. */ 5810 bitmap_set_bit (current_originators, INSN_UID (insn)); 5811 5812 if (!bitmap_clear_bit (current_copies, INSN_UID (insn))) 5813 { 5814 /* Note that original block needs to be rescheduled, as we pulled an 5815 instruction out of it. */ 5816 if (INSN_SCHED_TIMES (insn) > 0) 5817 bitmap_set_bit (blocks_to_reschedule, BLOCK_FOR_INSN (insn)->index); 5818 else if (INSN_UID (insn) < first_emitted_uid && !DEBUG_INSN_P (insn)) 5819 num_insns_scheduled++; 5820 } 5821 5822 /* For instructions we must immediately remove insn from the 5823 stream, so subsequent update_data_sets () won't include this 5824 insn into av_set. 5825 For expr we must make insn look like "INSN_REG (insn) := c_expr". */ 5826 if (INSN_UID (insn) > max_uid_before_move_op) 5827 stat_bookkeeping_copies--; 5828} 5829 5830/* Emit a register-register copy for INSN if needed. Return true if 5831 emitted one. PARAMS is the move_op static parameters. */ 5832static bool 5833maybe_emit_renaming_copy (rtx_insn *insn, 5834 moveop_static_params_p params) 5835{ 5836 bool insn_emitted = false; 5837 rtx cur_reg; 5838 5839 /* Bail out early when expression cannot be renamed at all. */ 5840 if (!EXPR_SEPARABLE_P (params->c_expr)) 5841 return false; 5842 5843 cur_reg = expr_dest_reg (params->c_expr); 5844 gcc_assert (cur_reg && params->dest && REG_P (params->dest)); 5845 5846 /* If original operation has expr and the register chosen for 5847 that expr is not original operation's dest reg, substitute 5848 operation's right hand side with the register chosen. */ 5849 if (REGNO (params->dest) != REGNO (cur_reg)) 5850 { 5851 insn_t reg_move_insn, reg_move_insn_rtx; 5852 5853 reg_move_insn_rtx = create_insn_rtx_with_rhs (INSN_VINSN (insn), 5854 params->dest); 5855 reg_move_insn = sel_gen_insn_from_rtx_after (reg_move_insn_rtx, 5856 INSN_EXPR (insn), 5857 INSN_SEQNO (insn), 5858 insn); 5859 EXPR_SPEC_DONE_DS (INSN_EXPR (reg_move_insn)) = 0; 5860 replace_dest_with_reg_in_expr (params->c_expr, params->dest); 5861 5862 insn_emitted = true; 5863 params->was_renamed = true; 5864 } 5865 5866 return insn_emitted; 5867} 5868 5869/* Emit a speculative check for INSN speculated as EXPR if needed. 5870 Return true if we've emitted one. PARAMS is the move_op static 5871 parameters. */ 5872static bool 5873maybe_emit_speculative_check (rtx_insn *insn, expr_t expr, 5874 moveop_static_params_p params) 5875{ 5876 bool insn_emitted = false; 5877 insn_t x; 5878 ds_t check_ds; 5879 5880 check_ds = get_spec_check_type_for_insn (insn, expr); 5881 if (check_ds != 0) 5882 { 5883 /* A speculation check should be inserted. */ 5884 x = create_speculation_check (params->c_expr, check_ds, insn); 5885 insn_emitted = true; 5886 } 5887 else 5888 { 5889 EXPR_SPEC_DONE_DS (INSN_EXPR (insn)) = 0; 5890 x = insn; 5891 } 5892 5893 gcc_assert (EXPR_SPEC_DONE_DS (INSN_EXPR (x)) == 0 5894 && EXPR_SPEC_TO_CHECK_DS (INSN_EXPR (x)) == 0); 5895 return insn_emitted; 5896} 5897 5898/* Handle transformations that leave an insn in place of original 5899 insn such as renaming/speculation. Return true if one of such 5900 transformations actually happened, and we have emitted this insn. */ 5901static bool 5902handle_emitting_transformations (rtx_insn *insn, expr_t expr, 5903 moveop_static_params_p params) 5904{ 5905 bool insn_emitted = false; 5906 5907 insn_emitted = maybe_emit_renaming_copy (insn, params); 5908 insn_emitted |= maybe_emit_speculative_check (insn, expr, params); 5909 5910 return insn_emitted; 5911} 5912 5913/* If INSN is the only insn in the basic block (not counting JUMP, 5914 which may be a jump to next insn, and DEBUG_INSNs), we want to 5915 leave a NOP there till the return to fill_insns. */ 5916 5917static bool 5918need_nop_to_preserve_insn_bb (rtx_insn *insn) 5919{ 5920 insn_t bb_head, bb_end, bb_next, in_next; 5921 basic_block bb = BLOCK_FOR_INSN (insn); 5922 5923 bb_head = sel_bb_head (bb); 5924 bb_end = sel_bb_end (bb); 5925 5926 if (bb_head == bb_end) 5927 return true; 5928 5929 while (bb_head != bb_end && DEBUG_INSN_P (bb_head)) 5930 bb_head = NEXT_INSN (bb_head); 5931 5932 if (bb_head == bb_end) 5933 return true; 5934 5935 while (bb_head != bb_end && DEBUG_INSN_P (bb_end)) 5936 bb_end = PREV_INSN (bb_end); 5937 5938 if (bb_head == bb_end) 5939 return true; 5940 5941 bb_next = NEXT_INSN (bb_head); 5942 while (bb_next != bb_end && DEBUG_INSN_P (bb_next)) 5943 bb_next = NEXT_INSN (bb_next); 5944 5945 if (bb_next == bb_end && JUMP_P (bb_end)) 5946 return true; 5947 5948 in_next = NEXT_INSN (insn); 5949 while (DEBUG_INSN_P (in_next)) 5950 in_next = NEXT_INSN (in_next); 5951 5952 if (IN_CURRENT_FENCE_P (in_next)) 5953 return true; 5954 5955 return false; 5956} 5957 5958/* Remove INSN from stream. When ONLY_DISCONNECT is true, its data 5959 is not removed but reused when INSN is re-emitted. */ 5960static void 5961remove_insn_from_stream (rtx_insn *insn, bool only_disconnect) 5962{ 5963 /* If there's only one insn in the BB, make sure that a nop is 5964 inserted into it, so the basic block won't disappear when we'll 5965 delete INSN below with sel_remove_insn. It should also survive 5966 till the return to fill_insns. */ 5967 if (need_nop_to_preserve_insn_bb (insn)) 5968 { 5969 insn_t nop = get_nop_from_pool (insn); 5970 gcc_assert (INSN_NOP_P (nop)); 5971 vec_temp_moveop_nops.safe_push (nop); 5972 } 5973 5974 sel_remove_insn (insn, only_disconnect, false); 5975} 5976 5977/* This function is called when original expr is found. 5978 INSN - current insn traversed, EXPR - the corresponding expr found. 5979 LPARAMS is the local parameters of code modion driver, STATIC_PARAMS 5980 is static parameters of move_op. */ 5981static void 5982move_op_orig_expr_found (insn_t insn, expr_t expr, 5983 cmpd_local_params_p lparams ATTRIBUTE_UNUSED, 5984 void *static_params) 5985{ 5986 bool only_disconnect; 5987 moveop_static_params_p params = (moveop_static_params_p) static_params; 5988 5989 copy_expr_onside (params->c_expr, INSN_EXPR (insn)); 5990 track_scheduled_insns_and_blocks (insn); 5991 handle_emitting_transformations (insn, expr, params); 5992 only_disconnect = params->uid == INSN_UID (insn); 5993 5994 /* Mark that we've disconnected an insn. */ 5995 if (only_disconnect) 5996 params->uid = -1; 5997 remove_insn_from_stream (insn, only_disconnect); 5998} 5999 6000/* The function is called when original expr is found. 6001 INSN - current insn traversed, EXPR - the corresponding expr found, 6002 crossed_call_abis and original_insns in STATIC_PARAMS are updated. */ 6003static void 6004fur_orig_expr_found (insn_t insn, expr_t expr ATTRIBUTE_UNUSED, 6005 cmpd_local_params_p lparams ATTRIBUTE_UNUSED, 6006 void *static_params) 6007{ 6008 fur_static_params_p params = (fur_static_params_p) static_params; 6009 regset tmp; 6010 6011 if (CALL_P (insn)) 6012 params->crossed_call_abis |= 1 << insn_callee_abi (insn).id (); 6013 6014 def_list_add (params->original_insns, insn, params->crossed_call_abis); 6015 6016 /* Mark the registers that do not meet the following condition: 6017 (2) not among the live registers of the point 6018 immediately following the first original operation on 6019 a given downward path, except for the original target 6020 register of the operation. */ 6021 tmp = get_clear_regset_from_pool (); 6022 compute_live_below_insn (insn, tmp); 6023 AND_COMPL_REG_SET (tmp, INSN_REG_SETS (insn)); 6024 AND_COMPL_REG_SET (tmp, INSN_REG_CLOBBERS (insn)); 6025 IOR_REG_SET (params->used_regs, tmp); 6026 return_regset_to_pool (tmp); 6027 6028 /* (*1) We need to add to USED_REGS registers that are read by 6029 INSN's lhs. This may lead to choosing wrong src register. 6030 E.g. (scheduling const expr enabled): 6031 6032 429: ax=0x0 <- Can't use AX for this expr (0x0) 6033 433: dx=[bp-0x18] 6034 427: [ax+dx+0x1]=ax 6035 REG_DEAD: ax 6036 168: di=dx 6037 REG_DEAD: dx 6038 */ 6039 /* FIXME: see comment above and enable MEM_P 6040 in vinsn_separable_p. */ 6041 gcc_assert (!VINSN_SEPARABLE_P (INSN_VINSN (insn)) 6042 || !MEM_P (INSN_LHS (insn))); 6043} 6044 6045/* This function is called on the ascending pass, before returning from 6046 current basic block. */ 6047static void 6048move_op_at_first_insn (insn_t insn, cmpd_local_params_p lparams, 6049 void *static_params) 6050{ 6051 moveop_static_params_p sparams = (moveop_static_params_p) static_params; 6052 basic_block book_block = NULL; 6053 6054 /* When we have removed the boundary insn for scheduling, which also 6055 happened to be the end insn in its bb, we don't need to update sets. */ 6056 if (!lparams->removed_last_insn 6057 && lparams->e1 6058 && sel_bb_head_p (insn)) 6059 { 6060 /* We should generate bookkeeping code only if we are not at the 6061 top level of the move_op. */ 6062 if (sel_num_cfg_preds_gt_1 (insn)) 6063 book_block = generate_bookkeeping_insn (sparams->c_expr, 6064 lparams->e1, lparams->e2); 6065 /* Update data sets for the current insn. */ 6066 update_data_sets (insn); 6067 } 6068 6069 /* If bookkeeping code was inserted, we need to update av sets of basic 6070 block that received bookkeeping. After generation of bookkeeping insn, 6071 bookkeeping block does not contain valid av set because we are not following 6072 the original algorithm in every detail with regards to e.g. renaming 6073 simple reg-reg copies. Consider example: 6074 6075 bookkeeping block scheduling fence 6076 \ / 6077 \ join / 6078 ---------- 6079 | | 6080 ---------- 6081 / \ 6082 / \ 6083 r1 := r2 r1 := r3 6084 6085 We try to schedule insn "r1 := r3" on the current 6086 scheduling fence. Also, note that av set of bookkeeping block 6087 contain both insns "r1 := r2" and "r1 := r3". When the insn has 6088 been scheduled, the CFG is as follows: 6089 6090 r1 := r3 r1 := r3 6091 bookkeeping block scheduling fence 6092 \ / 6093 \ join / 6094 ---------- 6095 | | 6096 ---------- 6097 / \ 6098 / \ 6099 r1 := r2 6100 6101 Here, insn "r1 := r3" was scheduled at the current scheduling point 6102 and bookkeeping code was generated at the bookeeping block. This 6103 way insn "r1 := r2" is no longer available as a whole instruction 6104 (but only as expr) ahead of insn "r1 := r3" in bookkeeping block. 6105 This situation is handled by calling update_data_sets. 6106 6107 Since update_data_sets is called only on the bookkeeping block, and 6108 it also may have predecessors with av_sets, containing instructions that 6109 are no longer available, we save all such expressions that become 6110 unavailable during data sets update on the bookkeeping block in 6111 VEC_BOOKKEEPING_BLOCKED_VINSNS. Later we avoid selecting such 6112 expressions for scheduling. This allows us to avoid recomputation of 6113 av_sets outside the code motion path. */ 6114 6115 if (book_block) 6116 update_and_record_unavailable_insns (book_block); 6117 6118 /* If INSN was previously marked for deletion, it's time to do it. */ 6119 if (lparams->removed_last_insn) 6120 insn = PREV_INSN (insn); 6121 6122 /* Do not tidy control flow at the topmost moveop, as we can erroneously 6123 kill a block with a single nop in which the insn should be emitted. */ 6124 if (lparams->e1) 6125 tidy_control_flow (BLOCK_FOR_INSN (insn), true); 6126} 6127 6128/* This function is called on the ascending pass, before returning from the 6129 current basic block. */ 6130static void 6131fur_at_first_insn (insn_t insn, 6132 cmpd_local_params_p lparams ATTRIBUTE_UNUSED, 6133 void *static_params ATTRIBUTE_UNUSED) 6134{ 6135 gcc_assert (!sel_bb_head_p (insn) || AV_SET_VALID_P (insn) 6136 || AV_LEVEL (insn) == -1); 6137} 6138 6139/* Called on the backward stage of recursion to call moveup_expr for insn 6140 and sparams->c_expr. */ 6141static void 6142move_op_ascend (insn_t insn, void *static_params) 6143{ 6144 enum MOVEUP_EXPR_CODE res; 6145 moveop_static_params_p sparams = (moveop_static_params_p) static_params; 6146 6147 if (! INSN_NOP_P (insn)) 6148 { 6149 res = moveup_expr_cached (sparams->c_expr, insn, false); 6150 gcc_assert (res != MOVEUP_EXPR_NULL); 6151 } 6152 6153 /* Update liveness for this insn as it was invalidated. */ 6154 update_liveness_on_insn (insn); 6155} 6156 6157/* This function is called on enter to the basic block. 6158 Returns TRUE if this block already have been visited and 6159 code_motion_path_driver should return 1, FALSE otherwise. */ 6160static int 6161fur_on_enter (insn_t insn ATTRIBUTE_UNUSED, cmpd_local_params_p local_params, 6162 void *static_params, bool visited_p) 6163{ 6164 fur_static_params_p sparams = (fur_static_params_p) static_params; 6165 6166 if (visited_p) 6167 { 6168 /* If we have found something below this block, there should be at 6169 least one insn in ORIGINAL_INSNS. */ 6170 gcc_assert (*sparams->original_insns); 6171 6172 /* Adjust CROSSED_CALL_ABIS, since we may have come to this block along 6173 different path. */ 6174 DEF_LIST_DEF (*sparams->original_insns)->crossed_call_abis 6175 |= sparams->crossed_call_abis; 6176 } 6177 else 6178 local_params->old_original_insns = *sparams->original_insns; 6179 6180 return 1; 6181} 6182 6183/* Same as above but for move_op. */ 6184static int 6185move_op_on_enter (insn_t insn ATTRIBUTE_UNUSED, 6186 cmpd_local_params_p local_params ATTRIBUTE_UNUSED, 6187 void *static_params ATTRIBUTE_UNUSED, bool visited_p) 6188{ 6189 if (visited_p) 6190 return -1; 6191 return 1; 6192} 6193 6194/* This function is called while descending current basic block if current 6195 insn is not the original EXPR we're searching for. 6196 6197 Return value: FALSE, if code_motion_path_driver should perform a local 6198 cleanup and return 0 itself; 6199 TRUE, if code_motion_path_driver should continue. */ 6200static bool 6201move_op_orig_expr_not_found (insn_t insn, av_set_t orig_ops ATTRIBUTE_UNUSED, 6202 void *static_params) 6203{ 6204 moveop_static_params_p sparams = (moveop_static_params_p) static_params; 6205 6206 sparams->failed_insn = insn; 6207 6208 /* If we're scheduling separate expr, in order to generate correct code 6209 we need to stop the search at bookkeeping code generated with the 6210 same destination register or memory. */ 6211 if (lhs_of_insn_equals_to_dest_p (insn, sparams->dest)) 6212 return false; 6213 return true; 6214} 6215 6216/* This function is called while descending current basic block if current 6217 insn is not the original EXPR we're searching for. 6218 6219 Return value: TRUE (code_motion_path_driver should continue). */ 6220static bool 6221fur_orig_expr_not_found (insn_t insn, av_set_t orig_ops, void *static_params) 6222{ 6223 bool mutexed; 6224 expr_t r; 6225 av_set_iterator avi; 6226 fur_static_params_p sparams = (fur_static_params_p) static_params; 6227 6228 if (CALL_P (insn)) 6229 sparams->crossed_call_abis |= 1 << insn_callee_abi (insn).id (); 6230 else if (DEBUG_INSN_P (insn)) 6231 return true; 6232 6233 /* If current insn we are looking at cannot be executed together 6234 with original insn, then we can skip it safely. 6235 6236 Example: ORIG_OPS = { (p6) r14 = sign_extend (r15); } 6237 INSN = (!p6) r14 = r14 + 1; 6238 6239 Here we can schedule ORIG_OP with lhs = r14, though only 6240 looking at the set of used and set registers of INSN we must 6241 forbid it. So, add set/used in INSN registers to the 6242 untouchable set only if there is an insn in ORIG_OPS that can 6243 affect INSN. */ 6244 mutexed = true; 6245 FOR_EACH_EXPR (r, avi, orig_ops) 6246 if (!sched_insns_conditions_mutex_p (insn, EXPR_INSN_RTX (r))) 6247 { 6248 mutexed = false; 6249 break; 6250 } 6251 6252 /* Mark all registers that do not meet the following condition: 6253 (1) Not set or read on any path from xi to an instance of the 6254 original operation. */ 6255 if (!mutexed) 6256 { 6257 IOR_REG_SET (sparams->used_regs, INSN_REG_SETS (insn)); 6258 IOR_REG_SET (sparams->used_regs, INSN_REG_USES (insn)); 6259 IOR_REG_SET (sparams->used_regs, INSN_REG_CLOBBERS (insn)); 6260 } 6261 6262 return true; 6263} 6264 6265/* Hooks and data to perform move_op operations with code_motion_path_driver. */ 6266struct code_motion_path_driver_info_def move_op_hooks = { 6267 move_op_on_enter, 6268 move_op_orig_expr_found, 6269 move_op_orig_expr_not_found, 6270 move_op_merge_succs, 6271 move_op_after_merge_succs, 6272 move_op_ascend, 6273 move_op_at_first_insn, 6274 SUCCS_NORMAL, 6275 "move_op" 6276}; 6277 6278/* Hooks and data to perform find_used_regs operations 6279 with code_motion_path_driver. */ 6280struct code_motion_path_driver_info_def fur_hooks = { 6281 fur_on_enter, 6282 fur_orig_expr_found, 6283 fur_orig_expr_not_found, 6284 fur_merge_succs, 6285 NULL, /* fur_after_merge_succs */ 6286 NULL, /* fur_ascend */ 6287 fur_at_first_insn, 6288 SUCCS_ALL, 6289 "find_used_regs" 6290}; 6291 6292/* Traverse all successors of INSN. For each successor that is SUCCS_NORMAL 6293 code_motion_path_driver is called recursively. Original operation 6294 was found at least on one path that is starting with one of INSN's 6295 successors (this fact is asserted). ORIG_OPS is expressions we're looking 6296 for, PATH is the path we've traversed, STATIC_PARAMS is the parameters 6297 of either move_op or find_used_regs depending on the caller. 6298 6299 Return 0 if we haven't found expression, 1 if we found it, -1 if we don't 6300 know for sure at this point. */ 6301static int 6302code_motion_process_successors (insn_t insn, av_set_t orig_ops, 6303 ilist_t path, void *static_params) 6304{ 6305 int res = 0; 6306 succ_iterator succ_i; 6307 insn_t succ; 6308 basic_block bb; 6309 int old_index; 6310 unsigned old_succs; 6311 6312 struct cmpd_local_params lparams; 6313 expr_def _x; 6314 6315 lparams.c_expr_local = &_x; 6316 lparams.c_expr_merged = NULL; 6317 6318 /* We need to process only NORMAL succs for move_op, and collect live 6319 registers from ALL branches (including those leading out of the 6320 region) for find_used_regs. 6321 6322 In move_op, there can be a case when insn's bb number has changed 6323 due to created bookkeeping. This happens very rare, as we need to 6324 move expression from the beginning to the end of the same block. 6325 Rescan successors in this case. */ 6326 6327 rescan: 6328 bb = BLOCK_FOR_INSN (insn); 6329 old_index = bb->index; 6330 old_succs = EDGE_COUNT (bb->succs); 6331 6332 FOR_EACH_SUCC_1 (succ, succ_i, insn, code_motion_path_driver_info->succ_flags) 6333 { 6334 int b; 6335 6336 lparams.e1 = succ_i.e1; 6337 lparams.e2 = succ_i.e2; 6338 6339 /* Go deep into recursion only for NORMAL edges (non-backedges within the 6340 current region). */ 6341 if (succ_i.current_flags == SUCCS_NORMAL) 6342 b = code_motion_path_driver (succ, orig_ops, path, &lparams, 6343 static_params); 6344 else 6345 b = 0; 6346 6347 /* Merge c_expres found or unify live register sets from different 6348 successors. */ 6349 code_motion_path_driver_info->merge_succs (insn, succ, b, &lparams, 6350 static_params); 6351 if (b == 1) 6352 res = b; 6353 else if (b == -1 && res != 1) 6354 res = b; 6355 6356 /* We have simplified the control flow below this point. In this case, 6357 the iterator becomes invalid. We need to try again. 6358 If we have removed the insn itself, it could be only an 6359 unconditional jump. Thus, do not rescan but break immediately -- 6360 we have already visited the only successor block. */ 6361 if (!BLOCK_FOR_INSN (insn)) 6362 { 6363 if (sched_verbose >= 6) 6364 sel_print ("Not doing rescan: already visited the only successor" 6365 " of block %d\n", old_index); 6366 break; 6367 } 6368 if (BLOCK_FOR_INSN (insn)->index != old_index 6369 || EDGE_COUNT (bb->succs) != old_succs) 6370 { 6371 if (sched_verbose >= 6) 6372 sel_print ("Rescan: CFG was simplified below insn %d, block %d\n", 6373 INSN_UID (insn), BLOCK_FOR_INSN (insn)->index); 6374 insn = sel_bb_end (BLOCK_FOR_INSN (insn)); 6375 goto rescan; 6376 } 6377 } 6378 6379 /* Here, RES==1 if original expr was found at least for one of the 6380 successors. After the loop, RES may happen to have zero value 6381 only if at some point the expr searched is present in av_set, but is 6382 not found below. In most cases, this situation is an error. 6383 The exception is when the original operation is blocked by 6384 bookkeeping generated for another fence or for another path in current 6385 move_op. */ 6386 gcc_checking_assert (res == 1 6387 || (res == 0 6388 && av_set_could_be_blocked_by_bookkeeping_p (orig_ops, static_params)) 6389 || res == -1); 6390 6391 /* Merge data, clean up, etc. */ 6392 if (res != -1 && code_motion_path_driver_info->after_merge_succs) 6393 code_motion_path_driver_info->after_merge_succs (&lparams, static_params); 6394 6395 return res; 6396} 6397 6398 6399/* Perform a cleanup when the driver is about to terminate. ORIG_OPS_P 6400 is the pointer to the av set with expressions we were looking for, 6401 PATH_P is the pointer to the traversed path. */ 6402static inline void 6403code_motion_path_driver_cleanup (av_set_t *orig_ops_p, ilist_t *path_p) 6404{ 6405 ilist_remove (path_p); 6406 av_set_clear (orig_ops_p); 6407} 6408 6409/* The driver function that implements move_op or find_used_regs 6410 functionality dependent whether code_motion_path_driver_INFO is set to 6411 &MOVE_OP_HOOKS or &FUR_HOOKS. This function implements the common parts 6412 of code (CFG traversal etc) that are shared among both functions. INSN 6413 is the insn we're starting the search from, ORIG_OPS are the expressions 6414 we're searching for, PATH is traversed path, LOCAL_PARAMS_IN are local 6415 parameters of the driver, and STATIC_PARAMS are static parameters of 6416 the caller. 6417 6418 Returns whether original instructions were found. Note that top-level 6419 code_motion_path_driver always returns true. */ 6420static int 6421code_motion_path_driver (insn_t insn, av_set_t orig_ops, ilist_t path, 6422 cmpd_local_params_p local_params_in, 6423 void *static_params) 6424{ 6425 expr_t expr = NULL; 6426 basic_block bb = BLOCK_FOR_INSN (insn); 6427 insn_t first_insn, original_insn, bb_tail, before_first; 6428 bool removed_last_insn = false; 6429 6430 if (sched_verbose >= 6) 6431 { 6432 sel_print ("%s (", code_motion_path_driver_info->routine_name); 6433 dump_insn (insn); 6434 sel_print (","); 6435 dump_av_set (orig_ops); 6436 sel_print (")\n"); 6437 } 6438 6439 gcc_assert (orig_ops); 6440 6441 /* If no original operations exist below this insn, return immediately. */ 6442 if (is_ineligible_successor (insn, path)) 6443 { 6444 if (sched_verbose >= 6) 6445 sel_print ("Insn %d is ineligible successor\n", INSN_UID (insn)); 6446 return false; 6447 } 6448 6449 /* The block can have invalid av set, in which case it was created earlier 6450 during move_op. Return immediately. */ 6451 if (sel_bb_head_p (insn)) 6452 { 6453 if (! AV_SET_VALID_P (insn)) 6454 { 6455 if (sched_verbose >= 6) 6456 sel_print ("Returned from block %d as it had invalid av set\n", 6457 bb->index); 6458 return false; 6459 } 6460 6461 if (bitmap_bit_p (code_motion_visited_blocks, bb->index)) 6462 { 6463 /* We have already found an original operation on this branch, do not 6464 go any further and just return TRUE here. If we don't stop here, 6465 function can have exponential behavior even on the small code 6466 with many different paths (e.g. with data speculation and 6467 recovery blocks). */ 6468 if (sched_verbose >= 6) 6469 sel_print ("Block %d already visited in this traversal\n", bb->index); 6470 if (code_motion_path_driver_info->on_enter) 6471 return code_motion_path_driver_info->on_enter (insn, 6472 local_params_in, 6473 static_params, 6474 true); 6475 } 6476 } 6477 6478 if (code_motion_path_driver_info->on_enter) 6479 code_motion_path_driver_info->on_enter (insn, local_params_in, 6480 static_params, false); 6481 orig_ops = av_set_copy (orig_ops); 6482 6483 /* Filter the orig_ops set. */ 6484 if (AV_SET_VALID_P (insn)) 6485 av_set_code_motion_filter (&orig_ops, AV_SET (insn)); 6486 6487 /* If no more original ops, return immediately. */ 6488 if (!orig_ops) 6489 { 6490 if (sched_verbose >= 6) 6491 sel_print ("No intersection with av set of block %d\n", bb->index); 6492 return false; 6493 } 6494 6495 /* For non-speculative insns we have to leave only one form of the 6496 original operation, because if we don't, we may end up with 6497 different C_EXPRes and, consequently, with bookkeepings for different 6498 expression forms along the same code motion path. That may lead to 6499 generation of incorrect code. So for each code motion we stick to 6500 the single form of the instruction, except for speculative insns 6501 which we need to keep in different forms with all speculation 6502 types. */ 6503 av_set_leave_one_nonspec (&orig_ops); 6504 6505 /* It is not possible that all ORIG_OPS are filtered out. */ 6506 gcc_assert (orig_ops); 6507 6508 /* It is enough to place only heads and tails of visited basic blocks into 6509 the PATH. */ 6510 ilist_add (&path, insn); 6511 first_insn = original_insn = insn; 6512 bb_tail = sel_bb_end (bb); 6513 6514 /* Descend the basic block in search of the original expr; this part 6515 corresponds to the part of the original move_op procedure executed 6516 before the recursive call. */ 6517 for (;;) 6518 { 6519 /* Look at the insn and decide if it could be an ancestor of currently 6520 scheduling operation. If it is so, then the insn "dest = op" could 6521 either be replaced with "dest = reg", because REG now holds the result 6522 of OP, or just removed, if we've scheduled the insn as a whole. 6523 6524 If this insn doesn't contain currently scheduling OP, then proceed 6525 with searching and look at its successors. Operations we're searching 6526 for could have changed when moving up through this insn via 6527 substituting. In this case, perform unsubstitution on them first. 6528 6529 When traversing the DAG below this insn is finished, insert 6530 bookkeeping code, if the insn is a joint point, and remove 6531 leftovers. */ 6532 6533 expr = av_set_lookup (orig_ops, INSN_VINSN (insn)); 6534 if (expr) 6535 { 6536 insn_t last_insn = PREV_INSN (insn); 6537 6538 /* We have found the original operation. */ 6539 if (sched_verbose >= 6) 6540 sel_print ("Found original operation at insn %d\n", INSN_UID (insn)); 6541 6542 code_motion_path_driver_info->orig_expr_found 6543 (insn, expr, local_params_in, static_params); 6544 6545 /* Step back, so on the way back we'll start traversing from the 6546 previous insn (or we'll see that it's bb_note and skip that 6547 loop). */ 6548 if (insn == first_insn) 6549 { 6550 first_insn = NEXT_INSN (last_insn); 6551 removed_last_insn = sel_bb_end_p (last_insn); 6552 } 6553 insn = last_insn; 6554 break; 6555 } 6556 else 6557 { 6558 /* We haven't found the original expr, continue descending the basic 6559 block. */ 6560 if (code_motion_path_driver_info->orig_expr_not_found 6561 (insn, orig_ops, static_params)) 6562 { 6563 /* Av set ops could have been changed when moving through this 6564 insn. To find them below it, we have to un-substitute them. */ 6565 undo_transformations (&orig_ops, insn); 6566 } 6567 else 6568 { 6569 /* Clean up and return, if the hook tells us to do so. It may 6570 happen if we've encountered the previously created 6571 bookkeeping. */ 6572 code_motion_path_driver_cleanup (&orig_ops, &path); 6573 return -1; 6574 } 6575 6576 gcc_assert (orig_ops); 6577 } 6578 6579 /* Stop at insn if we got to the end of BB. */ 6580 if (insn == bb_tail) 6581 break; 6582 6583 insn = NEXT_INSN (insn); 6584 } 6585 6586 /* Here INSN either points to the insn before the original insn (may be 6587 bb_note, if original insn was a bb_head) or to the bb_end. */ 6588 if (!expr) 6589 { 6590 int res; 6591 rtx_insn *last_insn = PREV_INSN (insn); 6592 bool added_to_path; 6593 6594 gcc_assert (insn == sel_bb_end (bb)); 6595 6596 /* Add bb tail to PATH (but it doesn't make any sense if it's a bb_head - 6597 it's already in PATH then). */ 6598 if (insn != first_insn) 6599 { 6600 ilist_add (&path, insn); 6601 added_to_path = true; 6602 } 6603 else 6604 added_to_path = false; 6605 6606 /* Process_successors should be able to find at least one 6607 successor for which code_motion_path_driver returns TRUE. */ 6608 res = code_motion_process_successors (insn, orig_ops, 6609 path, static_params); 6610 6611 /* Jump in the end of basic block could have been removed or replaced 6612 during code_motion_process_successors, so recompute insn as the 6613 last insn in bb. */ 6614 if (NEXT_INSN (last_insn) != insn) 6615 { 6616 insn = sel_bb_end (bb); 6617 first_insn = sel_bb_head (bb); 6618 if (first_insn != original_insn) 6619 first_insn = original_insn; 6620 } 6621 6622 /* Remove bb tail from path. */ 6623 if (added_to_path) 6624 ilist_remove (&path); 6625 6626 if (res != 1) 6627 { 6628 /* This is the case when one of the original expr is no longer available 6629 due to bookkeeping created on this branch with the same register. 6630 In the original algorithm, which doesn't have update_data_sets call 6631 on a bookkeeping block, it would simply result in returning 6632 FALSE when we've encountered a previously generated bookkeeping 6633 insn in moveop_orig_expr_not_found. */ 6634 code_motion_path_driver_cleanup (&orig_ops, &path); 6635 return res; 6636 } 6637 } 6638 6639 /* Don't need it any more. */ 6640 av_set_clear (&orig_ops); 6641 6642 /* Backward pass: now, when we have C_EXPR computed, we'll drag it to 6643 the beginning of the basic block. */ 6644 before_first = PREV_INSN (first_insn); 6645 while (insn != before_first) 6646 { 6647 if (code_motion_path_driver_info->ascend) 6648 code_motion_path_driver_info->ascend (insn, static_params); 6649 6650 insn = PREV_INSN (insn); 6651 } 6652 6653 /* Now we're at the bb head. */ 6654 insn = first_insn; 6655 ilist_remove (&path); 6656 local_params_in->removed_last_insn = removed_last_insn; 6657 code_motion_path_driver_info->at_first_insn (insn, local_params_in, static_params); 6658 6659 /* This should be the very last operation as at bb head we could change 6660 the numbering by creating bookkeeping blocks. */ 6661 if (removed_last_insn) 6662 insn = PREV_INSN (insn); 6663 6664 /* If we have simplified the control flow and removed the first jump insn, 6665 there's no point in marking this block in the visited blocks bitmap. */ 6666 if (BLOCK_FOR_INSN (insn)) 6667 bitmap_set_bit (code_motion_visited_blocks, BLOCK_FOR_INSN (insn)->index); 6668 return true; 6669} 6670 6671/* Move up the operations from ORIG_OPS set traversing the dag starting 6672 from INSN. PATH represents the edges traversed so far. 6673 DEST is the register chosen for scheduling the current expr. Insert 6674 bookkeeping code in the join points. EXPR_VLIW is the chosen expression, 6675 C_EXPR is how it looks like at the given cfg point. 6676 Set *SHOULD_MOVE to indicate whether we have only disconnected 6677 one of the insns found. 6678 6679 Returns whether original instructions were found, which is asserted 6680 to be true in the caller. */ 6681static bool 6682move_op (insn_t insn, av_set_t orig_ops, expr_t expr_vliw, 6683 rtx dest, expr_t c_expr, bool *should_move) 6684{ 6685 struct moveop_static_params sparams; 6686 struct cmpd_local_params lparams; 6687 int res; 6688 6689 /* Init params for code_motion_path_driver. */ 6690 sparams.dest = dest; 6691 sparams.c_expr = c_expr; 6692 sparams.uid = INSN_UID (EXPR_INSN_RTX (expr_vliw)); 6693 sparams.failed_insn = NULL; 6694 sparams.was_renamed = false; 6695 lparams.e1 = NULL; 6696 6697 /* We haven't visited any blocks yet. */ 6698 bitmap_clear (code_motion_visited_blocks); 6699 6700 /* Set appropriate hooks and data. */ 6701 code_motion_path_driver_info = &move_op_hooks; 6702 res = code_motion_path_driver (insn, orig_ops, NULL, &lparams, &sparams); 6703 6704 gcc_assert (res != -1); 6705 6706 if (sparams.was_renamed) 6707 EXPR_WAS_RENAMED (expr_vliw) = true; 6708 6709 *should_move = (sparams.uid == -1); 6710 6711 return res; 6712} 6713 6714 6715/* Functions that work with regions. */ 6716 6717/* Current number of seqno used in init_seqno and init_seqno_1. */ 6718static int cur_seqno; 6719 6720/* A helper for init_seqno. Traverse the region starting from BB and 6721 compute seqnos for visited insns, marking visited bbs in VISITED_BBS. 6722 Clear visited blocks from BLOCKS_TO_RESCHEDULE. */ 6723static void 6724init_seqno_1 (basic_block bb, sbitmap visited_bbs, bitmap blocks_to_reschedule) 6725{ 6726 int bbi = BLOCK_TO_BB (bb->index); 6727 insn_t insn; 6728 insn_t succ_insn; 6729 succ_iterator si; 6730 6731 rtx_note *note = bb_note (bb); 6732 bitmap_set_bit (visited_bbs, bbi); 6733 if (blocks_to_reschedule) 6734 bitmap_clear_bit (blocks_to_reschedule, bb->index); 6735 6736 FOR_EACH_SUCC_1 (succ_insn, si, BB_END (bb), 6737 SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS) 6738 { 6739 basic_block succ = BLOCK_FOR_INSN (succ_insn); 6740 int succ_bbi = BLOCK_TO_BB (succ->index); 6741 6742 gcc_assert (in_current_region_p (succ)); 6743 6744 if (!bitmap_bit_p (visited_bbs, succ_bbi)) 6745 { 6746 gcc_assert (succ_bbi > bbi); 6747 6748 init_seqno_1 (succ, visited_bbs, blocks_to_reschedule); 6749 } 6750 else if (blocks_to_reschedule) 6751 bitmap_set_bit (forced_ebb_heads, succ->index); 6752 } 6753 6754 for (insn = BB_END (bb); insn != note; insn = PREV_INSN (insn)) 6755 INSN_SEQNO (insn) = cur_seqno--; 6756} 6757 6758/* Initialize seqnos for the current region. BLOCKS_TO_RESCHEDULE contains 6759 blocks on which we're rescheduling when pipelining, FROM is the block where 6760 traversing region begins (it may not be the head of the region when 6761 pipelining, but the head of the loop instead). 6762 6763 Returns the maximal seqno found. */ 6764static int 6765init_seqno (bitmap blocks_to_reschedule, basic_block from) 6766{ 6767 bitmap_iterator bi; 6768 unsigned bbi; 6769 6770 auto_sbitmap visited_bbs (current_nr_blocks); 6771 6772 if (blocks_to_reschedule) 6773 { 6774 bitmap_ones (visited_bbs); 6775 EXECUTE_IF_SET_IN_BITMAP (blocks_to_reschedule, 0, bbi, bi) 6776 { 6777 gcc_assert (BLOCK_TO_BB (bbi) < current_nr_blocks); 6778 bitmap_clear_bit (visited_bbs, BLOCK_TO_BB (bbi)); 6779 } 6780 } 6781 else 6782 { 6783 bitmap_clear (visited_bbs); 6784 from = EBB_FIRST_BB (0); 6785 } 6786 6787 cur_seqno = sched_max_luid - 1; 6788 init_seqno_1 (from, visited_bbs, blocks_to_reschedule); 6789 6790 /* cur_seqno may be positive if the number of instructions is less than 6791 sched_max_luid - 1 (when rescheduling or if some instructions have been 6792 removed by the call to purge_empty_blocks in sel_sched_region_1). */ 6793 gcc_assert (cur_seqno >= 0); 6794 6795 return sched_max_luid - 1; 6796} 6797 6798/* Initialize scheduling parameters for current region. */ 6799static void 6800sel_setup_region_sched_flags (void) 6801{ 6802 enable_schedule_as_rhs_p = 1; 6803 bookkeeping_p = 1; 6804 pipelining_p = (bookkeeping_p 6805 && (flag_sel_sched_pipelining != 0) 6806 && current_loop_nest != NULL 6807 && loop_has_exit_edges (current_loop_nest)); 6808 max_insns_to_rename = param_selsched_insns_to_rename; 6809 max_ws = MAX_WS; 6810} 6811 6812/* Return true if all basic blocks of current region are empty. */ 6813static bool 6814current_region_empty_p (void) 6815{ 6816 int i; 6817 for (i = 0; i < current_nr_blocks; i++) 6818 if (! sel_bb_empty_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i)))) 6819 return false; 6820 6821 return true; 6822} 6823 6824/* Prepare and verify loop nest for pipelining. */ 6825static void 6826setup_current_loop_nest (int rgn, bb_vec_t *bbs) 6827{ 6828 current_loop_nest = get_loop_nest_for_rgn (rgn); 6829 6830 if (!current_loop_nest) 6831 return; 6832 6833 /* If this loop has any saved loop preheaders from nested loops, 6834 add these basic blocks to the current region. */ 6835 sel_add_loop_preheaders (bbs); 6836 6837 /* Check that we're starting with a valid information. */ 6838 gcc_assert (loop_latch_edge (current_loop_nest)); 6839 gcc_assert (LOOP_MARKED_FOR_PIPELINING_P (current_loop_nest)); 6840} 6841 6842/* Compute instruction priorities for current region. */ 6843static void 6844sel_compute_priorities (int rgn) 6845{ 6846 sched_rgn_compute_dependencies (rgn); 6847 6848 /* Compute insn priorities in haifa style. Then free haifa style 6849 dependencies that we've calculated for this. */ 6850 compute_priorities (); 6851 6852 if (sched_verbose >= 5) 6853 debug_rgn_dependencies (0); 6854 6855 free_rgn_deps (); 6856} 6857 6858/* Init scheduling data for RGN. Returns true when this region should not 6859 be scheduled. */ 6860static bool 6861sel_region_init (int rgn) 6862{ 6863 int i; 6864 bb_vec_t bbs; 6865 6866 rgn_setup_region (rgn); 6867 6868 /* Even if sched_is_disabled_for_current_region_p() is true, we still 6869 do region initialization here so the region can be bundled correctly, 6870 but we'll skip the scheduling in sel_sched_region (). */ 6871 if (current_region_empty_p ()) 6872 return true; 6873 6874 bbs.create (current_nr_blocks); 6875 6876 for (i = 0; i < current_nr_blocks; i++) 6877 bbs.quick_push (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i))); 6878 6879 sel_init_bbs (bbs); 6880 6881 if (flag_sel_sched_pipelining) 6882 setup_current_loop_nest (rgn, &bbs); 6883 6884 sel_setup_region_sched_flags (); 6885 6886 /* Initialize luids and dependence analysis which both sel-sched and haifa 6887 need. */ 6888 sched_init_luids (bbs); 6889 sched_deps_init (false); 6890 6891 /* Initialize haifa data. */ 6892 rgn_setup_sched_infos (); 6893 sel_set_sched_flags (); 6894 haifa_init_h_i_d (bbs); 6895 6896 sel_compute_priorities (rgn); 6897 init_deps_global (); 6898 6899 /* Main initialization. */ 6900 sel_setup_sched_infos (); 6901 sel_init_global_and_expr (bbs); 6902 6903 bbs.release (); 6904 6905 blocks_to_reschedule = BITMAP_ALLOC (NULL); 6906 6907 /* Init correct liveness sets on each instruction of a single-block loop. 6908 This is the only situation when we can't update liveness when calling 6909 compute_live for the first insn of the loop. */ 6910 if (current_loop_nest) 6911 { 6912 int header = 6913 (sel_is_loop_preheader_p (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (0))) 6914 ? 1 6915 : 0); 6916 6917 if (current_nr_blocks == header + 1) 6918 update_liveness_on_insn 6919 (sel_bb_head (BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (header)))); 6920 } 6921 6922 /* Set hooks so that no newly generated insn will go out unnoticed. */ 6923 sel_register_cfg_hooks (); 6924 6925 /* !!! We call target.sched.init () for the whole region, but we invoke 6926 targetm.sched.finish () for every ebb. */ 6927 if (targetm.sched.init) 6928 /* None of the arguments are actually used in any target. */ 6929 targetm.sched.init (sched_dump, sched_verbose, -1); 6930 6931 first_emitted_uid = get_max_uid () + 1; 6932 preheader_removed = false; 6933 6934 /* Reset register allocation ticks array. */ 6935 memset (reg_rename_tick, 0, sizeof reg_rename_tick); 6936 reg_rename_this_tick = 0; 6937 6938 forced_ebb_heads = BITMAP_ALLOC (NULL); 6939 6940 setup_nop_vinsn (); 6941 current_copies = BITMAP_ALLOC (NULL); 6942 current_originators = BITMAP_ALLOC (NULL); 6943 code_motion_visited_blocks = BITMAP_ALLOC (NULL); 6944 6945 return false; 6946} 6947 6948/* Simplify insns after the scheduling. */ 6949static void 6950simplify_changed_insns (void) 6951{ 6952 int i; 6953 6954 for (i = 0; i < current_nr_blocks; i++) 6955 { 6956 basic_block bb = BASIC_BLOCK_FOR_FN (cfun, BB_TO_BLOCK (i)); 6957 rtx_insn *insn; 6958 6959 FOR_BB_INSNS (bb, insn) 6960 if (INSN_P (insn)) 6961 { 6962 expr_t expr = INSN_EXPR (insn); 6963 6964 if (EXPR_WAS_SUBSTITUTED (expr)) 6965 validate_simplify_insn (insn); 6966 } 6967 } 6968} 6969 6970/* Find boundaries of the EBB starting from basic block BB, marking blocks of 6971 this EBB in SCHEDULED_BLOCKS and appropriately filling in HEAD, TAIL, 6972 PREV_HEAD, and NEXT_TAIL fields of CURRENT_SCHED_INFO structure. */ 6973static void 6974find_ebb_boundaries (basic_block bb, bitmap scheduled_blocks) 6975{ 6976 rtx_insn *head, *tail; 6977 basic_block bb1 = bb; 6978 if (sched_verbose >= 2) 6979 sel_print ("Finishing schedule in bbs: "); 6980 6981 do 6982 { 6983 bitmap_set_bit (scheduled_blocks, BLOCK_TO_BB (bb1->index)); 6984 6985 if (sched_verbose >= 2) 6986 sel_print ("%d; ", bb1->index); 6987 } 6988 while (!bb_ends_ebb_p (bb1) && (bb1 = bb_next_bb (bb1))); 6989 6990 if (sched_verbose >= 2) 6991 sel_print ("\n"); 6992 6993 get_ebb_head_tail (bb, bb1, &head, &tail); 6994 6995 current_sched_info->head = head; 6996 current_sched_info->tail = tail; 6997 current_sched_info->prev_head = PREV_INSN (head); 6998 current_sched_info->next_tail = NEXT_INSN (tail); 6999} 7000 7001/* Regenerate INSN_SCHED_CYCLEs for insns of current EBB. */ 7002static void 7003reset_sched_cycles_in_current_ebb (void) 7004{ 7005 int last_clock = 0; 7006 int haifa_last_clock = -1; 7007 int haifa_clock = 0; 7008 int issued_insns = 0; 7009 insn_t insn; 7010 7011 if (targetm.sched.init) 7012 { 7013 /* None of the arguments are actually used in any target. 7014 NB: We should have md_reset () hook for cases like this. */ 7015 targetm.sched.init (sched_dump, sched_verbose, -1); 7016 } 7017 7018 state_reset (curr_state); 7019 advance_state (curr_state); 7020 7021 for (insn = current_sched_info->head; 7022 insn != current_sched_info->next_tail; 7023 insn = NEXT_INSN (insn)) 7024 { 7025 int cost, haifa_cost; 7026 int sort_p; 7027 bool asm_p, real_insn, after_stall, all_issued; 7028 int clock; 7029 7030 if (!INSN_P (insn)) 7031 continue; 7032 7033 asm_p = false; 7034 real_insn = recog_memoized (insn) >= 0; 7035 clock = INSN_SCHED_CYCLE (insn); 7036 7037 cost = clock - last_clock; 7038 7039 /* Initialize HAIFA_COST. */ 7040 if (! real_insn) 7041 { 7042 asm_p = INSN_ASM_P (insn); 7043 7044 if (asm_p) 7045 /* This is asm insn which *had* to be scheduled first 7046 on the cycle. */ 7047 haifa_cost = 1; 7048 else 7049 /* This is a use/clobber insn. It should not change 7050 cost. */ 7051 haifa_cost = 0; 7052 } 7053 else 7054 haifa_cost = estimate_insn_cost (insn, curr_state); 7055 7056 /* Stall for whatever cycles we've stalled before. */ 7057 after_stall = 0; 7058 if (INSN_AFTER_STALL_P (insn) && cost > haifa_cost) 7059 { 7060 haifa_cost = cost; 7061 after_stall = 1; 7062 } 7063 all_issued = issued_insns == issue_rate; 7064 if (haifa_cost == 0 && all_issued) 7065 haifa_cost = 1; 7066 if (haifa_cost > 0) 7067 { 7068 int i = 0; 7069 7070 while (haifa_cost--) 7071 { 7072 advance_state (curr_state); 7073 issued_insns = 0; 7074 i++; 7075 7076 if (sched_verbose >= 2) 7077 { 7078 sel_print ("advance_state (state_transition)\n"); 7079 debug_state (curr_state); 7080 } 7081 7082 /* The DFA may report that e.g. insn requires 2 cycles to be 7083 issued, but on the next cycle it says that insn is ready 7084 to go. Check this here. */ 7085 if (!after_stall 7086 && real_insn 7087 && haifa_cost > 0 7088 && estimate_insn_cost (insn, curr_state) == 0) 7089 break; 7090 7091 /* When the data dependency stall is longer than the DFA stall, 7092 and when we have issued exactly issue_rate insns and stalled, 7093 it could be that after this longer stall the insn will again 7094 become unavailable to the DFA restrictions. Looks strange 7095 but happens e.g. on x86-64. So recheck DFA on the last 7096 iteration. */ 7097 if ((after_stall || all_issued) 7098 && real_insn 7099 && haifa_cost == 0) 7100 haifa_cost = estimate_insn_cost (insn, curr_state); 7101 } 7102 7103 haifa_clock += i; 7104 if (sched_verbose >= 2) 7105 sel_print ("haifa clock: %d\n", haifa_clock); 7106 } 7107 else 7108 gcc_assert (haifa_cost == 0); 7109 7110 if (sched_verbose >= 2) 7111 sel_print ("Haifa cost for insn %d: %d\n", INSN_UID (insn), haifa_cost); 7112 7113 if (targetm.sched.dfa_new_cycle) 7114 while (targetm.sched.dfa_new_cycle (sched_dump, sched_verbose, insn, 7115 haifa_last_clock, haifa_clock, 7116 &sort_p)) 7117 { 7118 advance_state (curr_state); 7119 issued_insns = 0; 7120 haifa_clock++; 7121 if (sched_verbose >= 2) 7122 { 7123 sel_print ("advance_state (dfa_new_cycle)\n"); 7124 debug_state (curr_state); 7125 sel_print ("haifa clock: %d\n", haifa_clock + 1); 7126 } 7127 } 7128 7129 if (real_insn) 7130 { 7131 static state_t temp = NULL; 7132 7133 if (!temp) 7134 temp = xmalloc (dfa_state_size); 7135 memcpy (temp, curr_state, dfa_state_size); 7136 7137 cost = state_transition (curr_state, insn); 7138 if (memcmp (temp, curr_state, dfa_state_size)) 7139 issued_insns++; 7140 7141 if (sched_verbose >= 2) 7142 { 7143 sel_print ("scheduled insn %d, clock %d\n", INSN_UID (insn), 7144 haifa_clock + 1); 7145 debug_state (curr_state); 7146 } 7147 gcc_assert (cost < 0); 7148 } 7149 7150 if (targetm.sched.variable_issue) 7151 targetm.sched.variable_issue (sched_dump, sched_verbose, insn, 0); 7152 7153 INSN_SCHED_CYCLE (insn) = haifa_clock; 7154 7155 last_clock = clock; 7156 haifa_last_clock = haifa_clock; 7157 } 7158} 7159 7160/* Put TImode markers on insns starting a new issue group. */ 7161static void 7162put_TImodes (void) 7163{ 7164 int last_clock = -1; 7165 insn_t insn; 7166 7167 for (insn = current_sched_info->head; insn != current_sched_info->next_tail; 7168 insn = NEXT_INSN (insn)) 7169 { 7170 int cost, clock; 7171 7172 if (!INSN_P (insn)) 7173 continue; 7174 7175 clock = INSN_SCHED_CYCLE (insn); 7176 cost = (last_clock == -1) ? 1 : clock - last_clock; 7177 7178 gcc_assert (cost >= 0); 7179 7180 if (issue_rate > 1 7181 && GET_CODE (PATTERN (insn)) != USE 7182 && GET_CODE (PATTERN (insn)) != CLOBBER) 7183 { 7184 if (reload_completed && cost > 0) 7185 PUT_MODE (insn, TImode); 7186 7187 last_clock = clock; 7188 } 7189 7190 if (sched_verbose >= 2) 7191 sel_print ("Cost for insn %d is %d\n", INSN_UID (insn), cost); 7192 } 7193} 7194 7195/* Perform MD_FINISH on EBBs comprising current region. When 7196 RESET_SCHED_CYCLES_P is true, run a pass emulating the scheduler 7197 to produce correct sched cycles on insns. */ 7198static void 7199sel_region_target_finish (bool reset_sched_cycles_p) 7200{ 7201 int i; 7202 bitmap scheduled_blocks = BITMAP_ALLOC (NULL); 7203 7204 for (i = 0; i < current_nr_blocks; i++) 7205 { 7206 if (bitmap_bit_p (scheduled_blocks, i)) 7207 continue; 7208 7209 /* While pipelining outer loops, skip bundling for loop 7210 preheaders. Those will be rescheduled in the outer loop. */ 7211 if (sel_is_loop_preheader_p (EBB_FIRST_BB (i))) 7212 continue; 7213 7214 find_ebb_boundaries (EBB_FIRST_BB (i), scheduled_blocks); 7215 7216 if (no_real_insns_p (current_sched_info->head, current_sched_info->tail)) 7217 continue; 7218 7219 if (reset_sched_cycles_p) 7220 reset_sched_cycles_in_current_ebb (); 7221 7222 if (targetm.sched.init) 7223 targetm.sched.init (sched_dump, sched_verbose, -1); 7224 7225 put_TImodes (); 7226 7227 if (targetm.sched.finish) 7228 { 7229 targetm.sched.finish (sched_dump, sched_verbose); 7230 7231 /* Extend luids so that insns generated by the target will 7232 get zero luid. */ 7233 sched_extend_luids (); 7234 } 7235 } 7236 7237 BITMAP_FREE (scheduled_blocks); 7238} 7239 7240/* Free the scheduling data for the current region. When RESET_SCHED_CYCLES_P 7241 is true, make an additional pass emulating scheduler to get correct insn 7242 cycles for md_finish calls. */ 7243static void 7244sel_region_finish (bool reset_sched_cycles_p) 7245{ 7246 simplify_changed_insns (); 7247 sched_finish_ready_list (); 7248 free_nop_pool (); 7249 7250 /* Free the vectors. */ 7251 vec_av_set.release (); 7252 BITMAP_FREE (current_copies); 7253 BITMAP_FREE (current_originators); 7254 BITMAP_FREE (code_motion_visited_blocks); 7255 vinsn_vec_free (vec_bookkeeping_blocked_vinsns); 7256 vinsn_vec_free (vec_target_unavailable_vinsns); 7257 7258 /* If LV_SET of the region head should be updated, do it now because 7259 there will be no other chance. */ 7260 { 7261 succ_iterator si; 7262 insn_t insn; 7263 7264 FOR_EACH_SUCC_1 (insn, si, bb_note (EBB_FIRST_BB (0)), 7265 SUCCS_NORMAL | SUCCS_SKIP_TO_LOOP_EXITS) 7266 { 7267 basic_block bb = BLOCK_FOR_INSN (insn); 7268 7269 if (!BB_LV_SET_VALID_P (bb)) 7270 compute_live (insn); 7271 } 7272 } 7273 7274 /* Emulate the Haifa scheduler for bundling. */ 7275 if (reload_completed) 7276 sel_region_target_finish (reset_sched_cycles_p); 7277 7278 sel_finish_global_and_expr (); 7279 7280 BITMAP_FREE (forced_ebb_heads); 7281 7282 free_nop_vinsn (); 7283 7284 finish_deps_global (); 7285 sched_finish_luids (); 7286 h_d_i_d.release (); 7287 7288 sel_finish_bbs (); 7289 BITMAP_FREE (blocks_to_reschedule); 7290 7291 sel_unregister_cfg_hooks (); 7292 7293 max_issue_size = 0; 7294} 7295 7296 7297/* Functions that implement the scheduler driver. */ 7298 7299/* Schedule a parallel instruction group on each of FENCES. MAX_SEQNO 7300 is the current maximum seqno. SCHEDULED_INSNS_TAILPP is the list 7301 of insns scheduled -- these would be postprocessed later. */ 7302static void 7303schedule_on_fences (flist_t fences, int max_seqno, 7304 ilist_t **scheduled_insns_tailpp) 7305{ 7306 flist_t old_fences = fences; 7307 7308 if (sched_verbose >= 1) 7309 { 7310 sel_print ("\nScheduling on fences: "); 7311 dump_flist (fences); 7312 sel_print ("\n"); 7313 } 7314 7315 scheduled_something_on_previous_fence = false; 7316 for (; fences; fences = FLIST_NEXT (fences)) 7317 { 7318 fence_t fence = NULL; 7319 int seqno = 0; 7320 flist_t fences2; 7321 bool first_p = true; 7322 7323 /* Choose the next fence group to schedule. 7324 The fact that insn can be scheduled only once 7325 on the cycle is guaranteed by two properties: 7326 1. seqnos of parallel groups decrease with each iteration. 7327 2. If is_ineligible_successor () sees the larger seqno, it 7328 checks if candidate insn is_in_current_fence_p (). */ 7329 for (fences2 = old_fences; fences2; fences2 = FLIST_NEXT (fences2)) 7330 { 7331 fence_t f = FLIST_FENCE (fences2); 7332 7333 if (!FENCE_PROCESSED_P (f)) 7334 { 7335 int i = INSN_SEQNO (FENCE_INSN (f)); 7336 7337 if (first_p || i > seqno) 7338 { 7339 seqno = i; 7340 fence = f; 7341 first_p = false; 7342 } 7343 else 7344 /* ??? Seqnos of different groups should be different. */ 7345 gcc_assert (1 || i != seqno); 7346 } 7347 } 7348 7349 gcc_assert (fence); 7350 7351 /* As FENCE is nonnull, SEQNO is initialized. */ 7352 seqno -= max_seqno + 1; 7353 fill_insns (fence, seqno, scheduled_insns_tailpp); 7354 FENCE_PROCESSED_P (fence) = true; 7355 } 7356 7357 /* All av_sets are invalidated by GLOBAL_LEVEL increase, thus we 7358 don't need to keep bookkeeping-invalidated and target-unavailable 7359 vinsns any more. */ 7360 vinsn_vec_clear (&vec_bookkeeping_blocked_vinsns); 7361 vinsn_vec_clear (&vec_target_unavailable_vinsns); 7362} 7363 7364/* Calculate MIN_SEQNO and MAX_SEQNO. */ 7365static void 7366find_min_max_seqno (flist_t fences, int *min_seqno, int *max_seqno) 7367{ 7368 *min_seqno = *max_seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences))); 7369 7370 /* The first element is already processed. */ 7371 while ((fences = FLIST_NEXT (fences))) 7372 { 7373 int seqno = INSN_SEQNO (FENCE_INSN (FLIST_FENCE (fences))); 7374 7375 if (*min_seqno > seqno) 7376 *min_seqno = seqno; 7377 else if (*max_seqno < seqno) 7378 *max_seqno = seqno; 7379 } 7380} 7381 7382/* Calculate new fences from FENCES. Write the current time to PTIME. */ 7383static flist_t 7384calculate_new_fences (flist_t fences, int orig_max_seqno, int *ptime) 7385{ 7386 flist_t old_fences = fences; 7387 struct flist_tail_def _new_fences, *new_fences = &_new_fences; 7388 int max_time = 0; 7389 7390 flist_tail_init (new_fences); 7391 for (; fences; fences = FLIST_NEXT (fences)) 7392 { 7393 fence_t fence = FLIST_FENCE (fences); 7394 insn_t insn; 7395 7396 if (!FENCE_BNDS (fence)) 7397 { 7398 /* This fence doesn't have any successors. */ 7399 if (!FENCE_SCHEDULED_P (fence)) 7400 { 7401 /* Nothing was scheduled on this fence. */ 7402 int seqno; 7403 7404 insn = FENCE_INSN (fence); 7405 seqno = INSN_SEQNO (insn); 7406 gcc_assert (seqno > 0 && seqno <= orig_max_seqno); 7407 7408 if (sched_verbose >= 1) 7409 sel_print ("Fence %d[%d] has not changed\n", 7410 INSN_UID (insn), 7411 BLOCK_NUM (insn)); 7412 move_fence_to_fences (fences, new_fences); 7413 } 7414 } 7415 else 7416 extract_new_fences_from (fences, new_fences, orig_max_seqno); 7417 max_time = MAX (max_time, FENCE_CYCLE (fence)); 7418 } 7419 7420 flist_clear (&old_fences); 7421 *ptime = max_time; 7422 return FLIST_TAIL_HEAD (new_fences); 7423} 7424 7425/* Update seqnos of insns given by PSCHEDULED_INSNS. MIN_SEQNO and MAX_SEQNO 7426 are the miminum and maximum seqnos of the group, HIGHEST_SEQNO_IN_USE is 7427 the highest seqno used in a region. Return the updated highest seqno. */ 7428static int 7429update_seqnos_and_stage (int min_seqno, int max_seqno, 7430 int highest_seqno_in_use, 7431 ilist_t *pscheduled_insns) 7432{ 7433 int new_hs; 7434 ilist_iterator ii; 7435 insn_t insn; 7436 7437 /* Actually, new_hs is the seqno of the instruction, that was 7438 scheduled first (i.e. it is the first one in SCHEDULED_INSNS). */ 7439 if (*pscheduled_insns) 7440 { 7441 new_hs = (INSN_SEQNO (ILIST_INSN (*pscheduled_insns)) 7442 + highest_seqno_in_use + max_seqno - min_seqno + 2); 7443 gcc_assert (new_hs > highest_seqno_in_use); 7444 } 7445 else 7446 new_hs = highest_seqno_in_use; 7447 7448 FOR_EACH_INSN (insn, ii, *pscheduled_insns) 7449 { 7450 gcc_assert (INSN_SEQNO (insn) < 0); 7451 INSN_SEQNO (insn) += highest_seqno_in_use + max_seqno - min_seqno + 2; 7452 gcc_assert (INSN_SEQNO (insn) <= new_hs); 7453 7454 /* When not pipelining, purge unneeded insn info on the scheduled insns. 7455 For example, having reg_last array of INSN_DEPS_CONTEXT in memory may 7456 require > 1GB of memory e.g. on limit-fnargs.c. */ 7457 if (! pipelining_p) 7458 free_data_for_scheduled_insn (insn); 7459 } 7460 7461 ilist_clear (pscheduled_insns); 7462 global_level++; 7463 7464 return new_hs; 7465} 7466 7467/* The main driver for scheduling a region. This function is responsible 7468 for correct propagation of fences (i.e. scheduling points) and creating 7469 a group of parallel insns at each of them. It also supports 7470 pipelining. ORIG_MAX_SEQNO is the maximal seqno before this pass 7471 of scheduling. */ 7472static void 7473sel_sched_region_2 (int orig_max_seqno) 7474{ 7475 int highest_seqno_in_use = orig_max_seqno; 7476 int max_time = 0; 7477 7478 stat_bookkeeping_copies = 0; 7479 stat_insns_needed_bookkeeping = 0; 7480 stat_renamed_scheduled = 0; 7481 stat_substitutions_total = 0; 7482 num_insns_scheduled = 0; 7483 7484 while (fences) 7485 { 7486 int min_seqno, max_seqno; 7487 ilist_t scheduled_insns = NULL; 7488 ilist_t *scheduled_insns_tailp = &scheduled_insns; 7489 7490 find_min_max_seqno (fences, &min_seqno, &max_seqno); 7491 schedule_on_fences (fences, max_seqno, &scheduled_insns_tailp); 7492 fences = calculate_new_fences (fences, orig_max_seqno, &max_time); 7493 highest_seqno_in_use = update_seqnos_and_stage (min_seqno, max_seqno, 7494 highest_seqno_in_use, 7495 &scheduled_insns); 7496 } 7497 7498 if (sched_verbose >= 1) 7499 { 7500 sel_print ("Total scheduling time: %d cycles\n", max_time); 7501 sel_print ("Scheduled %d bookkeeping copies, %d insns needed " 7502 "bookkeeping, %d insns renamed, %d insns substituted\n", 7503 stat_bookkeeping_copies, 7504 stat_insns_needed_bookkeeping, 7505 stat_renamed_scheduled, 7506 stat_substitutions_total); 7507 } 7508} 7509 7510/* Schedule a region. When pipelining, search for possibly never scheduled 7511 bookkeeping code and schedule it. Reschedule pipelined code without 7512 pipelining after. */ 7513static void 7514sel_sched_region_1 (void) 7515{ 7516 int orig_max_seqno; 7517 7518 /* Remove empty blocks that might be in the region from the beginning. */ 7519 purge_empty_blocks (); 7520 7521 orig_max_seqno = init_seqno (NULL, NULL); 7522 gcc_assert (orig_max_seqno >= 1); 7523 7524 /* When pipelining outer loops, create fences on the loop header, 7525 not preheader. */ 7526 fences = NULL; 7527 if (current_loop_nest) 7528 init_fences (BB_END (EBB_FIRST_BB (0))); 7529 else 7530 init_fences (bb_note (EBB_FIRST_BB (0))); 7531 global_level = 1; 7532 7533 sel_sched_region_2 (orig_max_seqno); 7534 7535 gcc_assert (fences == NULL); 7536 7537 if (pipelining_p) 7538 { 7539 int i; 7540 basic_block bb; 7541 struct flist_tail_def _new_fences; 7542 flist_tail_t new_fences = &_new_fences; 7543 bool do_p = true; 7544 7545 pipelining_p = false; 7546 max_ws = MIN (max_ws, issue_rate * 3 / 2); 7547 bookkeeping_p = false; 7548 enable_schedule_as_rhs_p = false; 7549 7550 /* Schedule newly created code, that has not been scheduled yet. */ 7551 do_p = true; 7552 7553 while (do_p) 7554 { 7555 do_p = false; 7556 7557 for (i = 0; i < current_nr_blocks; i++) 7558 { 7559 basic_block bb = EBB_FIRST_BB (i); 7560 7561 if (bitmap_bit_p (blocks_to_reschedule, bb->index)) 7562 { 7563 if (! bb_ends_ebb_p (bb)) 7564 bitmap_set_bit (blocks_to_reschedule, bb_next_bb (bb)->index); 7565 if (sel_bb_empty_p (bb)) 7566 { 7567 bitmap_clear_bit (blocks_to_reschedule, bb->index); 7568 continue; 7569 } 7570 clear_outdated_rtx_info (bb); 7571 if (sel_insn_is_speculation_check (BB_END (bb)) 7572 && JUMP_P (BB_END (bb))) 7573 bitmap_set_bit (blocks_to_reschedule, 7574 BRANCH_EDGE (bb)->dest->index); 7575 } 7576 else if (! sel_bb_empty_p (bb) 7577 && INSN_SCHED_TIMES (sel_bb_head (bb)) <= 0) 7578 bitmap_set_bit (blocks_to_reschedule, bb->index); 7579 } 7580 7581 for (i = 0; i < current_nr_blocks; i++) 7582 { 7583 bb = EBB_FIRST_BB (i); 7584 7585 /* While pipelining outer loops, skip bundling for loop 7586 preheaders. Those will be rescheduled in the outer 7587 loop. */ 7588 if (sel_is_loop_preheader_p (bb)) 7589 { 7590 clear_outdated_rtx_info (bb); 7591 continue; 7592 } 7593 7594 if (bitmap_bit_p (blocks_to_reschedule, bb->index)) 7595 { 7596 flist_tail_init (new_fences); 7597 7598 orig_max_seqno = init_seqno (blocks_to_reschedule, bb); 7599 7600 /* Mark BB as head of the new ebb. */ 7601 bitmap_set_bit (forced_ebb_heads, bb->index); 7602 7603 gcc_assert (fences == NULL); 7604 7605 init_fences (bb_note (bb)); 7606 7607 sel_sched_region_2 (orig_max_seqno); 7608 7609 do_p = true; 7610 break; 7611 } 7612 } 7613 } 7614 } 7615} 7616 7617/* Schedule the RGN region. */ 7618void 7619sel_sched_region (int rgn) 7620{ 7621 bool schedule_p; 7622 bool reset_sched_cycles_p; 7623 7624 if (sel_region_init (rgn)) 7625 return; 7626 7627 if (sched_verbose >= 1) 7628 sel_print ("Scheduling region %d\n", rgn); 7629 7630 schedule_p = (!sched_is_disabled_for_current_region_p () 7631 && dbg_cnt (sel_sched_region_cnt)); 7632 reset_sched_cycles_p = pipelining_p; 7633 if (schedule_p) 7634 sel_sched_region_1 (); 7635 else 7636 { 7637 /* Schedule always selecting the next insn to make the correct data 7638 for bundling or other later passes. */ 7639 pipelining_p = false; 7640 reset_sched_cycles_p = false; 7641 force_next_insn = 1; 7642 sel_sched_region_1 (); 7643 force_next_insn = 0; 7644 } 7645 sel_region_finish (reset_sched_cycles_p); 7646} 7647 7648/* Perform global init for the scheduler. */ 7649static void 7650sel_global_init (void) 7651{ 7652 /* Remove empty blocks: their presence can break assumptions elsewhere, 7653 e.g. the logic to invoke update_liveness_on_insn in sel_region_init. */ 7654 cleanup_cfg (0); 7655 7656 calculate_dominance_info (CDI_DOMINATORS); 7657 alloc_sched_pools (); 7658 7659 /* Setup the infos for sched_init. */ 7660 sel_setup_sched_infos (); 7661 setup_sched_dump (); 7662 7663 sched_rgn_init (false); 7664 sched_init (); 7665 7666 sched_init_bbs (); 7667 /* Reset AFTER_RECOVERY if it has been set by the 1st scheduler pass. */ 7668 after_recovery = 0; 7669 can_issue_more = issue_rate; 7670 7671 sched_extend_target (); 7672 sched_deps_init (true); 7673 setup_nop_and_exit_insns (); 7674 sel_extend_global_bb_info (); 7675 init_lv_sets (); 7676 init_hard_regs_data (); 7677} 7678 7679/* Free the global data of the scheduler. */ 7680static void 7681sel_global_finish (void) 7682{ 7683 free_bb_note_pool (); 7684 free_lv_sets (); 7685 sel_finish_global_bb_info (); 7686 7687 free_regset_pool (); 7688 free_nop_and_exit_insns (); 7689 7690 sched_rgn_finish (); 7691 sched_deps_finish (); 7692 sched_finish (); 7693 7694 if (current_loops) 7695 sel_finish_pipelining (); 7696 7697 free_sched_pools (); 7698 free_dominance_info (CDI_DOMINATORS); 7699} 7700 7701/* Return true when we need to skip selective scheduling. Used for debugging. */ 7702bool 7703maybe_skip_selective_scheduling (void) 7704{ 7705 return ! dbg_cnt (sel_sched_cnt); 7706} 7707 7708/* The entry point. */ 7709void 7710run_selective_scheduling (void) 7711{ 7712 int rgn; 7713 7714 if (n_basic_blocks_for_fn (cfun) == NUM_FIXED_BLOCKS) 7715 return; 7716 7717 sel_global_init (); 7718 7719 for (rgn = 0; rgn < nr_regions; rgn++) 7720 sel_sched_region (rgn); 7721 7722 sel_global_finish (); 7723} 7724 7725#endif 7726