1/* Lowering pass for OMP directives. Converts OMP directives into explicit 2 calls to the runtime library (libgomp), data marshalling to implement data 3 sharing and copying clauses, offloading to accelerators, and more. 4 5 Contributed by Diego Novillo <dnovillo@redhat.com> 6 7 Copyright (C) 2005-2015 Free Software Foundation, Inc. 8 9This file is part of GCC. 10 11GCC is free software; you can redistribute it and/or modify it under 12the terms of the GNU General Public License as published by the Free 13Software Foundation; either version 3, or (at your option) any later 14version. 15 16GCC is distributed in the hope that it will be useful, but WITHOUT ANY 17WARRANTY; without even the implied warranty of MERCHANTABILITY or 18FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 19for more details. 20 21You should have received a copy of the GNU General Public License 22along with GCC; see the file COPYING3. If not see 23<http://www.gnu.org/licenses/>. */ 24 25#include "config.h" 26#include "system.h" 27#include "coretypes.h" 28#include "tm.h" 29#include "hash-set.h" 30#include "machmode.h" 31#include "vec.h" 32#include "double-int.h" 33#include "input.h" 34#include "alias.h" 35#include "symtab.h" 36#include "wide-int.h" 37#include "inchash.h" 38#include "tree.h" 39#include "fold-const.h" 40#include "stringpool.h" 41#include "stor-layout.h" 42#include "rtl.h" 43#include "predict.h" 44#include "hard-reg-set.h" 45#include "function.h" 46#include "dominance.h" 47#include "cfg.h" 48#include "cfganal.h" 49#include "basic-block.h" 50#include "tree-ssa-alias.h" 51#include "internal-fn.h" 52#include "gimple-fold.h" 53#include "gimple-expr.h" 54#include "is-a.h" 55#include "gimple.h" 56#include "gimplify.h" 57#include "gimple-iterator.h" 58#include "gimplify-me.h" 59#include "gimple-walk.h" 60#include "tree-iterator.h" 61#include "tree-inline.h" 62#include "langhooks.h" 63#include "diagnostic-core.h" 64#include "gimple-ssa.h" 65#include "hash-map.h" 66#include "plugin-api.h" 67#include "ipa-ref.h" 68#include "cgraph.h" 69#include "tree-cfg.h" 70#include "tree-phinodes.h" 71#include "ssa-iterators.h" 72#include "tree-ssanames.h" 73#include "tree-into-ssa.h" 74#include "hashtab.h" 75#include "flags.h" 76#include "statistics.h" 77#include "real.h" 78#include "fixed-value.h" 79#include "insn-config.h" 80#include "expmed.h" 81#include "dojump.h" 82#include "explow.h" 83#include "calls.h" 84#include "emit-rtl.h" 85#include "varasm.h" 86#include "stmt.h" 87#include "expr.h" 88#include "tree-dfa.h" 89#include "tree-ssa.h" 90#include "tree-pass.h" 91#include "except.h" 92#include "splay-tree.h" 93#include "insn-codes.h" 94#include "optabs.h" 95#include "cfgloop.h" 96#include "target.h" 97#include "common/common-target.h" 98#include "omp-low.h" 99#include "gimple-low.h" 100#include "tree-cfgcleanup.h" 101#include "pretty-print.h" 102#include "alloc-pool.h" 103#include "symbol-summary.h" 104#include "ipa-prop.h" 105#include "tree-nested.h" 106#include "tree-eh.h" 107#include "cilk.h" 108#include "context.h" 109#include "lto-section-names.h" 110#include "gomp-constants.h" 111 112 113/* Lowering of OMP parallel and workshare constructs proceeds in two 114 phases. The first phase scans the function looking for OMP statements 115 and then for variables that must be replaced to satisfy data sharing 116 clauses. The second phase expands code for the constructs, as well as 117 re-gimplifying things when variables have been replaced with complex 118 expressions. 119 120 Final code generation is done by pass_expand_omp. The flowgraph is 121 scanned for regions which are then moved to a new 122 function, to be invoked by the thread library, or offloaded. */ 123 124/* OMP region information. Every parallel and workshare 125 directive is enclosed between two markers, the OMP_* directive 126 and a corresponding OMP_RETURN statement. */ 127 128struct omp_region 129{ 130 /* The enclosing region. */ 131 struct omp_region *outer; 132 133 /* First child region. */ 134 struct omp_region *inner; 135 136 /* Next peer region. */ 137 struct omp_region *next; 138 139 /* Block containing the omp directive as its last stmt. */ 140 basic_block entry; 141 142 /* Block containing the OMP_RETURN as its last stmt. */ 143 basic_block exit; 144 145 /* Block containing the OMP_CONTINUE as its last stmt. */ 146 basic_block cont; 147 148 /* If this is a combined parallel+workshare region, this is a list 149 of additional arguments needed by the combined parallel+workshare 150 library call. */ 151 vec<tree, va_gc> *ws_args; 152 153 /* The code for the omp directive of this region. */ 154 enum gimple_code type; 155 156 /* Schedule kind, only used for OMP_FOR type regions. */ 157 enum omp_clause_schedule_kind sched_kind; 158 159 /* True if this is a combined parallel+workshare region. */ 160 bool is_combined_parallel; 161}; 162 163/* Levels of parallelism as defined by OpenACC. Increasing numbers 164 correspond to deeper loop nesting levels. */ 165#define MASK_GANG 1 166#define MASK_WORKER 2 167#define MASK_VECTOR 4 168 169/* Context structure. Used to store information about each parallel 170 directive in the code. */ 171 172typedef struct omp_context 173{ 174 /* This field must be at the beginning, as we do "inheritance": Some 175 callback functions for tree-inline.c (e.g., omp_copy_decl) 176 receive a copy_body_data pointer that is up-casted to an 177 omp_context pointer. */ 178 copy_body_data cb; 179 180 /* The tree of contexts corresponding to the encountered constructs. */ 181 struct omp_context *outer; 182 gimple stmt; 183 184 /* Map variables to fields in a structure that allows communication 185 between sending and receiving threads. */ 186 splay_tree field_map; 187 tree record_type; 188 tree sender_decl; 189 tree receiver_decl; 190 191 /* These are used just by task contexts, if task firstprivate fn is 192 needed. srecord_type is used to communicate from the thread 193 that encountered the task construct to task firstprivate fn, 194 record_type is allocated by GOMP_task, initialized by task firstprivate 195 fn and passed to the task body fn. */ 196 splay_tree sfield_map; 197 tree srecord_type; 198 199 /* A chain of variables to add to the top-level block surrounding the 200 construct. In the case of a parallel, this is in the child function. */ 201 tree block_vars; 202 203 /* A map of reduction pointer variables. For accelerators, each 204 reduction variable is replaced with an array. Each thread, in turn, 205 is assigned to a slot on that array. */ 206 splay_tree reduction_map; 207 208 /* Label to which GOMP_cancel{,llation_point} and explicit and implicit 209 barriers should jump to during omplower pass. */ 210 tree cancel_label; 211 212 /* What to do with variables with implicitly determined sharing 213 attributes. */ 214 enum omp_clause_default_kind default_kind; 215 216 /* Nesting depth of this context. Used to beautify error messages re 217 invalid gotos. The outermost ctx is depth 1, with depth 0 being 218 reserved for the main body of the function. */ 219 int depth; 220 221 /* True if this parallel directive is nested within another. */ 222 bool is_nested; 223 224 /* True if this construct can be cancelled. */ 225 bool cancellable; 226 227 /* For OpenACC loops, a mask of gang, worker and vector used at 228 levels below this one. */ 229 int gwv_below; 230 /* For OpenACC loops, a mask of gang, worker and vector used at 231 this level and above. For parallel and kernels clauses, a mask 232 indicating which of num_gangs/num_workers/num_vectors was used. */ 233 int gwv_this; 234} omp_context; 235 236/* A structure holding the elements of: 237 for (V = N1; V cond N2; V += STEP) [...] */ 238 239struct omp_for_data_loop 240{ 241 tree v, n1, n2, step; 242 enum tree_code cond_code; 243}; 244 245/* A structure describing the main elements of a parallel loop. */ 246 247struct omp_for_data 248{ 249 struct omp_for_data_loop loop; 250 tree chunk_size; 251 gomp_for *for_stmt; 252 tree pre, iter_type; 253 int collapse; 254 bool have_nowait, have_ordered; 255 enum omp_clause_schedule_kind sched_kind; 256 struct omp_for_data_loop *loops; 257}; 258 259 260static splay_tree all_contexts; 261static int taskreg_nesting_level; 262static int target_nesting_level; 263static struct omp_region *root_omp_region; 264static bitmap task_shared_vars; 265static vec<omp_context *> taskreg_contexts; 266 267static void scan_omp (gimple_seq *, omp_context *); 268static tree scan_omp_1_op (tree *, int *, void *); 269 270#define WALK_SUBSTMTS \ 271 case GIMPLE_BIND: \ 272 case GIMPLE_TRY: \ 273 case GIMPLE_CATCH: \ 274 case GIMPLE_EH_FILTER: \ 275 case GIMPLE_TRANSACTION: \ 276 /* The sub-statements for these should be walked. */ \ 277 *handled_ops_p = false; \ 278 break; 279 280/* Helper function to get the name of the array containing the partial 281 reductions for OpenACC reductions. */ 282static const char * 283oacc_get_reduction_array_id (tree node) 284{ 285 const char *id = IDENTIFIER_POINTER (DECL_NAME (node)); 286 int len = strlen ("OACC") + strlen (id); 287 char *temp_name = XALLOCAVEC (char, len + 1); 288 snprintf (temp_name, len + 1, "OACC%s", id); 289 return IDENTIFIER_POINTER (get_identifier (temp_name)); 290} 291 292/* Determine the number of threads OpenACC threads used to determine the 293 size of the array of partial reductions. Currently, this is num_gangs 294 * vector_length. This value may be different than GOACC_GET_NUM_THREADS, 295 because it is independed of the device used. */ 296 297static tree 298oacc_max_threads (omp_context *ctx) 299{ 300 tree nthreads, vector_length, gangs, clauses; 301 302 gangs = fold_convert (sizetype, integer_one_node); 303 vector_length = gangs; 304 305 /* The reduction clause may be nested inside a loop directive. 306 Scan for the innermost vector_length clause. */ 307 for (omp_context *oc = ctx; oc; oc = oc->outer) 308 { 309 if (gimple_code (oc->stmt) != GIMPLE_OMP_TARGET 310 || (gimple_omp_target_kind (oc->stmt) 311 != GF_OMP_TARGET_KIND_OACC_PARALLEL)) 312 continue; 313 314 clauses = gimple_omp_target_clauses (oc->stmt); 315 316 vector_length = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); 317 if (vector_length) 318 vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (vector_length), 319 sizetype, 320 OMP_CLAUSE_VECTOR_LENGTH_EXPR 321 (vector_length)); 322 else 323 vector_length = fold_convert (sizetype, integer_one_node); 324 325 gangs = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); 326 if (gangs) 327 gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (gangs), sizetype, 328 OMP_CLAUSE_NUM_GANGS_EXPR (gangs)); 329 else 330 gangs = fold_convert (sizetype, integer_one_node); 331 332 break; 333 } 334 335 nthreads = fold_build2 (MULT_EXPR, sizetype, gangs, vector_length); 336 337 return nthreads; 338} 339 340/* Holds offload tables with decls. */ 341vec<tree, va_gc> *offload_funcs, *offload_vars; 342 343/* Convenience function for calling scan_omp_1_op on tree operands. */ 344 345static inline tree 346scan_omp_op (tree *tp, omp_context *ctx) 347{ 348 struct walk_stmt_info wi; 349 350 memset (&wi, 0, sizeof (wi)); 351 wi.info = ctx; 352 wi.want_locations = true; 353 354 return walk_tree (tp, scan_omp_1_op, &wi, NULL); 355} 356 357static void lower_omp (gimple_seq *, omp_context *); 358static tree lookup_decl_in_outer_ctx (tree, omp_context *); 359static tree maybe_lookup_decl_in_outer_ctx (tree, omp_context *); 360 361/* Find an OMP clause of type KIND within CLAUSES. */ 362 363tree 364find_omp_clause (tree clauses, enum omp_clause_code kind) 365{ 366 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) 367 if (OMP_CLAUSE_CODE (clauses) == kind) 368 return clauses; 369 370 return NULL_TREE; 371} 372 373/* Return true if CTX is for an omp parallel. */ 374 375static inline bool 376is_parallel_ctx (omp_context *ctx) 377{ 378 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL; 379} 380 381 382/* Return true if CTX is for an omp task. */ 383 384static inline bool 385is_task_ctx (omp_context *ctx) 386{ 387 return gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; 388} 389 390 391/* Return true if CTX is for an omp parallel or omp task. */ 392 393static inline bool 394is_taskreg_ctx (omp_context *ctx) 395{ 396 return gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL 397 || gimple_code (ctx->stmt) == GIMPLE_OMP_TASK; 398} 399 400 401/* Return true if REGION is a combined parallel+workshare region. */ 402 403static inline bool 404is_combined_parallel (struct omp_region *region) 405{ 406 return region->is_combined_parallel; 407} 408 409 410/* Extract the header elements of parallel loop FOR_STMT and store 411 them into *FD. */ 412 413static void 414extract_omp_for_data (gomp_for *for_stmt, struct omp_for_data *fd, 415 struct omp_for_data_loop *loops) 416{ 417 tree t, var, *collapse_iter, *collapse_count; 418 tree count = NULL_TREE, iter_type = long_integer_type_node; 419 struct omp_for_data_loop *loop; 420 int i; 421 struct omp_for_data_loop dummy_loop; 422 location_t loc = gimple_location (for_stmt); 423 bool simd = gimple_omp_for_kind (for_stmt) & GF_OMP_FOR_SIMD; 424 bool distribute = gimple_omp_for_kind (for_stmt) 425 == GF_OMP_FOR_KIND_DISTRIBUTE; 426 427 fd->for_stmt = for_stmt; 428 fd->pre = NULL; 429 fd->collapse = gimple_omp_for_collapse (for_stmt); 430 if (fd->collapse > 1) 431 fd->loops = loops; 432 else 433 fd->loops = &fd->loop; 434 435 fd->have_nowait = distribute || simd; 436 fd->have_ordered = false; 437 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; 438 fd->chunk_size = NULL_TREE; 439 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_CILKFOR) 440 fd->sched_kind = OMP_CLAUSE_SCHEDULE_CILKFOR; 441 collapse_iter = NULL; 442 collapse_count = NULL; 443 444 for (t = gimple_omp_for_clauses (for_stmt); t ; t = OMP_CLAUSE_CHAIN (t)) 445 switch (OMP_CLAUSE_CODE (t)) 446 { 447 case OMP_CLAUSE_NOWAIT: 448 fd->have_nowait = true; 449 break; 450 case OMP_CLAUSE_ORDERED: 451 fd->have_ordered = true; 452 break; 453 case OMP_CLAUSE_SCHEDULE: 454 gcc_assert (!distribute); 455 fd->sched_kind = OMP_CLAUSE_SCHEDULE_KIND (t); 456 fd->chunk_size = OMP_CLAUSE_SCHEDULE_CHUNK_EXPR (t); 457 break; 458 case OMP_CLAUSE_DIST_SCHEDULE: 459 gcc_assert (distribute); 460 fd->chunk_size = OMP_CLAUSE_DIST_SCHEDULE_CHUNK_EXPR (t); 461 break; 462 case OMP_CLAUSE_COLLAPSE: 463 if (fd->collapse > 1) 464 { 465 collapse_iter = &OMP_CLAUSE_COLLAPSE_ITERVAR (t); 466 collapse_count = &OMP_CLAUSE_COLLAPSE_COUNT (t); 467 } 468 break; 469 default: 470 break; 471 } 472 473 /* FIXME: for now map schedule(auto) to schedule(static). 474 There should be analysis to determine whether all iterations 475 are approximately the same amount of work (then schedule(static) 476 is best) or if it varies (then schedule(dynamic,N) is better). */ 477 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_AUTO) 478 { 479 fd->sched_kind = OMP_CLAUSE_SCHEDULE_STATIC; 480 gcc_assert (fd->chunk_size == NULL); 481 } 482 gcc_assert (fd->collapse == 1 || collapse_iter != NULL); 483 if (fd->sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) 484 gcc_assert (fd->chunk_size == NULL); 485 else if (fd->chunk_size == NULL) 486 { 487 /* We only need to compute a default chunk size for ordered 488 static loops and dynamic loops. */ 489 if (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC 490 || fd->have_ordered) 491 fd->chunk_size = (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) 492 ? integer_zero_node : integer_one_node; 493 } 494 495 for (i = 0; i < fd->collapse; i++) 496 { 497 if (fd->collapse == 1) 498 loop = &fd->loop; 499 else if (loops != NULL) 500 loop = loops + i; 501 else 502 loop = &dummy_loop; 503 504 loop->v = gimple_omp_for_index (for_stmt, i); 505 gcc_assert (SSA_VAR_P (loop->v)); 506 gcc_assert (TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE 507 || TREE_CODE (TREE_TYPE (loop->v)) == POINTER_TYPE); 508 var = TREE_CODE (loop->v) == SSA_NAME ? SSA_NAME_VAR (loop->v) : loop->v; 509 loop->n1 = gimple_omp_for_initial (for_stmt, i); 510 511 loop->cond_code = gimple_omp_for_cond (for_stmt, i); 512 loop->n2 = gimple_omp_for_final (for_stmt, i); 513 switch (loop->cond_code) 514 { 515 case LT_EXPR: 516 case GT_EXPR: 517 break; 518 case NE_EXPR: 519 gcc_assert (gimple_omp_for_kind (for_stmt) 520 == GF_OMP_FOR_KIND_CILKSIMD 521 || (gimple_omp_for_kind (for_stmt) 522 == GF_OMP_FOR_KIND_CILKFOR)); 523 break; 524 case LE_EXPR: 525 if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) 526 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, 1); 527 else 528 loop->n2 = fold_build2_loc (loc, 529 PLUS_EXPR, TREE_TYPE (loop->n2), loop->n2, 530 build_int_cst (TREE_TYPE (loop->n2), 1)); 531 loop->cond_code = LT_EXPR; 532 break; 533 case GE_EXPR: 534 if (POINTER_TYPE_P (TREE_TYPE (loop->n2))) 535 loop->n2 = fold_build_pointer_plus_hwi_loc (loc, loop->n2, -1); 536 else 537 loop->n2 = fold_build2_loc (loc, 538 MINUS_EXPR, TREE_TYPE (loop->n2), loop->n2, 539 build_int_cst (TREE_TYPE (loop->n2), 1)); 540 loop->cond_code = GT_EXPR; 541 break; 542 default: 543 gcc_unreachable (); 544 } 545 546 t = gimple_omp_for_incr (for_stmt, i); 547 gcc_assert (TREE_OPERAND (t, 0) == var); 548 switch (TREE_CODE (t)) 549 { 550 case PLUS_EXPR: 551 loop->step = TREE_OPERAND (t, 1); 552 break; 553 case POINTER_PLUS_EXPR: 554 loop->step = fold_convert (ssizetype, TREE_OPERAND (t, 1)); 555 break; 556 case MINUS_EXPR: 557 loop->step = TREE_OPERAND (t, 1); 558 loop->step = fold_build1_loc (loc, 559 NEGATE_EXPR, TREE_TYPE (loop->step), 560 loop->step); 561 break; 562 default: 563 gcc_unreachable (); 564 } 565 566 if (simd 567 || (fd->sched_kind == OMP_CLAUSE_SCHEDULE_STATIC 568 && !fd->have_ordered)) 569 { 570 if (fd->collapse == 1) 571 iter_type = TREE_TYPE (loop->v); 572 else if (i == 0 573 || TYPE_PRECISION (iter_type) 574 < TYPE_PRECISION (TREE_TYPE (loop->v))) 575 iter_type 576 = build_nonstandard_integer_type 577 (TYPE_PRECISION (TREE_TYPE (loop->v)), 1); 578 } 579 else if (iter_type != long_long_unsigned_type_node) 580 { 581 if (POINTER_TYPE_P (TREE_TYPE (loop->v))) 582 iter_type = long_long_unsigned_type_node; 583 else if (TYPE_UNSIGNED (TREE_TYPE (loop->v)) 584 && TYPE_PRECISION (TREE_TYPE (loop->v)) 585 >= TYPE_PRECISION (iter_type)) 586 { 587 tree n; 588 589 if (loop->cond_code == LT_EXPR) 590 n = fold_build2_loc (loc, 591 PLUS_EXPR, TREE_TYPE (loop->v), 592 loop->n2, loop->step); 593 else 594 n = loop->n1; 595 if (TREE_CODE (n) != INTEGER_CST 596 || tree_int_cst_lt (TYPE_MAX_VALUE (iter_type), n)) 597 iter_type = long_long_unsigned_type_node; 598 } 599 else if (TYPE_PRECISION (TREE_TYPE (loop->v)) 600 > TYPE_PRECISION (iter_type)) 601 { 602 tree n1, n2; 603 604 if (loop->cond_code == LT_EXPR) 605 { 606 n1 = loop->n1; 607 n2 = fold_build2_loc (loc, 608 PLUS_EXPR, TREE_TYPE (loop->v), 609 loop->n2, loop->step); 610 } 611 else 612 { 613 n1 = fold_build2_loc (loc, 614 MINUS_EXPR, TREE_TYPE (loop->v), 615 loop->n2, loop->step); 616 n2 = loop->n1; 617 } 618 if (TREE_CODE (n1) != INTEGER_CST 619 || TREE_CODE (n2) != INTEGER_CST 620 || !tree_int_cst_lt (TYPE_MIN_VALUE (iter_type), n1) 621 || !tree_int_cst_lt (n2, TYPE_MAX_VALUE (iter_type))) 622 iter_type = long_long_unsigned_type_node; 623 } 624 } 625 626 if (collapse_count && *collapse_count == NULL) 627 { 628 t = fold_binary (loop->cond_code, boolean_type_node, 629 fold_convert (TREE_TYPE (loop->v), loop->n1), 630 fold_convert (TREE_TYPE (loop->v), loop->n2)); 631 if (t && integer_zerop (t)) 632 count = build_zero_cst (long_long_unsigned_type_node); 633 else if ((i == 0 || count != NULL_TREE) 634 && TREE_CODE (TREE_TYPE (loop->v)) == INTEGER_TYPE 635 && TREE_CONSTANT (loop->n1) 636 && TREE_CONSTANT (loop->n2) 637 && TREE_CODE (loop->step) == INTEGER_CST) 638 { 639 tree itype = TREE_TYPE (loop->v); 640 641 if (POINTER_TYPE_P (itype)) 642 itype = signed_type_for (itype); 643 t = build_int_cst (itype, (loop->cond_code == LT_EXPR ? -1 : 1)); 644 t = fold_build2_loc (loc, 645 PLUS_EXPR, itype, 646 fold_convert_loc (loc, itype, loop->step), t); 647 t = fold_build2_loc (loc, PLUS_EXPR, itype, t, 648 fold_convert_loc (loc, itype, loop->n2)); 649 t = fold_build2_loc (loc, MINUS_EXPR, itype, t, 650 fold_convert_loc (loc, itype, loop->n1)); 651 if (TYPE_UNSIGNED (itype) && loop->cond_code == GT_EXPR) 652 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, 653 fold_build1_loc (loc, NEGATE_EXPR, itype, t), 654 fold_build1_loc (loc, NEGATE_EXPR, itype, 655 fold_convert_loc (loc, itype, 656 loop->step))); 657 else 658 t = fold_build2_loc (loc, TRUNC_DIV_EXPR, itype, t, 659 fold_convert_loc (loc, itype, loop->step)); 660 t = fold_convert_loc (loc, long_long_unsigned_type_node, t); 661 if (count != NULL_TREE) 662 count = fold_build2_loc (loc, 663 MULT_EXPR, long_long_unsigned_type_node, 664 count, t); 665 else 666 count = t; 667 if (TREE_CODE (count) != INTEGER_CST) 668 count = NULL_TREE; 669 } 670 else if (count && !integer_zerop (count)) 671 count = NULL_TREE; 672 } 673 } 674 675 if (count 676 && !simd 677 && (fd->sched_kind != OMP_CLAUSE_SCHEDULE_STATIC 678 || fd->have_ordered)) 679 { 680 if (!tree_int_cst_lt (count, TYPE_MAX_VALUE (long_integer_type_node))) 681 iter_type = long_long_unsigned_type_node; 682 else 683 iter_type = long_integer_type_node; 684 } 685 else if (collapse_iter && *collapse_iter != NULL) 686 iter_type = TREE_TYPE (*collapse_iter); 687 fd->iter_type = iter_type; 688 if (collapse_iter && *collapse_iter == NULL) 689 *collapse_iter = create_tmp_var (iter_type, ".iter"); 690 if (collapse_count && *collapse_count == NULL) 691 { 692 if (count) 693 *collapse_count = fold_convert_loc (loc, iter_type, count); 694 else 695 *collapse_count = create_tmp_var (iter_type, ".count"); 696 } 697 698 if (fd->collapse > 1) 699 { 700 fd->loop.v = *collapse_iter; 701 fd->loop.n1 = build_int_cst (TREE_TYPE (fd->loop.v), 0); 702 fd->loop.n2 = *collapse_count; 703 fd->loop.step = build_int_cst (TREE_TYPE (fd->loop.v), 1); 704 fd->loop.cond_code = LT_EXPR; 705 } 706 707 /* For OpenACC loops, force a chunk size of one, as this avoids the default 708 scheduling where several subsequent iterations are being executed by the 709 same thread. */ 710 if (gimple_omp_for_kind (for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 711 { 712 gcc_assert (fd->chunk_size == NULL_TREE); 713 fd->chunk_size = build_int_cst (TREE_TYPE (fd->loop.v), 1); 714 } 715} 716 717 718/* Given two blocks PAR_ENTRY_BB and WS_ENTRY_BB such that WS_ENTRY_BB 719 is the immediate dominator of PAR_ENTRY_BB, return true if there 720 are no data dependencies that would prevent expanding the parallel 721 directive at PAR_ENTRY_BB as a combined parallel+workshare region. 722 723 When expanding a combined parallel+workshare region, the call to 724 the child function may need additional arguments in the case of 725 GIMPLE_OMP_FOR regions. In some cases, these arguments are 726 computed out of variables passed in from the parent to the child 727 via 'struct .omp_data_s'. For instance: 728 729 #pragma omp parallel for schedule (guided, i * 4) 730 for (j ...) 731 732 Is lowered into: 733 734 # BLOCK 2 (PAR_ENTRY_BB) 735 .omp_data_o.i = i; 736 #pragma omp parallel [child fn: bar.omp_fn.0 ( ..., D.1598) 737 738 # BLOCK 3 (WS_ENTRY_BB) 739 .omp_data_i = &.omp_data_o; 740 D.1667 = .omp_data_i->i; 741 D.1598 = D.1667 * 4; 742 #pragma omp for schedule (guided, D.1598) 743 744 When we outline the parallel region, the call to the child function 745 'bar.omp_fn.0' will need the value D.1598 in its argument list, but 746 that value is computed *after* the call site. So, in principle we 747 cannot do the transformation. 748 749 To see whether the code in WS_ENTRY_BB blocks the combined 750 parallel+workshare call, we collect all the variables used in the 751 GIMPLE_OMP_FOR header check whether they appear on the LHS of any 752 statement in WS_ENTRY_BB. If so, then we cannot emit the combined 753 call. 754 755 FIXME. If we had the SSA form built at this point, we could merely 756 hoist the code in block 3 into block 2 and be done with it. But at 757 this point we don't have dataflow information and though we could 758 hack something up here, it is really not worth the aggravation. */ 759 760static bool 761workshare_safe_to_combine_p (basic_block ws_entry_bb) 762{ 763 struct omp_for_data fd; 764 gimple ws_stmt = last_stmt (ws_entry_bb); 765 766 if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) 767 return true; 768 769 gcc_assert (gimple_code (ws_stmt) == GIMPLE_OMP_FOR); 770 771 extract_omp_for_data (as_a <gomp_for *> (ws_stmt), &fd, NULL); 772 773 if (fd.collapse > 1 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 774 return false; 775 if (fd.iter_type != long_integer_type_node) 776 return false; 777 778 /* FIXME. We give up too easily here. If any of these arguments 779 are not constants, they will likely involve variables that have 780 been mapped into fields of .omp_data_s for sharing with the child 781 function. With appropriate data flow, it would be possible to 782 see through this. */ 783 if (!is_gimple_min_invariant (fd.loop.n1) 784 || !is_gimple_min_invariant (fd.loop.n2) 785 || !is_gimple_min_invariant (fd.loop.step) 786 || (fd.chunk_size && !is_gimple_min_invariant (fd.chunk_size))) 787 return false; 788 789 return true; 790} 791 792 793/* Collect additional arguments needed to emit a combined 794 parallel+workshare call. WS_STMT is the workshare directive being 795 expanded. */ 796 797static vec<tree, va_gc> * 798get_ws_args_for (gimple par_stmt, gimple ws_stmt) 799{ 800 tree t; 801 location_t loc = gimple_location (ws_stmt); 802 vec<tree, va_gc> *ws_args; 803 804 if (gomp_for *for_stmt = dyn_cast <gomp_for *> (ws_stmt)) 805 { 806 struct omp_for_data fd; 807 tree n1, n2; 808 809 extract_omp_for_data (for_stmt, &fd, NULL); 810 n1 = fd.loop.n1; 811 n2 = fd.loop.n2; 812 813 if (gimple_omp_for_combined_into_p (for_stmt)) 814 { 815 tree innerc 816 = find_omp_clause (gimple_omp_parallel_clauses (par_stmt), 817 OMP_CLAUSE__LOOPTEMP_); 818 gcc_assert (innerc); 819 n1 = OMP_CLAUSE_DECL (innerc); 820 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 821 OMP_CLAUSE__LOOPTEMP_); 822 gcc_assert (innerc); 823 n2 = OMP_CLAUSE_DECL (innerc); 824 } 825 826 vec_alloc (ws_args, 3 + (fd.chunk_size != 0)); 827 828 t = fold_convert_loc (loc, long_integer_type_node, n1); 829 ws_args->quick_push (t); 830 831 t = fold_convert_loc (loc, long_integer_type_node, n2); 832 ws_args->quick_push (t); 833 834 t = fold_convert_loc (loc, long_integer_type_node, fd.loop.step); 835 ws_args->quick_push (t); 836 837 if (fd.chunk_size) 838 { 839 t = fold_convert_loc (loc, long_integer_type_node, fd.chunk_size); 840 ws_args->quick_push (t); 841 } 842 843 return ws_args; 844 } 845 else if (gimple_code (ws_stmt) == GIMPLE_OMP_SECTIONS) 846 { 847 /* Number of sections is equal to the number of edges from the 848 GIMPLE_OMP_SECTIONS_SWITCH statement, except for the one to 849 the exit of the sections region. */ 850 basic_block bb = single_succ (gimple_bb (ws_stmt)); 851 t = build_int_cst (unsigned_type_node, EDGE_COUNT (bb->succs) - 1); 852 vec_alloc (ws_args, 1); 853 ws_args->quick_push (t); 854 return ws_args; 855 } 856 857 gcc_unreachable (); 858} 859 860 861/* Discover whether REGION is a combined parallel+workshare region. */ 862 863static void 864determine_parallel_type (struct omp_region *region) 865{ 866 basic_block par_entry_bb, par_exit_bb; 867 basic_block ws_entry_bb, ws_exit_bb; 868 869 if (region == NULL || region->inner == NULL 870 || region->exit == NULL || region->inner->exit == NULL 871 || region->inner->cont == NULL) 872 return; 873 874 /* We only support parallel+for and parallel+sections. */ 875 if (region->type != GIMPLE_OMP_PARALLEL 876 || (region->inner->type != GIMPLE_OMP_FOR 877 && region->inner->type != GIMPLE_OMP_SECTIONS)) 878 return; 879 880 /* Check for perfect nesting PAR_ENTRY_BB -> WS_ENTRY_BB and 881 WS_EXIT_BB -> PAR_EXIT_BB. */ 882 par_entry_bb = region->entry; 883 par_exit_bb = region->exit; 884 ws_entry_bb = region->inner->entry; 885 ws_exit_bb = region->inner->exit; 886 887 if (single_succ (par_entry_bb) == ws_entry_bb 888 && single_succ (ws_exit_bb) == par_exit_bb 889 && workshare_safe_to_combine_p (ws_entry_bb) 890 && (gimple_omp_parallel_combined_p (last_stmt (par_entry_bb)) 891 || (last_and_only_stmt (ws_entry_bb) 892 && last_and_only_stmt (par_exit_bb)))) 893 { 894 gimple par_stmt = last_stmt (par_entry_bb); 895 gimple ws_stmt = last_stmt (ws_entry_bb); 896 897 if (region->inner->type == GIMPLE_OMP_FOR) 898 { 899 /* If this is a combined parallel loop, we need to determine 900 whether or not to use the combined library calls. There 901 are two cases where we do not apply the transformation: 902 static loops and any kind of ordered loop. In the first 903 case, we already open code the loop so there is no need 904 to do anything else. In the latter case, the combined 905 parallel loop call would still need extra synchronization 906 to implement ordered semantics, so there would not be any 907 gain in using the combined call. */ 908 tree clauses = gimple_omp_for_clauses (ws_stmt); 909 tree c = find_omp_clause (clauses, OMP_CLAUSE_SCHEDULE); 910 if (c == NULL 911 || OMP_CLAUSE_SCHEDULE_KIND (c) == OMP_CLAUSE_SCHEDULE_STATIC 912 || find_omp_clause (clauses, OMP_CLAUSE_ORDERED)) 913 { 914 region->is_combined_parallel = false; 915 region->inner->is_combined_parallel = false; 916 return; 917 } 918 } 919 920 region->is_combined_parallel = true; 921 region->inner->is_combined_parallel = true; 922 region->ws_args = get_ws_args_for (par_stmt, ws_stmt); 923 } 924} 925 926 927/* Return true if EXPR is variable sized. */ 928 929static inline bool 930is_variable_sized (const_tree expr) 931{ 932 return !TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (expr))); 933} 934 935/* Return true if DECL is a reference type. */ 936 937static inline bool 938is_reference (tree decl) 939{ 940 return lang_hooks.decls.omp_privatize_by_reference (decl); 941} 942 943/* Return the type of a decl. If the decl is reference type, 944 return its base type. */ 945static inline tree 946get_base_type (tree decl) 947{ 948 tree type = TREE_TYPE (decl); 949 if (is_reference (decl)) 950 type = TREE_TYPE (type); 951 return type; 952} 953 954/* Lookup variables. The "maybe" form 955 allows for the variable form to not have been entered, otherwise we 956 assert that the variable must have been entered. */ 957 958static inline tree 959lookup_decl (tree var, omp_context *ctx) 960{ 961 tree *n = ctx->cb.decl_map->get (var); 962 return *n; 963} 964 965static inline tree 966maybe_lookup_decl (const_tree var, omp_context *ctx) 967{ 968 tree *n = ctx->cb.decl_map->get (const_cast<tree> (var)); 969 return n ? *n : NULL_TREE; 970} 971 972static inline tree 973lookup_field (tree var, omp_context *ctx) 974{ 975 splay_tree_node n; 976 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); 977 return (tree) n->value; 978} 979 980static inline tree 981lookup_sfield (tree var, omp_context *ctx) 982{ 983 splay_tree_node n; 984 n = splay_tree_lookup (ctx->sfield_map 985 ? ctx->sfield_map : ctx->field_map, 986 (splay_tree_key) var); 987 return (tree) n->value; 988} 989 990static inline tree 991maybe_lookup_field (tree var, omp_context *ctx) 992{ 993 splay_tree_node n; 994 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) var); 995 return n ? (tree) n->value : NULL_TREE; 996} 997 998static inline tree 999lookup_oacc_reduction (const char *id, omp_context *ctx) 1000{ 1001 splay_tree_node n; 1002 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) id); 1003 return (tree) n->value; 1004} 1005 1006static inline tree 1007maybe_lookup_oacc_reduction (tree var, omp_context *ctx) 1008{ 1009 splay_tree_node n = NULL; 1010 if (ctx->reduction_map) 1011 n = splay_tree_lookup (ctx->reduction_map, (splay_tree_key) var); 1012 return n ? (tree) n->value : NULL_TREE; 1013} 1014 1015/* Return true if DECL should be copied by pointer. SHARED_CTX is 1016 the parallel context if DECL is to be shared. */ 1017 1018static bool 1019use_pointer_for_field (tree decl, omp_context *shared_ctx) 1020{ 1021 if (AGGREGATE_TYPE_P (TREE_TYPE (decl))) 1022 return true; 1023 1024 /* We can only use copy-in/copy-out semantics for shared variables 1025 when we know the value is not accessible from an outer scope. */ 1026 if (shared_ctx) 1027 { 1028 gcc_assert (!is_gimple_omp_oacc (shared_ctx->stmt)); 1029 1030 /* ??? Trivially accessible from anywhere. But why would we even 1031 be passing an address in this case? Should we simply assert 1032 this to be false, or should we have a cleanup pass that removes 1033 these from the list of mappings? */ 1034 if (TREE_STATIC (decl) || DECL_EXTERNAL (decl)) 1035 return true; 1036 1037 /* For variables with DECL_HAS_VALUE_EXPR_P set, we cannot tell 1038 without analyzing the expression whether or not its location 1039 is accessible to anyone else. In the case of nested parallel 1040 regions it certainly may be. */ 1041 if (TREE_CODE (decl) != RESULT_DECL && DECL_HAS_VALUE_EXPR_P (decl)) 1042 return true; 1043 1044 /* Do not use copy-in/copy-out for variables that have their 1045 address taken. */ 1046 if (TREE_ADDRESSABLE (decl)) 1047 return true; 1048 1049 /* lower_send_shared_vars only uses copy-in, but not copy-out 1050 for these. */ 1051 if (TREE_READONLY (decl) 1052 || ((TREE_CODE (decl) == RESULT_DECL 1053 || TREE_CODE (decl) == PARM_DECL) 1054 && DECL_BY_REFERENCE (decl))) 1055 return false; 1056 1057 /* Disallow copy-in/out in nested parallel if 1058 decl is shared in outer parallel, otherwise 1059 each thread could store the shared variable 1060 in its own copy-in location, making the 1061 variable no longer really shared. */ 1062 if (shared_ctx->is_nested) 1063 { 1064 omp_context *up; 1065 1066 for (up = shared_ctx->outer; up; up = up->outer) 1067 if (is_taskreg_ctx (up) && maybe_lookup_decl (decl, up)) 1068 break; 1069 1070 if (up) 1071 { 1072 tree c; 1073 1074 for (c = gimple_omp_taskreg_clauses (up->stmt); 1075 c; c = OMP_CLAUSE_CHAIN (c)) 1076 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED 1077 && OMP_CLAUSE_DECL (c) == decl) 1078 break; 1079 1080 if (c) 1081 goto maybe_mark_addressable_and_ret; 1082 } 1083 } 1084 1085 /* For tasks avoid using copy-in/out. As tasks can be 1086 deferred or executed in different thread, when GOMP_task 1087 returns, the task hasn't necessarily terminated. */ 1088 if (is_task_ctx (shared_ctx)) 1089 { 1090 tree outer; 1091 maybe_mark_addressable_and_ret: 1092 outer = maybe_lookup_decl_in_outer_ctx (decl, shared_ctx); 1093 if (is_gimple_reg (outer)) 1094 { 1095 /* Taking address of OUTER in lower_send_shared_vars 1096 might need regimplification of everything that uses the 1097 variable. */ 1098 if (!task_shared_vars) 1099 task_shared_vars = BITMAP_ALLOC (NULL); 1100 bitmap_set_bit (task_shared_vars, DECL_UID (outer)); 1101 TREE_ADDRESSABLE (outer) = 1; 1102 } 1103 return true; 1104 } 1105 } 1106 1107 return false; 1108} 1109 1110/* Construct a new automatic decl similar to VAR. */ 1111 1112static tree 1113omp_copy_decl_2 (tree var, tree name, tree type, omp_context *ctx) 1114{ 1115 tree copy = copy_var_decl (var, name, type); 1116 1117 DECL_CONTEXT (copy) = current_function_decl; 1118 DECL_CHAIN (copy) = ctx->block_vars; 1119 ctx->block_vars = copy; 1120 1121 return copy; 1122} 1123 1124static tree 1125omp_copy_decl_1 (tree var, omp_context *ctx) 1126{ 1127 return omp_copy_decl_2 (var, DECL_NAME (var), TREE_TYPE (var), ctx); 1128} 1129 1130/* Build COMPONENT_REF and set TREE_THIS_VOLATILE and TREE_READONLY on it 1131 as appropriate. */ 1132static tree 1133omp_build_component_ref (tree obj, tree field) 1134{ 1135 tree ret = build3 (COMPONENT_REF, TREE_TYPE (field), obj, field, NULL); 1136 if (TREE_THIS_VOLATILE (field)) 1137 TREE_THIS_VOLATILE (ret) |= 1; 1138 if (TREE_READONLY (field)) 1139 TREE_READONLY (ret) |= 1; 1140 return ret; 1141} 1142 1143/* Build tree nodes to access the field for VAR on the receiver side. */ 1144 1145static tree 1146build_receiver_ref (tree var, bool by_ref, omp_context *ctx) 1147{ 1148 tree x, field = lookup_field (var, ctx); 1149 1150 /* If the receiver record type was remapped in the child function, 1151 remap the field into the new record type. */ 1152 x = maybe_lookup_field (field, ctx); 1153 if (x != NULL) 1154 field = x; 1155 1156 x = build_simple_mem_ref (ctx->receiver_decl); 1157 x = omp_build_component_ref (x, field); 1158 if (by_ref) 1159 x = build_simple_mem_ref (x); 1160 1161 return x; 1162} 1163 1164/* Build tree nodes to access VAR in the scope outer to CTX. In the case 1165 of a parallel, this is a component reference; for workshare constructs 1166 this is some variable. */ 1167 1168static tree 1169build_outer_var_ref (tree var, omp_context *ctx) 1170{ 1171 tree x; 1172 1173 if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, ctx))) 1174 x = var; 1175 else if (is_variable_sized (var)) 1176 { 1177 x = TREE_OPERAND (DECL_VALUE_EXPR (var), 0); 1178 x = build_outer_var_ref (x, ctx); 1179 x = build_simple_mem_ref (x); 1180 } 1181 else if (is_taskreg_ctx (ctx)) 1182 { 1183 bool by_ref = use_pointer_for_field (var, NULL); 1184 x = build_receiver_ref (var, by_ref, ctx); 1185 } 1186 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 1187 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 1188 { 1189 /* #pragma omp simd isn't a worksharing construct, and can reference even 1190 private vars in its linear etc. clauses. */ 1191 x = NULL_TREE; 1192 if (ctx->outer && is_taskreg_ctx (ctx)) 1193 x = lookup_decl (var, ctx->outer); 1194 else if (ctx->outer) 1195 x = maybe_lookup_decl_in_outer_ctx (var, ctx); 1196 if (x == NULL_TREE) 1197 x = var; 1198 } 1199 else if (ctx->outer) 1200 x = lookup_decl (var, ctx->outer); 1201 else if (is_reference (var)) 1202 /* This can happen with orphaned constructs. If var is reference, it is 1203 possible it is shared and as such valid. */ 1204 x = var; 1205 else 1206 gcc_unreachable (); 1207 1208 if (is_reference (var)) 1209 x = build_simple_mem_ref (x); 1210 1211 return x; 1212} 1213 1214/* Build tree nodes to access the field for VAR on the sender side. */ 1215 1216static tree 1217build_sender_ref (tree var, omp_context *ctx) 1218{ 1219 tree field = lookup_sfield (var, ctx); 1220 return omp_build_component_ref (ctx->sender_decl, field); 1221} 1222 1223/* Add a new field for VAR inside the structure CTX->SENDER_DECL. */ 1224 1225static void 1226install_var_field (tree var, bool by_ref, int mask, omp_context *ctx) 1227{ 1228 tree field, type, sfield = NULL_TREE; 1229 1230 gcc_assert ((mask & 1) == 0 1231 || !splay_tree_lookup (ctx->field_map, (splay_tree_key) var)); 1232 gcc_assert ((mask & 2) == 0 || !ctx->sfield_map 1233 || !splay_tree_lookup (ctx->sfield_map, (splay_tree_key) var)); 1234 gcc_assert ((mask & 3) == 3 1235 || !is_gimple_omp_oacc (ctx->stmt)); 1236 1237 type = TREE_TYPE (var); 1238 if (mask & 4) 1239 { 1240 gcc_assert (TREE_CODE (type) == ARRAY_TYPE); 1241 type = build_pointer_type (build_pointer_type (type)); 1242 } 1243 else if (by_ref) 1244 type = build_pointer_type (type); 1245 else if ((mask & 3) == 1 && is_reference (var)) 1246 type = TREE_TYPE (type); 1247 1248 field = build_decl (DECL_SOURCE_LOCATION (var), 1249 FIELD_DECL, DECL_NAME (var), type); 1250 1251 /* Remember what variable this field was created for. This does have a 1252 side effect of making dwarf2out ignore this member, so for helpful 1253 debugging we clear it later in delete_omp_context. */ 1254 DECL_ABSTRACT_ORIGIN (field) = var; 1255 if (type == TREE_TYPE (var)) 1256 { 1257 DECL_ALIGN (field) = DECL_ALIGN (var); 1258 DECL_USER_ALIGN (field) = DECL_USER_ALIGN (var); 1259 TREE_THIS_VOLATILE (field) = TREE_THIS_VOLATILE (var); 1260 } 1261 else 1262 DECL_ALIGN (field) = TYPE_ALIGN (type); 1263 1264 if ((mask & 3) == 3) 1265 { 1266 insert_field_into_struct (ctx->record_type, field); 1267 if (ctx->srecord_type) 1268 { 1269 sfield = build_decl (DECL_SOURCE_LOCATION (var), 1270 FIELD_DECL, DECL_NAME (var), type); 1271 DECL_ABSTRACT_ORIGIN (sfield) = var; 1272 DECL_ALIGN (sfield) = DECL_ALIGN (field); 1273 DECL_USER_ALIGN (sfield) = DECL_USER_ALIGN (field); 1274 TREE_THIS_VOLATILE (sfield) = TREE_THIS_VOLATILE (field); 1275 insert_field_into_struct (ctx->srecord_type, sfield); 1276 } 1277 } 1278 else 1279 { 1280 if (ctx->srecord_type == NULL_TREE) 1281 { 1282 tree t; 1283 1284 ctx->srecord_type = lang_hooks.types.make_type (RECORD_TYPE); 1285 ctx->sfield_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 1286 for (t = TYPE_FIELDS (ctx->record_type); t ; t = TREE_CHAIN (t)) 1287 { 1288 sfield = build_decl (DECL_SOURCE_LOCATION (var), 1289 FIELD_DECL, DECL_NAME (t), TREE_TYPE (t)); 1290 DECL_ABSTRACT_ORIGIN (sfield) = DECL_ABSTRACT_ORIGIN (t); 1291 insert_field_into_struct (ctx->srecord_type, sfield); 1292 splay_tree_insert (ctx->sfield_map, 1293 (splay_tree_key) DECL_ABSTRACT_ORIGIN (t), 1294 (splay_tree_value) sfield); 1295 } 1296 } 1297 sfield = field; 1298 insert_field_into_struct ((mask & 1) ? ctx->record_type 1299 : ctx->srecord_type, field); 1300 } 1301 1302 if (mask & 1) 1303 splay_tree_insert (ctx->field_map, (splay_tree_key) var, 1304 (splay_tree_value) field); 1305 if ((mask & 2) && ctx->sfield_map) 1306 splay_tree_insert (ctx->sfield_map, (splay_tree_key) var, 1307 (splay_tree_value) sfield); 1308} 1309 1310static tree 1311install_var_local (tree var, omp_context *ctx) 1312{ 1313 tree new_var = omp_copy_decl_1 (var, ctx); 1314 insert_decl_map (&ctx->cb, var, new_var); 1315 return new_var; 1316} 1317 1318/* Adjust the replacement for DECL in CTX for the new context. This means 1319 copying the DECL_VALUE_EXPR, and fixing up the type. */ 1320 1321static void 1322fixup_remapped_decl (tree decl, omp_context *ctx, bool private_debug) 1323{ 1324 tree new_decl, size; 1325 1326 new_decl = lookup_decl (decl, ctx); 1327 1328 TREE_TYPE (new_decl) = remap_type (TREE_TYPE (decl), &ctx->cb); 1329 1330 if ((!TREE_CONSTANT (DECL_SIZE (new_decl)) || private_debug) 1331 && DECL_HAS_VALUE_EXPR_P (decl)) 1332 { 1333 tree ve = DECL_VALUE_EXPR (decl); 1334 walk_tree (&ve, copy_tree_body_r, &ctx->cb, NULL); 1335 SET_DECL_VALUE_EXPR (new_decl, ve); 1336 DECL_HAS_VALUE_EXPR_P (new_decl) = 1; 1337 } 1338 1339 if (!TREE_CONSTANT (DECL_SIZE (new_decl))) 1340 { 1341 size = remap_decl (DECL_SIZE (decl), &ctx->cb); 1342 if (size == error_mark_node) 1343 size = TYPE_SIZE (TREE_TYPE (new_decl)); 1344 DECL_SIZE (new_decl) = size; 1345 1346 size = remap_decl (DECL_SIZE_UNIT (decl), &ctx->cb); 1347 if (size == error_mark_node) 1348 size = TYPE_SIZE_UNIT (TREE_TYPE (new_decl)); 1349 DECL_SIZE_UNIT (new_decl) = size; 1350 } 1351} 1352 1353/* The callback for remap_decl. Search all containing contexts for a 1354 mapping of the variable; this avoids having to duplicate the splay 1355 tree ahead of time. We know a mapping doesn't already exist in the 1356 given context. Create new mappings to implement default semantics. */ 1357 1358static tree 1359omp_copy_decl (tree var, copy_body_data *cb) 1360{ 1361 omp_context *ctx = (omp_context *) cb; 1362 tree new_var; 1363 1364 if (TREE_CODE (var) == LABEL_DECL) 1365 { 1366 new_var = create_artificial_label (DECL_SOURCE_LOCATION (var)); 1367 DECL_CONTEXT (new_var) = current_function_decl; 1368 insert_decl_map (&ctx->cb, var, new_var); 1369 return new_var; 1370 } 1371 1372 while (!is_taskreg_ctx (ctx)) 1373 { 1374 ctx = ctx->outer; 1375 if (ctx == NULL) 1376 return var; 1377 new_var = maybe_lookup_decl (var, ctx); 1378 if (new_var) 1379 return new_var; 1380 } 1381 1382 if (is_global_var (var) || decl_function_context (var) != ctx->cb.src_fn) 1383 return var; 1384 1385 return error_mark_node; 1386} 1387 1388 1389/* Debugging dumps for parallel regions. */ 1390void dump_omp_region (FILE *, struct omp_region *, int); 1391void debug_omp_region (struct omp_region *); 1392void debug_all_omp_regions (void); 1393 1394/* Dump the parallel region tree rooted at REGION. */ 1395 1396void 1397dump_omp_region (FILE *file, struct omp_region *region, int indent) 1398{ 1399 fprintf (file, "%*sbb %d: %s\n", indent, "", region->entry->index, 1400 gimple_code_name[region->type]); 1401 1402 if (region->inner) 1403 dump_omp_region (file, region->inner, indent + 4); 1404 1405 if (region->cont) 1406 { 1407 fprintf (file, "%*sbb %d: GIMPLE_OMP_CONTINUE\n", indent, "", 1408 region->cont->index); 1409 } 1410 1411 if (region->exit) 1412 fprintf (file, "%*sbb %d: GIMPLE_OMP_RETURN\n", indent, "", 1413 region->exit->index); 1414 else 1415 fprintf (file, "%*s[no exit marker]\n", indent, ""); 1416 1417 if (region->next) 1418 dump_omp_region (file, region->next, indent); 1419} 1420 1421DEBUG_FUNCTION void 1422debug_omp_region (struct omp_region *region) 1423{ 1424 dump_omp_region (stderr, region, 0); 1425} 1426 1427DEBUG_FUNCTION void 1428debug_all_omp_regions (void) 1429{ 1430 dump_omp_region (stderr, root_omp_region, 0); 1431} 1432 1433 1434/* Create a new parallel region starting at STMT inside region PARENT. */ 1435 1436static struct omp_region * 1437new_omp_region (basic_block bb, enum gimple_code type, 1438 struct omp_region *parent) 1439{ 1440 struct omp_region *region = XCNEW (struct omp_region); 1441 1442 region->outer = parent; 1443 region->entry = bb; 1444 region->type = type; 1445 1446 if (parent) 1447 { 1448 /* This is a nested region. Add it to the list of inner 1449 regions in PARENT. */ 1450 region->next = parent->inner; 1451 parent->inner = region; 1452 } 1453 else 1454 { 1455 /* This is a toplevel region. Add it to the list of toplevel 1456 regions in ROOT_OMP_REGION. */ 1457 region->next = root_omp_region; 1458 root_omp_region = region; 1459 } 1460 1461 return region; 1462} 1463 1464/* Release the memory associated with the region tree rooted at REGION. */ 1465 1466static void 1467free_omp_region_1 (struct omp_region *region) 1468{ 1469 struct omp_region *i, *n; 1470 1471 for (i = region->inner; i ; i = n) 1472 { 1473 n = i->next; 1474 free_omp_region_1 (i); 1475 } 1476 1477 free (region); 1478} 1479 1480/* Release the memory for the entire omp region tree. */ 1481 1482void 1483free_omp_regions (void) 1484{ 1485 struct omp_region *r, *n; 1486 for (r = root_omp_region; r ; r = n) 1487 { 1488 n = r->next; 1489 free_omp_region_1 (r); 1490 } 1491 root_omp_region = NULL; 1492} 1493 1494 1495/* Create a new context, with OUTER_CTX being the surrounding context. */ 1496 1497static omp_context * 1498new_omp_context (gimple stmt, omp_context *outer_ctx) 1499{ 1500 omp_context *ctx = XCNEW (omp_context); 1501 1502 splay_tree_insert (all_contexts, (splay_tree_key) stmt, 1503 (splay_tree_value) ctx); 1504 ctx->stmt = stmt; 1505 1506 if (outer_ctx) 1507 { 1508 ctx->outer = outer_ctx; 1509 ctx->cb = outer_ctx->cb; 1510 ctx->cb.block = NULL; 1511 ctx->depth = outer_ctx->depth + 1; 1512 ctx->reduction_map = outer_ctx->reduction_map; 1513 } 1514 else 1515 { 1516 ctx->cb.src_fn = current_function_decl; 1517 ctx->cb.dst_fn = current_function_decl; 1518 ctx->cb.src_node = cgraph_node::get (current_function_decl); 1519 gcc_checking_assert (ctx->cb.src_node); 1520 ctx->cb.dst_node = ctx->cb.src_node; 1521 ctx->cb.src_cfun = cfun; 1522 ctx->cb.copy_decl = omp_copy_decl; 1523 ctx->cb.eh_lp_nr = 0; 1524 ctx->cb.transform_call_graph_edges = CB_CGE_MOVE; 1525 ctx->depth = 1; 1526 } 1527 1528 ctx->cb.decl_map = new hash_map<tree, tree>; 1529 1530 return ctx; 1531} 1532 1533static gimple_seq maybe_catch_exception (gimple_seq); 1534 1535/* Finalize task copyfn. */ 1536 1537static void 1538finalize_task_copyfn (gomp_task *task_stmt) 1539{ 1540 struct function *child_cfun; 1541 tree child_fn; 1542 gimple_seq seq = NULL, new_seq; 1543 gbind *bind; 1544 1545 child_fn = gimple_omp_task_copy_fn (task_stmt); 1546 if (child_fn == NULL_TREE) 1547 return; 1548 1549 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 1550 DECL_STRUCT_FUNCTION (child_fn)->curr_properties = cfun->curr_properties; 1551 1552 push_cfun (child_cfun); 1553 bind = gimplify_body (child_fn, false); 1554 gimple_seq_add_stmt (&seq, bind); 1555 new_seq = maybe_catch_exception (seq); 1556 if (new_seq != seq) 1557 { 1558 bind = gimple_build_bind (NULL, new_seq, NULL); 1559 seq = NULL; 1560 gimple_seq_add_stmt (&seq, bind); 1561 } 1562 gimple_set_body (child_fn, seq); 1563 pop_cfun (); 1564 1565 /* Inform the callgraph about the new function. */ 1566 cgraph_node::add_new_function (child_fn, false); 1567 cgraph_node::get (child_fn)->parallelized_function = 1; 1568} 1569 1570/* Destroy a omp_context data structures. Called through the splay tree 1571 value delete callback. */ 1572 1573static void 1574delete_omp_context (splay_tree_value value) 1575{ 1576 omp_context *ctx = (omp_context *) value; 1577 1578 delete ctx->cb.decl_map; 1579 1580 if (ctx->field_map) 1581 splay_tree_delete (ctx->field_map); 1582 if (ctx->sfield_map) 1583 splay_tree_delete (ctx->sfield_map); 1584 /* Reduction map is copied to nested contexts, so only delete it in the 1585 owner. */ 1586 if (ctx->reduction_map 1587 && gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET 1588 && is_gimple_omp_offloaded (ctx->stmt) 1589 && is_gimple_omp_oacc (ctx->stmt)) 1590 splay_tree_delete (ctx->reduction_map); 1591 1592 /* We hijacked DECL_ABSTRACT_ORIGIN earlier. We need to clear it before 1593 it produces corrupt debug information. */ 1594 if (ctx->record_type) 1595 { 1596 tree t; 1597 for (t = TYPE_FIELDS (ctx->record_type); t ; t = DECL_CHAIN (t)) 1598 DECL_ABSTRACT_ORIGIN (t) = NULL; 1599 } 1600 if (ctx->srecord_type) 1601 { 1602 tree t; 1603 for (t = TYPE_FIELDS (ctx->srecord_type); t ; t = DECL_CHAIN (t)) 1604 DECL_ABSTRACT_ORIGIN (t) = NULL; 1605 } 1606 1607 if (is_task_ctx (ctx)) 1608 finalize_task_copyfn (as_a <gomp_task *> (ctx->stmt)); 1609 1610 XDELETE (ctx); 1611} 1612 1613/* Fix up RECEIVER_DECL with a type that has been remapped to the child 1614 context. */ 1615 1616static void 1617fixup_child_record_type (omp_context *ctx) 1618{ 1619 tree f, type = ctx->record_type; 1620 1621 /* ??? It isn't sufficient to just call remap_type here, because 1622 variably_modified_type_p doesn't work the way we expect for 1623 record types. Testing each field for whether it needs remapping 1624 and creating a new record by hand works, however. */ 1625 for (f = TYPE_FIELDS (type); f ; f = DECL_CHAIN (f)) 1626 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 1627 break; 1628 if (f) 1629 { 1630 tree name, new_fields = NULL; 1631 1632 type = lang_hooks.types.make_type (RECORD_TYPE); 1633 name = DECL_NAME (TYPE_NAME (ctx->record_type)); 1634 name = build_decl (DECL_SOURCE_LOCATION (ctx->receiver_decl), 1635 TYPE_DECL, name, type); 1636 TYPE_NAME (type) = name; 1637 1638 for (f = TYPE_FIELDS (ctx->record_type); f ; f = DECL_CHAIN (f)) 1639 { 1640 tree new_f = copy_node (f); 1641 DECL_CONTEXT (new_f) = type; 1642 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &ctx->cb); 1643 DECL_CHAIN (new_f) = new_fields; 1644 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &ctx->cb, NULL); 1645 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, 1646 &ctx->cb, NULL); 1647 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, 1648 &ctx->cb, NULL); 1649 new_fields = new_f; 1650 1651 /* Arrange to be able to look up the receiver field 1652 given the sender field. */ 1653 splay_tree_insert (ctx->field_map, (splay_tree_key) f, 1654 (splay_tree_value) new_f); 1655 } 1656 TYPE_FIELDS (type) = nreverse (new_fields); 1657 layout_type (type); 1658 } 1659 1660 TREE_TYPE (ctx->receiver_decl) 1661 = build_qualified_type (build_reference_type (type), TYPE_QUAL_RESTRICT); 1662} 1663 1664/* Instantiate decls as necessary in CTX to satisfy the data sharing 1665 specified by CLAUSES. */ 1666 1667static void 1668scan_sharing_clauses (tree clauses, omp_context *ctx) 1669{ 1670 tree c, decl; 1671 bool scan_array_reductions = false; 1672 1673 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 1674 { 1675 bool by_ref; 1676 1677 switch (OMP_CLAUSE_CODE (c)) 1678 { 1679 case OMP_CLAUSE_PRIVATE: 1680 decl = OMP_CLAUSE_DECL (c); 1681 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 1682 goto do_private; 1683 else if (!is_variable_sized (decl)) 1684 install_var_local (decl, ctx); 1685 break; 1686 1687 case OMP_CLAUSE_SHARED: 1688 decl = OMP_CLAUSE_DECL (c); 1689 /* Ignore shared directives in teams construct. */ 1690 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 1691 { 1692 /* Global variables don't need to be copied, 1693 the receiver side will use them directly. */ 1694 tree odecl = maybe_lookup_decl_in_outer_ctx (decl, ctx); 1695 if (is_global_var (odecl)) 1696 break; 1697 insert_decl_map (&ctx->cb, decl, odecl); 1698 break; 1699 } 1700 gcc_assert (is_taskreg_ctx (ctx)); 1701 gcc_assert (!COMPLETE_TYPE_P (TREE_TYPE (decl)) 1702 || !is_variable_sized (decl)); 1703 /* Global variables don't need to be copied, 1704 the receiver side will use them directly. */ 1705 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 1706 break; 1707 by_ref = use_pointer_for_field (decl, ctx); 1708 if (! TREE_READONLY (decl) 1709 || TREE_ADDRESSABLE (decl) 1710 || by_ref 1711 || is_reference (decl)) 1712 { 1713 install_var_field (decl, by_ref, 3, ctx); 1714 install_var_local (decl, ctx); 1715 break; 1716 } 1717 /* We don't need to copy const scalar vars back. */ 1718 OMP_CLAUSE_SET_CODE (c, OMP_CLAUSE_FIRSTPRIVATE); 1719 goto do_private; 1720 1721 case OMP_CLAUSE_LASTPRIVATE: 1722 /* Let the corresponding firstprivate clause create 1723 the variable. */ 1724 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 1725 break; 1726 /* FALLTHRU */ 1727 1728 case OMP_CLAUSE_FIRSTPRIVATE: 1729 if (is_gimple_omp_oacc (ctx->stmt)) 1730 { 1731 sorry ("clause not supported yet"); 1732 break; 1733 } 1734 /* FALLTHRU */ 1735 case OMP_CLAUSE_REDUCTION: 1736 case OMP_CLAUSE_LINEAR: 1737 decl = OMP_CLAUSE_DECL (c); 1738 do_private: 1739 if (is_variable_sized (decl)) 1740 { 1741 if (is_task_ctx (ctx)) 1742 install_var_field (decl, false, 1, ctx); 1743 break; 1744 } 1745 else if (is_taskreg_ctx (ctx)) 1746 { 1747 bool global 1748 = is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)); 1749 by_ref = use_pointer_for_field (decl, NULL); 1750 1751 if (is_task_ctx (ctx) 1752 && (global || by_ref || is_reference (decl))) 1753 { 1754 install_var_field (decl, false, 1, ctx); 1755 if (!global) 1756 install_var_field (decl, by_ref, 2, ctx); 1757 } 1758 else if (!global) 1759 install_var_field (decl, by_ref, 3, ctx); 1760 } 1761 install_var_local (decl, ctx); 1762 if (is_gimple_omp_oacc (ctx->stmt) 1763 && OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 1764 { 1765 /* Create a decl for the reduction array. */ 1766 tree var = OMP_CLAUSE_DECL (c); 1767 tree type = get_base_type (var); 1768 tree ptype = build_pointer_type (type); 1769 tree array = create_tmp_var (ptype, 1770 oacc_get_reduction_array_id (var)); 1771 omp_context *c = (ctx->field_map ? ctx : ctx->outer); 1772 install_var_field (array, true, 3, c); 1773 install_var_local (array, c); 1774 1775 /* Insert it into the current context. */ 1776 splay_tree_insert (ctx->reduction_map, (splay_tree_key) 1777 oacc_get_reduction_array_id (var), 1778 (splay_tree_value) array); 1779 splay_tree_insert (ctx->reduction_map, 1780 (splay_tree_key) array, 1781 (splay_tree_value) array); 1782 } 1783 break; 1784 1785 case OMP_CLAUSE__LOOPTEMP_: 1786 gcc_assert (is_parallel_ctx (ctx)); 1787 decl = OMP_CLAUSE_DECL (c); 1788 install_var_field (decl, false, 3, ctx); 1789 install_var_local (decl, ctx); 1790 break; 1791 1792 case OMP_CLAUSE_COPYPRIVATE: 1793 case OMP_CLAUSE_COPYIN: 1794 decl = OMP_CLAUSE_DECL (c); 1795 by_ref = use_pointer_for_field (decl, NULL); 1796 install_var_field (decl, by_ref, 3, ctx); 1797 break; 1798 1799 case OMP_CLAUSE_DEFAULT: 1800 ctx->default_kind = OMP_CLAUSE_DEFAULT_KIND (c); 1801 break; 1802 1803 case OMP_CLAUSE_FINAL: 1804 case OMP_CLAUSE_IF: 1805 case OMP_CLAUSE_NUM_THREADS: 1806 case OMP_CLAUSE_NUM_TEAMS: 1807 case OMP_CLAUSE_THREAD_LIMIT: 1808 case OMP_CLAUSE_DEVICE: 1809 case OMP_CLAUSE_SCHEDULE: 1810 case OMP_CLAUSE_DIST_SCHEDULE: 1811 case OMP_CLAUSE_DEPEND: 1812 case OMP_CLAUSE__CILK_FOR_COUNT_: 1813 case OMP_CLAUSE_NUM_GANGS: 1814 case OMP_CLAUSE_NUM_WORKERS: 1815 case OMP_CLAUSE_VECTOR_LENGTH: 1816 if (ctx->outer) 1817 scan_omp_op (&OMP_CLAUSE_OPERAND (c, 0), ctx->outer); 1818 break; 1819 1820 case OMP_CLAUSE_TO: 1821 case OMP_CLAUSE_FROM: 1822 case OMP_CLAUSE_MAP: 1823 if (ctx->outer) 1824 scan_omp_op (&OMP_CLAUSE_SIZE (c), ctx->outer); 1825 decl = OMP_CLAUSE_DECL (c); 1826 /* Global variables with "omp declare target" attribute 1827 don't need to be copied, the receiver side will use them 1828 directly. */ 1829 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1830 && DECL_P (decl) 1831 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) 1832 && varpool_node::get_create (decl)->offloadable) 1833 break; 1834 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1835 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER) 1836 { 1837 /* Ignore GOMP_MAP_POINTER kind for arrays in regions that are 1838 not offloaded; there is nothing to map for those. */ 1839 if (!is_gimple_omp_offloaded (ctx->stmt) 1840 && !POINTER_TYPE_P (TREE_TYPE (decl)) 1841 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 1842 break; 1843 } 1844 if (DECL_P (decl)) 1845 { 1846 if (DECL_SIZE (decl) 1847 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) 1848 { 1849 tree decl2 = DECL_VALUE_EXPR (decl); 1850 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); 1851 decl2 = TREE_OPERAND (decl2, 0); 1852 gcc_assert (DECL_P (decl2)); 1853 install_var_field (decl2, true, 3, ctx); 1854 install_var_local (decl2, ctx); 1855 install_var_local (decl, ctx); 1856 } 1857 else 1858 { 1859 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 1860 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 1861 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 1862 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) 1863 install_var_field (decl, true, 7, ctx); 1864 else 1865 install_var_field (decl, true, 3, ctx); 1866 if (is_gimple_omp_offloaded (ctx->stmt)) 1867 install_var_local (decl, ctx); 1868 } 1869 } 1870 else 1871 { 1872 tree base = get_base_address (decl); 1873 tree nc = OMP_CLAUSE_CHAIN (c); 1874 if (DECL_P (base) 1875 && nc != NULL_TREE 1876 && OMP_CLAUSE_CODE (nc) == OMP_CLAUSE_MAP 1877 && OMP_CLAUSE_DECL (nc) == base 1878 && OMP_CLAUSE_MAP_KIND (nc) == GOMP_MAP_POINTER 1879 && integer_zerop (OMP_CLAUSE_SIZE (nc))) 1880 { 1881 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) = 1; 1882 OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (nc) = 1; 1883 } 1884 else 1885 { 1886 if (ctx->outer) 1887 { 1888 scan_omp_op (&OMP_CLAUSE_DECL (c), ctx->outer); 1889 decl = OMP_CLAUSE_DECL (c); 1890 } 1891 gcc_assert (!splay_tree_lookup (ctx->field_map, 1892 (splay_tree_key) decl)); 1893 tree field 1894 = build_decl (OMP_CLAUSE_LOCATION (c), 1895 FIELD_DECL, NULL_TREE, ptr_type_node); 1896 DECL_ALIGN (field) = TYPE_ALIGN (ptr_type_node); 1897 insert_field_into_struct (ctx->record_type, field); 1898 splay_tree_insert (ctx->field_map, (splay_tree_key) decl, 1899 (splay_tree_value) field); 1900 } 1901 } 1902 break; 1903 1904 case OMP_CLAUSE_NOWAIT: 1905 case OMP_CLAUSE_ORDERED: 1906 case OMP_CLAUSE_COLLAPSE: 1907 case OMP_CLAUSE_UNTIED: 1908 case OMP_CLAUSE_MERGEABLE: 1909 case OMP_CLAUSE_PROC_BIND: 1910 case OMP_CLAUSE_SAFELEN: 1911 case OMP_CLAUSE_ASYNC: 1912 case OMP_CLAUSE_WAIT: 1913 case OMP_CLAUSE_GANG: 1914 case OMP_CLAUSE_WORKER: 1915 case OMP_CLAUSE_VECTOR: 1916 break; 1917 1918 case OMP_CLAUSE_ALIGNED: 1919 decl = OMP_CLAUSE_DECL (c); 1920 if (is_global_var (decl) 1921 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE) 1922 install_var_local (decl, ctx); 1923 break; 1924 1925 case OMP_CLAUSE_DEVICE_RESIDENT: 1926 case OMP_CLAUSE_USE_DEVICE: 1927 case OMP_CLAUSE__CACHE_: 1928 case OMP_CLAUSE_INDEPENDENT: 1929 case OMP_CLAUSE_AUTO: 1930 case OMP_CLAUSE_SEQ: 1931 sorry ("Clause not supported yet"); 1932 break; 1933 1934 default: 1935 gcc_unreachable (); 1936 } 1937 } 1938 1939 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 1940 { 1941 switch (OMP_CLAUSE_CODE (c)) 1942 { 1943 case OMP_CLAUSE_LASTPRIVATE: 1944 /* Let the corresponding firstprivate clause create 1945 the variable. */ 1946 if (OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 1947 scan_array_reductions = true; 1948 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 1949 break; 1950 /* FALLTHRU */ 1951 1952 case OMP_CLAUSE_FIRSTPRIVATE: 1953 if (is_gimple_omp_oacc (ctx->stmt)) 1954 { 1955 sorry ("clause not supported yet"); 1956 break; 1957 } 1958 /* FALLTHRU */ 1959 case OMP_CLAUSE_PRIVATE: 1960 case OMP_CLAUSE_REDUCTION: 1961 case OMP_CLAUSE_LINEAR: 1962 decl = OMP_CLAUSE_DECL (c); 1963 if (is_variable_sized (decl)) 1964 install_var_local (decl, ctx); 1965 fixup_remapped_decl (decl, ctx, 1966 OMP_CLAUSE_CODE (c) == OMP_CLAUSE_PRIVATE 1967 && OMP_CLAUSE_PRIVATE_DEBUG (c)); 1968 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION 1969 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 1970 scan_array_reductions = true; 1971 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 1972 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 1973 scan_array_reductions = true; 1974 break; 1975 1976 case OMP_CLAUSE_SHARED: 1977 /* Ignore shared directives in teams construct. */ 1978 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 1979 break; 1980 decl = OMP_CLAUSE_DECL (c); 1981 if (! is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 1982 fixup_remapped_decl (decl, ctx, false); 1983 break; 1984 1985 case OMP_CLAUSE_MAP: 1986 if (!is_gimple_omp_offloaded (ctx->stmt)) 1987 break; 1988 decl = OMP_CLAUSE_DECL (c); 1989 if (DECL_P (decl) 1990 && is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx)) 1991 && varpool_node::get_create (decl)->offloadable) 1992 break; 1993 if (DECL_P (decl)) 1994 { 1995 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 1996 && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE 1997 && !COMPLETE_TYPE_P (TREE_TYPE (decl))) 1998 { 1999 tree new_decl = lookup_decl (decl, ctx); 2000 TREE_TYPE (new_decl) 2001 = remap_type (TREE_TYPE (decl), &ctx->cb); 2002 } 2003 else if (DECL_SIZE (decl) 2004 && TREE_CODE (DECL_SIZE (decl)) != INTEGER_CST) 2005 { 2006 tree decl2 = DECL_VALUE_EXPR (decl); 2007 gcc_assert (TREE_CODE (decl2) == INDIRECT_REF); 2008 decl2 = TREE_OPERAND (decl2, 0); 2009 gcc_assert (DECL_P (decl2)); 2010 fixup_remapped_decl (decl2, ctx, false); 2011 fixup_remapped_decl (decl, ctx, true); 2012 } 2013 else 2014 fixup_remapped_decl (decl, ctx, false); 2015 } 2016 break; 2017 2018 case OMP_CLAUSE_COPYPRIVATE: 2019 case OMP_CLAUSE_COPYIN: 2020 case OMP_CLAUSE_DEFAULT: 2021 case OMP_CLAUSE_IF: 2022 case OMP_CLAUSE_NUM_THREADS: 2023 case OMP_CLAUSE_NUM_TEAMS: 2024 case OMP_CLAUSE_THREAD_LIMIT: 2025 case OMP_CLAUSE_DEVICE: 2026 case OMP_CLAUSE_SCHEDULE: 2027 case OMP_CLAUSE_DIST_SCHEDULE: 2028 case OMP_CLAUSE_NOWAIT: 2029 case OMP_CLAUSE_ORDERED: 2030 case OMP_CLAUSE_COLLAPSE: 2031 case OMP_CLAUSE_UNTIED: 2032 case OMP_CLAUSE_FINAL: 2033 case OMP_CLAUSE_MERGEABLE: 2034 case OMP_CLAUSE_PROC_BIND: 2035 case OMP_CLAUSE_SAFELEN: 2036 case OMP_CLAUSE_ALIGNED: 2037 case OMP_CLAUSE_DEPEND: 2038 case OMP_CLAUSE__LOOPTEMP_: 2039 case OMP_CLAUSE_TO: 2040 case OMP_CLAUSE_FROM: 2041 case OMP_CLAUSE__CILK_FOR_COUNT_: 2042 case OMP_CLAUSE_ASYNC: 2043 case OMP_CLAUSE_WAIT: 2044 case OMP_CLAUSE_NUM_GANGS: 2045 case OMP_CLAUSE_NUM_WORKERS: 2046 case OMP_CLAUSE_VECTOR_LENGTH: 2047 case OMP_CLAUSE_GANG: 2048 case OMP_CLAUSE_WORKER: 2049 case OMP_CLAUSE_VECTOR: 2050 break; 2051 2052 case OMP_CLAUSE_DEVICE_RESIDENT: 2053 case OMP_CLAUSE_USE_DEVICE: 2054 case OMP_CLAUSE__CACHE_: 2055 case OMP_CLAUSE_INDEPENDENT: 2056 case OMP_CLAUSE_AUTO: 2057 case OMP_CLAUSE_SEQ: 2058 sorry ("Clause not supported yet"); 2059 break; 2060 2061 default: 2062 gcc_unreachable (); 2063 } 2064 } 2065 2066 gcc_checking_assert (!scan_array_reductions 2067 || !is_gimple_omp_oacc (ctx->stmt)); 2068 if (scan_array_reductions) 2069 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2070 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION 2071 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 2072 { 2073 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c), ctx); 2074 scan_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); 2075 } 2076 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 2077 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 2078 scan_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); 2079 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 2080 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 2081 scan_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); 2082} 2083 2084/* Create a new name for omp child function. Returns an identifier. If 2085 IS_CILK_FOR is true then the suffix for the child function is 2086 "_cilk_for_fn." */ 2087 2088static tree 2089create_omp_child_function_name (bool task_copy, bool is_cilk_for) 2090{ 2091 if (is_cilk_for) 2092 return clone_function_name (current_function_decl, "_cilk_for_fn"); 2093 return clone_function_name (current_function_decl, 2094 task_copy ? "_omp_cpyfn" : "_omp_fn"); 2095} 2096 2097/* Returns the type of the induction variable for the child function for 2098 _Cilk_for and the types for _high and _low variables based on TYPE. */ 2099 2100static tree 2101cilk_for_check_loop_diff_type (tree type) 2102{ 2103 if (TYPE_PRECISION (type) <= TYPE_PRECISION (uint32_type_node)) 2104 { 2105 if (TYPE_UNSIGNED (type)) 2106 return uint32_type_node; 2107 else 2108 return integer_type_node; 2109 } 2110 else 2111 { 2112 if (TYPE_UNSIGNED (type)) 2113 return uint64_type_node; 2114 else 2115 return long_long_integer_type_node; 2116 } 2117} 2118 2119/* Build a decl for the omp child function. It'll not contain a body 2120 yet, just the bare decl. */ 2121 2122static void 2123create_omp_child_function (omp_context *ctx, bool task_copy) 2124{ 2125 tree decl, type, name, t; 2126 2127 tree cilk_for_count 2128 = (flag_cilkplus && gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) 2129 ? find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 2130 OMP_CLAUSE__CILK_FOR_COUNT_) : NULL_TREE; 2131 tree cilk_var_type = NULL_TREE; 2132 2133 name = create_omp_child_function_name (task_copy, 2134 cilk_for_count != NULL_TREE); 2135 if (task_copy) 2136 type = build_function_type_list (void_type_node, ptr_type_node, 2137 ptr_type_node, NULL_TREE); 2138 else if (cilk_for_count) 2139 { 2140 type = TREE_TYPE (OMP_CLAUSE_OPERAND (cilk_for_count, 0)); 2141 cilk_var_type = cilk_for_check_loop_diff_type (type); 2142 type = build_function_type_list (void_type_node, ptr_type_node, 2143 cilk_var_type, cilk_var_type, NULL_TREE); 2144 } 2145 else 2146 type = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE); 2147 2148 decl = build_decl (gimple_location (ctx->stmt), FUNCTION_DECL, name, type); 2149 2150 gcc_checking_assert (!is_gimple_omp_oacc (ctx->stmt) 2151 || !task_copy); 2152 if (!task_copy) 2153 ctx->cb.dst_fn = decl; 2154 else 2155 gimple_omp_task_set_copy_fn (ctx->stmt, decl); 2156 2157 TREE_STATIC (decl) = 1; 2158 TREE_USED (decl) = 1; 2159 DECL_ARTIFICIAL (decl) = 1; 2160 DECL_IGNORED_P (decl) = 0; 2161 TREE_PUBLIC (decl) = 0; 2162 DECL_UNINLINABLE (decl) = 1; 2163 DECL_EXTERNAL (decl) = 0; 2164 DECL_CONTEXT (decl) = NULL_TREE; 2165 DECL_INITIAL (decl) = make_node (BLOCK); 2166 if (cgraph_node::get (current_function_decl)->offloadable) 2167 cgraph_node::get_create (decl)->offloadable = 1; 2168 else 2169 { 2170 omp_context *octx; 2171 for (octx = ctx; octx; octx = octx->outer) 2172 if (is_gimple_omp_offloaded (octx->stmt)) 2173 { 2174 cgraph_node::get_create (decl)->offloadable = 1; 2175#ifdef ENABLE_OFFLOADING 2176 g->have_offload = true; 2177#endif 2178 break; 2179 } 2180 } 2181 2182 if (cgraph_node::get_create (decl)->offloadable 2183 && !lookup_attribute ("omp declare target", 2184 DECL_ATTRIBUTES (current_function_decl))) 2185 DECL_ATTRIBUTES (decl) 2186 = tree_cons (get_identifier ("omp target entrypoint"), 2187 NULL_TREE, DECL_ATTRIBUTES (decl)); 2188 2189 t = build_decl (DECL_SOURCE_LOCATION (decl), 2190 RESULT_DECL, NULL_TREE, void_type_node); 2191 DECL_ARTIFICIAL (t) = 1; 2192 DECL_IGNORED_P (t) = 1; 2193 DECL_CONTEXT (t) = decl; 2194 DECL_RESULT (decl) = t; 2195 2196 /* _Cilk_for's child function requires two extra parameters called 2197 __low and __high that are set the by Cilk runtime when it calls this 2198 function. */ 2199 if (cilk_for_count) 2200 { 2201 t = build_decl (DECL_SOURCE_LOCATION (decl), 2202 PARM_DECL, get_identifier ("__high"), cilk_var_type); 2203 DECL_ARTIFICIAL (t) = 1; 2204 DECL_NAMELESS (t) = 1; 2205 DECL_ARG_TYPE (t) = ptr_type_node; 2206 DECL_CONTEXT (t) = current_function_decl; 2207 TREE_USED (t) = 1; 2208 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2209 DECL_ARGUMENTS (decl) = t; 2210 2211 t = build_decl (DECL_SOURCE_LOCATION (decl), 2212 PARM_DECL, get_identifier ("__low"), cilk_var_type); 2213 DECL_ARTIFICIAL (t) = 1; 2214 DECL_NAMELESS (t) = 1; 2215 DECL_ARG_TYPE (t) = ptr_type_node; 2216 DECL_CONTEXT (t) = current_function_decl; 2217 TREE_USED (t) = 1; 2218 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2219 DECL_ARGUMENTS (decl) = t; 2220 } 2221 2222 tree data_name = get_identifier (".omp_data_i"); 2223 t = build_decl (DECL_SOURCE_LOCATION (decl), PARM_DECL, data_name, 2224 ptr_type_node); 2225 DECL_ARTIFICIAL (t) = 1; 2226 DECL_NAMELESS (t) = 1; 2227 DECL_ARG_TYPE (t) = ptr_type_node; 2228 DECL_CONTEXT (t) = current_function_decl; 2229 TREE_USED (t) = 1; 2230 if (cilk_for_count) 2231 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2232 DECL_ARGUMENTS (decl) = t; 2233 if (!task_copy) 2234 ctx->receiver_decl = t; 2235 else 2236 { 2237 t = build_decl (DECL_SOURCE_LOCATION (decl), 2238 PARM_DECL, get_identifier (".omp_data_o"), 2239 ptr_type_node); 2240 DECL_ARTIFICIAL (t) = 1; 2241 DECL_NAMELESS (t) = 1; 2242 DECL_ARG_TYPE (t) = ptr_type_node; 2243 DECL_CONTEXT (t) = current_function_decl; 2244 TREE_USED (t) = 1; 2245 TREE_ADDRESSABLE (t) = 1; 2246 DECL_CHAIN (t) = DECL_ARGUMENTS (decl); 2247 DECL_ARGUMENTS (decl) = t; 2248 } 2249 2250 /* Allocate memory for the function structure. The call to 2251 allocate_struct_function clobbers CFUN, so we need to restore 2252 it afterward. */ 2253 push_struct_function (decl); 2254 cfun->function_end_locus = gimple_location (ctx->stmt); 2255 pop_cfun (); 2256} 2257 2258/* Callback for walk_gimple_seq. Check if combined parallel 2259 contains gimple_omp_for_combined_into_p OMP_FOR. */ 2260 2261static tree 2262find_combined_for (gimple_stmt_iterator *gsi_p, 2263 bool *handled_ops_p, 2264 struct walk_stmt_info *wi) 2265{ 2266 gimple stmt = gsi_stmt (*gsi_p); 2267 2268 *handled_ops_p = true; 2269 switch (gimple_code (stmt)) 2270 { 2271 WALK_SUBSTMTS; 2272 2273 case GIMPLE_OMP_FOR: 2274 if (gimple_omp_for_combined_into_p (stmt) 2275 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR) 2276 { 2277 wi->info = stmt; 2278 return integer_zero_node; 2279 } 2280 break; 2281 default: 2282 break; 2283 } 2284 return NULL; 2285} 2286 2287/* Scan an OpenMP parallel directive. */ 2288 2289static void 2290scan_omp_parallel (gimple_stmt_iterator *gsi, omp_context *outer_ctx) 2291{ 2292 omp_context *ctx; 2293 tree name; 2294 gomp_parallel *stmt = as_a <gomp_parallel *> (gsi_stmt (*gsi)); 2295 2296 /* Ignore parallel directives with empty bodies, unless there 2297 are copyin clauses. */ 2298 if (optimize > 0 2299 && empty_body_p (gimple_omp_body (stmt)) 2300 && find_omp_clause (gimple_omp_parallel_clauses (stmt), 2301 OMP_CLAUSE_COPYIN) == NULL) 2302 { 2303 gsi_replace (gsi, gimple_build_nop (), false); 2304 return; 2305 } 2306 2307 if (gimple_omp_parallel_combined_p (stmt)) 2308 { 2309 struct walk_stmt_info wi; 2310 2311 memset (&wi, 0, sizeof (wi)); 2312 wi.val_only = true; 2313 walk_gimple_seq (gimple_omp_body (stmt), 2314 find_combined_for, NULL, &wi); 2315 if (wi.info) 2316 { 2317 gomp_for *for_stmt = as_a <gomp_for *> ((gimple) wi.info); 2318 struct omp_for_data fd; 2319 extract_omp_for_data (for_stmt, &fd, NULL); 2320 /* We need two temporaries with fd.loop.v type (istart/iend) 2321 and then (fd.collapse - 1) temporaries with the same 2322 type for count2 ... countN-1 vars if not constant. */ 2323 size_t count = 2, i; 2324 tree type = fd.iter_type; 2325 if (fd.collapse > 1 2326 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 2327 count += fd.collapse - 1; 2328 for (i = 0; i < count; i++) 2329 { 2330 tree temp = create_tmp_var (type); 2331 tree c = build_omp_clause (UNKNOWN_LOCATION, 2332 OMP_CLAUSE__LOOPTEMP_); 2333 insert_decl_map (&outer_ctx->cb, temp, temp); 2334 OMP_CLAUSE_DECL (c) = temp; 2335 OMP_CLAUSE_CHAIN (c) = gimple_omp_parallel_clauses (stmt); 2336 gimple_omp_parallel_set_clauses (stmt, c); 2337 } 2338 } 2339 } 2340 2341 ctx = new_omp_context (stmt, outer_ctx); 2342 taskreg_contexts.safe_push (ctx); 2343 if (taskreg_nesting_level > 1) 2344 ctx->is_nested = true; 2345 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2346 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2347 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2348 name = create_tmp_var_name (".omp_data_s"); 2349 name = build_decl (gimple_location (stmt), 2350 TYPE_DECL, name, ctx->record_type); 2351 DECL_ARTIFICIAL (name) = 1; 2352 DECL_NAMELESS (name) = 1; 2353 TYPE_NAME (ctx->record_type) = name; 2354 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2355 create_omp_child_function (ctx, false); 2356 gimple_omp_parallel_set_child_fn (stmt, ctx->cb.dst_fn); 2357 2358 scan_sharing_clauses (gimple_omp_parallel_clauses (stmt), ctx); 2359 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2360 2361 if (TYPE_FIELDS (ctx->record_type) == NULL) 2362 ctx->record_type = ctx->receiver_decl = NULL; 2363} 2364 2365/* Scan an OpenMP task directive. */ 2366 2367static void 2368scan_omp_task (gimple_stmt_iterator *gsi, omp_context *outer_ctx) 2369{ 2370 omp_context *ctx; 2371 tree name, t; 2372 gomp_task *stmt = as_a <gomp_task *> (gsi_stmt (*gsi)); 2373 2374 /* Ignore task directives with empty bodies. */ 2375 if (optimize > 0 2376 && empty_body_p (gimple_omp_body (stmt))) 2377 { 2378 gsi_replace (gsi, gimple_build_nop (), false); 2379 return; 2380 } 2381 2382 ctx = new_omp_context (stmt, outer_ctx); 2383 taskreg_contexts.safe_push (ctx); 2384 if (taskreg_nesting_level > 1) 2385 ctx->is_nested = true; 2386 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2387 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2388 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2389 name = create_tmp_var_name (".omp_data_s"); 2390 name = build_decl (gimple_location (stmt), 2391 TYPE_DECL, name, ctx->record_type); 2392 DECL_ARTIFICIAL (name) = 1; 2393 DECL_NAMELESS (name) = 1; 2394 TYPE_NAME (ctx->record_type) = name; 2395 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2396 create_omp_child_function (ctx, false); 2397 gimple_omp_task_set_child_fn (stmt, ctx->cb.dst_fn); 2398 2399 scan_sharing_clauses (gimple_omp_task_clauses (stmt), ctx); 2400 2401 if (ctx->srecord_type) 2402 { 2403 name = create_tmp_var_name (".omp_data_a"); 2404 name = build_decl (gimple_location (stmt), 2405 TYPE_DECL, name, ctx->srecord_type); 2406 DECL_ARTIFICIAL (name) = 1; 2407 DECL_NAMELESS (name) = 1; 2408 TYPE_NAME (ctx->srecord_type) = name; 2409 TYPE_ARTIFICIAL (ctx->srecord_type) = 1; 2410 create_omp_child_function (ctx, true); 2411 } 2412 2413 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2414 2415 if (TYPE_FIELDS (ctx->record_type) == NULL) 2416 { 2417 ctx->record_type = ctx->receiver_decl = NULL; 2418 t = build_int_cst (long_integer_type_node, 0); 2419 gimple_omp_task_set_arg_size (stmt, t); 2420 t = build_int_cst (long_integer_type_node, 1); 2421 gimple_omp_task_set_arg_align (stmt, t); 2422 } 2423} 2424 2425 2426/* If any decls have been made addressable during scan_omp, 2427 adjust their fields if needed, and layout record types 2428 of parallel/task constructs. */ 2429 2430static void 2431finish_taskreg_scan (omp_context *ctx) 2432{ 2433 if (ctx->record_type == NULL_TREE) 2434 return; 2435 2436 /* If any task_shared_vars were needed, verify all 2437 OMP_CLAUSE_SHARED clauses on GIMPLE_OMP_{PARALLEL,TASK} 2438 statements if use_pointer_for_field hasn't changed 2439 because of that. If it did, update field types now. */ 2440 if (task_shared_vars) 2441 { 2442 tree c; 2443 2444 for (c = gimple_omp_taskreg_clauses (ctx->stmt); 2445 c; c = OMP_CLAUSE_CHAIN (c)) 2446 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_SHARED) 2447 { 2448 tree decl = OMP_CLAUSE_DECL (c); 2449 2450 /* Global variables don't need to be copied, 2451 the receiver side will use them directly. */ 2452 if (is_global_var (maybe_lookup_decl_in_outer_ctx (decl, ctx))) 2453 continue; 2454 if (!bitmap_bit_p (task_shared_vars, DECL_UID (decl)) 2455 || !use_pointer_for_field (decl, ctx)) 2456 continue; 2457 tree field = lookup_field (decl, ctx); 2458 if (TREE_CODE (TREE_TYPE (field)) == POINTER_TYPE 2459 && TREE_TYPE (TREE_TYPE (field)) == TREE_TYPE (decl)) 2460 continue; 2461 TREE_TYPE (field) = build_pointer_type (TREE_TYPE (decl)); 2462 TREE_THIS_VOLATILE (field) = 0; 2463 DECL_USER_ALIGN (field) = 0; 2464 DECL_ALIGN (field) = TYPE_ALIGN (TREE_TYPE (field)); 2465 if (TYPE_ALIGN (ctx->record_type) < DECL_ALIGN (field)) 2466 TYPE_ALIGN (ctx->record_type) = DECL_ALIGN (field); 2467 if (ctx->srecord_type) 2468 { 2469 tree sfield = lookup_sfield (decl, ctx); 2470 TREE_TYPE (sfield) = TREE_TYPE (field); 2471 TREE_THIS_VOLATILE (sfield) = 0; 2472 DECL_USER_ALIGN (sfield) = 0; 2473 DECL_ALIGN (sfield) = DECL_ALIGN (field); 2474 if (TYPE_ALIGN (ctx->srecord_type) < DECL_ALIGN (sfield)) 2475 TYPE_ALIGN (ctx->srecord_type) = DECL_ALIGN (sfield); 2476 } 2477 } 2478 } 2479 2480 if (gimple_code (ctx->stmt) == GIMPLE_OMP_PARALLEL) 2481 { 2482 layout_type (ctx->record_type); 2483 fixup_child_record_type (ctx); 2484 } 2485 else 2486 { 2487 location_t loc = gimple_location (ctx->stmt); 2488 tree *p, vla_fields = NULL_TREE, *q = &vla_fields; 2489 /* Move VLA fields to the end. */ 2490 p = &TYPE_FIELDS (ctx->record_type); 2491 while (*p) 2492 if (!TYPE_SIZE_UNIT (TREE_TYPE (*p)) 2493 || ! TREE_CONSTANT (TYPE_SIZE_UNIT (TREE_TYPE (*p)))) 2494 { 2495 *q = *p; 2496 *p = TREE_CHAIN (*p); 2497 TREE_CHAIN (*q) = NULL_TREE; 2498 q = &TREE_CHAIN (*q); 2499 } 2500 else 2501 p = &DECL_CHAIN (*p); 2502 *p = vla_fields; 2503 layout_type (ctx->record_type); 2504 fixup_child_record_type (ctx); 2505 if (ctx->srecord_type) 2506 layout_type (ctx->srecord_type); 2507 tree t = fold_convert_loc (loc, long_integer_type_node, 2508 TYPE_SIZE_UNIT (ctx->record_type)); 2509 gimple_omp_task_set_arg_size (ctx->stmt, t); 2510 t = build_int_cst (long_integer_type_node, 2511 TYPE_ALIGN_UNIT (ctx->record_type)); 2512 gimple_omp_task_set_arg_align (ctx->stmt, t); 2513 } 2514} 2515 2516 2517static omp_context * 2518enclosing_target_ctx (omp_context *ctx) 2519{ 2520 while (ctx != NULL 2521 && gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) 2522 ctx = ctx->outer; 2523 gcc_assert (ctx != NULL); 2524 return ctx; 2525} 2526 2527static bool 2528oacc_loop_or_target_p (gimple stmt) 2529{ 2530 enum gimple_code outer_type = gimple_code (stmt); 2531 return ((outer_type == GIMPLE_OMP_TARGET 2532 && ((gimple_omp_target_kind (stmt) 2533 == GF_OMP_TARGET_KIND_OACC_PARALLEL) 2534 || (gimple_omp_target_kind (stmt) 2535 == GF_OMP_TARGET_KIND_OACC_KERNELS))) 2536 || (outer_type == GIMPLE_OMP_FOR 2537 && gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_OACC_LOOP)); 2538} 2539 2540/* Scan a GIMPLE_OMP_FOR. */ 2541 2542static void 2543scan_omp_for (gomp_for *stmt, omp_context *outer_ctx) 2544{ 2545 enum gimple_code outer_type = GIMPLE_ERROR_MARK; 2546 omp_context *ctx; 2547 size_t i; 2548 tree clauses = gimple_omp_for_clauses (stmt); 2549 2550 if (outer_ctx) 2551 outer_type = gimple_code (outer_ctx->stmt); 2552 2553 ctx = new_omp_context (stmt, outer_ctx); 2554 2555 if (is_gimple_omp_oacc (stmt)) 2556 { 2557 if (outer_ctx && outer_type == GIMPLE_OMP_FOR) 2558 ctx->gwv_this = outer_ctx->gwv_this; 2559 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2560 { 2561 int val; 2562 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_GANG) 2563 val = MASK_GANG; 2564 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WORKER) 2565 val = MASK_WORKER; 2566 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR) 2567 val = MASK_VECTOR; 2568 else 2569 continue; 2570 ctx->gwv_this |= val; 2571 if (!outer_ctx) 2572 { 2573 /* Skip; not nested inside a region. */ 2574 continue; 2575 } 2576 if (!oacc_loop_or_target_p (outer_ctx->stmt)) 2577 { 2578 /* Skip; not nested inside an OpenACC region. */ 2579 continue; 2580 } 2581 if (outer_type == GIMPLE_OMP_FOR) 2582 outer_ctx->gwv_below |= val; 2583 if (OMP_CLAUSE_OPERAND (c, 0) != NULL_TREE) 2584 { 2585 omp_context *enclosing = enclosing_target_ctx (outer_ctx); 2586 if (gimple_omp_target_kind (enclosing->stmt) 2587 == GF_OMP_TARGET_KIND_OACC_PARALLEL) 2588 error_at (gimple_location (stmt), 2589 "no arguments allowed to gang, worker and vector clauses inside parallel"); 2590 } 2591 } 2592 } 2593 2594 scan_sharing_clauses (clauses, ctx); 2595 2596 scan_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); 2597 for (i = 0; i < gimple_omp_for_collapse (stmt); i++) 2598 { 2599 scan_omp_op (gimple_omp_for_index_ptr (stmt, i), ctx); 2600 scan_omp_op (gimple_omp_for_initial_ptr (stmt, i), ctx); 2601 scan_omp_op (gimple_omp_for_final_ptr (stmt, i), ctx); 2602 scan_omp_op (gimple_omp_for_incr_ptr (stmt, i), ctx); 2603 } 2604 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2605 2606 if (is_gimple_omp_oacc (stmt)) 2607 { 2608 if (ctx->gwv_this & ctx->gwv_below) 2609 error_at (gimple_location (stmt), 2610 "gang, worker and vector may occur only once in a loop nest"); 2611 else if (ctx->gwv_below != 0 2612 && ctx->gwv_this > ctx->gwv_below) 2613 error_at (gimple_location (stmt), 2614 "gang, worker and vector must occur in this order in a loop nest"); 2615 if (outer_ctx && outer_type == GIMPLE_OMP_FOR) 2616 outer_ctx->gwv_below |= ctx->gwv_below; 2617 } 2618} 2619 2620/* Scan an OpenMP sections directive. */ 2621 2622static void 2623scan_omp_sections (gomp_sections *stmt, omp_context *outer_ctx) 2624{ 2625 omp_context *ctx; 2626 2627 ctx = new_omp_context (stmt, outer_ctx); 2628 scan_sharing_clauses (gimple_omp_sections_clauses (stmt), ctx); 2629 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2630} 2631 2632/* Scan an OpenMP single directive. */ 2633 2634static void 2635scan_omp_single (gomp_single *stmt, omp_context *outer_ctx) 2636{ 2637 omp_context *ctx; 2638 tree name; 2639 2640 ctx = new_omp_context (stmt, outer_ctx); 2641 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2642 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2643 name = create_tmp_var_name (".omp_copy_s"); 2644 name = build_decl (gimple_location (stmt), 2645 TYPE_DECL, name, ctx->record_type); 2646 TYPE_NAME (ctx->record_type) = name; 2647 2648 scan_sharing_clauses (gimple_omp_single_clauses (stmt), ctx); 2649 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2650 2651 if (TYPE_FIELDS (ctx->record_type) == NULL) 2652 ctx->record_type = NULL; 2653 else 2654 layout_type (ctx->record_type); 2655} 2656 2657/* Scan a GIMPLE_OMP_TARGET. */ 2658 2659static void 2660scan_omp_target (gomp_target *stmt, omp_context *outer_ctx) 2661{ 2662 omp_context *ctx; 2663 tree name; 2664 bool offloaded = is_gimple_omp_offloaded (stmt); 2665 tree clauses = gimple_omp_target_clauses (stmt); 2666 2667 ctx = new_omp_context (stmt, outer_ctx); 2668 ctx->field_map = splay_tree_new (splay_tree_compare_pointers, 0, 0); 2669 ctx->default_kind = OMP_CLAUSE_DEFAULT_SHARED; 2670 ctx->record_type = lang_hooks.types.make_type (RECORD_TYPE); 2671 name = create_tmp_var_name (".omp_data_t"); 2672 name = build_decl (gimple_location (stmt), 2673 TYPE_DECL, name, ctx->record_type); 2674 DECL_ARTIFICIAL (name) = 1; 2675 DECL_NAMELESS (name) = 1; 2676 TYPE_NAME (ctx->record_type) = name; 2677 TYPE_ARTIFICIAL (ctx->record_type) = 1; 2678 if (offloaded) 2679 { 2680 if (is_gimple_omp_oacc (stmt)) 2681 ctx->reduction_map = splay_tree_new (splay_tree_compare_pointers, 2682 0, 0); 2683 2684 create_omp_child_function (ctx, false); 2685 gimple_omp_target_set_child_fn (stmt, ctx->cb.dst_fn); 2686 } 2687 2688 if (is_gimple_omp_oacc (stmt)) 2689 { 2690 for (tree c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 2691 { 2692 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_GANGS) 2693 ctx->gwv_this |= MASK_GANG; 2694 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_NUM_WORKERS) 2695 ctx->gwv_this |= MASK_WORKER; 2696 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_VECTOR_LENGTH) 2697 ctx->gwv_this |= MASK_VECTOR; 2698 } 2699 } 2700 2701 scan_sharing_clauses (clauses, ctx); 2702 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2703 2704 if (TYPE_FIELDS (ctx->record_type) == NULL) 2705 ctx->record_type = ctx->receiver_decl = NULL; 2706 else 2707 { 2708 TYPE_FIELDS (ctx->record_type) 2709 = nreverse (TYPE_FIELDS (ctx->record_type)); 2710#ifdef ENABLE_CHECKING 2711 tree field; 2712 unsigned int align = DECL_ALIGN (TYPE_FIELDS (ctx->record_type)); 2713 for (field = TYPE_FIELDS (ctx->record_type); 2714 field; 2715 field = DECL_CHAIN (field)) 2716 gcc_assert (DECL_ALIGN (field) == align); 2717#endif 2718 layout_type (ctx->record_type); 2719 if (offloaded) 2720 fixup_child_record_type (ctx); 2721 } 2722} 2723 2724/* Scan an OpenMP teams directive. */ 2725 2726static void 2727scan_omp_teams (gomp_teams *stmt, omp_context *outer_ctx) 2728{ 2729 omp_context *ctx = new_omp_context (stmt, outer_ctx); 2730 scan_sharing_clauses (gimple_omp_teams_clauses (stmt), ctx); 2731 scan_omp (gimple_omp_body_ptr (stmt), ctx); 2732} 2733 2734/* Check nesting restrictions. */ 2735static bool 2736check_omp_nesting_restrictions (gimple stmt, omp_context *ctx) 2737{ 2738 /* No nesting of non-OpenACC STMT (that is, an OpenMP one, or a GOMP builtin) 2739 inside an OpenACC CTX. */ 2740 if (!(is_gimple_omp (stmt) 2741 && is_gimple_omp_oacc (stmt))) 2742 { 2743 for (omp_context *ctx_ = ctx; ctx_ != NULL; ctx_ = ctx_->outer) 2744 if (is_gimple_omp (ctx_->stmt) 2745 && is_gimple_omp_oacc (ctx_->stmt)) 2746 { 2747 error_at (gimple_location (stmt), 2748 "non-OpenACC construct inside of OpenACC region"); 2749 return false; 2750 } 2751 } 2752 2753 if (ctx != NULL) 2754 { 2755 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 2756 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 2757 { 2758 error_at (gimple_location (stmt), 2759 "OpenMP constructs may not be nested inside simd region"); 2760 return false; 2761 } 2762 else if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 2763 { 2764 if ((gimple_code (stmt) != GIMPLE_OMP_FOR 2765 || (gimple_omp_for_kind (stmt) 2766 != GF_OMP_FOR_KIND_DISTRIBUTE)) 2767 && gimple_code (stmt) != GIMPLE_OMP_PARALLEL) 2768 { 2769 error_at (gimple_location (stmt), 2770 "only distribute or parallel constructs are allowed to " 2771 "be closely nested inside teams construct"); 2772 return false; 2773 } 2774 } 2775 } 2776 switch (gimple_code (stmt)) 2777 { 2778 case GIMPLE_OMP_FOR: 2779 if (gimple_omp_for_kind (stmt) & GF_OMP_FOR_SIMD) 2780 return true; 2781 if (gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_DISTRIBUTE) 2782 { 2783 if (ctx != NULL && gimple_code (ctx->stmt) != GIMPLE_OMP_TEAMS) 2784 { 2785 error_at (gimple_location (stmt), 2786 "distribute construct must be closely nested inside " 2787 "teams construct"); 2788 return false; 2789 } 2790 return true; 2791 } 2792 /* FALLTHRU */ 2793 case GIMPLE_CALL: 2794 if (is_gimple_call (stmt) 2795 && (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2796 == BUILT_IN_GOMP_CANCEL 2797 || DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2798 == BUILT_IN_GOMP_CANCELLATION_POINT)) 2799 { 2800 const char *bad = NULL; 2801 const char *kind = NULL; 2802 if (ctx == NULL) 2803 { 2804 error_at (gimple_location (stmt), "orphaned %qs construct", 2805 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2806 == BUILT_IN_GOMP_CANCEL 2807 ? "#pragma omp cancel" 2808 : "#pragma omp cancellation point"); 2809 return false; 2810 } 2811 switch (tree_fits_shwi_p (gimple_call_arg (stmt, 0)) 2812 ? tree_to_shwi (gimple_call_arg (stmt, 0)) 2813 : 0) 2814 { 2815 case 1: 2816 if (gimple_code (ctx->stmt) != GIMPLE_OMP_PARALLEL) 2817 bad = "#pragma omp parallel"; 2818 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2819 == BUILT_IN_GOMP_CANCEL 2820 && !integer_zerop (gimple_call_arg (stmt, 1))) 2821 ctx->cancellable = true; 2822 kind = "parallel"; 2823 break; 2824 case 2: 2825 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR 2826 || gimple_omp_for_kind (ctx->stmt) != GF_OMP_FOR_KIND_FOR) 2827 bad = "#pragma omp for"; 2828 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2829 == BUILT_IN_GOMP_CANCEL 2830 && !integer_zerop (gimple_call_arg (stmt, 1))) 2831 { 2832 ctx->cancellable = true; 2833 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2834 OMP_CLAUSE_NOWAIT)) 2835 warning_at (gimple_location (stmt), 0, 2836 "%<#pragma omp cancel for%> inside " 2837 "%<nowait%> for construct"); 2838 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2839 OMP_CLAUSE_ORDERED)) 2840 warning_at (gimple_location (stmt), 0, 2841 "%<#pragma omp cancel for%> inside " 2842 "%<ordered%> for construct"); 2843 } 2844 kind = "for"; 2845 break; 2846 case 4: 2847 if (gimple_code (ctx->stmt) != GIMPLE_OMP_SECTIONS 2848 && gimple_code (ctx->stmt) != GIMPLE_OMP_SECTION) 2849 bad = "#pragma omp sections"; 2850 else if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2851 == BUILT_IN_GOMP_CANCEL 2852 && !integer_zerop (gimple_call_arg (stmt, 1))) 2853 { 2854 if (gimple_code (ctx->stmt) == GIMPLE_OMP_SECTIONS) 2855 { 2856 ctx->cancellable = true; 2857 if (find_omp_clause (gimple_omp_sections_clauses 2858 (ctx->stmt), 2859 OMP_CLAUSE_NOWAIT)) 2860 warning_at (gimple_location (stmt), 0, 2861 "%<#pragma omp cancel sections%> inside " 2862 "%<nowait%> sections construct"); 2863 } 2864 else 2865 { 2866 gcc_assert (ctx->outer 2867 && gimple_code (ctx->outer->stmt) 2868 == GIMPLE_OMP_SECTIONS); 2869 ctx->outer->cancellable = true; 2870 if (find_omp_clause (gimple_omp_sections_clauses 2871 (ctx->outer->stmt), 2872 OMP_CLAUSE_NOWAIT)) 2873 warning_at (gimple_location (stmt), 0, 2874 "%<#pragma omp cancel sections%> inside " 2875 "%<nowait%> sections construct"); 2876 } 2877 } 2878 kind = "sections"; 2879 break; 2880 case 8: 2881 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TASK) 2882 bad = "#pragma omp task"; 2883 else 2884 ctx->cancellable = true; 2885 kind = "taskgroup"; 2886 break; 2887 default: 2888 error_at (gimple_location (stmt), "invalid arguments"); 2889 return false; 2890 } 2891 if (bad) 2892 { 2893 error_at (gimple_location (stmt), 2894 "%<%s %s%> construct not closely nested inside of %qs", 2895 DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2896 == BUILT_IN_GOMP_CANCEL 2897 ? "#pragma omp cancel" 2898 : "#pragma omp cancellation point", kind, bad); 2899 return false; 2900 } 2901 } 2902 /* FALLTHRU */ 2903 case GIMPLE_OMP_SECTIONS: 2904 case GIMPLE_OMP_SINGLE: 2905 for (; ctx != NULL; ctx = ctx->outer) 2906 switch (gimple_code (ctx->stmt)) 2907 { 2908 case GIMPLE_OMP_FOR: 2909 case GIMPLE_OMP_SECTIONS: 2910 case GIMPLE_OMP_SINGLE: 2911 case GIMPLE_OMP_ORDERED: 2912 case GIMPLE_OMP_MASTER: 2913 case GIMPLE_OMP_TASK: 2914 case GIMPLE_OMP_CRITICAL: 2915 if (is_gimple_call (stmt)) 2916 { 2917 if (DECL_FUNCTION_CODE (gimple_call_fndecl (stmt)) 2918 != BUILT_IN_GOMP_BARRIER) 2919 return true; 2920 error_at (gimple_location (stmt), 2921 "barrier region may not be closely nested inside " 2922 "of work-sharing, critical, ordered, master or " 2923 "explicit task region"); 2924 return false; 2925 } 2926 error_at (gimple_location (stmt), 2927 "work-sharing region may not be closely nested inside " 2928 "of work-sharing, critical, ordered, master or explicit " 2929 "task region"); 2930 return false; 2931 case GIMPLE_OMP_PARALLEL: 2932 return true; 2933 default: 2934 break; 2935 } 2936 break; 2937 case GIMPLE_OMP_MASTER: 2938 for (; ctx != NULL; ctx = ctx->outer) 2939 switch (gimple_code (ctx->stmt)) 2940 { 2941 case GIMPLE_OMP_FOR: 2942 case GIMPLE_OMP_SECTIONS: 2943 case GIMPLE_OMP_SINGLE: 2944 case GIMPLE_OMP_TASK: 2945 error_at (gimple_location (stmt), 2946 "master region may not be closely nested inside " 2947 "of work-sharing or explicit task region"); 2948 return false; 2949 case GIMPLE_OMP_PARALLEL: 2950 return true; 2951 default: 2952 break; 2953 } 2954 break; 2955 case GIMPLE_OMP_ORDERED: 2956 for (; ctx != NULL; ctx = ctx->outer) 2957 switch (gimple_code (ctx->stmt)) 2958 { 2959 case GIMPLE_OMP_CRITICAL: 2960 case GIMPLE_OMP_TASK: 2961 error_at (gimple_location (stmt), 2962 "ordered region may not be closely nested inside " 2963 "of critical or explicit task region"); 2964 return false; 2965 case GIMPLE_OMP_FOR: 2966 if (find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 2967 OMP_CLAUSE_ORDERED) == NULL) 2968 { 2969 error_at (gimple_location (stmt), 2970 "ordered region must be closely nested inside " 2971 "a loop region with an ordered clause"); 2972 return false; 2973 } 2974 return true; 2975 case GIMPLE_OMP_PARALLEL: 2976 error_at (gimple_location (stmt), 2977 "ordered region must be closely nested inside " 2978 "a loop region with an ordered clause"); 2979 return false; 2980 default: 2981 break; 2982 } 2983 break; 2984 case GIMPLE_OMP_CRITICAL: 2985 { 2986 tree this_stmt_name 2987 = gimple_omp_critical_name (as_a <gomp_critical *> (stmt)); 2988 for (; ctx != NULL; ctx = ctx->outer) 2989 if (gomp_critical *other_crit 2990 = dyn_cast <gomp_critical *> (ctx->stmt)) 2991 if (this_stmt_name == gimple_omp_critical_name (other_crit)) 2992 { 2993 error_at (gimple_location (stmt), 2994 "critical region may not be nested inside a critical " 2995 "region with the same name"); 2996 return false; 2997 } 2998 } 2999 break; 3000 case GIMPLE_OMP_TEAMS: 3001 if (ctx == NULL 3002 || gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET 3003 || gimple_omp_target_kind (ctx->stmt) != GF_OMP_TARGET_KIND_REGION) 3004 { 3005 error_at (gimple_location (stmt), 3006 "teams construct not closely nested inside of target " 3007 "region"); 3008 return false; 3009 } 3010 break; 3011 case GIMPLE_OMP_TARGET: 3012 for (; ctx != NULL; ctx = ctx->outer) 3013 { 3014 if (gimple_code (ctx->stmt) != GIMPLE_OMP_TARGET) 3015 { 3016 if (is_gimple_omp (stmt) 3017 && is_gimple_omp_oacc (stmt) 3018 && is_gimple_omp (ctx->stmt)) 3019 { 3020 error_at (gimple_location (stmt), 3021 "OpenACC construct inside of non-OpenACC region"); 3022 return false; 3023 } 3024 continue; 3025 } 3026 3027 const char *stmt_name, *ctx_stmt_name; 3028 switch (gimple_omp_target_kind (stmt)) 3029 { 3030 case GF_OMP_TARGET_KIND_REGION: stmt_name = "target"; break; 3031 case GF_OMP_TARGET_KIND_DATA: stmt_name = "target data"; break; 3032 case GF_OMP_TARGET_KIND_UPDATE: stmt_name = "target update"; break; 3033 case GF_OMP_TARGET_KIND_OACC_PARALLEL: stmt_name = "parallel"; break; 3034 case GF_OMP_TARGET_KIND_OACC_KERNELS: stmt_name = "kernels"; break; 3035 case GF_OMP_TARGET_KIND_OACC_DATA: stmt_name = "data"; break; 3036 case GF_OMP_TARGET_KIND_OACC_UPDATE: stmt_name = "update"; break; 3037 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: stmt_name = "enter/exit data"; break; 3038 default: gcc_unreachable (); 3039 } 3040 switch (gimple_omp_target_kind (ctx->stmt)) 3041 { 3042 case GF_OMP_TARGET_KIND_REGION: ctx_stmt_name = "target"; break; 3043 case GF_OMP_TARGET_KIND_DATA: ctx_stmt_name = "target data"; break; 3044 case GF_OMP_TARGET_KIND_OACC_PARALLEL: ctx_stmt_name = "parallel"; break; 3045 case GF_OMP_TARGET_KIND_OACC_KERNELS: ctx_stmt_name = "kernels"; break; 3046 case GF_OMP_TARGET_KIND_OACC_DATA: ctx_stmt_name = "data"; break; 3047 default: gcc_unreachable (); 3048 } 3049 3050 /* OpenACC/OpenMP mismatch? */ 3051 if (is_gimple_omp_oacc (stmt) 3052 != is_gimple_omp_oacc (ctx->stmt)) 3053 { 3054 error_at (gimple_location (stmt), 3055 "%s %s construct inside of %s %s region", 3056 (is_gimple_omp_oacc (stmt) 3057 ? "OpenACC" : "OpenMP"), stmt_name, 3058 (is_gimple_omp_oacc (ctx->stmt) 3059 ? "OpenACC" : "OpenMP"), ctx_stmt_name); 3060 return false; 3061 } 3062 if (is_gimple_omp_offloaded (ctx->stmt)) 3063 { 3064 /* No GIMPLE_OMP_TARGET inside offloaded OpenACC CTX. */ 3065 if (is_gimple_omp_oacc (ctx->stmt)) 3066 { 3067 error_at (gimple_location (stmt), 3068 "%s construct inside of %s region", 3069 stmt_name, ctx_stmt_name); 3070 return false; 3071 } 3072 else 3073 { 3074 gcc_checking_assert (!is_gimple_omp_oacc (stmt)); 3075 warning_at (gimple_location (stmt), 0, 3076 "%s construct inside of %s region", 3077 stmt_name, ctx_stmt_name); 3078 } 3079 } 3080 } 3081 break; 3082 default: 3083 break; 3084 } 3085 return true; 3086} 3087 3088 3089/* Helper function scan_omp. 3090 3091 Callback for walk_tree or operators in walk_gimple_stmt used to 3092 scan for OMP directives in TP. */ 3093 3094static tree 3095scan_omp_1_op (tree *tp, int *walk_subtrees, void *data) 3096{ 3097 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 3098 omp_context *ctx = (omp_context *) wi->info; 3099 tree t = *tp; 3100 3101 switch (TREE_CODE (t)) 3102 { 3103 case VAR_DECL: 3104 case PARM_DECL: 3105 case LABEL_DECL: 3106 case RESULT_DECL: 3107 if (ctx) 3108 *tp = remap_decl (t, &ctx->cb); 3109 break; 3110 3111 default: 3112 if (ctx && TYPE_P (t)) 3113 *tp = remap_type (t, &ctx->cb); 3114 else if (!DECL_P (t)) 3115 { 3116 *walk_subtrees = 1; 3117 if (ctx) 3118 { 3119 tree tem = remap_type (TREE_TYPE (t), &ctx->cb); 3120 if (tem != TREE_TYPE (t)) 3121 { 3122 if (TREE_CODE (t) == INTEGER_CST) 3123 *tp = wide_int_to_tree (tem, t); 3124 else 3125 TREE_TYPE (t) = tem; 3126 } 3127 } 3128 } 3129 break; 3130 } 3131 3132 return NULL_TREE; 3133} 3134 3135/* Return true if FNDECL is a setjmp or a longjmp. */ 3136 3137static bool 3138setjmp_or_longjmp_p (const_tree fndecl) 3139{ 3140 if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL 3141 && (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_SETJMP 3142 || DECL_FUNCTION_CODE (fndecl) == BUILT_IN_LONGJMP)) 3143 return true; 3144 3145 tree declname = DECL_NAME (fndecl); 3146 if (!declname) 3147 return false; 3148 const char *name = IDENTIFIER_POINTER (declname); 3149 return !strcmp (name, "setjmp") || !strcmp (name, "longjmp"); 3150} 3151 3152 3153/* Helper function for scan_omp. 3154 3155 Callback for walk_gimple_stmt used to scan for OMP directives in 3156 the current statement in GSI. */ 3157 3158static tree 3159scan_omp_1_stmt (gimple_stmt_iterator *gsi, bool *handled_ops_p, 3160 struct walk_stmt_info *wi) 3161{ 3162 gimple stmt = gsi_stmt (*gsi); 3163 omp_context *ctx = (omp_context *) wi->info; 3164 3165 if (gimple_has_location (stmt)) 3166 input_location = gimple_location (stmt); 3167 3168 /* Check the nesting restrictions. */ 3169 bool remove = false; 3170 if (is_gimple_omp (stmt)) 3171 remove = !check_omp_nesting_restrictions (stmt, ctx); 3172 else if (is_gimple_call (stmt)) 3173 { 3174 tree fndecl = gimple_call_fndecl (stmt); 3175 if (fndecl) 3176 { 3177 if (setjmp_or_longjmp_p (fndecl) 3178 && ctx 3179 && gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 3180 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 3181 { 3182 remove = true; 3183 error_at (gimple_location (stmt), 3184 "setjmp/longjmp inside simd construct"); 3185 } 3186 else if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 3187 switch (DECL_FUNCTION_CODE (fndecl)) 3188 { 3189 case BUILT_IN_GOMP_BARRIER: 3190 case BUILT_IN_GOMP_CANCEL: 3191 case BUILT_IN_GOMP_CANCELLATION_POINT: 3192 case BUILT_IN_GOMP_TASKYIELD: 3193 case BUILT_IN_GOMP_TASKWAIT: 3194 case BUILT_IN_GOMP_TASKGROUP_START: 3195 case BUILT_IN_GOMP_TASKGROUP_END: 3196 remove = !check_omp_nesting_restrictions (stmt, ctx); 3197 break; 3198 default: 3199 break; 3200 } 3201 } 3202 } 3203 if (remove) 3204 { 3205 stmt = gimple_build_nop (); 3206 gsi_replace (gsi, stmt, false); 3207 } 3208 3209 *handled_ops_p = true; 3210 3211 switch (gimple_code (stmt)) 3212 { 3213 case GIMPLE_OMP_PARALLEL: 3214 taskreg_nesting_level++; 3215 scan_omp_parallel (gsi, ctx); 3216 taskreg_nesting_level--; 3217 break; 3218 3219 case GIMPLE_OMP_TASK: 3220 taskreg_nesting_level++; 3221 scan_omp_task (gsi, ctx); 3222 taskreg_nesting_level--; 3223 break; 3224 3225 case GIMPLE_OMP_FOR: 3226 scan_omp_for (as_a <gomp_for *> (stmt), ctx); 3227 break; 3228 3229 case GIMPLE_OMP_SECTIONS: 3230 scan_omp_sections (as_a <gomp_sections *> (stmt), ctx); 3231 break; 3232 3233 case GIMPLE_OMP_SINGLE: 3234 scan_omp_single (as_a <gomp_single *> (stmt), ctx); 3235 break; 3236 3237 case GIMPLE_OMP_SECTION: 3238 case GIMPLE_OMP_MASTER: 3239 case GIMPLE_OMP_TASKGROUP: 3240 case GIMPLE_OMP_ORDERED: 3241 case GIMPLE_OMP_CRITICAL: 3242 ctx = new_omp_context (stmt, ctx); 3243 scan_omp (gimple_omp_body_ptr (stmt), ctx); 3244 break; 3245 3246 case GIMPLE_OMP_TARGET: 3247 scan_omp_target (as_a <gomp_target *> (stmt), ctx); 3248 break; 3249 3250 case GIMPLE_OMP_TEAMS: 3251 scan_omp_teams (as_a <gomp_teams *> (stmt), ctx); 3252 break; 3253 3254 case GIMPLE_BIND: 3255 { 3256 tree var; 3257 3258 *handled_ops_p = false; 3259 if (ctx) 3260 for (var = gimple_bind_vars (as_a <gbind *> (stmt)); 3261 var ; 3262 var = DECL_CHAIN (var)) 3263 insert_decl_map (&ctx->cb, var, var); 3264 } 3265 break; 3266 default: 3267 *handled_ops_p = false; 3268 break; 3269 } 3270 3271 return NULL_TREE; 3272} 3273 3274 3275/* Scan all the statements starting at the current statement. CTX 3276 contains context information about the OMP directives and 3277 clauses found during the scan. */ 3278 3279static void 3280scan_omp (gimple_seq *body_p, omp_context *ctx) 3281{ 3282 location_t saved_location; 3283 struct walk_stmt_info wi; 3284 3285 memset (&wi, 0, sizeof (wi)); 3286 wi.info = ctx; 3287 wi.want_locations = true; 3288 3289 saved_location = input_location; 3290 walk_gimple_seq_mod (body_p, scan_omp_1_stmt, scan_omp_1_op, &wi); 3291 input_location = saved_location; 3292} 3293 3294/* Re-gimplification and code generation routines. */ 3295 3296/* Build a call to GOMP_barrier. */ 3297 3298static gimple 3299build_omp_barrier (tree lhs) 3300{ 3301 tree fndecl = builtin_decl_explicit (lhs ? BUILT_IN_GOMP_BARRIER_CANCEL 3302 : BUILT_IN_GOMP_BARRIER); 3303 gcall *g = gimple_build_call (fndecl, 0); 3304 if (lhs) 3305 gimple_call_set_lhs (g, lhs); 3306 return g; 3307} 3308 3309/* If a context was created for STMT when it was scanned, return it. */ 3310 3311static omp_context * 3312maybe_lookup_ctx (gimple stmt) 3313{ 3314 splay_tree_node n; 3315 n = splay_tree_lookup (all_contexts, (splay_tree_key) stmt); 3316 return n ? (omp_context *) n->value : NULL; 3317} 3318 3319 3320/* Find the mapping for DECL in CTX or the immediately enclosing 3321 context that has a mapping for DECL. 3322 3323 If CTX is a nested parallel directive, we may have to use the decl 3324 mappings created in CTX's parent context. Suppose that we have the 3325 following parallel nesting (variable UIDs showed for clarity): 3326 3327 iD.1562 = 0; 3328 #omp parallel shared(iD.1562) -> outer parallel 3329 iD.1562 = iD.1562 + 1; 3330 3331 #omp parallel shared (iD.1562) -> inner parallel 3332 iD.1562 = iD.1562 - 1; 3333 3334 Each parallel structure will create a distinct .omp_data_s structure 3335 for copying iD.1562 in/out of the directive: 3336 3337 outer parallel .omp_data_s.1.i -> iD.1562 3338 inner parallel .omp_data_s.2.i -> iD.1562 3339 3340 A shared variable mapping will produce a copy-out operation before 3341 the parallel directive and a copy-in operation after it. So, in 3342 this case we would have: 3343 3344 iD.1562 = 0; 3345 .omp_data_o.1.i = iD.1562; 3346 #omp parallel shared(iD.1562) -> outer parallel 3347 .omp_data_i.1 = &.omp_data_o.1 3348 .omp_data_i.1->i = .omp_data_i.1->i + 1; 3349 3350 .omp_data_o.2.i = iD.1562; -> ** 3351 #omp parallel shared(iD.1562) -> inner parallel 3352 .omp_data_i.2 = &.omp_data_o.2 3353 .omp_data_i.2->i = .omp_data_i.2->i - 1; 3354 3355 3356 ** This is a problem. The symbol iD.1562 cannot be referenced 3357 inside the body of the outer parallel region. But since we are 3358 emitting this copy operation while expanding the inner parallel 3359 directive, we need to access the CTX structure of the outer 3360 parallel directive to get the correct mapping: 3361 3362 .omp_data_o.2.i = .omp_data_i.1->i 3363 3364 Since there may be other workshare or parallel directives enclosing 3365 the parallel directive, it may be necessary to walk up the context 3366 parent chain. This is not a problem in general because nested 3367 parallelism happens only rarely. */ 3368 3369static tree 3370lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) 3371{ 3372 tree t; 3373 omp_context *up; 3374 3375 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) 3376 t = maybe_lookup_decl (decl, up); 3377 3378 gcc_assert (!ctx->is_nested || t || is_global_var (decl)); 3379 3380 return t ? t : decl; 3381} 3382 3383 3384/* Similar to lookup_decl_in_outer_ctx, but return DECL if not found 3385 in outer contexts. */ 3386 3387static tree 3388maybe_lookup_decl_in_outer_ctx (tree decl, omp_context *ctx) 3389{ 3390 tree t = NULL; 3391 omp_context *up; 3392 3393 for (up = ctx->outer, t = NULL; up && t == NULL; up = up->outer) 3394 t = maybe_lookup_decl (decl, up); 3395 3396 return t ? t : decl; 3397} 3398 3399 3400/* Construct the initialization value for reduction CLAUSE. */ 3401 3402tree 3403omp_reduction_init (tree clause, tree type) 3404{ 3405 location_t loc = OMP_CLAUSE_LOCATION (clause); 3406 switch (OMP_CLAUSE_REDUCTION_CODE (clause)) 3407 { 3408 case PLUS_EXPR: 3409 case MINUS_EXPR: 3410 case BIT_IOR_EXPR: 3411 case BIT_XOR_EXPR: 3412 case TRUTH_OR_EXPR: 3413 case TRUTH_ORIF_EXPR: 3414 case TRUTH_XOR_EXPR: 3415 case NE_EXPR: 3416 return build_zero_cst (type); 3417 3418 case MULT_EXPR: 3419 case TRUTH_AND_EXPR: 3420 case TRUTH_ANDIF_EXPR: 3421 case EQ_EXPR: 3422 return fold_convert_loc (loc, type, integer_one_node); 3423 3424 case BIT_AND_EXPR: 3425 return fold_convert_loc (loc, type, integer_minus_one_node); 3426 3427 case MAX_EXPR: 3428 if (SCALAR_FLOAT_TYPE_P (type)) 3429 { 3430 REAL_VALUE_TYPE max, min; 3431 if (HONOR_INFINITIES (type)) 3432 { 3433 real_inf (&max); 3434 real_arithmetic (&min, NEGATE_EXPR, &max, NULL); 3435 } 3436 else 3437 real_maxval (&min, 1, TYPE_MODE (type)); 3438 return build_real (type, min); 3439 } 3440 else 3441 { 3442 gcc_assert (INTEGRAL_TYPE_P (type)); 3443 return TYPE_MIN_VALUE (type); 3444 } 3445 3446 case MIN_EXPR: 3447 if (SCALAR_FLOAT_TYPE_P (type)) 3448 { 3449 REAL_VALUE_TYPE max; 3450 if (HONOR_INFINITIES (type)) 3451 real_inf (&max); 3452 else 3453 real_maxval (&max, 0, TYPE_MODE (type)); 3454 return build_real (type, max); 3455 } 3456 else 3457 { 3458 gcc_assert (INTEGRAL_TYPE_P (type)); 3459 return TYPE_MAX_VALUE (type); 3460 } 3461 3462 default: 3463 gcc_unreachable (); 3464 } 3465} 3466 3467/* Return alignment to be assumed for var in CLAUSE, which should be 3468 OMP_CLAUSE_ALIGNED. */ 3469 3470static tree 3471omp_clause_aligned_alignment (tree clause) 3472{ 3473 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (clause)) 3474 return OMP_CLAUSE_ALIGNED_ALIGNMENT (clause); 3475 3476 /* Otherwise return implementation defined alignment. */ 3477 unsigned int al = 1; 3478 machine_mode mode, vmode; 3479 int vs = targetm.vectorize.autovectorize_vector_sizes (); 3480 if (vs) 3481 vs = 1 << floor_log2 (vs); 3482 static enum mode_class classes[] 3483 = { MODE_INT, MODE_VECTOR_INT, MODE_FLOAT, MODE_VECTOR_FLOAT }; 3484 for (int i = 0; i < 4; i += 2) 3485 for (mode = GET_CLASS_NARROWEST_MODE (classes[i]); 3486 mode != VOIDmode; 3487 mode = GET_MODE_WIDER_MODE (mode)) 3488 { 3489 vmode = targetm.vectorize.preferred_simd_mode (mode); 3490 if (GET_MODE_CLASS (vmode) != classes[i + 1]) 3491 continue; 3492 while (vs 3493 && GET_MODE_SIZE (vmode) < vs 3494 && GET_MODE_2XWIDER_MODE (vmode) != VOIDmode) 3495 vmode = GET_MODE_2XWIDER_MODE (vmode); 3496 3497 tree type = lang_hooks.types.type_for_mode (mode, 1); 3498 if (type == NULL_TREE || TYPE_MODE (type) != mode) 3499 continue; 3500 type = build_vector_type (type, GET_MODE_SIZE (vmode) 3501 / GET_MODE_SIZE (mode)); 3502 if (TYPE_MODE (type) != vmode) 3503 continue; 3504 if (TYPE_ALIGN_UNIT (type) > al) 3505 al = TYPE_ALIGN_UNIT (type); 3506 } 3507 return build_int_cst (integer_type_node, al); 3508} 3509 3510/* Return maximum possible vectorization factor for the target. */ 3511 3512static int 3513omp_max_vf (void) 3514{ 3515 if (!optimize 3516 || optimize_debug 3517 || !flag_tree_loop_optimize 3518 || (!flag_tree_loop_vectorize 3519 && (global_options_set.x_flag_tree_loop_vectorize 3520 || global_options_set.x_flag_tree_vectorize))) 3521 return 1; 3522 3523 int vs = targetm.vectorize.autovectorize_vector_sizes (); 3524 if (vs) 3525 { 3526 vs = 1 << floor_log2 (vs); 3527 return vs; 3528 } 3529 machine_mode vqimode = targetm.vectorize.preferred_simd_mode (QImode); 3530 if (GET_MODE_CLASS (vqimode) == MODE_VECTOR_INT) 3531 return GET_MODE_NUNITS (vqimode); 3532 return 1; 3533} 3534 3535/* Helper function of lower_rec_input_clauses, used for #pragma omp simd 3536 privatization. */ 3537 3538static bool 3539lower_rec_simd_input_clauses (tree new_var, omp_context *ctx, int &max_vf, 3540 tree &idx, tree &lane, tree &ivar, tree &lvar) 3541{ 3542 if (max_vf == 0) 3543 { 3544 max_vf = omp_max_vf (); 3545 if (max_vf > 1) 3546 { 3547 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 3548 OMP_CLAUSE_SAFELEN); 3549 if (c && TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) != INTEGER_CST) 3550 max_vf = 1; 3551 else if (c && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), 3552 max_vf) == -1) 3553 max_vf = tree_to_shwi (OMP_CLAUSE_SAFELEN_EXPR (c)); 3554 } 3555 if (max_vf > 1) 3556 { 3557 idx = create_tmp_var (unsigned_type_node); 3558 lane = create_tmp_var (unsigned_type_node); 3559 } 3560 } 3561 if (max_vf == 1) 3562 return false; 3563 3564 tree atype = build_array_type_nelts (TREE_TYPE (new_var), max_vf); 3565 tree avar = create_tmp_var_raw (atype); 3566 if (TREE_ADDRESSABLE (new_var)) 3567 TREE_ADDRESSABLE (avar) = 1; 3568 DECL_ATTRIBUTES (avar) 3569 = tree_cons (get_identifier ("omp simd array"), NULL, 3570 DECL_ATTRIBUTES (avar)); 3571 gimple_add_tmp_var (avar); 3572 ivar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, idx, 3573 NULL_TREE, NULL_TREE); 3574 lvar = build4 (ARRAY_REF, TREE_TYPE (new_var), avar, lane, 3575 NULL_TREE, NULL_TREE); 3576 if (DECL_P (new_var)) 3577 { 3578 SET_DECL_VALUE_EXPR (new_var, lvar); 3579 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3580 } 3581 return true; 3582} 3583 3584/* Helper function of lower_rec_input_clauses. For a reference 3585 in simd reduction, add an underlying variable it will reference. */ 3586 3587static void 3588handle_simd_reference (location_t loc, tree new_vard, gimple_seq *ilist) 3589{ 3590 tree z = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_vard))); 3591 if (TREE_CONSTANT (z)) 3592 { 3593 const char *name = NULL; 3594 if (DECL_NAME (new_vard)) 3595 name = IDENTIFIER_POINTER (DECL_NAME (new_vard)); 3596 3597 z = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_vard)), name); 3598 gimple_add_tmp_var (z); 3599 TREE_ADDRESSABLE (z) = 1; 3600 z = build_fold_addr_expr_loc (loc, z); 3601 gimplify_assign (new_vard, z, ilist); 3602 } 3603} 3604 3605/* Generate code to implement the input clauses, FIRSTPRIVATE and COPYIN, 3606 from the receiver (aka child) side and initializers for REFERENCE_TYPE 3607 private variables. Initialization statements go in ILIST, while calls 3608 to destructors go in DLIST. */ 3609 3610static void 3611lower_rec_input_clauses (tree clauses, gimple_seq *ilist, gimple_seq *dlist, 3612 omp_context *ctx, struct omp_for_data *fd) 3613{ 3614 tree c, dtor, copyin_seq, x, ptr; 3615 bool copyin_by_ref = false; 3616 bool lastprivate_firstprivate = false; 3617 bool reduction_omp_orig_ref = false; 3618 int pass; 3619 bool is_simd = (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 3620 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD); 3621 int max_vf = 0; 3622 tree lane = NULL_TREE, idx = NULL_TREE; 3623 tree ivar = NULL_TREE, lvar = NULL_TREE; 3624 gimple_seq llist[2] = { NULL, NULL }; 3625 3626 copyin_seq = NULL; 3627 3628 /* Set max_vf=1 (which will later enforce safelen=1) in simd loops 3629 with data sharing clauses referencing variable sized vars. That 3630 is unnecessarily hard to support and very unlikely to result in 3631 vectorized code anyway. */ 3632 if (is_simd) 3633 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 3634 switch (OMP_CLAUSE_CODE (c)) 3635 { 3636 case OMP_CLAUSE_LINEAR: 3637 if (OMP_CLAUSE_LINEAR_ARRAY (c)) 3638 max_vf = 1; 3639 /* FALLTHRU */ 3640 case OMP_CLAUSE_REDUCTION: 3641 case OMP_CLAUSE_PRIVATE: 3642 case OMP_CLAUSE_FIRSTPRIVATE: 3643 case OMP_CLAUSE_LASTPRIVATE: 3644 if (is_variable_sized (OMP_CLAUSE_DECL (c))) 3645 max_vf = 1; 3646 break; 3647 default: 3648 continue; 3649 } 3650 3651 /* Do all the fixed sized types in the first pass, and the variable sized 3652 types in the second pass. This makes sure that the scalar arguments to 3653 the variable sized types are processed before we use them in the 3654 variable sized operations. */ 3655 for (pass = 0; pass < 2; ++pass) 3656 { 3657 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 3658 { 3659 enum omp_clause_code c_kind = OMP_CLAUSE_CODE (c); 3660 tree var, new_var; 3661 bool by_ref; 3662 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 3663 3664 switch (c_kind) 3665 { 3666 case OMP_CLAUSE_PRIVATE: 3667 if (OMP_CLAUSE_PRIVATE_DEBUG (c)) 3668 continue; 3669 break; 3670 case OMP_CLAUSE_SHARED: 3671 /* Ignore shared directives in teams construct. */ 3672 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 3673 continue; 3674 if (maybe_lookup_decl (OMP_CLAUSE_DECL (c), ctx) == NULL) 3675 { 3676 gcc_assert (is_global_var (OMP_CLAUSE_DECL (c))); 3677 continue; 3678 } 3679 case OMP_CLAUSE_FIRSTPRIVATE: 3680 case OMP_CLAUSE_COPYIN: 3681 case OMP_CLAUSE_LINEAR: 3682 break; 3683 case OMP_CLAUSE_REDUCTION: 3684 if (OMP_CLAUSE_REDUCTION_OMP_ORIG_REF (c)) 3685 reduction_omp_orig_ref = true; 3686 break; 3687 case OMP_CLAUSE__LOOPTEMP_: 3688 /* Handle _looptemp_ clauses only on parallel. */ 3689 if (fd) 3690 continue; 3691 break; 3692 case OMP_CLAUSE_LASTPRIVATE: 3693 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 3694 { 3695 lastprivate_firstprivate = true; 3696 if (pass != 0) 3697 continue; 3698 } 3699 /* Even without corresponding firstprivate, if 3700 decl is Fortran allocatable, it needs outer var 3701 reference. */ 3702 else if (pass == 0 3703 && lang_hooks.decls.omp_private_outer_ref 3704 (OMP_CLAUSE_DECL (c))) 3705 lastprivate_firstprivate = true; 3706 break; 3707 case OMP_CLAUSE_ALIGNED: 3708 if (pass == 0) 3709 continue; 3710 var = OMP_CLAUSE_DECL (c); 3711 if (TREE_CODE (TREE_TYPE (var)) == POINTER_TYPE 3712 && !is_global_var (var)) 3713 { 3714 new_var = maybe_lookup_decl (var, ctx); 3715 if (new_var == NULL_TREE) 3716 new_var = maybe_lookup_decl_in_outer_ctx (var, ctx); 3717 x = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 3718 x = build_call_expr_loc (clause_loc, x, 2, new_var, 3719 omp_clause_aligned_alignment (c)); 3720 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); 3721 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); 3722 gimplify_and_add (x, ilist); 3723 } 3724 else if (TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE 3725 && is_global_var (var)) 3726 { 3727 tree ptype = build_pointer_type (TREE_TYPE (var)), t, t2; 3728 new_var = lookup_decl (var, ctx); 3729 t = maybe_lookup_decl_in_outer_ctx (var, ctx); 3730 t = build_fold_addr_expr_loc (clause_loc, t); 3731 t2 = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 3732 t = build_call_expr_loc (clause_loc, t2, 2, t, 3733 omp_clause_aligned_alignment (c)); 3734 t = fold_convert_loc (clause_loc, ptype, t); 3735 x = create_tmp_var (ptype); 3736 t = build2 (MODIFY_EXPR, ptype, x, t); 3737 gimplify_and_add (t, ilist); 3738 t = build_simple_mem_ref_loc (clause_loc, x); 3739 SET_DECL_VALUE_EXPR (new_var, t); 3740 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3741 } 3742 continue; 3743 default: 3744 continue; 3745 } 3746 3747 new_var = var = OMP_CLAUSE_DECL (c); 3748 if (c_kind != OMP_CLAUSE_COPYIN) 3749 new_var = lookup_decl (var, ctx); 3750 3751 if (c_kind == OMP_CLAUSE_SHARED || c_kind == OMP_CLAUSE_COPYIN) 3752 { 3753 if (pass != 0) 3754 continue; 3755 } 3756 else if (is_variable_sized (var)) 3757 { 3758 /* For variable sized types, we need to allocate the 3759 actual storage here. Call alloca and store the 3760 result in the pointer decl that we created elsewhere. */ 3761 if (pass == 0) 3762 continue; 3763 3764 if (c_kind != OMP_CLAUSE_FIRSTPRIVATE || !is_task_ctx (ctx)) 3765 { 3766 gcall *stmt; 3767 tree tmp, atmp; 3768 3769 ptr = DECL_VALUE_EXPR (new_var); 3770 gcc_assert (TREE_CODE (ptr) == INDIRECT_REF); 3771 ptr = TREE_OPERAND (ptr, 0); 3772 gcc_assert (DECL_P (ptr)); 3773 x = TYPE_SIZE_UNIT (TREE_TYPE (new_var)); 3774 3775 /* void *tmp = __builtin_alloca */ 3776 atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); 3777 stmt = gimple_build_call (atmp, 1, x); 3778 tmp = create_tmp_var_raw (ptr_type_node); 3779 gimple_add_tmp_var (tmp); 3780 gimple_call_set_lhs (stmt, tmp); 3781 3782 gimple_seq_add_stmt (ilist, stmt); 3783 3784 x = fold_convert_loc (clause_loc, TREE_TYPE (ptr), tmp); 3785 gimplify_assign (ptr, x, ilist); 3786 } 3787 } 3788 else if (is_reference (var)) 3789 { 3790 /* For references that are being privatized for Fortran, 3791 allocate new backing storage for the new pointer 3792 variable. This allows us to avoid changing all the 3793 code that expects a pointer to something that expects 3794 a direct variable. */ 3795 if (pass == 0) 3796 continue; 3797 3798 x = TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (new_var))); 3799 if (c_kind == OMP_CLAUSE_FIRSTPRIVATE && is_task_ctx (ctx)) 3800 { 3801 x = build_receiver_ref (var, false, ctx); 3802 x = build_fold_addr_expr_loc (clause_loc, x); 3803 } 3804 else if (TREE_CONSTANT (x)) 3805 { 3806 /* For reduction in SIMD loop, defer adding the 3807 initialization of the reference, because if we decide 3808 to use SIMD array for it, the initilization could cause 3809 expansion ICE. */ 3810 if (c_kind == OMP_CLAUSE_REDUCTION && is_simd) 3811 x = NULL_TREE; 3812 else 3813 { 3814 const char *name = NULL; 3815 if (DECL_NAME (var)) 3816 name = IDENTIFIER_POINTER (DECL_NAME (new_var)); 3817 3818 x = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (new_var)), 3819 name); 3820 gimple_add_tmp_var (x); 3821 TREE_ADDRESSABLE (x) = 1; 3822 x = build_fold_addr_expr_loc (clause_loc, x); 3823 } 3824 } 3825 else 3826 { 3827 tree atmp = builtin_decl_explicit (BUILT_IN_ALLOCA); 3828 x = build_call_expr_loc (clause_loc, atmp, 1, x); 3829 } 3830 3831 if (x) 3832 { 3833 x = fold_convert_loc (clause_loc, TREE_TYPE (new_var), x); 3834 gimplify_assign (new_var, x, ilist); 3835 } 3836 3837 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 3838 } 3839 else if (c_kind == OMP_CLAUSE_REDUCTION 3840 && OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 3841 { 3842 if (pass == 0) 3843 continue; 3844 } 3845 else if (pass != 0) 3846 continue; 3847 3848 switch (OMP_CLAUSE_CODE (c)) 3849 { 3850 case OMP_CLAUSE_SHARED: 3851 /* Ignore shared directives in teams construct. */ 3852 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TEAMS) 3853 continue; 3854 /* Shared global vars are just accessed directly. */ 3855 if (is_global_var (new_var)) 3856 break; 3857 /* Set up the DECL_VALUE_EXPR for shared variables now. This 3858 needs to be delayed until after fixup_child_record_type so 3859 that we get the correct type during the dereference. */ 3860 by_ref = use_pointer_for_field (var, ctx); 3861 x = build_receiver_ref (var, by_ref, ctx); 3862 SET_DECL_VALUE_EXPR (new_var, x); 3863 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3864 3865 /* ??? If VAR is not passed by reference, and the variable 3866 hasn't been initialized yet, then we'll get a warning for 3867 the store into the omp_data_s structure. Ideally, we'd be 3868 able to notice this and not store anything at all, but 3869 we're generating code too early. Suppress the warning. */ 3870 if (!by_ref) 3871 TREE_NO_WARNING (var) = 1; 3872 break; 3873 3874 case OMP_CLAUSE_LASTPRIVATE: 3875 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 3876 break; 3877 /* FALLTHRU */ 3878 3879 case OMP_CLAUSE_PRIVATE: 3880 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_PRIVATE) 3881 x = build_outer_var_ref (var, ctx); 3882 else if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 3883 { 3884 if (is_task_ctx (ctx)) 3885 x = build_receiver_ref (var, false, ctx); 3886 else 3887 x = build_outer_var_ref (var, ctx); 3888 } 3889 else 3890 x = NULL; 3891 do_private: 3892 tree nx; 3893 nx = lang_hooks.decls.omp_clause_default_ctor (c, new_var, x); 3894 if (is_simd) 3895 { 3896 tree y = lang_hooks.decls.omp_clause_dtor (c, new_var); 3897 if ((TREE_ADDRESSABLE (new_var) || nx || y 3898 || OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE) 3899 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 3900 idx, lane, ivar, lvar)) 3901 { 3902 if (nx) 3903 x = lang_hooks.decls.omp_clause_default_ctor 3904 (c, unshare_expr (ivar), x); 3905 if (nx && x) 3906 gimplify_and_add (x, &llist[0]); 3907 if (y) 3908 { 3909 y = lang_hooks.decls.omp_clause_dtor (c, ivar); 3910 if (y) 3911 { 3912 gimple_seq tseq = NULL; 3913 3914 dtor = y; 3915 gimplify_stmt (&dtor, &tseq); 3916 gimple_seq_add_seq (&llist[1], tseq); 3917 } 3918 } 3919 break; 3920 } 3921 } 3922 if (nx) 3923 gimplify_and_add (nx, ilist); 3924 /* FALLTHRU */ 3925 3926 do_dtor: 3927 x = lang_hooks.decls.omp_clause_dtor (c, new_var); 3928 if (x) 3929 { 3930 gimple_seq tseq = NULL; 3931 3932 dtor = x; 3933 gimplify_stmt (&dtor, &tseq); 3934 gimple_seq_add_seq (dlist, tseq); 3935 } 3936 break; 3937 3938 case OMP_CLAUSE_LINEAR: 3939 if (!OMP_CLAUSE_LINEAR_NO_COPYIN (c)) 3940 goto do_firstprivate; 3941 if (OMP_CLAUSE_LINEAR_NO_COPYOUT (c)) 3942 x = NULL; 3943 else 3944 x = build_outer_var_ref (var, ctx); 3945 goto do_private; 3946 3947 case OMP_CLAUSE_FIRSTPRIVATE: 3948 if (is_task_ctx (ctx)) 3949 { 3950 if (is_reference (var) || is_variable_sized (var)) 3951 goto do_dtor; 3952 else if (is_global_var (maybe_lookup_decl_in_outer_ctx (var, 3953 ctx)) 3954 || use_pointer_for_field (var, NULL)) 3955 { 3956 x = build_receiver_ref (var, false, ctx); 3957 SET_DECL_VALUE_EXPR (new_var, x); 3958 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 3959 goto do_dtor; 3960 } 3961 } 3962 do_firstprivate: 3963 x = build_outer_var_ref (var, ctx); 3964 if (is_simd) 3965 { 3966 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 3967 && gimple_omp_for_combined_into_p (ctx->stmt)) 3968 { 3969 tree t = OMP_CLAUSE_LINEAR_STEP (c); 3970 tree stept = TREE_TYPE (t); 3971 tree ct = find_omp_clause (clauses, 3972 OMP_CLAUSE__LOOPTEMP_); 3973 gcc_assert (ct); 3974 tree l = OMP_CLAUSE_DECL (ct); 3975 tree n1 = fd->loop.n1; 3976 tree step = fd->loop.step; 3977 tree itype = TREE_TYPE (l); 3978 if (POINTER_TYPE_P (itype)) 3979 itype = signed_type_for (itype); 3980 l = fold_build2 (MINUS_EXPR, itype, l, n1); 3981 if (TYPE_UNSIGNED (itype) 3982 && fd->loop.cond_code == GT_EXPR) 3983 l = fold_build2 (TRUNC_DIV_EXPR, itype, 3984 fold_build1 (NEGATE_EXPR, itype, l), 3985 fold_build1 (NEGATE_EXPR, 3986 itype, step)); 3987 else 3988 l = fold_build2 (TRUNC_DIV_EXPR, itype, l, step); 3989 t = fold_build2 (MULT_EXPR, stept, 3990 fold_convert (stept, l), t); 3991 3992 if (OMP_CLAUSE_LINEAR_ARRAY (c)) 3993 { 3994 x = lang_hooks.decls.omp_clause_linear_ctor 3995 (c, new_var, x, t); 3996 gimplify_and_add (x, ilist); 3997 goto do_dtor; 3998 } 3999 4000 if (POINTER_TYPE_P (TREE_TYPE (x))) 4001 x = fold_build2 (POINTER_PLUS_EXPR, 4002 TREE_TYPE (x), x, t); 4003 else 4004 x = fold_build2 (PLUS_EXPR, TREE_TYPE (x), x, t); 4005 } 4006 4007 if ((OMP_CLAUSE_CODE (c) != OMP_CLAUSE_LINEAR 4008 || TREE_ADDRESSABLE (new_var)) 4009 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4010 idx, lane, ivar, lvar)) 4011 { 4012 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR) 4013 { 4014 tree iv = create_tmp_var (TREE_TYPE (new_var)); 4015 x = lang_hooks.decls.omp_clause_copy_ctor (c, iv, x); 4016 gimplify_and_add (x, ilist); 4017 gimple_stmt_iterator gsi 4018 = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); 4019 gassign *g 4020 = gimple_build_assign (unshare_expr (lvar), iv); 4021 gsi_insert_before_without_update (&gsi, g, 4022 GSI_SAME_STMT); 4023 tree t = OMP_CLAUSE_LINEAR_STEP (c); 4024 enum tree_code code = PLUS_EXPR; 4025 if (POINTER_TYPE_P (TREE_TYPE (new_var))) 4026 code = POINTER_PLUS_EXPR; 4027 g = gimple_build_assign (iv, code, iv, t); 4028 gsi_insert_before_without_update (&gsi, g, 4029 GSI_SAME_STMT); 4030 break; 4031 } 4032 x = lang_hooks.decls.omp_clause_copy_ctor 4033 (c, unshare_expr (ivar), x); 4034 gimplify_and_add (x, &llist[0]); 4035 x = lang_hooks.decls.omp_clause_dtor (c, ivar); 4036 if (x) 4037 { 4038 gimple_seq tseq = NULL; 4039 4040 dtor = x; 4041 gimplify_stmt (&dtor, &tseq); 4042 gimple_seq_add_seq (&llist[1], tseq); 4043 } 4044 break; 4045 } 4046 } 4047 x = lang_hooks.decls.omp_clause_copy_ctor (c, new_var, x); 4048 gimplify_and_add (x, ilist); 4049 goto do_dtor; 4050 4051 case OMP_CLAUSE__LOOPTEMP_: 4052 gcc_assert (is_parallel_ctx (ctx)); 4053 x = build_outer_var_ref (var, ctx); 4054 x = build2 (MODIFY_EXPR, TREE_TYPE (new_var), new_var, x); 4055 gimplify_and_add (x, ilist); 4056 break; 4057 4058 case OMP_CLAUSE_COPYIN: 4059 by_ref = use_pointer_for_field (var, NULL); 4060 x = build_receiver_ref (var, by_ref, ctx); 4061 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, x); 4062 append_to_statement_list (x, ©in_seq); 4063 copyin_by_ref |= by_ref; 4064 break; 4065 4066 case OMP_CLAUSE_REDUCTION: 4067 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4068 { 4069 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); 4070 gimple tseq; 4071 x = build_outer_var_ref (var, ctx); 4072 4073 if (is_reference (var) 4074 && !useless_type_conversion_p (TREE_TYPE (placeholder), 4075 TREE_TYPE (x))) 4076 x = build_fold_addr_expr_loc (clause_loc, x); 4077 SET_DECL_VALUE_EXPR (placeholder, x); 4078 DECL_HAS_VALUE_EXPR_P (placeholder) = 1; 4079 tree new_vard = new_var; 4080 if (is_reference (var)) 4081 { 4082 gcc_assert (TREE_CODE (new_var) == MEM_REF); 4083 new_vard = TREE_OPERAND (new_var, 0); 4084 gcc_assert (DECL_P (new_vard)); 4085 } 4086 if (is_simd 4087 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4088 idx, lane, ivar, lvar)) 4089 { 4090 if (new_vard == new_var) 4091 { 4092 gcc_assert (DECL_VALUE_EXPR (new_var) == lvar); 4093 SET_DECL_VALUE_EXPR (new_var, ivar); 4094 } 4095 else 4096 { 4097 SET_DECL_VALUE_EXPR (new_vard, 4098 build_fold_addr_expr (ivar)); 4099 DECL_HAS_VALUE_EXPR_P (new_vard) = 1; 4100 } 4101 x = lang_hooks.decls.omp_clause_default_ctor 4102 (c, unshare_expr (ivar), 4103 build_outer_var_ref (var, ctx)); 4104 if (x) 4105 gimplify_and_add (x, &llist[0]); 4106 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) 4107 { 4108 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); 4109 lower_omp (&tseq, ctx); 4110 gimple_seq_add_seq (&llist[0], tseq); 4111 } 4112 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; 4113 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); 4114 lower_omp (&tseq, ctx); 4115 gimple_seq_add_seq (&llist[1], tseq); 4116 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4117 DECL_HAS_VALUE_EXPR_P (placeholder) = 0; 4118 if (new_vard == new_var) 4119 SET_DECL_VALUE_EXPR (new_var, lvar); 4120 else 4121 SET_DECL_VALUE_EXPR (new_vard, 4122 build_fold_addr_expr (lvar)); 4123 x = lang_hooks.decls.omp_clause_dtor (c, ivar); 4124 if (x) 4125 { 4126 tseq = NULL; 4127 dtor = x; 4128 gimplify_stmt (&dtor, &tseq); 4129 gimple_seq_add_seq (&llist[1], tseq); 4130 } 4131 break; 4132 } 4133 /* If this is a reference to constant size reduction var 4134 with placeholder, we haven't emitted the initializer 4135 for it because it is undesirable if SIMD arrays are used. 4136 But if they aren't used, we need to emit the deferred 4137 initialization now. */ 4138 else if (is_reference (var) && is_simd) 4139 handle_simd_reference (clause_loc, new_vard, ilist); 4140 x = lang_hooks.decls.omp_clause_default_ctor 4141 (c, unshare_expr (new_var), 4142 build_outer_var_ref (var, ctx)); 4143 if (x) 4144 gimplify_and_add (x, ilist); 4145 if (OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c)) 4146 { 4147 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c); 4148 lower_omp (&tseq, ctx); 4149 gimple_seq_add_seq (ilist, tseq); 4150 } 4151 OMP_CLAUSE_REDUCTION_GIMPLE_INIT (c) = NULL; 4152 if (is_simd) 4153 { 4154 tseq = OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c); 4155 lower_omp (&tseq, ctx); 4156 gimple_seq_add_seq (dlist, tseq); 4157 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4158 } 4159 DECL_HAS_VALUE_EXPR_P (placeholder) = 0; 4160 goto do_dtor; 4161 } 4162 else 4163 { 4164 x = omp_reduction_init (c, TREE_TYPE (new_var)); 4165 gcc_assert (TREE_CODE (TREE_TYPE (new_var)) != ARRAY_TYPE); 4166 enum tree_code code = OMP_CLAUSE_REDUCTION_CODE (c); 4167 4168 /* reduction(-:var) sums up the partial results, so it 4169 acts identically to reduction(+:var). */ 4170 if (code == MINUS_EXPR) 4171 code = PLUS_EXPR; 4172 4173 tree new_vard = new_var; 4174 if (is_simd && is_reference (var)) 4175 { 4176 gcc_assert (TREE_CODE (new_var) == MEM_REF); 4177 new_vard = TREE_OPERAND (new_var, 0); 4178 gcc_assert (DECL_P (new_vard)); 4179 } 4180 if (is_simd 4181 && lower_rec_simd_input_clauses (new_var, ctx, max_vf, 4182 idx, lane, ivar, lvar)) 4183 { 4184 tree ref = build_outer_var_ref (var, ctx); 4185 4186 gimplify_assign (unshare_expr (ivar), x, &llist[0]); 4187 4188 x = build2 (code, TREE_TYPE (ref), ref, ivar); 4189 ref = build_outer_var_ref (var, ctx); 4190 gimplify_assign (ref, x, &llist[1]); 4191 4192 if (new_vard != new_var) 4193 { 4194 SET_DECL_VALUE_EXPR (new_vard, 4195 build_fold_addr_expr (lvar)); 4196 DECL_HAS_VALUE_EXPR_P (new_vard) = 1; 4197 } 4198 } 4199 else 4200 { 4201 if (is_reference (var) && is_simd) 4202 handle_simd_reference (clause_loc, new_vard, ilist); 4203 gimplify_assign (new_var, x, ilist); 4204 if (is_simd) 4205 { 4206 tree ref = build_outer_var_ref (var, ctx); 4207 4208 x = build2 (code, TREE_TYPE (ref), ref, new_var); 4209 ref = build_outer_var_ref (var, ctx); 4210 gimplify_assign (ref, x, dlist); 4211 } 4212 } 4213 } 4214 break; 4215 4216 default: 4217 gcc_unreachable (); 4218 } 4219 } 4220 } 4221 4222 if (lane) 4223 { 4224 tree uid = create_tmp_var (ptr_type_node, "simduid"); 4225 /* Don't want uninit warnings on simduid, it is always uninitialized, 4226 but we use it not for the value, but for the DECL_UID only. */ 4227 TREE_NO_WARNING (uid) = 1; 4228 gimple g 4229 = gimple_build_call_internal (IFN_GOMP_SIMD_LANE, 1, uid); 4230 gimple_call_set_lhs (g, lane); 4231 gimple_stmt_iterator gsi = gsi_start_1 (gimple_omp_body_ptr (ctx->stmt)); 4232 gsi_insert_before_without_update (&gsi, g, GSI_SAME_STMT); 4233 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__SIMDUID_); 4234 OMP_CLAUSE__SIMDUID__DECL (c) = uid; 4235 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); 4236 gimple_omp_for_set_clauses (ctx->stmt, c); 4237 g = gimple_build_assign (lane, INTEGER_CST, 4238 build_int_cst (unsigned_type_node, 0)); 4239 gimple_seq_add_stmt (ilist, g); 4240 for (int i = 0; i < 2; i++) 4241 if (llist[i]) 4242 { 4243 tree vf = create_tmp_var (unsigned_type_node); 4244 g = gimple_build_call_internal (IFN_GOMP_SIMD_VF, 1, uid); 4245 gimple_call_set_lhs (g, vf); 4246 gimple_seq *seq = i == 0 ? ilist : dlist; 4247 gimple_seq_add_stmt (seq, g); 4248 tree t = build_int_cst (unsigned_type_node, 0); 4249 g = gimple_build_assign (idx, INTEGER_CST, t); 4250 gimple_seq_add_stmt (seq, g); 4251 tree body = create_artificial_label (UNKNOWN_LOCATION); 4252 tree header = create_artificial_label (UNKNOWN_LOCATION); 4253 tree end = create_artificial_label (UNKNOWN_LOCATION); 4254 gimple_seq_add_stmt (seq, gimple_build_goto (header)); 4255 gimple_seq_add_stmt (seq, gimple_build_label (body)); 4256 gimple_seq_add_seq (seq, llist[i]); 4257 t = build_int_cst (unsigned_type_node, 1); 4258 g = gimple_build_assign (idx, PLUS_EXPR, idx, t); 4259 gimple_seq_add_stmt (seq, g); 4260 gimple_seq_add_stmt (seq, gimple_build_label (header)); 4261 g = gimple_build_cond (LT_EXPR, idx, vf, body, end); 4262 gimple_seq_add_stmt (seq, g); 4263 gimple_seq_add_stmt (seq, gimple_build_label (end)); 4264 } 4265 } 4266 4267 /* The copyin sequence is not to be executed by the main thread, since 4268 that would result in self-copies. Perhaps not visible to scalars, 4269 but it certainly is to C++ operator=. */ 4270 if (copyin_seq) 4271 { 4272 x = build_call_expr (builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM), 4273 0); 4274 x = build2 (NE_EXPR, boolean_type_node, x, 4275 build_int_cst (TREE_TYPE (x), 0)); 4276 x = build3 (COND_EXPR, void_type_node, x, copyin_seq, NULL); 4277 gimplify_and_add (x, ilist); 4278 } 4279 4280 /* If any copyin variable is passed by reference, we must ensure the 4281 master thread doesn't modify it before it is copied over in all 4282 threads. Similarly for variables in both firstprivate and 4283 lastprivate clauses we need to ensure the lastprivate copying 4284 happens after firstprivate copying in all threads. And similarly 4285 for UDRs if initializer expression refers to omp_orig. */ 4286 if (copyin_by_ref || lastprivate_firstprivate || reduction_omp_orig_ref) 4287 { 4288 /* Don't add any barrier for #pragma omp simd or 4289 #pragma omp distribute. */ 4290 if (gimple_code (ctx->stmt) != GIMPLE_OMP_FOR 4291 || gimple_omp_for_kind (ctx->stmt) == GF_OMP_FOR_KIND_FOR) 4292 gimple_seq_add_stmt (ilist, build_omp_barrier (NULL_TREE)); 4293 } 4294 4295 /* If max_vf is non-zero, then we can use only a vectorization factor 4296 up to the max_vf we chose. So stick it into the safelen clause. */ 4297 if (max_vf) 4298 { 4299 tree c = find_omp_clause (gimple_omp_for_clauses (ctx->stmt), 4300 OMP_CLAUSE_SAFELEN); 4301 if (c == NULL_TREE 4302 || (TREE_CODE (OMP_CLAUSE_SAFELEN_EXPR (c)) == INTEGER_CST 4303 && compare_tree_int (OMP_CLAUSE_SAFELEN_EXPR (c), 4304 max_vf) == 1)) 4305 { 4306 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_SAFELEN); 4307 OMP_CLAUSE_SAFELEN_EXPR (c) = build_int_cst (integer_type_node, 4308 max_vf); 4309 OMP_CLAUSE_CHAIN (c) = gimple_omp_for_clauses (ctx->stmt); 4310 gimple_omp_for_set_clauses (ctx->stmt, c); 4311 } 4312 } 4313} 4314 4315 4316/* Generate code to implement the LASTPRIVATE clauses. This is used for 4317 both parallel and workshare constructs. PREDICATE may be NULL if it's 4318 always true. */ 4319 4320static void 4321lower_lastprivate_clauses (tree clauses, tree predicate, gimple_seq *stmt_list, 4322 omp_context *ctx) 4323{ 4324 tree x, c, label = NULL, orig_clauses = clauses; 4325 bool par_clauses = false; 4326 tree simduid = NULL, lastlane = NULL; 4327 4328 /* Early exit if there are no lastprivate or linear clauses. */ 4329 for (; clauses ; clauses = OMP_CLAUSE_CHAIN (clauses)) 4330 if (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LASTPRIVATE 4331 || (OMP_CLAUSE_CODE (clauses) == OMP_CLAUSE_LINEAR 4332 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (clauses))) 4333 break; 4334 if (clauses == NULL) 4335 { 4336 /* If this was a workshare clause, see if it had been combined 4337 with its parallel. In that case, look for the clauses on the 4338 parallel statement itself. */ 4339 if (is_parallel_ctx (ctx)) 4340 return; 4341 4342 ctx = ctx->outer; 4343 if (ctx == NULL || !is_parallel_ctx (ctx)) 4344 return; 4345 4346 clauses = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 4347 OMP_CLAUSE_LASTPRIVATE); 4348 if (clauses == NULL) 4349 return; 4350 par_clauses = true; 4351 } 4352 4353 if (predicate) 4354 { 4355 gcond *stmt; 4356 tree label_true, arm1, arm2; 4357 4358 label = create_artificial_label (UNKNOWN_LOCATION); 4359 label_true = create_artificial_label (UNKNOWN_LOCATION); 4360 arm1 = TREE_OPERAND (predicate, 0); 4361 arm2 = TREE_OPERAND (predicate, 1); 4362 gimplify_expr (&arm1, stmt_list, NULL, is_gimple_val, fb_rvalue); 4363 gimplify_expr (&arm2, stmt_list, NULL, is_gimple_val, fb_rvalue); 4364 stmt = gimple_build_cond (TREE_CODE (predicate), arm1, arm2, 4365 label_true, label); 4366 gimple_seq_add_stmt (stmt_list, stmt); 4367 gimple_seq_add_stmt (stmt_list, gimple_build_label (label_true)); 4368 } 4369 4370 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 4371 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 4372 { 4373 simduid = find_omp_clause (orig_clauses, OMP_CLAUSE__SIMDUID_); 4374 if (simduid) 4375 simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); 4376 } 4377 4378 for (c = clauses; c ;) 4379 { 4380 tree var, new_var; 4381 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4382 4383 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 4384 || (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 4385 && !OMP_CLAUSE_LINEAR_NO_COPYOUT (c))) 4386 { 4387 var = OMP_CLAUSE_DECL (c); 4388 new_var = lookup_decl (var, ctx); 4389 4390 if (simduid && DECL_HAS_VALUE_EXPR_P (new_var)) 4391 { 4392 tree val = DECL_VALUE_EXPR (new_var); 4393 if (TREE_CODE (val) == ARRAY_REF 4394 && VAR_P (TREE_OPERAND (val, 0)) 4395 && lookup_attribute ("omp simd array", 4396 DECL_ATTRIBUTES (TREE_OPERAND (val, 4397 0)))) 4398 { 4399 if (lastlane == NULL) 4400 { 4401 lastlane = create_tmp_var (unsigned_type_node); 4402 gcall *g 4403 = gimple_build_call_internal (IFN_GOMP_SIMD_LAST_LANE, 4404 2, simduid, 4405 TREE_OPERAND (val, 1)); 4406 gimple_call_set_lhs (g, lastlane); 4407 gimple_seq_add_stmt (stmt_list, g); 4408 } 4409 new_var = build4 (ARRAY_REF, TREE_TYPE (val), 4410 TREE_OPERAND (val, 0), lastlane, 4411 NULL_TREE, NULL_TREE); 4412 } 4413 } 4414 4415 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LASTPRIVATE 4416 && OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)) 4417 { 4418 lower_omp (&OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c), ctx); 4419 gimple_seq_add_seq (stmt_list, 4420 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c)); 4421 OMP_CLAUSE_LASTPRIVATE_GIMPLE_SEQ (c) = NULL; 4422 } 4423 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_LINEAR 4424 && OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)) 4425 { 4426 lower_omp (&OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c), ctx); 4427 gimple_seq_add_seq (stmt_list, 4428 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c)); 4429 OMP_CLAUSE_LINEAR_GIMPLE_SEQ (c) = NULL; 4430 } 4431 4432 x = build_outer_var_ref (var, ctx); 4433 if (is_reference (var)) 4434 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4435 x = lang_hooks.decls.omp_clause_assign_op (c, x, new_var); 4436 gimplify_and_add (x, stmt_list); 4437 } 4438 c = OMP_CLAUSE_CHAIN (c); 4439 if (c == NULL && !par_clauses) 4440 { 4441 /* If this was a workshare clause, see if it had been combined 4442 with its parallel. In that case, continue looking for the 4443 clauses also on the parallel statement itself. */ 4444 if (is_parallel_ctx (ctx)) 4445 break; 4446 4447 ctx = ctx->outer; 4448 if (ctx == NULL || !is_parallel_ctx (ctx)) 4449 break; 4450 4451 c = find_omp_clause (gimple_omp_parallel_clauses (ctx->stmt), 4452 OMP_CLAUSE_LASTPRIVATE); 4453 par_clauses = true; 4454 } 4455 } 4456 4457 if (label) 4458 gimple_seq_add_stmt (stmt_list, gimple_build_label (label)); 4459} 4460 4461static void 4462oacc_lower_reduction_var_helper (gimple_seq *stmt_seqp, omp_context *ctx, 4463 tree tid, tree var, tree new_var) 4464{ 4465 /* The atomic add at the end of the sum creates unnecessary 4466 write contention on accelerators. To work around this, 4467 create an array to store the partial reductions. Later, in 4468 lower_omp_for (for openacc), the values of array will be 4469 combined. */ 4470 4471 tree t = NULL_TREE, array, x; 4472 tree type = get_base_type (var); 4473 gimple stmt; 4474 4475 /* Now insert the partial reductions into the array. */ 4476 4477 /* Find the reduction array. */ 4478 4479 tree ptype = build_pointer_type (type); 4480 4481 t = lookup_oacc_reduction (oacc_get_reduction_array_id (var), ctx); 4482 t = build_receiver_ref (t, false, ctx->outer); 4483 4484 array = create_tmp_var (ptype); 4485 gimplify_assign (array, t, stmt_seqp); 4486 4487 tree ptr = create_tmp_var (TREE_TYPE (array)); 4488 4489 /* Find the reduction array. */ 4490 4491 /* testing a unary conversion. */ 4492 tree offset = create_tmp_var (sizetype); 4493 gimplify_assign (offset, TYPE_SIZE_UNIT (type), 4494 stmt_seqp); 4495 t = create_tmp_var (sizetype); 4496 gimplify_assign (t, unshare_expr (fold_build1 (NOP_EXPR, sizetype, tid)), 4497 stmt_seqp); 4498 stmt = gimple_build_assign (offset, MULT_EXPR, offset, t); 4499 gimple_seq_add_stmt (stmt_seqp, stmt); 4500 4501 /* Offset expression. Does the POINTER_PLUS_EXPR take care 4502 of adding sizeof(var) to the array? */ 4503 ptr = create_tmp_var (ptype); 4504 stmt = gimple_build_assign (unshare_expr (ptr), POINTER_PLUS_EXPR, array, 4505 offset); 4506 gimple_seq_add_stmt (stmt_seqp, stmt); 4507 4508 /* Move the local sum to gfc$sum[i]. */ 4509 x = unshare_expr (build_simple_mem_ref (ptr)); 4510 stmt = gimplify_assign (x, new_var, stmt_seqp); 4511} 4512 4513/* Generate code to implement the REDUCTION clauses. */ 4514 4515static void 4516lower_reduction_clauses (tree clauses, gimple_seq *stmt_seqp, omp_context *ctx) 4517{ 4518 gimple_seq sub_seq = NULL; 4519 gimple stmt; 4520 tree x, c, tid = NULL_TREE; 4521 int count = 0; 4522 4523 /* SIMD reductions are handled in lower_rec_input_clauses. */ 4524 if (gimple_code (ctx->stmt) == GIMPLE_OMP_FOR 4525 && gimple_omp_for_kind (ctx->stmt) & GF_OMP_FOR_SIMD) 4526 return; 4527 4528 /* First see if there is exactly one reduction clause. Use OMP_ATOMIC 4529 update in that case, otherwise use a lock. */ 4530 for (c = clauses; c && count < 2; c = OMP_CLAUSE_CHAIN (c)) 4531 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 4532 { 4533 if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4534 { 4535 /* Never use OMP_ATOMIC for array reductions or UDRs. */ 4536 count = -1; 4537 break; 4538 } 4539 count++; 4540 } 4541 4542 if (count == 0) 4543 return; 4544 4545 /* Initialize thread info for OpenACC. */ 4546 if (is_gimple_omp_oacc (ctx->stmt)) 4547 { 4548 /* Get the current thread id. */ 4549 tree call = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 4550 tid = create_tmp_var (TREE_TYPE (TREE_TYPE (call))); 4551 gimple stmt = gimple_build_call (call, 0); 4552 gimple_call_set_lhs (stmt, tid); 4553 gimple_seq_add_stmt (stmt_seqp, stmt); 4554 } 4555 4556 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4557 { 4558 tree var, ref, new_var; 4559 enum tree_code code; 4560 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4561 4562 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 4563 continue; 4564 4565 var = OMP_CLAUSE_DECL (c); 4566 new_var = lookup_decl (var, ctx); 4567 if (is_reference (var)) 4568 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4569 ref = build_outer_var_ref (var, ctx); 4570 code = OMP_CLAUSE_REDUCTION_CODE (c); 4571 4572 /* reduction(-:var) sums up the partial results, so it acts 4573 identically to reduction(+:var). */ 4574 if (code == MINUS_EXPR) 4575 code = PLUS_EXPR; 4576 4577 if (is_gimple_omp_oacc (ctx->stmt)) 4578 { 4579 gcc_checking_assert (!OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)); 4580 4581 oacc_lower_reduction_var_helper (stmt_seqp, ctx, tid, var, new_var); 4582 } 4583 else if (count == 1) 4584 { 4585 tree addr = build_fold_addr_expr_loc (clause_loc, ref); 4586 4587 addr = save_expr (addr); 4588 ref = build1 (INDIRECT_REF, TREE_TYPE (TREE_TYPE (addr)), addr); 4589 x = fold_build2_loc (clause_loc, code, TREE_TYPE (ref), ref, new_var); 4590 x = build2 (OMP_ATOMIC, void_type_node, addr, x); 4591 gimplify_and_add (x, stmt_seqp); 4592 return; 4593 } 4594 else if (OMP_CLAUSE_REDUCTION_PLACEHOLDER (c)) 4595 { 4596 tree placeholder = OMP_CLAUSE_REDUCTION_PLACEHOLDER (c); 4597 4598 if (is_reference (var) 4599 && !useless_type_conversion_p (TREE_TYPE (placeholder), 4600 TREE_TYPE (ref))) 4601 ref = build_fold_addr_expr_loc (clause_loc, ref); 4602 SET_DECL_VALUE_EXPR (placeholder, ref); 4603 DECL_HAS_VALUE_EXPR_P (placeholder) = 1; 4604 lower_omp (&OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c), ctx); 4605 gimple_seq_add_seq (&sub_seq, OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c)); 4606 OMP_CLAUSE_REDUCTION_GIMPLE_MERGE (c) = NULL; 4607 OMP_CLAUSE_REDUCTION_PLACEHOLDER (c) = NULL; 4608 } 4609 else 4610 { 4611 x = build2 (code, TREE_TYPE (ref), ref, new_var); 4612 ref = build_outer_var_ref (var, ctx); 4613 gimplify_assign (ref, x, &sub_seq); 4614 } 4615 } 4616 4617 if (is_gimple_omp_oacc (ctx->stmt)) 4618 return; 4619 4620 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START), 4621 0); 4622 gimple_seq_add_stmt (stmt_seqp, stmt); 4623 4624 gimple_seq_add_seq (stmt_seqp, sub_seq); 4625 4626 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END), 4627 0); 4628 gimple_seq_add_stmt (stmt_seqp, stmt); 4629} 4630 4631 4632/* Generate code to implement the COPYPRIVATE clauses. */ 4633 4634static void 4635lower_copyprivate_clauses (tree clauses, gimple_seq *slist, gimple_seq *rlist, 4636 omp_context *ctx) 4637{ 4638 tree c; 4639 4640 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4641 { 4642 tree var, new_var, ref, x; 4643 bool by_ref; 4644 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4645 4646 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYPRIVATE) 4647 continue; 4648 4649 var = OMP_CLAUSE_DECL (c); 4650 by_ref = use_pointer_for_field (var, NULL); 4651 4652 ref = build_sender_ref (var, ctx); 4653 x = new_var = lookup_decl_in_outer_ctx (var, ctx); 4654 if (by_ref) 4655 { 4656 x = build_fold_addr_expr_loc (clause_loc, new_var); 4657 x = fold_convert_loc (clause_loc, TREE_TYPE (ref), x); 4658 } 4659 gimplify_assign (ref, x, slist); 4660 4661 ref = build_receiver_ref (var, false, ctx); 4662 if (by_ref) 4663 { 4664 ref = fold_convert_loc (clause_loc, 4665 build_pointer_type (TREE_TYPE (new_var)), 4666 ref); 4667 ref = build_fold_indirect_ref_loc (clause_loc, ref); 4668 } 4669 if (is_reference (var)) 4670 { 4671 ref = fold_convert_loc (clause_loc, TREE_TYPE (new_var), ref); 4672 ref = build_simple_mem_ref_loc (clause_loc, ref); 4673 new_var = build_simple_mem_ref_loc (clause_loc, new_var); 4674 } 4675 x = lang_hooks.decls.omp_clause_assign_op (c, new_var, ref); 4676 gimplify_and_add (x, rlist); 4677 } 4678} 4679 4680 4681/* Generate code to implement the clauses, FIRSTPRIVATE, COPYIN, LASTPRIVATE, 4682 and REDUCTION from the sender (aka parent) side. */ 4683 4684static void 4685lower_send_clauses (tree clauses, gimple_seq *ilist, gimple_seq *olist, 4686 omp_context *ctx) 4687{ 4688 tree c; 4689 4690 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 4691 { 4692 tree val, ref, x, var; 4693 bool by_ref, do_in = false, do_out = false; 4694 location_t clause_loc = OMP_CLAUSE_LOCATION (c); 4695 4696 switch (OMP_CLAUSE_CODE (c)) 4697 { 4698 case OMP_CLAUSE_PRIVATE: 4699 if (OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 4700 break; 4701 continue; 4702 case OMP_CLAUSE_FIRSTPRIVATE: 4703 case OMP_CLAUSE_COPYIN: 4704 case OMP_CLAUSE_LASTPRIVATE: 4705 case OMP_CLAUSE_REDUCTION: 4706 case OMP_CLAUSE__LOOPTEMP_: 4707 break; 4708 default: 4709 continue; 4710 } 4711 4712 val = OMP_CLAUSE_DECL (c); 4713 var = lookup_decl_in_outer_ctx (val, ctx); 4714 4715 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_COPYIN 4716 && is_global_var (var)) 4717 continue; 4718 if (is_variable_sized (val)) 4719 continue; 4720 by_ref = use_pointer_for_field (val, NULL); 4721 4722 switch (OMP_CLAUSE_CODE (c)) 4723 { 4724 case OMP_CLAUSE_PRIVATE: 4725 case OMP_CLAUSE_FIRSTPRIVATE: 4726 case OMP_CLAUSE_COPYIN: 4727 case OMP_CLAUSE__LOOPTEMP_: 4728 do_in = true; 4729 break; 4730 4731 case OMP_CLAUSE_LASTPRIVATE: 4732 if (by_ref || is_reference (val)) 4733 { 4734 if (OMP_CLAUSE_LASTPRIVATE_FIRSTPRIVATE (c)) 4735 continue; 4736 do_in = true; 4737 } 4738 else 4739 { 4740 do_out = true; 4741 if (lang_hooks.decls.omp_private_outer_ref (val)) 4742 do_in = true; 4743 } 4744 break; 4745 4746 case OMP_CLAUSE_REDUCTION: 4747 do_in = true; 4748 do_out = !(by_ref || is_reference (val)); 4749 break; 4750 4751 default: 4752 gcc_unreachable (); 4753 } 4754 4755 if (do_in) 4756 { 4757 ref = build_sender_ref (val, ctx); 4758 x = by_ref ? build_fold_addr_expr_loc (clause_loc, var) : var; 4759 gimplify_assign (ref, x, ilist); 4760 if (is_task_ctx (ctx)) 4761 DECL_ABSTRACT_ORIGIN (TREE_OPERAND (ref, 1)) = NULL; 4762 } 4763 4764 if (do_out) 4765 { 4766 ref = build_sender_ref (val, ctx); 4767 gimplify_assign (var, ref, olist); 4768 } 4769 } 4770} 4771 4772/* Generate code to implement SHARED from the sender (aka parent) 4773 side. This is trickier, since GIMPLE_OMP_PARALLEL_CLAUSES doesn't 4774 list things that got automatically shared. */ 4775 4776static void 4777lower_send_shared_vars (gimple_seq *ilist, gimple_seq *olist, omp_context *ctx) 4778{ 4779 tree var, ovar, nvar, f, x, record_type; 4780 4781 if (ctx->record_type == NULL) 4782 return; 4783 4784 record_type = ctx->srecord_type ? ctx->srecord_type : ctx->record_type; 4785 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) 4786 { 4787 ovar = DECL_ABSTRACT_ORIGIN (f); 4788 nvar = maybe_lookup_decl (ovar, ctx); 4789 if (!nvar || !DECL_HAS_VALUE_EXPR_P (nvar)) 4790 continue; 4791 4792 /* If CTX is a nested parallel directive. Find the immediately 4793 enclosing parallel or workshare construct that contains a 4794 mapping for OVAR. */ 4795 var = lookup_decl_in_outer_ctx (ovar, ctx); 4796 4797 if (use_pointer_for_field (ovar, ctx)) 4798 { 4799 x = build_sender_ref (ovar, ctx); 4800 var = build_fold_addr_expr (var); 4801 gimplify_assign (x, var, ilist); 4802 } 4803 else 4804 { 4805 x = build_sender_ref (ovar, ctx); 4806 gimplify_assign (x, var, ilist); 4807 4808 if (!TREE_READONLY (var) 4809 /* We don't need to receive a new reference to a result 4810 or parm decl. In fact we may not store to it as we will 4811 invalidate any pending RSO and generate wrong gimple 4812 during inlining. */ 4813 && !((TREE_CODE (var) == RESULT_DECL 4814 || TREE_CODE (var) == PARM_DECL) 4815 && DECL_BY_REFERENCE (var))) 4816 { 4817 x = build_sender_ref (ovar, ctx); 4818 gimplify_assign (var, x, olist); 4819 } 4820 } 4821 } 4822} 4823 4824 4825/* A convenience function to build an empty GIMPLE_COND with just the 4826 condition. */ 4827 4828static gcond * 4829gimple_build_cond_empty (tree cond) 4830{ 4831 enum tree_code pred_code; 4832 tree lhs, rhs; 4833 4834 gimple_cond_get_ops_from_tree (cond, &pred_code, &lhs, &rhs); 4835 return gimple_build_cond (pred_code, lhs, rhs, NULL_TREE, NULL_TREE); 4836} 4837 4838 4839/* Build the function calls to GOMP_parallel_start etc to actually 4840 generate the parallel operation. REGION is the parallel region 4841 being expanded. BB is the block where to insert the code. WS_ARGS 4842 will be set if this is a call to a combined parallel+workshare 4843 construct, it contains the list of additional arguments needed by 4844 the workshare construct. */ 4845 4846static void 4847expand_parallel_call (struct omp_region *region, basic_block bb, 4848 gomp_parallel *entry_stmt, 4849 vec<tree, va_gc> *ws_args) 4850{ 4851 tree t, t1, t2, val, cond, c, clauses, flags; 4852 gimple_stmt_iterator gsi; 4853 gimple stmt; 4854 enum built_in_function start_ix; 4855 int start_ix2; 4856 location_t clause_loc; 4857 vec<tree, va_gc> *args; 4858 4859 clauses = gimple_omp_parallel_clauses (entry_stmt); 4860 4861 /* Determine what flavor of GOMP_parallel we will be 4862 emitting. */ 4863 start_ix = BUILT_IN_GOMP_PARALLEL; 4864 if (is_combined_parallel (region)) 4865 { 4866 switch (region->inner->type) 4867 { 4868 case GIMPLE_OMP_FOR: 4869 gcc_assert (region->inner->sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); 4870 start_ix2 = ((int)BUILT_IN_GOMP_PARALLEL_LOOP_STATIC 4871 + (region->inner->sched_kind 4872 == OMP_CLAUSE_SCHEDULE_RUNTIME 4873 ? 3 : region->inner->sched_kind)); 4874 start_ix = (enum built_in_function)start_ix2; 4875 break; 4876 case GIMPLE_OMP_SECTIONS: 4877 start_ix = BUILT_IN_GOMP_PARALLEL_SECTIONS; 4878 break; 4879 default: 4880 gcc_unreachable (); 4881 } 4882 } 4883 4884 /* By default, the value of NUM_THREADS is zero (selected at run time) 4885 and there is no conditional. */ 4886 cond = NULL_TREE; 4887 val = build_int_cst (unsigned_type_node, 0); 4888 flags = build_int_cst (unsigned_type_node, 0); 4889 4890 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 4891 if (c) 4892 cond = OMP_CLAUSE_IF_EXPR (c); 4893 4894 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_THREADS); 4895 if (c) 4896 { 4897 val = OMP_CLAUSE_NUM_THREADS_EXPR (c); 4898 clause_loc = OMP_CLAUSE_LOCATION (c); 4899 } 4900 else 4901 clause_loc = gimple_location (entry_stmt); 4902 4903 c = find_omp_clause (clauses, OMP_CLAUSE_PROC_BIND); 4904 if (c) 4905 flags = build_int_cst (unsigned_type_node, OMP_CLAUSE_PROC_BIND_KIND (c)); 4906 4907 /* Ensure 'val' is of the correct type. */ 4908 val = fold_convert_loc (clause_loc, unsigned_type_node, val); 4909 4910 /* If we found the clause 'if (cond)', build either 4911 (cond != 0) or (cond ? val : 1u). */ 4912 if (cond) 4913 { 4914 cond = gimple_boolify (cond); 4915 4916 if (integer_zerop (val)) 4917 val = fold_build2_loc (clause_loc, 4918 EQ_EXPR, unsigned_type_node, cond, 4919 build_int_cst (TREE_TYPE (cond), 0)); 4920 else 4921 { 4922 basic_block cond_bb, then_bb, else_bb; 4923 edge e, e_then, e_else; 4924 tree tmp_then, tmp_else, tmp_join, tmp_var; 4925 4926 tmp_var = create_tmp_var (TREE_TYPE (val)); 4927 if (gimple_in_ssa_p (cfun)) 4928 { 4929 tmp_then = make_ssa_name (tmp_var); 4930 tmp_else = make_ssa_name (tmp_var); 4931 tmp_join = make_ssa_name (tmp_var); 4932 } 4933 else 4934 { 4935 tmp_then = tmp_var; 4936 tmp_else = tmp_var; 4937 tmp_join = tmp_var; 4938 } 4939 4940 e = split_block (bb, NULL); 4941 cond_bb = e->src; 4942 bb = e->dest; 4943 remove_edge (e); 4944 4945 then_bb = create_empty_bb (cond_bb); 4946 else_bb = create_empty_bb (then_bb); 4947 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); 4948 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); 4949 4950 stmt = gimple_build_cond_empty (cond); 4951 gsi = gsi_start_bb (cond_bb); 4952 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4953 4954 gsi = gsi_start_bb (then_bb); 4955 stmt = gimple_build_assign (tmp_then, val); 4956 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4957 4958 gsi = gsi_start_bb (else_bb); 4959 stmt = gimple_build_assign 4960 (tmp_else, build_int_cst (unsigned_type_node, 1)); 4961 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 4962 4963 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); 4964 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); 4965 add_bb_to_loop (then_bb, cond_bb->loop_father); 4966 add_bb_to_loop (else_bb, cond_bb->loop_father); 4967 e_then = make_edge (then_bb, bb, EDGE_FALLTHRU); 4968 e_else = make_edge (else_bb, bb, EDGE_FALLTHRU); 4969 4970 if (gimple_in_ssa_p (cfun)) 4971 { 4972 gphi *phi = create_phi_node (tmp_join, bb); 4973 add_phi_arg (phi, tmp_then, e_then, UNKNOWN_LOCATION); 4974 add_phi_arg (phi, tmp_else, e_else, UNKNOWN_LOCATION); 4975 } 4976 4977 val = tmp_join; 4978 } 4979 4980 gsi = gsi_start_bb (bb); 4981 val = force_gimple_operand_gsi (&gsi, val, true, NULL_TREE, 4982 false, GSI_CONTINUE_LINKING); 4983 } 4984 4985 gsi = gsi_last_bb (bb); 4986 t = gimple_omp_parallel_data_arg (entry_stmt); 4987 if (t == NULL) 4988 t1 = null_pointer_node; 4989 else 4990 t1 = build_fold_addr_expr (t); 4991 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); 4992 4993 vec_alloc (args, 4 + vec_safe_length (ws_args)); 4994 args->quick_push (t2); 4995 args->quick_push (t1); 4996 args->quick_push (val); 4997 if (ws_args) 4998 args->splice (*ws_args); 4999 args->quick_push (flags); 5000 5001 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, 5002 builtin_decl_explicit (start_ix), args); 5003 5004 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 5005 false, GSI_CONTINUE_LINKING); 5006} 5007 5008/* Insert a function call whose name is FUNC_NAME with the information from 5009 ENTRY_STMT into the basic_block BB. */ 5010 5011static void 5012expand_cilk_for_call (basic_block bb, gomp_parallel *entry_stmt, 5013 vec <tree, va_gc> *ws_args) 5014{ 5015 tree t, t1, t2; 5016 gimple_stmt_iterator gsi; 5017 vec <tree, va_gc> *args; 5018 5019 gcc_assert (vec_safe_length (ws_args) == 2); 5020 tree func_name = (*ws_args)[0]; 5021 tree grain = (*ws_args)[1]; 5022 5023 tree clauses = gimple_omp_parallel_clauses (entry_stmt); 5024 tree count = find_omp_clause (clauses, OMP_CLAUSE__CILK_FOR_COUNT_); 5025 gcc_assert (count != NULL_TREE); 5026 count = OMP_CLAUSE_OPERAND (count, 0); 5027 5028 gsi = gsi_last_bb (bb); 5029 t = gimple_omp_parallel_data_arg (entry_stmt); 5030 if (t == NULL) 5031 t1 = null_pointer_node; 5032 else 5033 t1 = build_fold_addr_expr (t); 5034 t2 = build_fold_addr_expr (gimple_omp_parallel_child_fn (entry_stmt)); 5035 5036 vec_alloc (args, 4); 5037 args->quick_push (t2); 5038 args->quick_push (t1); 5039 args->quick_push (count); 5040 args->quick_push (grain); 5041 t = build_call_expr_loc_vec (UNKNOWN_LOCATION, func_name, args); 5042 5043 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, false, 5044 GSI_CONTINUE_LINKING); 5045} 5046 5047/* Build the function call to GOMP_task to actually 5048 generate the task operation. BB is the block where to insert the code. */ 5049 5050static void 5051expand_task_call (basic_block bb, gomp_task *entry_stmt) 5052{ 5053 tree t, t1, t2, t3, flags, cond, c, c2, clauses, depend; 5054 gimple_stmt_iterator gsi; 5055 location_t loc = gimple_location (entry_stmt); 5056 5057 clauses = gimple_omp_task_clauses (entry_stmt); 5058 5059 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 5060 if (c) 5061 cond = gimple_boolify (OMP_CLAUSE_IF_EXPR (c)); 5062 else 5063 cond = boolean_true_node; 5064 5065 c = find_omp_clause (clauses, OMP_CLAUSE_UNTIED); 5066 c2 = find_omp_clause (clauses, OMP_CLAUSE_MERGEABLE); 5067 depend = find_omp_clause (clauses, OMP_CLAUSE_DEPEND); 5068 flags = build_int_cst (unsigned_type_node, 5069 (c ? 1 : 0) + (c2 ? 4 : 0) + (depend ? 8 : 0)); 5070 5071 c = find_omp_clause (clauses, OMP_CLAUSE_FINAL); 5072 if (c) 5073 { 5074 c = gimple_boolify (OMP_CLAUSE_FINAL_EXPR (c)); 5075 c = fold_build3_loc (loc, COND_EXPR, unsigned_type_node, c, 5076 build_int_cst (unsigned_type_node, 2), 5077 build_int_cst (unsigned_type_node, 0)); 5078 flags = fold_build2_loc (loc, PLUS_EXPR, unsigned_type_node, flags, c); 5079 } 5080 if (depend) 5081 depend = OMP_CLAUSE_DECL (depend); 5082 else 5083 depend = build_int_cst (ptr_type_node, 0); 5084 5085 gsi = gsi_last_bb (bb); 5086 t = gimple_omp_task_data_arg (entry_stmt); 5087 if (t == NULL) 5088 t2 = null_pointer_node; 5089 else 5090 t2 = build_fold_addr_expr_loc (loc, t); 5091 t1 = build_fold_addr_expr_loc (loc, gimple_omp_task_child_fn (entry_stmt)); 5092 t = gimple_omp_task_copy_fn (entry_stmt); 5093 if (t == NULL) 5094 t3 = null_pointer_node; 5095 else 5096 t3 = build_fold_addr_expr_loc (loc, t); 5097 5098 t = build_call_expr (builtin_decl_explicit (BUILT_IN_GOMP_TASK), 5099 8, t1, t2, t3, 5100 gimple_omp_task_arg_size (entry_stmt), 5101 gimple_omp_task_arg_align (entry_stmt), cond, flags, 5102 depend); 5103 5104 force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 5105 false, GSI_CONTINUE_LINKING); 5106} 5107 5108 5109/* If exceptions are enabled, wrap the statements in BODY in a MUST_NOT_THROW 5110 catch handler and return it. This prevents programs from violating the 5111 structured block semantics with throws. */ 5112 5113static gimple_seq 5114maybe_catch_exception (gimple_seq body) 5115{ 5116 gimple g; 5117 tree decl; 5118 5119 if (!flag_exceptions) 5120 return body; 5121 5122 if (lang_hooks.eh_protect_cleanup_actions != NULL) 5123 decl = lang_hooks.eh_protect_cleanup_actions (); 5124 else 5125 decl = builtin_decl_explicit (BUILT_IN_TRAP); 5126 5127 g = gimple_build_eh_must_not_throw (decl); 5128 g = gimple_build_try (body, gimple_seq_alloc_with_stmt (g), 5129 GIMPLE_TRY_CATCH); 5130 5131 return gimple_seq_alloc_with_stmt (g); 5132} 5133 5134/* Chain all the DECLs in LIST by their TREE_CHAIN fields. */ 5135 5136static tree 5137vec2chain (vec<tree, va_gc> *v) 5138{ 5139 tree chain = NULL_TREE, t; 5140 unsigned ix; 5141 5142 FOR_EACH_VEC_SAFE_ELT_REVERSE (v, ix, t) 5143 { 5144 DECL_CHAIN (t) = chain; 5145 chain = t; 5146 } 5147 5148 return chain; 5149} 5150 5151 5152/* Remove barriers in REGION->EXIT's block. Note that this is only 5153 valid for GIMPLE_OMP_PARALLEL regions. Since the end of a parallel region 5154 is an implicit barrier, any workshare inside the GIMPLE_OMP_PARALLEL that 5155 left a barrier at the end of the GIMPLE_OMP_PARALLEL region can now be 5156 removed. */ 5157 5158static void 5159remove_exit_barrier (struct omp_region *region) 5160{ 5161 gimple_stmt_iterator gsi; 5162 basic_block exit_bb; 5163 edge_iterator ei; 5164 edge e; 5165 gimple stmt; 5166 int any_addressable_vars = -1; 5167 5168 exit_bb = region->exit; 5169 5170 /* If the parallel region doesn't return, we don't have REGION->EXIT 5171 block at all. */ 5172 if (! exit_bb) 5173 return; 5174 5175 /* The last insn in the block will be the parallel's GIMPLE_OMP_RETURN. The 5176 workshare's GIMPLE_OMP_RETURN will be in a preceding block. The kinds of 5177 statements that can appear in between are extremely limited -- no 5178 memory operations at all. Here, we allow nothing at all, so the 5179 only thing we allow to precede this GIMPLE_OMP_RETURN is a label. */ 5180 gsi = gsi_last_bb (exit_bb); 5181 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 5182 gsi_prev (&gsi); 5183 if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) != GIMPLE_LABEL) 5184 return; 5185 5186 FOR_EACH_EDGE (e, ei, exit_bb->preds) 5187 { 5188 gsi = gsi_last_bb (e->src); 5189 if (gsi_end_p (gsi)) 5190 continue; 5191 stmt = gsi_stmt (gsi); 5192 if (gimple_code (stmt) == GIMPLE_OMP_RETURN 5193 && !gimple_omp_return_nowait_p (stmt)) 5194 { 5195 /* OpenMP 3.0 tasks unfortunately prevent this optimization 5196 in many cases. If there could be tasks queued, the barrier 5197 might be needed to let the tasks run before some local 5198 variable of the parallel that the task uses as shared 5199 runs out of scope. The task can be spawned either 5200 from within current function (this would be easy to check) 5201 or from some function it calls and gets passed an address 5202 of such a variable. */ 5203 if (any_addressable_vars < 0) 5204 { 5205 gomp_parallel *parallel_stmt 5206 = as_a <gomp_parallel *> (last_stmt (region->entry)); 5207 tree child_fun = gimple_omp_parallel_child_fn (parallel_stmt); 5208 tree local_decls, block, decl; 5209 unsigned ix; 5210 5211 any_addressable_vars = 0; 5212 FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (child_fun), ix, decl) 5213 if (TREE_ADDRESSABLE (decl)) 5214 { 5215 any_addressable_vars = 1; 5216 break; 5217 } 5218 for (block = gimple_block (stmt); 5219 !any_addressable_vars 5220 && block 5221 && TREE_CODE (block) == BLOCK; 5222 block = BLOCK_SUPERCONTEXT (block)) 5223 { 5224 for (local_decls = BLOCK_VARS (block); 5225 local_decls; 5226 local_decls = DECL_CHAIN (local_decls)) 5227 if (TREE_ADDRESSABLE (local_decls)) 5228 { 5229 any_addressable_vars = 1; 5230 break; 5231 } 5232 if (block == gimple_block (parallel_stmt)) 5233 break; 5234 } 5235 } 5236 if (!any_addressable_vars) 5237 gimple_omp_return_set_nowait (stmt); 5238 } 5239 } 5240} 5241 5242static void 5243remove_exit_barriers (struct omp_region *region) 5244{ 5245 if (region->type == GIMPLE_OMP_PARALLEL) 5246 remove_exit_barrier (region); 5247 5248 if (region->inner) 5249 { 5250 region = region->inner; 5251 remove_exit_barriers (region); 5252 while (region->next) 5253 { 5254 region = region->next; 5255 remove_exit_barriers (region); 5256 } 5257 } 5258} 5259 5260/* Optimize omp_get_thread_num () and omp_get_num_threads () 5261 calls. These can't be declared as const functions, but 5262 within one parallel body they are constant, so they can be 5263 transformed there into __builtin_omp_get_{thread_num,num_threads} () 5264 which are declared const. Similarly for task body, except 5265 that in untied task omp_get_thread_num () can change at any task 5266 scheduling point. */ 5267 5268static void 5269optimize_omp_library_calls (gimple entry_stmt) 5270{ 5271 basic_block bb; 5272 gimple_stmt_iterator gsi; 5273 tree thr_num_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 5274 tree thr_num_id = DECL_ASSEMBLER_NAME (thr_num_tree); 5275 tree num_thr_tree = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 5276 tree num_thr_id = DECL_ASSEMBLER_NAME (num_thr_tree); 5277 bool untied_task = (gimple_code (entry_stmt) == GIMPLE_OMP_TASK 5278 && find_omp_clause (gimple_omp_task_clauses (entry_stmt), 5279 OMP_CLAUSE_UNTIED) != NULL); 5280 5281 FOR_EACH_BB_FN (bb, cfun) 5282 for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 5283 { 5284 gimple call = gsi_stmt (gsi); 5285 tree decl; 5286 5287 if (is_gimple_call (call) 5288 && (decl = gimple_call_fndecl (call)) 5289 && DECL_EXTERNAL (decl) 5290 && TREE_PUBLIC (decl) 5291 && DECL_INITIAL (decl) == NULL) 5292 { 5293 tree built_in; 5294 5295 if (DECL_NAME (decl) == thr_num_id) 5296 { 5297 /* In #pragma omp task untied omp_get_thread_num () can change 5298 during the execution of the task region. */ 5299 if (untied_task) 5300 continue; 5301 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 5302 } 5303 else if (DECL_NAME (decl) == num_thr_id) 5304 built_in = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 5305 else 5306 continue; 5307 5308 if (DECL_ASSEMBLER_NAME (decl) != DECL_ASSEMBLER_NAME (built_in) 5309 || gimple_call_num_args (call) != 0) 5310 continue; 5311 5312 if (flag_exceptions && !TREE_NOTHROW (decl)) 5313 continue; 5314 5315 if (TREE_CODE (TREE_TYPE (decl)) != FUNCTION_TYPE 5316 || !types_compatible_p (TREE_TYPE (TREE_TYPE (decl)), 5317 TREE_TYPE (TREE_TYPE (built_in)))) 5318 continue; 5319 5320 gimple_call_set_fndecl (call, built_in); 5321 } 5322 } 5323} 5324 5325/* Callback for expand_omp_build_assign. Return non-NULL if *tp needs to be 5326 regimplified. */ 5327 5328static tree 5329expand_omp_regimplify_p (tree *tp, int *walk_subtrees, void *) 5330{ 5331 tree t = *tp; 5332 5333 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ 5334 if (TREE_CODE (t) == VAR_DECL && DECL_HAS_VALUE_EXPR_P (t)) 5335 return t; 5336 5337 if (TREE_CODE (t) == ADDR_EXPR) 5338 recompute_tree_invariant_for_addr_expr (t); 5339 5340 *walk_subtrees = !TYPE_P (t) && !DECL_P (t); 5341 return NULL_TREE; 5342} 5343 5344/* Prepend TO = FROM assignment before *GSI_P. */ 5345 5346static void 5347expand_omp_build_assign (gimple_stmt_iterator *gsi_p, tree to, tree from) 5348{ 5349 bool simple_p = DECL_P (to) && TREE_ADDRESSABLE (to); 5350 from = force_gimple_operand_gsi (gsi_p, from, simple_p, NULL_TREE, 5351 true, GSI_SAME_STMT); 5352 gimple stmt = gimple_build_assign (to, from); 5353 gsi_insert_before (gsi_p, stmt, GSI_SAME_STMT); 5354 if (walk_tree (&from, expand_omp_regimplify_p, NULL, NULL) 5355 || walk_tree (&to, expand_omp_regimplify_p, NULL, NULL)) 5356 { 5357 gimple_stmt_iterator gsi = gsi_for_stmt (stmt); 5358 gimple_regimplify_operands (stmt, &gsi); 5359 } 5360} 5361 5362/* Expand the OpenMP parallel or task directive starting at REGION. */ 5363 5364static void 5365expand_omp_taskreg (struct omp_region *region) 5366{ 5367 basic_block entry_bb, exit_bb, new_bb; 5368 struct function *child_cfun; 5369 tree child_fn, block, t; 5370 gimple_stmt_iterator gsi; 5371 gimple entry_stmt, stmt; 5372 edge e; 5373 vec<tree, va_gc> *ws_args; 5374 5375 entry_stmt = last_stmt (region->entry); 5376 child_fn = gimple_omp_taskreg_child_fn (entry_stmt); 5377 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 5378 5379 entry_bb = region->entry; 5380 if (gimple_code (entry_stmt) == GIMPLE_OMP_TASK) 5381 exit_bb = region->cont; 5382 else 5383 exit_bb = region->exit; 5384 5385 bool is_cilk_for 5386 = (flag_cilkplus 5387 && gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL 5388 && find_omp_clause (gimple_omp_parallel_clauses (entry_stmt), 5389 OMP_CLAUSE__CILK_FOR_COUNT_) != NULL_TREE); 5390 5391 if (is_cilk_for) 5392 /* If it is a _Cilk_for statement, it is modelled *like* a parallel for, 5393 and the inner statement contains the name of the built-in function 5394 and grain. */ 5395 ws_args = region->inner->ws_args; 5396 else if (is_combined_parallel (region)) 5397 ws_args = region->ws_args; 5398 else 5399 ws_args = NULL; 5400 5401 if (child_cfun->cfg) 5402 { 5403 /* Due to inlining, it may happen that we have already outlined 5404 the region, in which case all we need to do is make the 5405 sub-graph unreachable and emit the parallel call. */ 5406 edge entry_succ_e, exit_succ_e; 5407 5408 entry_succ_e = single_succ_edge (entry_bb); 5409 5410 gsi = gsi_last_bb (entry_bb); 5411 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_PARALLEL 5412 || gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_TASK); 5413 gsi_remove (&gsi, true); 5414 5415 new_bb = entry_bb; 5416 if (exit_bb) 5417 { 5418 exit_succ_e = single_succ_edge (exit_bb); 5419 make_edge (new_bb, exit_succ_e->dest, EDGE_FALLTHRU); 5420 } 5421 remove_edge_and_dominated_blocks (entry_succ_e); 5422 } 5423 else 5424 { 5425 unsigned srcidx, dstidx, num; 5426 5427 /* If the parallel region needs data sent from the parent 5428 function, then the very first statement (except possible 5429 tree profile counter updates) of the parallel body 5430 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since 5431 &.OMP_DATA_O is passed as an argument to the child function, 5432 we need to replace it with the argument as seen by the child 5433 function. 5434 5435 In most cases, this will end up being the identity assignment 5436 .OMP_DATA_I = .OMP_DATA_I. However, if the parallel body had 5437 a function call that has been inlined, the original PARM_DECL 5438 .OMP_DATA_I may have been converted into a different local 5439 variable. In which case, we need to keep the assignment. */ 5440 if (gimple_omp_taskreg_data_arg (entry_stmt)) 5441 { 5442 basic_block entry_succ_bb 5443 = single_succ_p (entry_bb) ? single_succ (entry_bb) 5444 : FALLTHRU_EDGE (entry_bb)->dest; 5445 tree arg, narg; 5446 gimple parcopy_stmt = NULL; 5447 5448 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) 5449 { 5450 gimple stmt; 5451 5452 gcc_assert (!gsi_end_p (gsi)); 5453 stmt = gsi_stmt (gsi); 5454 if (gimple_code (stmt) != GIMPLE_ASSIGN) 5455 continue; 5456 5457 if (gimple_num_ops (stmt) == 2) 5458 { 5459 tree arg = gimple_assign_rhs1 (stmt); 5460 5461 /* We're ignore the subcode because we're 5462 effectively doing a STRIP_NOPS. */ 5463 5464 if (TREE_CODE (arg) == ADDR_EXPR 5465 && TREE_OPERAND (arg, 0) 5466 == gimple_omp_taskreg_data_arg (entry_stmt)) 5467 { 5468 parcopy_stmt = stmt; 5469 break; 5470 } 5471 } 5472 } 5473 5474 gcc_assert (parcopy_stmt != NULL); 5475 arg = DECL_ARGUMENTS (child_fn); 5476 5477 if (!gimple_in_ssa_p (cfun)) 5478 { 5479 if (gimple_assign_lhs (parcopy_stmt) == arg) 5480 gsi_remove (&gsi, true); 5481 else 5482 { 5483 /* ?? Is setting the subcode really necessary ?? */ 5484 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (arg)); 5485 gimple_assign_set_rhs1 (parcopy_stmt, arg); 5486 } 5487 } 5488 else 5489 { 5490 /* If we are in ssa form, we must load the value from the default 5491 definition of the argument. That should not be defined now, 5492 since the argument is not used uninitialized. */ 5493 gcc_assert (ssa_default_def (cfun, arg) == NULL); 5494 narg = make_ssa_name (arg, gimple_build_nop ()); 5495 set_ssa_default_def (cfun, arg, narg); 5496 /* ?? Is setting the subcode really necessary ?? */ 5497 gimple_omp_set_subcode (parcopy_stmt, TREE_CODE (narg)); 5498 gimple_assign_set_rhs1 (parcopy_stmt, narg); 5499 update_stmt (parcopy_stmt); 5500 } 5501 } 5502 5503 /* Declare local variables needed in CHILD_CFUN. */ 5504 block = DECL_INITIAL (child_fn); 5505 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); 5506 /* The gimplifier could record temporaries in parallel/task block 5507 rather than in containing function's local_decls chain, 5508 which would mean cgraph missed finalizing them. Do it now. */ 5509 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) 5510 if (TREE_CODE (t) == VAR_DECL 5511 && TREE_STATIC (t) 5512 && !DECL_EXTERNAL (t)) 5513 varpool_node::finalize_decl (t); 5514 DECL_SAVED_TREE (child_fn) = NULL; 5515 /* We'll create a CFG for child_fn, so no gimple body is needed. */ 5516 gimple_set_body (child_fn, NULL); 5517 TREE_USED (block) = 1; 5518 5519 /* Reset DECL_CONTEXT on function arguments. */ 5520 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 5521 DECL_CONTEXT (t) = child_fn; 5522 5523 /* Split ENTRY_BB at GIMPLE_OMP_PARALLEL or GIMPLE_OMP_TASK, 5524 so that it can be moved to the child function. */ 5525 gsi = gsi_last_bb (entry_bb); 5526 stmt = gsi_stmt (gsi); 5527 gcc_assert (stmt && (gimple_code (stmt) == GIMPLE_OMP_PARALLEL 5528 || gimple_code (stmt) == GIMPLE_OMP_TASK)); 5529 e = split_block (entry_bb, stmt); 5530 gsi_remove (&gsi, true); 5531 entry_bb = e->dest; 5532 edge e2 = NULL; 5533 if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL) 5534 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 5535 else 5536 { 5537 e2 = make_edge (e->src, BRANCH_EDGE (entry_bb)->dest, EDGE_ABNORMAL); 5538 gcc_assert (e2->dest == region->exit); 5539 remove_edge (BRANCH_EDGE (entry_bb)); 5540 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e->src); 5541 gsi = gsi_last_bb (region->exit); 5542 gcc_assert (!gsi_end_p (gsi) 5543 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 5544 gsi_remove (&gsi, true); 5545 } 5546 5547 /* Convert GIMPLE_OMP_{RETURN,CONTINUE} into a RETURN_EXPR. */ 5548 if (exit_bb) 5549 { 5550 gsi = gsi_last_bb (exit_bb); 5551 gcc_assert (!gsi_end_p (gsi) 5552 && (gimple_code (gsi_stmt (gsi)) 5553 == (e2 ? GIMPLE_OMP_CONTINUE : GIMPLE_OMP_RETURN))); 5554 stmt = gimple_build_return (NULL); 5555 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); 5556 gsi_remove (&gsi, true); 5557 } 5558 5559 /* Move the parallel region into CHILD_CFUN. */ 5560 5561 if (gimple_in_ssa_p (cfun)) 5562 { 5563 init_tree_ssa (child_cfun); 5564 init_ssa_operands (child_cfun); 5565 child_cfun->gimple_df->in_ssa_p = true; 5566 block = NULL_TREE; 5567 } 5568 else 5569 block = gimple_block (entry_stmt); 5570 5571 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); 5572 if (exit_bb) 5573 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; 5574 if (e2) 5575 { 5576 basic_block dest_bb = e2->dest; 5577 if (!exit_bb) 5578 make_edge (new_bb, dest_bb, EDGE_FALLTHRU); 5579 remove_edge (e2); 5580 set_immediate_dominator (CDI_DOMINATORS, dest_bb, new_bb); 5581 } 5582 /* When the OMP expansion process cannot guarantee an up-to-date 5583 loop tree arrange for the child function to fixup loops. */ 5584 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) 5585 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; 5586 5587 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ 5588 num = vec_safe_length (child_cfun->local_decls); 5589 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) 5590 { 5591 t = (*child_cfun->local_decls)[srcidx]; 5592 if (DECL_CONTEXT (t) == cfun->decl) 5593 continue; 5594 if (srcidx != dstidx) 5595 (*child_cfun->local_decls)[dstidx] = t; 5596 dstidx++; 5597 } 5598 if (dstidx != num) 5599 vec_safe_truncate (child_cfun->local_decls, dstidx); 5600 5601 /* Inform the callgraph about the new function. */ 5602 child_cfun->curr_properties = cfun->curr_properties; 5603 child_cfun->has_simduid_loops |= cfun->has_simduid_loops; 5604 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops; 5605 cgraph_node::add_new_function (child_fn, true); 5606 cgraph_node::get (child_fn)->parallelized_function = 1; 5607 5608 /* Fix the callgraph edges for child_cfun. Those for cfun will be 5609 fixed in a following pass. */ 5610 push_cfun (child_cfun); 5611 if (optimize) 5612 optimize_omp_library_calls (entry_stmt); 5613 cgraph_edge::rebuild_edges (); 5614 5615 /* Some EH regions might become dead, see PR34608. If 5616 pass_cleanup_cfg isn't the first pass to happen with the 5617 new child, these dead EH edges might cause problems. 5618 Clean them up now. */ 5619 if (flag_exceptions) 5620 { 5621 basic_block bb; 5622 bool changed = false; 5623 5624 FOR_EACH_BB_FN (bb, cfun) 5625 changed |= gimple_purge_dead_eh_edges (bb); 5626 if (changed) 5627 cleanup_tree_cfg (); 5628 } 5629 if (gimple_in_ssa_p (cfun)) 5630 update_ssa (TODO_update_ssa); 5631 pop_cfun (); 5632 } 5633 5634 /* Emit a library call to launch the children threads. */ 5635 if (is_cilk_for) 5636 expand_cilk_for_call (new_bb, 5637 as_a <gomp_parallel *> (entry_stmt), ws_args); 5638 else if (gimple_code (entry_stmt) == GIMPLE_OMP_PARALLEL) 5639 expand_parallel_call (region, new_bb, 5640 as_a <gomp_parallel *> (entry_stmt), ws_args); 5641 else 5642 expand_task_call (new_bb, as_a <gomp_task *> (entry_stmt)); 5643 if (gimple_in_ssa_p (cfun)) 5644 update_ssa (TODO_update_ssa_only_virtuals); 5645} 5646 5647 5648/* Helper function for expand_omp_{for_*,simd}. If this is the outermost 5649 of the combined collapse > 1 loop constructs, generate code like: 5650 if (__builtin_expect (N32 cond3 N31, 0)) goto ZERO_ITER_BB; 5651 if (cond3 is <) 5652 adj = STEP3 - 1; 5653 else 5654 adj = STEP3 + 1; 5655 count3 = (adj + N32 - N31) / STEP3; 5656 if (__builtin_expect (N22 cond2 N21, 0)) goto ZERO_ITER_BB; 5657 if (cond2 is <) 5658 adj = STEP2 - 1; 5659 else 5660 adj = STEP2 + 1; 5661 count2 = (adj + N22 - N21) / STEP2; 5662 if (__builtin_expect (N12 cond1 N11, 0)) goto ZERO_ITER_BB; 5663 if (cond1 is <) 5664 adj = STEP1 - 1; 5665 else 5666 adj = STEP1 + 1; 5667 count1 = (adj + N12 - N11) / STEP1; 5668 count = count1 * count2 * count3; 5669 Furthermore, if ZERO_ITER_BB is NULL, create a BB which does: 5670 count = 0; 5671 and set ZERO_ITER_BB to that bb. If this isn't the outermost 5672 of the combined loop constructs, just initialize COUNTS array 5673 from the _looptemp_ clauses. */ 5674 5675/* NOTE: It *could* be better to moosh all of the BBs together, 5676 creating one larger BB with all the computation and the unexpected 5677 jump at the end. I.e. 5678 5679 bool zero3, zero2, zero1, zero; 5680 5681 zero3 = N32 c3 N31; 5682 count3 = (N32 - N31) /[cl] STEP3; 5683 zero2 = N22 c2 N21; 5684 count2 = (N22 - N21) /[cl] STEP2; 5685 zero1 = N12 c1 N11; 5686 count1 = (N12 - N11) /[cl] STEP1; 5687 zero = zero3 || zero2 || zero1; 5688 count = count1 * count2 * count3; 5689 if (__builtin_expect(zero, false)) goto zero_iter_bb; 5690 5691 After all, we expect the zero=false, and thus we expect to have to 5692 evaluate all of the comparison expressions, so short-circuiting 5693 oughtn't be a win. Since the condition isn't protecting a 5694 denominator, we're not concerned about divide-by-zero, so we can 5695 fully evaluate count even if a numerator turned out to be wrong. 5696 5697 It seems like putting this all together would create much better 5698 scheduling opportunities, and less pressure on the chip's branch 5699 predictor. */ 5700 5701static void 5702expand_omp_for_init_counts (struct omp_for_data *fd, gimple_stmt_iterator *gsi, 5703 basic_block &entry_bb, tree *counts, 5704 basic_block &zero_iter_bb, int &first_zero_iter, 5705 basic_block &l2_dom_bb) 5706{ 5707 tree t, type = TREE_TYPE (fd->loop.v); 5708 edge e, ne; 5709 int i; 5710 5711 /* Collapsed loops need work for expansion into SSA form. */ 5712 gcc_assert (!gimple_in_ssa_p (cfun)); 5713 5714 if (gimple_omp_for_combined_into_p (fd->for_stmt) 5715 && TREE_CODE (fd->loop.n2) != INTEGER_CST) 5716 { 5717 /* First two _looptemp_ clauses are for istart/iend, counts[0] 5718 isn't supposed to be handled, as the inner loop doesn't 5719 use it. */ 5720 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 5721 OMP_CLAUSE__LOOPTEMP_); 5722 gcc_assert (innerc); 5723 for (i = 0; i < fd->collapse; i++) 5724 { 5725 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 5726 OMP_CLAUSE__LOOPTEMP_); 5727 gcc_assert (innerc); 5728 if (i) 5729 counts[i] = OMP_CLAUSE_DECL (innerc); 5730 else 5731 counts[0] = NULL_TREE; 5732 } 5733 return; 5734 } 5735 5736 for (i = 0; i < fd->collapse; i++) 5737 { 5738 tree itype = TREE_TYPE (fd->loops[i].v); 5739 5740 if (SSA_VAR_P (fd->loop.n2) 5741 && ((t = fold_binary (fd->loops[i].cond_code, boolean_type_node, 5742 fold_convert (itype, fd->loops[i].n1), 5743 fold_convert (itype, fd->loops[i].n2))) 5744 == NULL_TREE || !integer_onep (t))) 5745 { 5746 gcond *cond_stmt; 5747 tree n1, n2; 5748 n1 = fold_convert (itype, unshare_expr (fd->loops[i].n1)); 5749 n1 = force_gimple_operand_gsi (gsi, n1, true, NULL_TREE, 5750 true, GSI_SAME_STMT); 5751 n2 = fold_convert (itype, unshare_expr (fd->loops[i].n2)); 5752 n2 = force_gimple_operand_gsi (gsi, n2, true, NULL_TREE, 5753 true, GSI_SAME_STMT); 5754 cond_stmt = gimple_build_cond (fd->loops[i].cond_code, n1, n2, 5755 NULL_TREE, NULL_TREE); 5756 gsi_insert_before (gsi, cond_stmt, GSI_SAME_STMT); 5757 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 5758 expand_omp_regimplify_p, NULL, NULL) 5759 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 5760 expand_omp_regimplify_p, NULL, NULL)) 5761 { 5762 *gsi = gsi_for_stmt (cond_stmt); 5763 gimple_regimplify_operands (cond_stmt, gsi); 5764 } 5765 e = split_block (entry_bb, cond_stmt); 5766 if (zero_iter_bb == NULL) 5767 { 5768 gassign *assign_stmt; 5769 first_zero_iter = i; 5770 zero_iter_bb = create_empty_bb (entry_bb); 5771 add_bb_to_loop (zero_iter_bb, entry_bb->loop_father); 5772 *gsi = gsi_after_labels (zero_iter_bb); 5773 assign_stmt = gimple_build_assign (fd->loop.n2, 5774 build_zero_cst (type)); 5775 gsi_insert_before (gsi, assign_stmt, GSI_SAME_STMT); 5776 set_immediate_dominator (CDI_DOMINATORS, zero_iter_bb, 5777 entry_bb); 5778 } 5779 ne = make_edge (entry_bb, zero_iter_bb, EDGE_FALSE_VALUE); 5780 ne->probability = REG_BR_PROB_BASE / 2000 - 1; 5781 e->flags = EDGE_TRUE_VALUE; 5782 e->probability = REG_BR_PROB_BASE - ne->probability; 5783 if (l2_dom_bb == NULL) 5784 l2_dom_bb = entry_bb; 5785 entry_bb = e->dest; 5786 *gsi = gsi_last_bb (entry_bb); 5787 } 5788 5789 if (POINTER_TYPE_P (itype)) 5790 itype = signed_type_for (itype); 5791 t = build_int_cst (itype, (fd->loops[i].cond_code == LT_EXPR 5792 ? -1 : 1)); 5793 t = fold_build2 (PLUS_EXPR, itype, 5794 fold_convert (itype, fd->loops[i].step), t); 5795 t = fold_build2 (PLUS_EXPR, itype, t, 5796 fold_convert (itype, fd->loops[i].n2)); 5797 t = fold_build2 (MINUS_EXPR, itype, t, 5798 fold_convert (itype, fd->loops[i].n1)); 5799 /* ?? We could probably use CEIL_DIV_EXPR instead of 5800 TRUNC_DIV_EXPR and adjusting by hand. Unless we can't 5801 generate the same code in the end because generically we 5802 don't know that the values involved must be negative for 5803 GT?? */ 5804 if (TYPE_UNSIGNED (itype) && fd->loops[i].cond_code == GT_EXPR) 5805 t = fold_build2 (TRUNC_DIV_EXPR, itype, 5806 fold_build1 (NEGATE_EXPR, itype, t), 5807 fold_build1 (NEGATE_EXPR, itype, 5808 fold_convert (itype, 5809 fd->loops[i].step))); 5810 else 5811 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, 5812 fold_convert (itype, fd->loops[i].step)); 5813 t = fold_convert (type, t); 5814 if (TREE_CODE (t) == INTEGER_CST) 5815 counts[i] = t; 5816 else 5817 { 5818 counts[i] = create_tmp_reg (type, ".count"); 5819 expand_omp_build_assign (gsi, counts[i], t); 5820 } 5821 if (SSA_VAR_P (fd->loop.n2)) 5822 { 5823 if (i == 0) 5824 t = counts[0]; 5825 else 5826 t = fold_build2 (MULT_EXPR, type, fd->loop.n2, counts[i]); 5827 expand_omp_build_assign (gsi, fd->loop.n2, t); 5828 } 5829 } 5830} 5831 5832 5833/* Helper function for expand_omp_{for_*,simd}. Generate code like: 5834 T = V; 5835 V3 = N31 + (T % count3) * STEP3; 5836 T = T / count3; 5837 V2 = N21 + (T % count2) * STEP2; 5838 T = T / count2; 5839 V1 = N11 + T * STEP1; 5840 if this loop doesn't have an inner loop construct combined with it. 5841 If it does have an inner loop construct combined with it and the 5842 iteration count isn't known constant, store values from counts array 5843 into its _looptemp_ temporaries instead. */ 5844 5845static void 5846expand_omp_for_init_vars (struct omp_for_data *fd, gimple_stmt_iterator *gsi, 5847 tree *counts, gimple inner_stmt, tree startvar) 5848{ 5849 int i; 5850 if (gimple_omp_for_combined_p (fd->for_stmt)) 5851 { 5852 /* If fd->loop.n2 is constant, then no propagation of the counts 5853 is needed, they are constant. */ 5854 if (TREE_CODE (fd->loop.n2) == INTEGER_CST) 5855 return; 5856 5857 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 5858 ? gimple_omp_parallel_clauses (inner_stmt) 5859 : gimple_omp_for_clauses (inner_stmt); 5860 /* First two _looptemp_ clauses are for istart/iend, counts[0] 5861 isn't supposed to be handled, as the inner loop doesn't 5862 use it. */ 5863 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 5864 gcc_assert (innerc); 5865 for (i = 0; i < fd->collapse; i++) 5866 { 5867 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 5868 OMP_CLAUSE__LOOPTEMP_); 5869 gcc_assert (innerc); 5870 if (i) 5871 { 5872 tree tem = OMP_CLAUSE_DECL (innerc); 5873 tree t = fold_convert (TREE_TYPE (tem), counts[i]); 5874 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, 5875 false, GSI_CONTINUE_LINKING); 5876 gassign *stmt = gimple_build_assign (tem, t); 5877 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5878 } 5879 } 5880 return; 5881 } 5882 5883 tree type = TREE_TYPE (fd->loop.v); 5884 tree tem = create_tmp_reg (type, ".tem"); 5885 gassign *stmt = gimple_build_assign (tem, startvar); 5886 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5887 5888 for (i = fd->collapse - 1; i >= 0; i--) 5889 { 5890 tree vtype = TREE_TYPE (fd->loops[i].v), itype, t; 5891 itype = vtype; 5892 if (POINTER_TYPE_P (vtype)) 5893 itype = signed_type_for (vtype); 5894 if (i != 0) 5895 t = fold_build2 (TRUNC_MOD_EXPR, type, tem, counts[i]); 5896 else 5897 t = tem; 5898 t = fold_convert (itype, t); 5899 t = fold_build2 (MULT_EXPR, itype, t, 5900 fold_convert (itype, fd->loops[i].step)); 5901 if (POINTER_TYPE_P (vtype)) 5902 t = fold_build_pointer_plus (fd->loops[i].n1, t); 5903 else 5904 t = fold_build2 (PLUS_EXPR, itype, fd->loops[i].n1, t); 5905 t = force_gimple_operand_gsi (gsi, t, 5906 DECL_P (fd->loops[i].v) 5907 && TREE_ADDRESSABLE (fd->loops[i].v), 5908 NULL_TREE, false, 5909 GSI_CONTINUE_LINKING); 5910 stmt = gimple_build_assign (fd->loops[i].v, t); 5911 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5912 if (i != 0) 5913 { 5914 t = fold_build2 (TRUNC_DIV_EXPR, type, tem, counts[i]); 5915 t = force_gimple_operand_gsi (gsi, t, false, NULL_TREE, 5916 false, GSI_CONTINUE_LINKING); 5917 stmt = gimple_build_assign (tem, t); 5918 gsi_insert_after (gsi, stmt, GSI_CONTINUE_LINKING); 5919 } 5920 } 5921} 5922 5923 5924/* Helper function for expand_omp_for_*. Generate code like: 5925 L10: 5926 V3 += STEP3; 5927 if (V3 cond3 N32) goto BODY_BB; else goto L11; 5928 L11: 5929 V3 = N31; 5930 V2 += STEP2; 5931 if (V2 cond2 N22) goto BODY_BB; else goto L12; 5932 L12: 5933 V2 = N21; 5934 V1 += STEP1; 5935 goto BODY_BB; */ 5936 5937static basic_block 5938extract_omp_for_update_vars (struct omp_for_data *fd, basic_block cont_bb, 5939 basic_block body_bb) 5940{ 5941 basic_block last_bb, bb, collapse_bb = NULL; 5942 int i; 5943 gimple_stmt_iterator gsi; 5944 edge e; 5945 tree t; 5946 gimple stmt; 5947 5948 last_bb = cont_bb; 5949 for (i = fd->collapse - 1; i >= 0; i--) 5950 { 5951 tree vtype = TREE_TYPE (fd->loops[i].v); 5952 5953 bb = create_empty_bb (last_bb); 5954 add_bb_to_loop (bb, last_bb->loop_father); 5955 gsi = gsi_start_bb (bb); 5956 5957 if (i < fd->collapse - 1) 5958 { 5959 e = make_edge (last_bb, bb, EDGE_FALSE_VALUE); 5960 e->probability = REG_BR_PROB_BASE / 8; 5961 5962 t = fd->loops[i + 1].n1; 5963 t = force_gimple_operand_gsi (&gsi, t, 5964 DECL_P (fd->loops[i + 1].v) 5965 && TREE_ADDRESSABLE (fd->loops[i 5966 + 1].v), 5967 NULL_TREE, false, 5968 GSI_CONTINUE_LINKING); 5969 stmt = gimple_build_assign (fd->loops[i + 1].v, t); 5970 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 5971 } 5972 else 5973 collapse_bb = bb; 5974 5975 set_immediate_dominator (CDI_DOMINATORS, bb, last_bb); 5976 5977 if (POINTER_TYPE_P (vtype)) 5978 t = fold_build_pointer_plus (fd->loops[i].v, fd->loops[i].step); 5979 else 5980 t = fold_build2 (PLUS_EXPR, vtype, fd->loops[i].v, fd->loops[i].step); 5981 t = force_gimple_operand_gsi (&gsi, t, 5982 DECL_P (fd->loops[i].v) 5983 && TREE_ADDRESSABLE (fd->loops[i].v), 5984 NULL_TREE, false, GSI_CONTINUE_LINKING); 5985 stmt = gimple_build_assign (fd->loops[i].v, t); 5986 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 5987 5988 if (i > 0) 5989 { 5990 t = fd->loops[i].n2; 5991 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 5992 false, GSI_CONTINUE_LINKING); 5993 tree v = fd->loops[i].v; 5994 if (DECL_P (v) && TREE_ADDRESSABLE (v)) 5995 v = force_gimple_operand_gsi (&gsi, v, true, NULL_TREE, 5996 false, GSI_CONTINUE_LINKING); 5997 t = fold_build2 (fd->loops[i].cond_code, boolean_type_node, v, t); 5998 stmt = gimple_build_cond_empty (t); 5999 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 6000 e = make_edge (bb, body_bb, EDGE_TRUE_VALUE); 6001 e->probability = REG_BR_PROB_BASE * 7 / 8; 6002 } 6003 else 6004 make_edge (bb, body_bb, EDGE_FALLTHRU); 6005 last_bb = bb; 6006 } 6007 6008 return collapse_bb; 6009} 6010 6011 6012/* A subroutine of expand_omp_for. Generate code for a parallel 6013 loop with any schedule. Given parameters: 6014 6015 for (V = N1; V cond N2; V += STEP) BODY; 6016 6017 where COND is "<" or ">", we generate pseudocode 6018 6019 more = GOMP_loop_foo_start (N1, N2, STEP, CHUNK, &istart0, &iend0); 6020 if (more) goto L0; else goto L3; 6021 L0: 6022 V = istart0; 6023 iend = iend0; 6024 L1: 6025 BODY; 6026 V += STEP; 6027 if (V cond iend) goto L1; else goto L2; 6028 L2: 6029 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; 6030 L3: 6031 6032 If this is a combined omp parallel loop, instead of the call to 6033 GOMP_loop_foo_start, we call GOMP_loop_foo_next. 6034 If this is gimple_omp_for_combined_p loop, then instead of assigning 6035 V and iend in L0 we assign the first two _looptemp_ clause decls of the 6036 inner GIMPLE_OMP_FOR and V += STEP; and 6037 if (V cond iend) goto L1; else goto L2; are removed. 6038 6039 For collapsed loops, given parameters: 6040 collapse(3) 6041 for (V1 = N11; V1 cond1 N12; V1 += STEP1) 6042 for (V2 = N21; V2 cond2 N22; V2 += STEP2) 6043 for (V3 = N31; V3 cond3 N32; V3 += STEP3) 6044 BODY; 6045 6046 we generate pseudocode 6047 6048 if (__builtin_expect (N32 cond3 N31, 0)) goto Z0; 6049 if (cond3 is <) 6050 adj = STEP3 - 1; 6051 else 6052 adj = STEP3 + 1; 6053 count3 = (adj + N32 - N31) / STEP3; 6054 if (__builtin_expect (N22 cond2 N21, 0)) goto Z0; 6055 if (cond2 is <) 6056 adj = STEP2 - 1; 6057 else 6058 adj = STEP2 + 1; 6059 count2 = (adj + N22 - N21) / STEP2; 6060 if (__builtin_expect (N12 cond1 N11, 0)) goto Z0; 6061 if (cond1 is <) 6062 adj = STEP1 - 1; 6063 else 6064 adj = STEP1 + 1; 6065 count1 = (adj + N12 - N11) / STEP1; 6066 count = count1 * count2 * count3; 6067 goto Z1; 6068 Z0: 6069 count = 0; 6070 Z1: 6071 more = GOMP_loop_foo_start (0, count, 1, CHUNK, &istart0, &iend0); 6072 if (more) goto L0; else goto L3; 6073 L0: 6074 V = istart0; 6075 T = V; 6076 V3 = N31 + (T % count3) * STEP3; 6077 T = T / count3; 6078 V2 = N21 + (T % count2) * STEP2; 6079 T = T / count2; 6080 V1 = N11 + T * STEP1; 6081 iend = iend0; 6082 L1: 6083 BODY; 6084 V += 1; 6085 if (V < iend) goto L10; else goto L2; 6086 L10: 6087 V3 += STEP3; 6088 if (V3 cond3 N32) goto L1; else goto L11; 6089 L11: 6090 V3 = N31; 6091 V2 += STEP2; 6092 if (V2 cond2 N22) goto L1; else goto L12; 6093 L12: 6094 V2 = N21; 6095 V1 += STEP1; 6096 goto L1; 6097 L2: 6098 if (GOMP_loop_foo_next (&istart0, &iend0)) goto L0; else goto L3; 6099 L3: 6100 6101 */ 6102 6103static void 6104expand_omp_for_generic (struct omp_region *region, 6105 struct omp_for_data *fd, 6106 enum built_in_function start_fn, 6107 enum built_in_function next_fn, 6108 gimple inner_stmt) 6109{ 6110 tree type, istart0, iend0, iend; 6111 tree t, vmain, vback, bias = NULL_TREE; 6112 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, collapse_bb; 6113 basic_block l2_bb = NULL, l3_bb = NULL; 6114 gimple_stmt_iterator gsi; 6115 gassign *assign_stmt; 6116 bool in_combined_parallel = is_combined_parallel (region); 6117 bool broken_loop = region->cont == NULL; 6118 edge e, ne; 6119 tree *counts = NULL; 6120 int i; 6121 6122 gcc_assert (!broken_loop || !in_combined_parallel); 6123 gcc_assert (fd->iter_type == long_integer_type_node 6124 || !in_combined_parallel); 6125 6126 type = TREE_TYPE (fd->loop.v); 6127 istart0 = create_tmp_var (fd->iter_type, ".istart0"); 6128 iend0 = create_tmp_var (fd->iter_type, ".iend0"); 6129 TREE_ADDRESSABLE (istart0) = 1; 6130 TREE_ADDRESSABLE (iend0) = 1; 6131 6132 /* See if we need to bias by LLONG_MIN. */ 6133 if (fd->iter_type == long_long_unsigned_type_node 6134 && TREE_CODE (type) == INTEGER_TYPE 6135 && !TYPE_UNSIGNED (type)) 6136 { 6137 tree n1, n2; 6138 6139 if (fd->loop.cond_code == LT_EXPR) 6140 { 6141 n1 = fd->loop.n1; 6142 n2 = fold_build2 (PLUS_EXPR, type, fd->loop.n2, fd->loop.step); 6143 } 6144 else 6145 { 6146 n1 = fold_build2 (MINUS_EXPR, type, fd->loop.n2, fd->loop.step); 6147 n2 = fd->loop.n1; 6148 } 6149 if (TREE_CODE (n1) != INTEGER_CST 6150 || TREE_CODE (n2) != INTEGER_CST 6151 || ((tree_int_cst_sgn (n1) < 0) ^ (tree_int_cst_sgn (n2) < 0))) 6152 bias = fold_convert (fd->iter_type, TYPE_MIN_VALUE (type)); 6153 } 6154 6155 entry_bb = region->entry; 6156 cont_bb = region->cont; 6157 collapse_bb = NULL; 6158 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 6159 gcc_assert (broken_loop 6160 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 6161 l0_bb = split_edge (FALLTHRU_EDGE (entry_bb)); 6162 l1_bb = single_succ (l0_bb); 6163 if (!broken_loop) 6164 { 6165 l2_bb = create_empty_bb (cont_bb); 6166 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l1_bb); 6167 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6168 } 6169 else 6170 l2_bb = NULL; 6171 l3_bb = BRANCH_EDGE (entry_bb)->dest; 6172 exit_bb = region->exit; 6173 6174 gsi = gsi_last_bb (entry_bb); 6175 6176 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6177 if (fd->collapse > 1) 6178 { 6179 int first_zero_iter = -1; 6180 basic_block zero_iter_bb = NULL, l2_dom_bb = NULL; 6181 6182 counts = XALLOCAVEC (tree, fd->collapse); 6183 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6184 zero_iter_bb, first_zero_iter, 6185 l2_dom_bb); 6186 6187 if (zero_iter_bb) 6188 { 6189 /* Some counts[i] vars might be uninitialized if 6190 some loop has zero iterations. But the body shouldn't 6191 be executed in that case, so just avoid uninit warnings. */ 6192 for (i = first_zero_iter; i < fd->collapse; i++) 6193 if (SSA_VAR_P (counts[i])) 6194 TREE_NO_WARNING (counts[i]) = 1; 6195 gsi_prev (&gsi); 6196 e = split_block (entry_bb, gsi_stmt (gsi)); 6197 entry_bb = e->dest; 6198 make_edge (zero_iter_bb, entry_bb, EDGE_FALLTHRU); 6199 gsi = gsi_last_bb (entry_bb); 6200 set_immediate_dominator (CDI_DOMINATORS, entry_bb, 6201 get_immediate_dominator (CDI_DOMINATORS, 6202 zero_iter_bb)); 6203 } 6204 } 6205 if (in_combined_parallel) 6206 { 6207 /* In a combined parallel loop, emit a call to 6208 GOMP_loop_foo_next. */ 6209 t = build_call_expr (builtin_decl_explicit (next_fn), 2, 6210 build_fold_addr_expr (istart0), 6211 build_fold_addr_expr (iend0)); 6212 } 6213 else 6214 { 6215 tree t0, t1, t2, t3, t4; 6216 /* If this is not a combined parallel loop, emit a call to 6217 GOMP_loop_foo_start in ENTRY_BB. */ 6218 t4 = build_fold_addr_expr (iend0); 6219 t3 = build_fold_addr_expr (istart0); 6220 t2 = fold_convert (fd->iter_type, fd->loop.step); 6221 t1 = fd->loop.n2; 6222 t0 = fd->loop.n1; 6223 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6224 { 6225 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 6226 OMP_CLAUSE__LOOPTEMP_); 6227 gcc_assert (innerc); 6228 t0 = OMP_CLAUSE_DECL (innerc); 6229 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6230 OMP_CLAUSE__LOOPTEMP_); 6231 gcc_assert (innerc); 6232 t1 = OMP_CLAUSE_DECL (innerc); 6233 } 6234 if (POINTER_TYPE_P (TREE_TYPE (t0)) 6235 && TYPE_PRECISION (TREE_TYPE (t0)) 6236 != TYPE_PRECISION (fd->iter_type)) 6237 { 6238 /* Avoid casting pointers to integer of a different size. */ 6239 tree itype = signed_type_for (type); 6240 t1 = fold_convert (fd->iter_type, fold_convert (itype, t1)); 6241 t0 = fold_convert (fd->iter_type, fold_convert (itype, t0)); 6242 } 6243 else 6244 { 6245 t1 = fold_convert (fd->iter_type, t1); 6246 t0 = fold_convert (fd->iter_type, t0); 6247 } 6248 if (bias) 6249 { 6250 t1 = fold_build2 (PLUS_EXPR, fd->iter_type, t1, bias); 6251 t0 = fold_build2 (PLUS_EXPR, fd->iter_type, t0, bias); 6252 } 6253 if (fd->iter_type == long_integer_type_node) 6254 { 6255 if (fd->chunk_size) 6256 { 6257 t = fold_convert (fd->iter_type, fd->chunk_size); 6258 t = build_call_expr (builtin_decl_explicit (start_fn), 6259 6, t0, t1, t2, t, t3, t4); 6260 } 6261 else 6262 t = build_call_expr (builtin_decl_explicit (start_fn), 6263 5, t0, t1, t2, t3, t4); 6264 } 6265 else 6266 { 6267 tree t5; 6268 tree c_bool_type; 6269 tree bfn_decl; 6270 6271 /* The GOMP_loop_ull_*start functions have additional boolean 6272 argument, true for < loops and false for > loops. 6273 In Fortran, the C bool type can be different from 6274 boolean_type_node. */ 6275 bfn_decl = builtin_decl_explicit (start_fn); 6276 c_bool_type = TREE_TYPE (TREE_TYPE (bfn_decl)); 6277 t5 = build_int_cst (c_bool_type, 6278 fd->loop.cond_code == LT_EXPR ? 1 : 0); 6279 if (fd->chunk_size) 6280 { 6281 tree bfn_decl = builtin_decl_explicit (start_fn); 6282 t = fold_convert (fd->iter_type, fd->chunk_size); 6283 t = build_call_expr (bfn_decl, 7, t5, t0, t1, t2, t, t3, t4); 6284 } 6285 else 6286 t = build_call_expr (builtin_decl_explicit (start_fn), 6287 6, t5, t0, t1, t2, t3, t4); 6288 } 6289 } 6290 if (TREE_TYPE (t) != boolean_type_node) 6291 t = fold_build2 (NE_EXPR, boolean_type_node, 6292 t, build_int_cst (TREE_TYPE (t), 0)); 6293 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6294 true, GSI_SAME_STMT); 6295 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6296 6297 /* Remove the GIMPLE_OMP_FOR statement. */ 6298 gsi_remove (&gsi, true); 6299 6300 /* Iteration setup for sequential loop goes in L0_BB. */ 6301 tree startvar = fd->loop.v; 6302 tree endvar = NULL_TREE; 6303 6304 if (gimple_omp_for_combined_p (fd->for_stmt)) 6305 { 6306 gcc_assert (gimple_code (inner_stmt) == GIMPLE_OMP_FOR 6307 && gimple_omp_for_kind (inner_stmt) 6308 == GF_OMP_FOR_KIND_SIMD); 6309 tree innerc = find_omp_clause (gimple_omp_for_clauses (inner_stmt), 6310 OMP_CLAUSE__LOOPTEMP_); 6311 gcc_assert (innerc); 6312 startvar = OMP_CLAUSE_DECL (innerc); 6313 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6314 OMP_CLAUSE__LOOPTEMP_); 6315 gcc_assert (innerc); 6316 endvar = OMP_CLAUSE_DECL (innerc); 6317 } 6318 6319 gsi = gsi_start_bb (l0_bb); 6320 t = istart0; 6321 if (bias) 6322 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); 6323 if (POINTER_TYPE_P (TREE_TYPE (startvar))) 6324 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); 6325 t = fold_convert (TREE_TYPE (startvar), t); 6326 t = force_gimple_operand_gsi (&gsi, t, 6327 DECL_P (startvar) 6328 && TREE_ADDRESSABLE (startvar), 6329 NULL_TREE, false, GSI_CONTINUE_LINKING); 6330 assign_stmt = gimple_build_assign (startvar, t); 6331 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6332 6333 t = iend0; 6334 if (bias) 6335 t = fold_build2 (MINUS_EXPR, fd->iter_type, t, bias); 6336 if (POINTER_TYPE_P (TREE_TYPE (startvar))) 6337 t = fold_convert (signed_type_for (TREE_TYPE (startvar)), t); 6338 t = fold_convert (TREE_TYPE (startvar), t); 6339 iend = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6340 false, GSI_CONTINUE_LINKING); 6341 if (endvar) 6342 { 6343 assign_stmt = gimple_build_assign (endvar, iend); 6344 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6345 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (iend))) 6346 assign_stmt = gimple_build_assign (fd->loop.v, iend); 6347 else 6348 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, iend); 6349 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6350 } 6351 if (fd->collapse > 1) 6352 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 6353 6354 if (!broken_loop) 6355 { 6356 /* Code to control the increment and predicate for the sequential 6357 loop goes in the CONT_BB. */ 6358 gsi = gsi_last_bb (cont_bb); 6359 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 6360 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); 6361 vmain = gimple_omp_continue_control_use (cont_stmt); 6362 vback = gimple_omp_continue_control_def (cont_stmt); 6363 6364 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6365 { 6366 if (POINTER_TYPE_P (type)) 6367 t = fold_build_pointer_plus (vmain, fd->loop.step); 6368 else 6369 t = fold_build2 (PLUS_EXPR, type, vmain, fd->loop.step); 6370 t = force_gimple_operand_gsi (&gsi, t, 6371 DECL_P (vback) 6372 && TREE_ADDRESSABLE (vback), 6373 NULL_TREE, true, GSI_SAME_STMT); 6374 assign_stmt = gimple_build_assign (vback, t); 6375 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6376 6377 t = build2 (fd->loop.cond_code, boolean_type_node, 6378 DECL_P (vback) && TREE_ADDRESSABLE (vback) ? t : vback, 6379 iend); 6380 gcond *cond_stmt = gimple_build_cond_empty (t); 6381 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6382 } 6383 6384 /* Remove GIMPLE_OMP_CONTINUE. */ 6385 gsi_remove (&gsi, true); 6386 6387 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 6388 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, l1_bb); 6389 6390 /* Emit code to get the next parallel iteration in L2_BB. */ 6391 gsi = gsi_start_bb (l2_bb); 6392 6393 t = build_call_expr (builtin_decl_explicit (next_fn), 2, 6394 build_fold_addr_expr (istart0), 6395 build_fold_addr_expr (iend0)); 6396 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6397 false, GSI_CONTINUE_LINKING); 6398 if (TREE_TYPE (t) != boolean_type_node) 6399 t = fold_build2 (NE_EXPR, boolean_type_node, 6400 t, build_int_cst (TREE_TYPE (t), 0)); 6401 gcond *cond_stmt = gimple_build_cond_empty (t); 6402 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); 6403 } 6404 6405 /* Add the loop cleanup function. */ 6406 gsi = gsi_last_bb (exit_bb); 6407 if (gimple_omp_return_nowait_p (gsi_stmt (gsi))) 6408 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_NOWAIT); 6409 else if (gimple_omp_return_lhs (gsi_stmt (gsi))) 6410 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END_CANCEL); 6411 else 6412 t = builtin_decl_explicit (BUILT_IN_GOMP_LOOP_END); 6413 gcall *call_stmt = gimple_build_call (t, 0); 6414 if (gimple_omp_return_lhs (gsi_stmt (gsi))) 6415 gimple_call_set_lhs (call_stmt, gimple_omp_return_lhs (gsi_stmt (gsi))); 6416 gsi_insert_after (&gsi, call_stmt, GSI_SAME_STMT); 6417 gsi_remove (&gsi, true); 6418 6419 /* Connect the new blocks. */ 6420 find_edge (entry_bb, l0_bb)->flags = EDGE_TRUE_VALUE; 6421 find_edge (entry_bb, l3_bb)->flags = EDGE_FALSE_VALUE; 6422 6423 if (!broken_loop) 6424 { 6425 gimple_seq phis; 6426 6427 e = find_edge (cont_bb, l3_bb); 6428 ne = make_edge (l2_bb, l3_bb, EDGE_FALSE_VALUE); 6429 6430 phis = phi_nodes (l3_bb); 6431 for (gsi = gsi_start (phis); !gsi_end_p (gsi); gsi_next (&gsi)) 6432 { 6433 gimple phi = gsi_stmt (gsi); 6434 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, ne), 6435 PHI_ARG_DEF_FROM_EDGE (phi, e)); 6436 } 6437 remove_edge (e); 6438 6439 make_edge (cont_bb, l2_bb, EDGE_FALSE_VALUE); 6440 add_bb_to_loop (l2_bb, cont_bb->loop_father); 6441 e = find_edge (cont_bb, l1_bb); 6442 if (gimple_omp_for_combined_p (fd->for_stmt)) 6443 { 6444 remove_edge (e); 6445 e = NULL; 6446 } 6447 else if (fd->collapse > 1) 6448 { 6449 remove_edge (e); 6450 e = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 6451 } 6452 else 6453 e->flags = EDGE_TRUE_VALUE; 6454 if (e) 6455 { 6456 e->probability = REG_BR_PROB_BASE * 7 / 8; 6457 find_edge (cont_bb, l2_bb)->probability = REG_BR_PROB_BASE / 8; 6458 } 6459 else 6460 { 6461 e = find_edge (cont_bb, l2_bb); 6462 e->flags = EDGE_FALLTHRU; 6463 } 6464 make_edge (l2_bb, l0_bb, EDGE_TRUE_VALUE); 6465 6466 set_immediate_dominator (CDI_DOMINATORS, l2_bb, 6467 recompute_dominator (CDI_DOMINATORS, l2_bb)); 6468 set_immediate_dominator (CDI_DOMINATORS, l3_bb, 6469 recompute_dominator (CDI_DOMINATORS, l3_bb)); 6470 set_immediate_dominator (CDI_DOMINATORS, l0_bb, 6471 recompute_dominator (CDI_DOMINATORS, l0_bb)); 6472 set_immediate_dominator (CDI_DOMINATORS, l1_bb, 6473 recompute_dominator (CDI_DOMINATORS, l1_bb)); 6474 6475 struct loop *outer_loop = alloc_loop (); 6476 outer_loop->header = l0_bb; 6477 outer_loop->latch = l2_bb; 6478 add_loop (outer_loop, l0_bb->loop_father); 6479 6480 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6481 { 6482 struct loop *loop = alloc_loop (); 6483 loop->header = l1_bb; 6484 /* The loop may have multiple latches. */ 6485 add_loop (loop, outer_loop); 6486 } 6487 } 6488} 6489 6490 6491/* A subroutine of expand_omp_for. Generate code for a parallel 6492 loop with static schedule and no specified chunk size. Given 6493 parameters: 6494 6495 for (V = N1; V cond N2; V += STEP) BODY; 6496 6497 where COND is "<" or ">", we generate pseudocode 6498 6499 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; 6500 if (cond is <) 6501 adj = STEP - 1; 6502 else 6503 adj = STEP + 1; 6504 if ((__typeof (V)) -1 > 0 && cond is >) 6505 n = -(adj + N2 - N1) / -STEP; 6506 else 6507 n = (adj + N2 - N1) / STEP; 6508 q = n / nthreads; 6509 tt = n % nthreads; 6510 if (threadid < tt) goto L3; else goto L4; 6511 L3: 6512 tt = 0; 6513 q = q + 1; 6514 L4: 6515 s0 = q * threadid + tt; 6516 e0 = s0 + q; 6517 V = s0 * STEP + N1; 6518 if (s0 >= e0) goto L2; else goto L0; 6519 L0: 6520 e = e0 * STEP + N1; 6521 L1: 6522 BODY; 6523 V += STEP; 6524 if (V cond e) goto L1; 6525 L2: 6526*/ 6527 6528static void 6529expand_omp_for_static_nochunk (struct omp_region *region, 6530 struct omp_for_data *fd, 6531 gimple inner_stmt) 6532{ 6533 tree n, q, s0, e0, e, t, tt, nthreads, threadid; 6534 tree type, itype, vmain, vback; 6535 basic_block entry_bb, second_bb, third_bb, exit_bb, seq_start_bb; 6536 basic_block body_bb, cont_bb, collapse_bb = NULL; 6537 basic_block fin_bb; 6538 gimple_stmt_iterator gsi; 6539 edge ep; 6540 bool broken_loop = region->cont == NULL; 6541 tree *counts = NULL; 6542 tree n1, n2, step; 6543 6544 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) 6545 != GF_OMP_FOR_KIND_OACC_LOOP) 6546 || !inner_stmt); 6547 6548 itype = type = TREE_TYPE (fd->loop.v); 6549 if (POINTER_TYPE_P (type)) 6550 itype = signed_type_for (type); 6551 6552 entry_bb = region->entry; 6553 cont_bb = region->cont; 6554 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 6555 fin_bb = BRANCH_EDGE (entry_bb)->dest; 6556 gcc_assert (broken_loop 6557 || (fin_bb == FALLTHRU_EDGE (cont_bb)->dest)); 6558 seq_start_bb = split_edge (FALLTHRU_EDGE (entry_bb)); 6559 body_bb = single_succ (seq_start_bb); 6560 if (!broken_loop) 6561 { 6562 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); 6563 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6564 } 6565 exit_bb = region->exit; 6566 6567 /* Iteration space partitioning goes in ENTRY_BB. */ 6568 gsi = gsi_last_bb (entry_bb); 6569 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6570 6571 if (fd->collapse > 1) 6572 { 6573 int first_zero_iter = -1; 6574 basic_block l2_dom_bb = NULL; 6575 6576 counts = XALLOCAVEC (tree, fd->collapse); 6577 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6578 fin_bb, first_zero_iter, 6579 l2_dom_bb); 6580 t = NULL_TREE; 6581 } 6582 else if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6583 t = integer_one_node; 6584 else 6585 t = fold_binary (fd->loop.cond_code, boolean_type_node, 6586 fold_convert (type, fd->loop.n1), 6587 fold_convert (type, fd->loop.n2)); 6588 if (fd->collapse == 1 6589 && TYPE_UNSIGNED (type) 6590 && (t == NULL_TREE || !integer_onep (t))) 6591 { 6592 n1 = fold_convert (type, unshare_expr (fd->loop.n1)); 6593 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, 6594 true, GSI_SAME_STMT); 6595 n2 = fold_convert (type, unshare_expr (fd->loop.n2)); 6596 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, 6597 true, GSI_SAME_STMT); 6598 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, 6599 NULL_TREE, NULL_TREE); 6600 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6601 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 6602 expand_omp_regimplify_p, NULL, NULL) 6603 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 6604 expand_omp_regimplify_p, NULL, NULL)) 6605 { 6606 gsi = gsi_for_stmt (cond_stmt); 6607 gimple_regimplify_operands (cond_stmt, &gsi); 6608 } 6609 ep = split_block (entry_bb, cond_stmt); 6610 ep->flags = EDGE_TRUE_VALUE; 6611 entry_bb = ep->dest; 6612 ep->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); 6613 ep = make_edge (ep->src, fin_bb, EDGE_FALSE_VALUE); 6614 ep->probability = REG_BR_PROB_BASE / 2000 - 1; 6615 if (gimple_in_ssa_p (cfun)) 6616 { 6617 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; 6618 for (gphi_iterator gpi = gsi_start_phis (fin_bb); 6619 !gsi_end_p (gpi); gsi_next (&gpi)) 6620 { 6621 gphi *phi = gpi.phi (); 6622 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), 6623 ep, UNKNOWN_LOCATION); 6624 } 6625 } 6626 gsi = gsi_last_bb (entry_bb); 6627 } 6628 6629 switch (gimple_omp_for_kind (fd->for_stmt)) 6630 { 6631 case GF_OMP_FOR_KIND_FOR: 6632 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 6633 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 6634 break; 6635 case GF_OMP_FOR_KIND_DISTRIBUTE: 6636 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); 6637 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); 6638 break; 6639 case GF_OMP_FOR_KIND_OACC_LOOP: 6640 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); 6641 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 6642 break; 6643 default: 6644 gcc_unreachable (); 6645 } 6646 nthreads = build_call_expr (nthreads, 0); 6647 nthreads = fold_convert (itype, nthreads); 6648 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, 6649 true, GSI_SAME_STMT); 6650 threadid = build_call_expr (threadid, 0); 6651 threadid = fold_convert (itype, threadid); 6652 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, 6653 true, GSI_SAME_STMT); 6654 6655 n1 = fd->loop.n1; 6656 n2 = fd->loop.n2; 6657 step = fd->loop.step; 6658 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6659 { 6660 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 6661 OMP_CLAUSE__LOOPTEMP_); 6662 gcc_assert (innerc); 6663 n1 = OMP_CLAUSE_DECL (innerc); 6664 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6665 OMP_CLAUSE__LOOPTEMP_); 6666 gcc_assert (innerc); 6667 n2 = OMP_CLAUSE_DECL (innerc); 6668 } 6669 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), 6670 true, NULL_TREE, true, GSI_SAME_STMT); 6671 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), 6672 true, NULL_TREE, true, GSI_SAME_STMT); 6673 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), 6674 true, NULL_TREE, true, GSI_SAME_STMT); 6675 6676 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); 6677 t = fold_build2 (PLUS_EXPR, itype, step, t); 6678 t = fold_build2 (PLUS_EXPR, itype, t, n2); 6679 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); 6680 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) 6681 t = fold_build2 (TRUNC_DIV_EXPR, itype, 6682 fold_build1 (NEGATE_EXPR, itype, t), 6683 fold_build1 (NEGATE_EXPR, itype, step)); 6684 else 6685 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); 6686 t = fold_convert (itype, t); 6687 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6688 6689 q = create_tmp_reg (itype, "q"); 6690 t = fold_build2 (TRUNC_DIV_EXPR, itype, n, nthreads); 6691 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); 6692 gsi_insert_before (&gsi, gimple_build_assign (q, t), GSI_SAME_STMT); 6693 6694 tt = create_tmp_reg (itype, "tt"); 6695 t = fold_build2 (TRUNC_MOD_EXPR, itype, n, nthreads); 6696 t = force_gimple_operand_gsi (&gsi, t, false, NULL_TREE, true, GSI_SAME_STMT); 6697 gsi_insert_before (&gsi, gimple_build_assign (tt, t), GSI_SAME_STMT); 6698 6699 t = build2 (LT_EXPR, boolean_type_node, threadid, tt); 6700 gcond *cond_stmt = gimple_build_cond_empty (t); 6701 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6702 6703 second_bb = split_block (entry_bb, cond_stmt)->dest; 6704 gsi = gsi_last_bb (second_bb); 6705 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6706 6707 gsi_insert_before (&gsi, gimple_build_assign (tt, build_int_cst (itype, 0)), 6708 GSI_SAME_STMT); 6709 gassign *assign_stmt 6710 = gimple_build_assign (q, PLUS_EXPR, q, build_int_cst (itype, 1)); 6711 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6712 6713 third_bb = split_block (second_bb, assign_stmt)->dest; 6714 gsi = gsi_last_bb (third_bb); 6715 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6716 6717 t = build2 (MULT_EXPR, itype, q, threadid); 6718 t = build2 (PLUS_EXPR, itype, t, tt); 6719 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6720 6721 t = fold_build2 (PLUS_EXPR, itype, s0, q); 6722 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, true, GSI_SAME_STMT); 6723 6724 t = build2 (GE_EXPR, boolean_type_node, s0, e0); 6725 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6726 6727 /* Remove the GIMPLE_OMP_FOR statement. */ 6728 gsi_remove (&gsi, true); 6729 6730 /* Setup code for sequential iteration goes in SEQ_START_BB. */ 6731 gsi = gsi_start_bb (seq_start_bb); 6732 6733 tree startvar = fd->loop.v; 6734 tree endvar = NULL_TREE; 6735 6736 if (gimple_omp_for_combined_p (fd->for_stmt)) 6737 { 6738 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 6739 ? gimple_omp_parallel_clauses (inner_stmt) 6740 : gimple_omp_for_clauses (inner_stmt); 6741 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 6742 gcc_assert (innerc); 6743 startvar = OMP_CLAUSE_DECL (innerc); 6744 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 6745 OMP_CLAUSE__LOOPTEMP_); 6746 gcc_assert (innerc); 6747 endvar = OMP_CLAUSE_DECL (innerc); 6748 } 6749 t = fold_convert (itype, s0); 6750 t = fold_build2 (MULT_EXPR, itype, t, step); 6751 if (POINTER_TYPE_P (type)) 6752 t = fold_build_pointer_plus (n1, t); 6753 else 6754 t = fold_build2 (PLUS_EXPR, type, t, n1); 6755 t = fold_convert (TREE_TYPE (startvar), t); 6756 t = force_gimple_operand_gsi (&gsi, t, 6757 DECL_P (startvar) 6758 && TREE_ADDRESSABLE (startvar), 6759 NULL_TREE, false, GSI_CONTINUE_LINKING); 6760 assign_stmt = gimple_build_assign (startvar, t); 6761 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6762 6763 t = fold_convert (itype, e0); 6764 t = fold_build2 (MULT_EXPR, itype, t, step); 6765 if (POINTER_TYPE_P (type)) 6766 t = fold_build_pointer_plus (n1, t); 6767 else 6768 t = fold_build2 (PLUS_EXPR, type, t, n1); 6769 t = fold_convert (TREE_TYPE (startvar), t); 6770 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 6771 false, GSI_CONTINUE_LINKING); 6772 if (endvar) 6773 { 6774 assign_stmt = gimple_build_assign (endvar, e); 6775 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6776 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) 6777 assign_stmt = gimple_build_assign (fd->loop.v, e); 6778 else 6779 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); 6780 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 6781 } 6782 if (fd->collapse > 1) 6783 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 6784 6785 if (!broken_loop) 6786 { 6787 /* The code controlling the sequential loop replaces the 6788 GIMPLE_OMP_CONTINUE. */ 6789 gsi = gsi_last_bb (cont_bb); 6790 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 6791 gcc_assert (gimple_code (cont_stmt) == GIMPLE_OMP_CONTINUE); 6792 vmain = gimple_omp_continue_control_use (cont_stmt); 6793 vback = gimple_omp_continue_control_def (cont_stmt); 6794 6795 if (!gimple_omp_for_combined_p (fd->for_stmt)) 6796 { 6797 if (POINTER_TYPE_P (type)) 6798 t = fold_build_pointer_plus (vmain, step); 6799 else 6800 t = fold_build2 (PLUS_EXPR, type, vmain, step); 6801 t = force_gimple_operand_gsi (&gsi, t, 6802 DECL_P (vback) 6803 && TREE_ADDRESSABLE (vback), 6804 NULL_TREE, true, GSI_SAME_STMT); 6805 assign_stmt = gimple_build_assign (vback, t); 6806 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 6807 6808 t = build2 (fd->loop.cond_code, boolean_type_node, 6809 DECL_P (vback) && TREE_ADDRESSABLE (vback) 6810 ? t : vback, e); 6811 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 6812 } 6813 6814 /* Remove the GIMPLE_OMP_CONTINUE statement. */ 6815 gsi_remove (&gsi, true); 6816 6817 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 6818 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); 6819 } 6820 6821 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ 6822 gsi = gsi_last_bb (exit_bb); 6823 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) 6824 { 6825 t = gimple_omp_return_lhs (gsi_stmt (gsi)); 6826 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 6827 gcc_checking_assert (t == NULL_TREE); 6828 else 6829 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); 6830 } 6831 gsi_remove (&gsi, true); 6832 6833 /* Connect all the blocks. */ 6834 ep = make_edge (entry_bb, third_bb, EDGE_FALSE_VALUE); 6835 ep->probability = REG_BR_PROB_BASE / 4 * 3; 6836 ep = find_edge (entry_bb, second_bb); 6837 ep->flags = EDGE_TRUE_VALUE; 6838 ep->probability = REG_BR_PROB_BASE / 4; 6839 find_edge (third_bb, seq_start_bb)->flags = EDGE_FALSE_VALUE; 6840 find_edge (third_bb, fin_bb)->flags = EDGE_TRUE_VALUE; 6841 6842 if (!broken_loop) 6843 { 6844 ep = find_edge (cont_bb, body_bb); 6845 if (gimple_omp_for_combined_p (fd->for_stmt)) 6846 { 6847 remove_edge (ep); 6848 ep = NULL; 6849 } 6850 else if (fd->collapse > 1) 6851 { 6852 remove_edge (ep); 6853 ep = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 6854 } 6855 else 6856 ep->flags = EDGE_TRUE_VALUE; 6857 find_edge (cont_bb, fin_bb)->flags 6858 = ep ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; 6859 } 6860 6861 set_immediate_dominator (CDI_DOMINATORS, second_bb, entry_bb); 6862 set_immediate_dominator (CDI_DOMINATORS, third_bb, entry_bb); 6863 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, third_bb); 6864 6865 set_immediate_dominator (CDI_DOMINATORS, body_bb, 6866 recompute_dominator (CDI_DOMINATORS, body_bb)); 6867 set_immediate_dominator (CDI_DOMINATORS, fin_bb, 6868 recompute_dominator (CDI_DOMINATORS, fin_bb)); 6869 6870 if (!broken_loop && !gimple_omp_for_combined_p (fd->for_stmt)) 6871 { 6872 struct loop *loop = alloc_loop (); 6873 loop->header = body_bb; 6874 if (collapse_bb == NULL) 6875 loop->latch = cont_bb; 6876 add_loop (loop, body_bb->loop_father); 6877 } 6878} 6879 6880 6881/* A subroutine of expand_omp_for. Generate code for a parallel 6882 loop with static schedule and a specified chunk size. Given 6883 parameters: 6884 6885 for (V = N1; V cond N2; V += STEP) BODY; 6886 6887 where COND is "<" or ">", we generate pseudocode 6888 6889 if ((__typeof (V)) -1 > 0 && N2 cond N1) goto L2; 6890 if (cond is <) 6891 adj = STEP - 1; 6892 else 6893 adj = STEP + 1; 6894 if ((__typeof (V)) -1 > 0 && cond is >) 6895 n = -(adj + N2 - N1) / -STEP; 6896 else 6897 n = (adj + N2 - N1) / STEP; 6898 trip = 0; 6899 V = threadid * CHUNK * STEP + N1; -- this extra definition of V is 6900 here so that V is defined 6901 if the loop is not entered 6902 L0: 6903 s0 = (trip * nthreads + threadid) * CHUNK; 6904 e0 = min(s0 + CHUNK, n); 6905 if (s0 < n) goto L1; else goto L4; 6906 L1: 6907 V = s0 * STEP + N1; 6908 e = e0 * STEP + N1; 6909 L2: 6910 BODY; 6911 V += STEP; 6912 if (V cond e) goto L2; else goto L3; 6913 L3: 6914 trip += 1; 6915 goto L0; 6916 L4: 6917*/ 6918 6919static void 6920expand_omp_for_static_chunk (struct omp_region *region, 6921 struct omp_for_data *fd, gimple inner_stmt) 6922{ 6923 tree n, s0, e0, e, t; 6924 tree trip_var, trip_init, trip_main, trip_back, nthreads, threadid; 6925 tree type, itype, vmain, vback, vextra; 6926 basic_block entry_bb, exit_bb, body_bb, seq_start_bb, iter_part_bb; 6927 basic_block trip_update_bb = NULL, cont_bb, collapse_bb = NULL, fin_bb; 6928 gimple_stmt_iterator gsi; 6929 edge se; 6930 bool broken_loop = region->cont == NULL; 6931 tree *counts = NULL; 6932 tree n1, n2, step; 6933 6934 gcc_checking_assert ((gimple_omp_for_kind (fd->for_stmt) 6935 != GF_OMP_FOR_KIND_OACC_LOOP) 6936 || !inner_stmt); 6937 6938 itype = type = TREE_TYPE (fd->loop.v); 6939 if (POINTER_TYPE_P (type)) 6940 itype = signed_type_for (type); 6941 6942 entry_bb = region->entry; 6943 se = split_block (entry_bb, last_stmt (entry_bb)); 6944 entry_bb = se->src; 6945 iter_part_bb = se->dest; 6946 cont_bb = region->cont; 6947 gcc_assert (EDGE_COUNT (iter_part_bb->succs) == 2); 6948 fin_bb = BRANCH_EDGE (iter_part_bb)->dest; 6949 gcc_assert (broken_loop 6950 || fin_bb == FALLTHRU_EDGE (cont_bb)->dest); 6951 seq_start_bb = split_edge (FALLTHRU_EDGE (iter_part_bb)); 6952 body_bb = single_succ (seq_start_bb); 6953 if (!broken_loop) 6954 { 6955 gcc_assert (BRANCH_EDGE (cont_bb)->dest == body_bb); 6956 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 6957 trip_update_bb = split_edge (FALLTHRU_EDGE (cont_bb)); 6958 } 6959 exit_bb = region->exit; 6960 6961 /* Trip and adjustment setup goes in ENTRY_BB. */ 6962 gsi = gsi_last_bb (entry_bb); 6963 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 6964 6965 if (fd->collapse > 1) 6966 { 6967 int first_zero_iter = -1; 6968 basic_block l2_dom_bb = NULL; 6969 6970 counts = XALLOCAVEC (tree, fd->collapse); 6971 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 6972 fin_bb, first_zero_iter, 6973 l2_dom_bb); 6974 t = NULL_TREE; 6975 } 6976 else if (gimple_omp_for_combined_into_p (fd->for_stmt)) 6977 t = integer_one_node; 6978 else 6979 t = fold_binary (fd->loop.cond_code, boolean_type_node, 6980 fold_convert (type, fd->loop.n1), 6981 fold_convert (type, fd->loop.n2)); 6982 if (fd->collapse == 1 6983 && TYPE_UNSIGNED (type) 6984 && (t == NULL_TREE || !integer_onep (t))) 6985 { 6986 n1 = fold_convert (type, unshare_expr (fd->loop.n1)); 6987 n1 = force_gimple_operand_gsi (&gsi, n1, true, NULL_TREE, 6988 true, GSI_SAME_STMT); 6989 n2 = fold_convert (type, unshare_expr (fd->loop.n2)); 6990 n2 = force_gimple_operand_gsi (&gsi, n2, true, NULL_TREE, 6991 true, GSI_SAME_STMT); 6992 gcond *cond_stmt = gimple_build_cond (fd->loop.cond_code, n1, n2, 6993 NULL_TREE, NULL_TREE); 6994 gsi_insert_before (&gsi, cond_stmt, GSI_SAME_STMT); 6995 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 6996 expand_omp_regimplify_p, NULL, NULL) 6997 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 6998 expand_omp_regimplify_p, NULL, NULL)) 6999 { 7000 gsi = gsi_for_stmt (cond_stmt); 7001 gimple_regimplify_operands (cond_stmt, &gsi); 7002 } 7003 se = split_block (entry_bb, cond_stmt); 7004 se->flags = EDGE_TRUE_VALUE; 7005 entry_bb = se->dest; 7006 se->probability = REG_BR_PROB_BASE - (REG_BR_PROB_BASE / 2000 - 1); 7007 se = make_edge (se->src, fin_bb, EDGE_FALSE_VALUE); 7008 se->probability = REG_BR_PROB_BASE / 2000 - 1; 7009 if (gimple_in_ssa_p (cfun)) 7010 { 7011 int dest_idx = find_edge (entry_bb, fin_bb)->dest_idx; 7012 for (gphi_iterator gpi = gsi_start_phis (fin_bb); 7013 !gsi_end_p (gpi); gsi_next (&gpi)) 7014 { 7015 gphi *phi = gpi.phi (); 7016 add_phi_arg (phi, gimple_phi_arg_def (phi, dest_idx), 7017 se, UNKNOWN_LOCATION); 7018 } 7019 } 7020 gsi = gsi_last_bb (entry_bb); 7021 } 7022 7023 switch (gimple_omp_for_kind (fd->for_stmt)) 7024 { 7025 case GF_OMP_FOR_KIND_FOR: 7026 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_THREADS); 7027 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 7028 break; 7029 case GF_OMP_FOR_KIND_DISTRIBUTE: 7030 nthreads = builtin_decl_explicit (BUILT_IN_OMP_GET_NUM_TEAMS); 7031 threadid = builtin_decl_explicit (BUILT_IN_OMP_GET_TEAM_NUM); 7032 break; 7033 case GF_OMP_FOR_KIND_OACC_LOOP: 7034 nthreads = builtin_decl_explicit (BUILT_IN_GOACC_GET_NUM_THREADS); 7035 threadid = builtin_decl_explicit (BUILT_IN_GOACC_GET_THREAD_NUM); 7036 break; 7037 default: 7038 gcc_unreachable (); 7039 } 7040 nthreads = build_call_expr (nthreads, 0); 7041 nthreads = fold_convert (itype, nthreads); 7042 nthreads = force_gimple_operand_gsi (&gsi, nthreads, true, NULL_TREE, 7043 true, GSI_SAME_STMT); 7044 threadid = build_call_expr (threadid, 0); 7045 threadid = fold_convert (itype, threadid); 7046 threadid = force_gimple_operand_gsi (&gsi, threadid, true, NULL_TREE, 7047 true, GSI_SAME_STMT); 7048 7049 n1 = fd->loop.n1; 7050 n2 = fd->loop.n2; 7051 step = fd->loop.step; 7052 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 7053 { 7054 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7055 OMP_CLAUSE__LOOPTEMP_); 7056 gcc_assert (innerc); 7057 n1 = OMP_CLAUSE_DECL (innerc); 7058 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7059 OMP_CLAUSE__LOOPTEMP_); 7060 gcc_assert (innerc); 7061 n2 = OMP_CLAUSE_DECL (innerc); 7062 } 7063 n1 = force_gimple_operand_gsi (&gsi, fold_convert (type, n1), 7064 true, NULL_TREE, true, GSI_SAME_STMT); 7065 n2 = force_gimple_operand_gsi (&gsi, fold_convert (itype, n2), 7066 true, NULL_TREE, true, GSI_SAME_STMT); 7067 step = force_gimple_operand_gsi (&gsi, fold_convert (itype, step), 7068 true, NULL_TREE, true, GSI_SAME_STMT); 7069 fd->chunk_size 7070 = force_gimple_operand_gsi (&gsi, fold_convert (itype, fd->chunk_size), 7071 true, NULL_TREE, true, GSI_SAME_STMT); 7072 7073 t = build_int_cst (itype, (fd->loop.cond_code == LT_EXPR ? -1 : 1)); 7074 t = fold_build2 (PLUS_EXPR, itype, step, t); 7075 t = fold_build2 (PLUS_EXPR, itype, t, n2); 7076 t = fold_build2 (MINUS_EXPR, itype, t, fold_convert (itype, n1)); 7077 if (TYPE_UNSIGNED (itype) && fd->loop.cond_code == GT_EXPR) 7078 t = fold_build2 (TRUNC_DIV_EXPR, itype, 7079 fold_build1 (NEGATE_EXPR, itype, t), 7080 fold_build1 (NEGATE_EXPR, itype, step)); 7081 else 7082 t = fold_build2 (TRUNC_DIV_EXPR, itype, t, step); 7083 t = fold_convert (itype, t); 7084 n = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7085 true, GSI_SAME_STMT); 7086 7087 trip_var = create_tmp_reg (itype, ".trip"); 7088 if (gimple_in_ssa_p (cfun)) 7089 { 7090 trip_init = make_ssa_name (trip_var); 7091 trip_main = make_ssa_name (trip_var); 7092 trip_back = make_ssa_name (trip_var); 7093 } 7094 else 7095 { 7096 trip_init = trip_var; 7097 trip_main = trip_var; 7098 trip_back = trip_var; 7099 } 7100 7101 gassign *assign_stmt 7102 = gimple_build_assign (trip_init, build_int_cst (itype, 0)); 7103 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 7104 7105 t = fold_build2 (MULT_EXPR, itype, threadid, fd->chunk_size); 7106 t = fold_build2 (MULT_EXPR, itype, t, step); 7107 if (POINTER_TYPE_P (type)) 7108 t = fold_build_pointer_plus (n1, t); 7109 else 7110 t = fold_build2 (PLUS_EXPR, type, t, n1); 7111 vextra = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7112 true, GSI_SAME_STMT); 7113 7114 /* Remove the GIMPLE_OMP_FOR. */ 7115 gsi_remove (&gsi, true); 7116 7117 /* Iteration space partitioning goes in ITER_PART_BB. */ 7118 gsi = gsi_last_bb (iter_part_bb); 7119 7120 t = fold_build2 (MULT_EXPR, itype, trip_main, nthreads); 7121 t = fold_build2 (PLUS_EXPR, itype, t, threadid); 7122 t = fold_build2 (MULT_EXPR, itype, t, fd->chunk_size); 7123 s0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7124 false, GSI_CONTINUE_LINKING); 7125 7126 t = fold_build2 (PLUS_EXPR, itype, s0, fd->chunk_size); 7127 t = fold_build2 (MIN_EXPR, itype, t, n); 7128 e0 = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7129 false, GSI_CONTINUE_LINKING); 7130 7131 t = build2 (LT_EXPR, boolean_type_node, s0, n); 7132 gsi_insert_after (&gsi, gimple_build_cond_empty (t), GSI_CONTINUE_LINKING); 7133 7134 /* Setup code for sequential iteration goes in SEQ_START_BB. */ 7135 gsi = gsi_start_bb (seq_start_bb); 7136 7137 tree startvar = fd->loop.v; 7138 tree endvar = NULL_TREE; 7139 7140 if (gimple_omp_for_combined_p (fd->for_stmt)) 7141 { 7142 tree clauses = gimple_code (inner_stmt) == GIMPLE_OMP_PARALLEL 7143 ? gimple_omp_parallel_clauses (inner_stmt) 7144 : gimple_omp_for_clauses (inner_stmt); 7145 tree innerc = find_omp_clause (clauses, OMP_CLAUSE__LOOPTEMP_); 7146 gcc_assert (innerc); 7147 startvar = OMP_CLAUSE_DECL (innerc); 7148 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7149 OMP_CLAUSE__LOOPTEMP_); 7150 gcc_assert (innerc); 7151 endvar = OMP_CLAUSE_DECL (innerc); 7152 } 7153 7154 t = fold_convert (itype, s0); 7155 t = fold_build2 (MULT_EXPR, itype, t, step); 7156 if (POINTER_TYPE_P (type)) 7157 t = fold_build_pointer_plus (n1, t); 7158 else 7159 t = fold_build2 (PLUS_EXPR, type, t, n1); 7160 t = fold_convert (TREE_TYPE (startvar), t); 7161 t = force_gimple_operand_gsi (&gsi, t, 7162 DECL_P (startvar) 7163 && TREE_ADDRESSABLE (startvar), 7164 NULL_TREE, false, GSI_CONTINUE_LINKING); 7165 assign_stmt = gimple_build_assign (startvar, t); 7166 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7167 7168 t = fold_convert (itype, e0); 7169 t = fold_build2 (MULT_EXPR, itype, t, step); 7170 if (POINTER_TYPE_P (type)) 7171 t = fold_build_pointer_plus (n1, t); 7172 else 7173 t = fold_build2 (PLUS_EXPR, type, t, n1); 7174 t = fold_convert (TREE_TYPE (startvar), t); 7175 e = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7176 false, GSI_CONTINUE_LINKING); 7177 if (endvar) 7178 { 7179 assign_stmt = gimple_build_assign (endvar, e); 7180 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7181 if (useless_type_conversion_p (TREE_TYPE (fd->loop.v), TREE_TYPE (e))) 7182 assign_stmt = gimple_build_assign (fd->loop.v, e); 7183 else 7184 assign_stmt = gimple_build_assign (fd->loop.v, NOP_EXPR, e); 7185 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7186 } 7187 if (fd->collapse > 1) 7188 expand_omp_for_init_vars (fd, &gsi, counts, inner_stmt, startvar); 7189 7190 if (!broken_loop) 7191 { 7192 /* The code controlling the sequential loop goes in CONT_BB, 7193 replacing the GIMPLE_OMP_CONTINUE. */ 7194 gsi = gsi_last_bb (cont_bb); 7195 gomp_continue *cont_stmt = as_a <gomp_continue *> (gsi_stmt (gsi)); 7196 vmain = gimple_omp_continue_control_use (cont_stmt); 7197 vback = gimple_omp_continue_control_def (cont_stmt); 7198 7199 if (!gimple_omp_for_combined_p (fd->for_stmt)) 7200 { 7201 if (POINTER_TYPE_P (type)) 7202 t = fold_build_pointer_plus (vmain, step); 7203 else 7204 t = fold_build2 (PLUS_EXPR, type, vmain, step); 7205 if (DECL_P (vback) && TREE_ADDRESSABLE (vback)) 7206 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7207 true, GSI_SAME_STMT); 7208 assign_stmt = gimple_build_assign (vback, t); 7209 gsi_insert_before (&gsi, assign_stmt, GSI_SAME_STMT); 7210 7211 t = build2 (fd->loop.cond_code, boolean_type_node, 7212 DECL_P (vback) && TREE_ADDRESSABLE (vback) 7213 ? t : vback, e); 7214 gsi_insert_before (&gsi, gimple_build_cond_empty (t), GSI_SAME_STMT); 7215 } 7216 7217 /* Remove GIMPLE_OMP_CONTINUE. */ 7218 gsi_remove (&gsi, true); 7219 7220 if (fd->collapse > 1 && !gimple_omp_for_combined_p (fd->for_stmt)) 7221 collapse_bb = extract_omp_for_update_vars (fd, cont_bb, body_bb); 7222 7223 /* Trip update code goes into TRIP_UPDATE_BB. */ 7224 gsi = gsi_start_bb (trip_update_bb); 7225 7226 t = build_int_cst (itype, 1); 7227 t = build2 (PLUS_EXPR, itype, trip_main, t); 7228 assign_stmt = gimple_build_assign (trip_back, t); 7229 gsi_insert_after (&gsi, assign_stmt, GSI_CONTINUE_LINKING); 7230 } 7231 7232 /* Replace the GIMPLE_OMP_RETURN with a barrier, or nothing. */ 7233 gsi = gsi_last_bb (exit_bb); 7234 if (!gimple_omp_return_nowait_p (gsi_stmt (gsi))) 7235 { 7236 t = gimple_omp_return_lhs (gsi_stmt (gsi)); 7237 if (gimple_omp_for_kind (fd->for_stmt) == GF_OMP_FOR_KIND_OACC_LOOP) 7238 gcc_checking_assert (t == NULL_TREE); 7239 else 7240 gsi_insert_after (&gsi, build_omp_barrier (t), GSI_SAME_STMT); 7241 } 7242 gsi_remove (&gsi, true); 7243 7244 /* Connect the new blocks. */ 7245 find_edge (iter_part_bb, seq_start_bb)->flags = EDGE_TRUE_VALUE; 7246 find_edge (iter_part_bb, fin_bb)->flags = EDGE_FALSE_VALUE; 7247 7248 if (!broken_loop) 7249 { 7250 se = find_edge (cont_bb, body_bb); 7251 if (gimple_omp_for_combined_p (fd->for_stmt)) 7252 { 7253 remove_edge (se); 7254 se = NULL; 7255 } 7256 else if (fd->collapse > 1) 7257 { 7258 remove_edge (se); 7259 se = make_edge (cont_bb, collapse_bb, EDGE_TRUE_VALUE); 7260 } 7261 else 7262 se->flags = EDGE_TRUE_VALUE; 7263 find_edge (cont_bb, trip_update_bb)->flags 7264 = se ? EDGE_FALSE_VALUE : EDGE_FALLTHRU; 7265 7266 redirect_edge_and_branch (single_succ_edge (trip_update_bb), iter_part_bb); 7267 } 7268 7269 if (gimple_in_ssa_p (cfun)) 7270 { 7271 gphi_iterator psi; 7272 gphi *phi; 7273 edge re, ene; 7274 edge_var_map *vm; 7275 size_t i; 7276 7277 gcc_assert (fd->collapse == 1 && !broken_loop); 7278 7279 /* When we redirect the edge from trip_update_bb to iter_part_bb, we 7280 remove arguments of the phi nodes in fin_bb. We need to create 7281 appropriate phi nodes in iter_part_bb instead. */ 7282 se = single_pred_edge (fin_bb); 7283 re = single_succ_edge (trip_update_bb); 7284 vec<edge_var_map> *head = redirect_edge_var_map_vector (re); 7285 ene = single_succ_edge (entry_bb); 7286 7287 psi = gsi_start_phis (fin_bb); 7288 for (i = 0; !gsi_end_p (psi) && head->iterate (i, &vm); 7289 gsi_next (&psi), ++i) 7290 { 7291 gphi *nphi; 7292 source_location locus; 7293 7294 phi = psi.phi (); 7295 t = gimple_phi_result (phi); 7296 gcc_assert (t == redirect_edge_var_map_result (vm)); 7297 nphi = create_phi_node (t, iter_part_bb); 7298 7299 t = PHI_ARG_DEF_FROM_EDGE (phi, se); 7300 locus = gimple_phi_arg_location_from_edge (phi, se); 7301 7302 /* A special case -- fd->loop.v is not yet computed in 7303 iter_part_bb, we need to use vextra instead. */ 7304 if (t == fd->loop.v) 7305 t = vextra; 7306 add_phi_arg (nphi, t, ene, locus); 7307 locus = redirect_edge_var_map_location (vm); 7308 add_phi_arg (nphi, redirect_edge_var_map_def (vm), re, locus); 7309 } 7310 gcc_assert (gsi_end_p (psi) && i == head->length ()); 7311 redirect_edge_var_map_clear (re); 7312 while (1) 7313 { 7314 psi = gsi_start_phis (fin_bb); 7315 if (gsi_end_p (psi)) 7316 break; 7317 remove_phi_node (&psi, false); 7318 } 7319 7320 /* Make phi node for trip. */ 7321 phi = create_phi_node (trip_main, iter_part_bb); 7322 add_phi_arg (phi, trip_back, single_succ_edge (trip_update_bb), 7323 UNKNOWN_LOCATION); 7324 add_phi_arg (phi, trip_init, single_succ_edge (entry_bb), 7325 UNKNOWN_LOCATION); 7326 } 7327 7328 if (!broken_loop) 7329 set_immediate_dominator (CDI_DOMINATORS, trip_update_bb, cont_bb); 7330 set_immediate_dominator (CDI_DOMINATORS, iter_part_bb, 7331 recompute_dominator (CDI_DOMINATORS, iter_part_bb)); 7332 set_immediate_dominator (CDI_DOMINATORS, fin_bb, 7333 recompute_dominator (CDI_DOMINATORS, fin_bb)); 7334 set_immediate_dominator (CDI_DOMINATORS, seq_start_bb, 7335 recompute_dominator (CDI_DOMINATORS, seq_start_bb)); 7336 set_immediate_dominator (CDI_DOMINATORS, body_bb, 7337 recompute_dominator (CDI_DOMINATORS, body_bb)); 7338 7339 if (!broken_loop) 7340 { 7341 struct loop *trip_loop = alloc_loop (); 7342 trip_loop->header = iter_part_bb; 7343 trip_loop->latch = trip_update_bb; 7344 add_loop (trip_loop, iter_part_bb->loop_father); 7345 7346 if (!gimple_omp_for_combined_p (fd->for_stmt)) 7347 { 7348 struct loop *loop = alloc_loop (); 7349 loop->header = body_bb; 7350 if (collapse_bb == NULL) 7351 loop->latch = cont_bb; 7352 add_loop (loop, trip_loop); 7353 } 7354 } 7355} 7356 7357/* A subroutine of expand_omp_for. Generate code for _Cilk_for loop. 7358 Given parameters: 7359 for (V = N1; V cond N2; V += STEP) BODY; 7360 7361 where COND is "<" or ">" or "!=", we generate pseudocode 7362 7363 for (ind_var = low; ind_var < high; ind_var++) 7364 { 7365 V = n1 + (ind_var * STEP) 7366 7367 <BODY> 7368 } 7369 7370 In the above pseudocode, low and high are function parameters of the 7371 child function. In the function below, we are inserting a temp. 7372 variable that will be making a call to two OMP functions that will not be 7373 found in the body of _Cilk_for (since OMP_FOR cannot be mixed 7374 with _Cilk_for). These functions are replaced with low and high 7375 by the function that handles taskreg. */ 7376 7377 7378static void 7379expand_cilk_for (struct omp_region *region, struct omp_for_data *fd) 7380{ 7381 bool broken_loop = region->cont == NULL; 7382 basic_block entry_bb = region->entry; 7383 basic_block cont_bb = region->cont; 7384 7385 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 7386 gcc_assert (broken_loop 7387 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 7388 basic_block l0_bb = FALLTHRU_EDGE (entry_bb)->dest; 7389 basic_block l1_bb, l2_bb; 7390 7391 if (!broken_loop) 7392 { 7393 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); 7394 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 7395 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; 7396 l2_bb = BRANCH_EDGE (entry_bb)->dest; 7397 } 7398 else 7399 { 7400 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; 7401 l1_bb = split_edge (BRANCH_EDGE (entry_bb)); 7402 l2_bb = single_succ (l1_bb); 7403 } 7404 basic_block exit_bb = region->exit; 7405 basic_block l2_dom_bb = NULL; 7406 7407 gimple_stmt_iterator gsi = gsi_last_bb (entry_bb); 7408 7409 /* Below statements until the "tree high_val = ..." are pseudo statements 7410 used to pass information to be used by expand_omp_taskreg. 7411 low_val and high_val will be replaced by the __low and __high 7412 parameter from the child function. 7413 7414 The call_exprs part is a place-holder, it is mainly used 7415 to distinctly identify to the top-level part that this is 7416 where we should put low and high (reasoning given in header 7417 comment). */ 7418 7419 tree child_fndecl 7420 = gimple_omp_parallel_child_fn ( 7421 as_a <gomp_parallel *> (last_stmt (region->outer->entry))); 7422 tree t, low_val = NULL_TREE, high_val = NULL_TREE; 7423 for (t = DECL_ARGUMENTS (child_fndecl); t; t = TREE_CHAIN (t)) 7424 { 7425 if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__high")) 7426 high_val = t; 7427 else if (!strcmp (IDENTIFIER_POINTER (DECL_NAME (t)), "__low")) 7428 low_val = t; 7429 } 7430 gcc_assert (low_val && high_val); 7431 7432 tree type = TREE_TYPE (low_val); 7433 tree ind_var = create_tmp_reg (type, "__cilk_ind_var"); 7434 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 7435 7436 /* Not needed in SSA form right now. */ 7437 gcc_assert (!gimple_in_ssa_p (cfun)); 7438 if (l2_dom_bb == NULL) 7439 l2_dom_bb = l1_bb; 7440 7441 tree n1 = low_val; 7442 tree n2 = high_val; 7443 7444 gimple stmt = gimple_build_assign (ind_var, n1); 7445 7446 /* Replace the GIMPLE_OMP_FOR statement. */ 7447 gsi_replace (&gsi, stmt, true); 7448 7449 if (!broken_loop) 7450 { 7451 /* Code to control the increment goes in the CONT_BB. */ 7452 gsi = gsi_last_bb (cont_bb); 7453 stmt = gsi_stmt (gsi); 7454 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); 7455 stmt = gimple_build_assign (ind_var, PLUS_EXPR, ind_var, 7456 build_one_cst (type)); 7457 7458 /* Replace GIMPLE_OMP_CONTINUE. */ 7459 gsi_replace (&gsi, stmt, true); 7460 } 7461 7462 /* Emit the condition in L1_BB. */ 7463 gsi = gsi_after_labels (l1_bb); 7464 t = fold_build2 (MULT_EXPR, TREE_TYPE (fd->loop.step), 7465 fold_convert (TREE_TYPE (fd->loop.step), ind_var), 7466 fd->loop.step); 7467 if (POINTER_TYPE_P (TREE_TYPE (fd->loop.n1))) 7468 t = fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (fd->loop.n1), 7469 fd->loop.n1, fold_convert (sizetype, t)); 7470 else 7471 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loop.n1), 7472 fd->loop.n1, fold_convert (TREE_TYPE (fd->loop.n1), t)); 7473 t = fold_convert (TREE_TYPE (fd->loop.v), t); 7474 expand_omp_build_assign (&gsi, fd->loop.v, t); 7475 7476 /* The condition is always '<' since the runtime will fill in the low 7477 and high values. */ 7478 stmt = gimple_build_cond (LT_EXPR, ind_var, n2, NULL_TREE, NULL_TREE); 7479 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 7480 7481 /* Remove GIMPLE_OMP_RETURN. */ 7482 gsi = gsi_last_bb (exit_bb); 7483 gsi_remove (&gsi, true); 7484 7485 /* Connect the new blocks. */ 7486 remove_edge (FALLTHRU_EDGE (entry_bb)); 7487 7488 edge e, ne; 7489 if (!broken_loop) 7490 { 7491 remove_edge (BRANCH_EDGE (entry_bb)); 7492 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); 7493 7494 e = BRANCH_EDGE (l1_bb); 7495 ne = FALLTHRU_EDGE (l1_bb); 7496 e->flags = EDGE_TRUE_VALUE; 7497 } 7498 else 7499 { 7500 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 7501 7502 ne = single_succ_edge (l1_bb); 7503 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); 7504 7505 } 7506 ne->flags = EDGE_FALSE_VALUE; 7507 e->probability = REG_BR_PROB_BASE * 7 / 8; 7508 ne->probability = REG_BR_PROB_BASE / 8; 7509 7510 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); 7511 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); 7512 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); 7513 7514 if (!broken_loop) 7515 { 7516 struct loop *loop = alloc_loop (); 7517 loop->header = l1_bb; 7518 loop->latch = cont_bb; 7519 add_loop (loop, l1_bb->loop_father); 7520 loop->safelen = INT_MAX; 7521 } 7522 7523 /* Pick the correct library function based on the precision of the 7524 induction variable type. */ 7525 tree lib_fun = NULL_TREE; 7526 if (TYPE_PRECISION (type) == 32) 7527 lib_fun = cilk_for_32_fndecl; 7528 else if (TYPE_PRECISION (type) == 64) 7529 lib_fun = cilk_for_64_fndecl; 7530 else 7531 gcc_unreachable (); 7532 7533 gcc_assert (fd->sched_kind == OMP_CLAUSE_SCHEDULE_CILKFOR); 7534 7535 /* WS_ARGS contains the library function flavor to call: 7536 __libcilkrts_cilk_for_64 or __libcilkrts_cilk_for_32), and the 7537 user-defined grain value. If the user does not define one, then zero 7538 is passed in by the parser. */ 7539 vec_alloc (region->ws_args, 2); 7540 region->ws_args->quick_push (lib_fun); 7541 region->ws_args->quick_push (fd->chunk_size); 7542} 7543 7544/* A subroutine of expand_omp_for. Generate code for a simd non-worksharing 7545 loop. Given parameters: 7546 7547 for (V = N1; V cond N2; V += STEP) BODY; 7548 7549 where COND is "<" or ">", we generate pseudocode 7550 7551 V = N1; 7552 goto L1; 7553 L0: 7554 BODY; 7555 V += STEP; 7556 L1: 7557 if (V cond N2) goto L0; else goto L2; 7558 L2: 7559 7560 For collapsed loops, given parameters: 7561 collapse(3) 7562 for (V1 = N11; V1 cond1 N12; V1 += STEP1) 7563 for (V2 = N21; V2 cond2 N22; V2 += STEP2) 7564 for (V3 = N31; V3 cond3 N32; V3 += STEP3) 7565 BODY; 7566 7567 we generate pseudocode 7568 7569 if (cond3 is <) 7570 adj = STEP3 - 1; 7571 else 7572 adj = STEP3 + 1; 7573 count3 = (adj + N32 - N31) / STEP3; 7574 if (cond2 is <) 7575 adj = STEP2 - 1; 7576 else 7577 adj = STEP2 + 1; 7578 count2 = (adj + N22 - N21) / STEP2; 7579 if (cond1 is <) 7580 adj = STEP1 - 1; 7581 else 7582 adj = STEP1 + 1; 7583 count1 = (adj + N12 - N11) / STEP1; 7584 count = count1 * count2 * count3; 7585 V = 0; 7586 V1 = N11; 7587 V2 = N21; 7588 V3 = N31; 7589 goto L1; 7590 L0: 7591 BODY; 7592 V += 1; 7593 V3 += STEP3; 7594 V2 += (V3 cond3 N32) ? 0 : STEP2; 7595 V3 = (V3 cond3 N32) ? V3 : N31; 7596 V1 += (V2 cond2 N22) ? 0 : STEP1; 7597 V2 = (V2 cond2 N22) ? V2 : N21; 7598 L1: 7599 if (V < count) goto L0; else goto L2; 7600 L2: 7601 7602 */ 7603 7604static void 7605expand_omp_simd (struct omp_region *region, struct omp_for_data *fd) 7606{ 7607 tree type, t; 7608 basic_block entry_bb, cont_bb, exit_bb, l0_bb, l1_bb, l2_bb, l2_dom_bb; 7609 gimple_stmt_iterator gsi; 7610 gimple stmt; 7611 gcond *cond_stmt; 7612 bool broken_loop = region->cont == NULL; 7613 edge e, ne; 7614 tree *counts = NULL; 7615 int i; 7616 tree safelen = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7617 OMP_CLAUSE_SAFELEN); 7618 tree simduid = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7619 OMP_CLAUSE__SIMDUID_); 7620 tree n1, n2; 7621 7622 type = TREE_TYPE (fd->loop.v); 7623 entry_bb = region->entry; 7624 cont_bb = region->cont; 7625 gcc_assert (EDGE_COUNT (entry_bb->succs) == 2); 7626 gcc_assert (broken_loop 7627 || BRANCH_EDGE (entry_bb)->dest == FALLTHRU_EDGE (cont_bb)->dest); 7628 l0_bb = FALLTHRU_EDGE (entry_bb)->dest; 7629 if (!broken_loop) 7630 { 7631 gcc_assert (BRANCH_EDGE (cont_bb)->dest == l0_bb); 7632 gcc_assert (EDGE_COUNT (cont_bb->succs) == 2); 7633 l1_bb = split_block (cont_bb, last_stmt (cont_bb))->dest; 7634 l2_bb = BRANCH_EDGE (entry_bb)->dest; 7635 } 7636 else 7637 { 7638 BRANCH_EDGE (entry_bb)->flags &= ~EDGE_ABNORMAL; 7639 l1_bb = split_edge (BRANCH_EDGE (entry_bb)); 7640 l2_bb = single_succ (l1_bb); 7641 } 7642 exit_bb = region->exit; 7643 l2_dom_bb = NULL; 7644 7645 gsi = gsi_last_bb (entry_bb); 7646 7647 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_FOR); 7648 /* Not needed in SSA form right now. */ 7649 gcc_assert (!gimple_in_ssa_p (cfun)); 7650 if (fd->collapse > 1) 7651 { 7652 int first_zero_iter = -1; 7653 basic_block zero_iter_bb = l2_bb; 7654 7655 counts = XALLOCAVEC (tree, fd->collapse); 7656 expand_omp_for_init_counts (fd, &gsi, entry_bb, counts, 7657 zero_iter_bb, first_zero_iter, 7658 l2_dom_bb); 7659 } 7660 if (l2_dom_bb == NULL) 7661 l2_dom_bb = l1_bb; 7662 7663 n1 = fd->loop.n1; 7664 n2 = fd->loop.n2; 7665 if (gimple_omp_for_combined_into_p (fd->for_stmt)) 7666 { 7667 tree innerc = find_omp_clause (gimple_omp_for_clauses (fd->for_stmt), 7668 OMP_CLAUSE__LOOPTEMP_); 7669 gcc_assert (innerc); 7670 n1 = OMP_CLAUSE_DECL (innerc); 7671 innerc = find_omp_clause (OMP_CLAUSE_CHAIN (innerc), 7672 OMP_CLAUSE__LOOPTEMP_); 7673 gcc_assert (innerc); 7674 n2 = OMP_CLAUSE_DECL (innerc); 7675 expand_omp_build_assign (&gsi, fd->loop.v, 7676 fold_convert (type, n1)); 7677 if (fd->collapse > 1) 7678 { 7679 gsi_prev (&gsi); 7680 expand_omp_for_init_vars (fd, &gsi, counts, NULL, n1); 7681 gsi_next (&gsi); 7682 } 7683 } 7684 else 7685 { 7686 expand_omp_build_assign (&gsi, fd->loop.v, 7687 fold_convert (type, fd->loop.n1)); 7688 if (fd->collapse > 1) 7689 for (i = 0; i < fd->collapse; i++) 7690 { 7691 tree itype = TREE_TYPE (fd->loops[i].v); 7692 if (POINTER_TYPE_P (itype)) 7693 itype = signed_type_for (itype); 7694 t = fold_convert (TREE_TYPE (fd->loops[i].v), fd->loops[i].n1); 7695 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7696 } 7697 } 7698 7699 /* Remove the GIMPLE_OMP_FOR statement. */ 7700 gsi_remove (&gsi, true); 7701 7702 if (!broken_loop) 7703 { 7704 /* Code to control the increment goes in the CONT_BB. */ 7705 gsi = gsi_last_bb (cont_bb); 7706 stmt = gsi_stmt (gsi); 7707 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_CONTINUE); 7708 7709 if (POINTER_TYPE_P (type)) 7710 t = fold_build_pointer_plus (fd->loop.v, fd->loop.step); 7711 else 7712 t = fold_build2 (PLUS_EXPR, type, fd->loop.v, fd->loop.step); 7713 expand_omp_build_assign (&gsi, fd->loop.v, t); 7714 7715 if (fd->collapse > 1) 7716 { 7717 i = fd->collapse - 1; 7718 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i].v))) 7719 { 7720 t = fold_convert (sizetype, fd->loops[i].step); 7721 t = fold_build_pointer_plus (fd->loops[i].v, t); 7722 } 7723 else 7724 { 7725 t = fold_convert (TREE_TYPE (fd->loops[i].v), 7726 fd->loops[i].step); 7727 t = fold_build2 (PLUS_EXPR, TREE_TYPE (fd->loops[i].v), 7728 fd->loops[i].v, t); 7729 } 7730 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7731 7732 for (i = fd->collapse - 1; i > 0; i--) 7733 { 7734 tree itype = TREE_TYPE (fd->loops[i].v); 7735 tree itype2 = TREE_TYPE (fd->loops[i - 1].v); 7736 if (POINTER_TYPE_P (itype2)) 7737 itype2 = signed_type_for (itype2); 7738 t = build3 (COND_EXPR, itype2, 7739 build2 (fd->loops[i].cond_code, boolean_type_node, 7740 fd->loops[i].v, 7741 fold_convert (itype, fd->loops[i].n2)), 7742 build_int_cst (itype2, 0), 7743 fold_convert (itype2, fd->loops[i - 1].step)); 7744 if (POINTER_TYPE_P (TREE_TYPE (fd->loops[i - 1].v))) 7745 t = fold_build_pointer_plus (fd->loops[i - 1].v, t); 7746 else 7747 t = fold_build2 (PLUS_EXPR, itype2, fd->loops[i - 1].v, t); 7748 expand_omp_build_assign (&gsi, fd->loops[i - 1].v, t); 7749 7750 t = build3 (COND_EXPR, itype, 7751 build2 (fd->loops[i].cond_code, boolean_type_node, 7752 fd->loops[i].v, 7753 fold_convert (itype, fd->loops[i].n2)), 7754 fd->loops[i].v, 7755 fold_convert (itype, fd->loops[i].n1)); 7756 expand_omp_build_assign (&gsi, fd->loops[i].v, t); 7757 } 7758 } 7759 7760 /* Remove GIMPLE_OMP_CONTINUE. */ 7761 gsi_remove (&gsi, true); 7762 } 7763 7764 /* Emit the condition in L1_BB. */ 7765 gsi = gsi_start_bb (l1_bb); 7766 7767 t = fold_convert (type, n2); 7768 t = force_gimple_operand_gsi (&gsi, t, true, NULL_TREE, 7769 false, GSI_CONTINUE_LINKING); 7770 t = build2 (fd->loop.cond_code, boolean_type_node, fd->loop.v, t); 7771 cond_stmt = gimple_build_cond_empty (t); 7772 gsi_insert_after (&gsi, cond_stmt, GSI_CONTINUE_LINKING); 7773 if (walk_tree (gimple_cond_lhs_ptr (cond_stmt), expand_omp_regimplify_p, 7774 NULL, NULL) 7775 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), expand_omp_regimplify_p, 7776 NULL, NULL)) 7777 { 7778 gsi = gsi_for_stmt (cond_stmt); 7779 gimple_regimplify_operands (cond_stmt, &gsi); 7780 } 7781 7782 /* Remove GIMPLE_OMP_RETURN. */ 7783 gsi = gsi_last_bb (exit_bb); 7784 gsi_remove (&gsi, true); 7785 7786 /* Connect the new blocks. */ 7787 remove_edge (FALLTHRU_EDGE (entry_bb)); 7788 7789 if (!broken_loop) 7790 { 7791 remove_edge (BRANCH_EDGE (entry_bb)); 7792 make_edge (entry_bb, l1_bb, EDGE_FALLTHRU); 7793 7794 e = BRANCH_EDGE (l1_bb); 7795 ne = FALLTHRU_EDGE (l1_bb); 7796 e->flags = EDGE_TRUE_VALUE; 7797 } 7798 else 7799 { 7800 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 7801 7802 ne = single_succ_edge (l1_bb); 7803 e = make_edge (l1_bb, l0_bb, EDGE_TRUE_VALUE); 7804 7805 } 7806 ne->flags = EDGE_FALSE_VALUE; 7807 e->probability = REG_BR_PROB_BASE * 7 / 8; 7808 ne->probability = REG_BR_PROB_BASE / 8; 7809 7810 set_immediate_dominator (CDI_DOMINATORS, l1_bb, entry_bb); 7811 set_immediate_dominator (CDI_DOMINATORS, l2_bb, l2_dom_bb); 7812 set_immediate_dominator (CDI_DOMINATORS, l0_bb, l1_bb); 7813 7814 if (!broken_loop) 7815 { 7816 struct loop *loop = alloc_loop (); 7817 loop->header = l1_bb; 7818 loop->latch = cont_bb; 7819 add_loop (loop, l1_bb->loop_father); 7820 if (safelen == NULL_TREE) 7821 loop->safelen = INT_MAX; 7822 else 7823 { 7824 safelen = OMP_CLAUSE_SAFELEN_EXPR (safelen); 7825 if (TREE_CODE (safelen) != INTEGER_CST) 7826 loop->safelen = 0; 7827 else if (!tree_fits_uhwi_p (safelen) 7828 || tree_to_uhwi (safelen) > INT_MAX) 7829 loop->safelen = INT_MAX; 7830 else 7831 loop->safelen = tree_to_uhwi (safelen); 7832 if (loop->safelen == 1) 7833 loop->safelen = 0; 7834 } 7835 if (simduid) 7836 { 7837 loop->simduid = OMP_CLAUSE__SIMDUID__DECL (simduid); 7838 cfun->has_simduid_loops = true; 7839 } 7840 /* If not -fno-tree-loop-vectorize, hint that we want to vectorize 7841 the loop. */ 7842 if ((flag_tree_loop_vectorize 7843 || (!global_options_set.x_flag_tree_loop_vectorize 7844 && !global_options_set.x_flag_tree_vectorize)) 7845 && flag_tree_loop_optimize 7846 && loop->safelen > 1) 7847 { 7848 loop->force_vectorize = true; 7849 cfun->has_force_vectorize_loops = true; 7850 } 7851 } 7852 else if (simduid) 7853 cfun->has_simduid_loops = true; 7854} 7855 7856 7857/* Expand the OMP loop defined by REGION. */ 7858 7859static void 7860expand_omp_for (struct omp_region *region, gimple inner_stmt) 7861{ 7862 struct omp_for_data fd; 7863 struct omp_for_data_loop *loops; 7864 7865 loops 7866 = (struct omp_for_data_loop *) 7867 alloca (gimple_omp_for_collapse (last_stmt (region->entry)) 7868 * sizeof (struct omp_for_data_loop)); 7869 extract_omp_for_data (as_a <gomp_for *> (last_stmt (region->entry)), 7870 &fd, loops); 7871 region->sched_kind = fd.sched_kind; 7872 7873 gcc_assert (EDGE_COUNT (region->entry->succs) == 2); 7874 BRANCH_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; 7875 FALLTHRU_EDGE (region->entry)->flags &= ~EDGE_ABNORMAL; 7876 if (region->cont) 7877 { 7878 gcc_assert (EDGE_COUNT (region->cont->succs) == 2); 7879 BRANCH_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; 7880 FALLTHRU_EDGE (region->cont)->flags &= ~EDGE_ABNORMAL; 7881 } 7882 else 7883 /* If there isn't a continue then this is a degerate case where 7884 the introduction of abnormal edges during lowering will prevent 7885 original loops from being detected. Fix that up. */ 7886 loops_state_set (LOOPS_NEED_FIXUP); 7887 7888 if (gimple_omp_for_kind (fd.for_stmt) & GF_OMP_FOR_SIMD) 7889 expand_omp_simd (region, &fd); 7890 else if (gimple_omp_for_kind (fd.for_stmt) == GF_OMP_FOR_KIND_CILKFOR) 7891 expand_cilk_for (region, &fd); 7892 else if (fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC 7893 && !fd.have_ordered) 7894 { 7895 if (fd.chunk_size == NULL) 7896 expand_omp_for_static_nochunk (region, &fd, inner_stmt); 7897 else 7898 expand_omp_for_static_chunk (region, &fd, inner_stmt); 7899 } 7900 else 7901 { 7902 int fn_index, start_ix, next_ix; 7903 7904 gcc_assert (gimple_omp_for_kind (fd.for_stmt) 7905 == GF_OMP_FOR_KIND_FOR); 7906 if (fd.chunk_size == NULL 7907 && fd.sched_kind == OMP_CLAUSE_SCHEDULE_STATIC) 7908 fd.chunk_size = integer_zero_node; 7909 gcc_assert (fd.sched_kind != OMP_CLAUSE_SCHEDULE_AUTO); 7910 fn_index = (fd.sched_kind == OMP_CLAUSE_SCHEDULE_RUNTIME) 7911 ? 3 : fd.sched_kind; 7912 fn_index += fd.have_ordered * 4; 7913 start_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_START) + fn_index; 7914 next_ix = ((int)BUILT_IN_GOMP_LOOP_STATIC_NEXT) + fn_index; 7915 if (fd.iter_type == long_long_unsigned_type_node) 7916 { 7917 start_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_START 7918 - (int)BUILT_IN_GOMP_LOOP_STATIC_START); 7919 next_ix += ((int)BUILT_IN_GOMP_LOOP_ULL_STATIC_NEXT 7920 - (int)BUILT_IN_GOMP_LOOP_STATIC_NEXT); 7921 } 7922 expand_omp_for_generic (region, &fd, (enum built_in_function) start_ix, 7923 (enum built_in_function) next_ix, inner_stmt); 7924 } 7925 7926 if (gimple_in_ssa_p (cfun)) 7927 update_ssa (TODO_update_ssa_only_virtuals); 7928} 7929 7930 7931/* Expand code for an OpenMP sections directive. In pseudo code, we generate 7932 7933 v = GOMP_sections_start (n); 7934 L0: 7935 switch (v) 7936 { 7937 case 0: 7938 goto L2; 7939 case 1: 7940 section 1; 7941 goto L1; 7942 case 2: 7943 ... 7944 case n: 7945 ... 7946 default: 7947 abort (); 7948 } 7949 L1: 7950 v = GOMP_sections_next (); 7951 goto L0; 7952 L2: 7953 reduction; 7954 7955 If this is a combined parallel sections, replace the call to 7956 GOMP_sections_start with call to GOMP_sections_next. */ 7957 7958static void 7959expand_omp_sections (struct omp_region *region) 7960{ 7961 tree t, u, vin = NULL, vmain, vnext, l2; 7962 unsigned len; 7963 basic_block entry_bb, l0_bb, l1_bb, l2_bb, default_bb; 7964 gimple_stmt_iterator si, switch_si; 7965 gomp_sections *sections_stmt; 7966 gimple stmt; 7967 gomp_continue *cont; 7968 edge_iterator ei; 7969 edge e; 7970 struct omp_region *inner; 7971 unsigned i, casei; 7972 bool exit_reachable = region->cont != NULL; 7973 7974 gcc_assert (region->exit != NULL); 7975 entry_bb = region->entry; 7976 l0_bb = single_succ (entry_bb); 7977 l1_bb = region->cont; 7978 l2_bb = region->exit; 7979 if (single_pred_p (l2_bb) && single_pred (l2_bb) == l0_bb) 7980 l2 = gimple_block_label (l2_bb); 7981 else 7982 { 7983 /* This can happen if there are reductions. */ 7984 len = EDGE_COUNT (l0_bb->succs); 7985 gcc_assert (len > 0); 7986 e = EDGE_SUCC (l0_bb, len - 1); 7987 si = gsi_last_bb (e->dest); 7988 l2 = NULL_TREE; 7989 if (gsi_end_p (si) 7990 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) 7991 l2 = gimple_block_label (e->dest); 7992 else 7993 FOR_EACH_EDGE (e, ei, l0_bb->succs) 7994 { 7995 si = gsi_last_bb (e->dest); 7996 if (gsi_end_p (si) 7997 || gimple_code (gsi_stmt (si)) != GIMPLE_OMP_SECTION) 7998 { 7999 l2 = gimple_block_label (e->dest); 8000 break; 8001 } 8002 } 8003 } 8004 if (exit_reachable) 8005 default_bb = create_empty_bb (l1_bb->prev_bb); 8006 else 8007 default_bb = create_empty_bb (l0_bb); 8008 8009 /* We will build a switch() with enough cases for all the 8010 GIMPLE_OMP_SECTION regions, a '0' case to handle the end of more work 8011 and a default case to abort if something goes wrong. */ 8012 len = EDGE_COUNT (l0_bb->succs); 8013 8014 /* Use vec::quick_push on label_vec throughout, since we know the size 8015 in advance. */ 8016 auto_vec<tree> label_vec (len); 8017 8018 /* The call to GOMP_sections_start goes in ENTRY_BB, replacing the 8019 GIMPLE_OMP_SECTIONS statement. */ 8020 si = gsi_last_bb (entry_bb); 8021 sections_stmt = as_a <gomp_sections *> (gsi_stmt (si)); 8022 gcc_assert (gimple_code (sections_stmt) == GIMPLE_OMP_SECTIONS); 8023 vin = gimple_omp_sections_control (sections_stmt); 8024 if (!is_combined_parallel (region)) 8025 { 8026 /* If we are not inside a combined parallel+sections region, 8027 call GOMP_sections_start. */ 8028 t = build_int_cst (unsigned_type_node, len - 1); 8029 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_START); 8030 stmt = gimple_build_call (u, 1, t); 8031 } 8032 else 8033 { 8034 /* Otherwise, call GOMP_sections_next. */ 8035 u = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); 8036 stmt = gimple_build_call (u, 0); 8037 } 8038 gimple_call_set_lhs (stmt, vin); 8039 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8040 gsi_remove (&si, true); 8041 8042 /* The switch() statement replacing GIMPLE_OMP_SECTIONS_SWITCH goes in 8043 L0_BB. */ 8044 switch_si = gsi_last_bb (l0_bb); 8045 gcc_assert (gimple_code (gsi_stmt (switch_si)) == GIMPLE_OMP_SECTIONS_SWITCH); 8046 if (exit_reachable) 8047 { 8048 cont = as_a <gomp_continue *> (last_stmt (l1_bb)); 8049 gcc_assert (gimple_code (cont) == GIMPLE_OMP_CONTINUE); 8050 vmain = gimple_omp_continue_control_use (cont); 8051 vnext = gimple_omp_continue_control_def (cont); 8052 } 8053 else 8054 { 8055 vmain = vin; 8056 vnext = NULL_TREE; 8057 } 8058 8059 t = build_case_label (build_int_cst (unsigned_type_node, 0), NULL, l2); 8060 label_vec.quick_push (t); 8061 i = 1; 8062 8063 /* Convert each GIMPLE_OMP_SECTION into a CASE_LABEL_EXPR. */ 8064 for (inner = region->inner, casei = 1; 8065 inner; 8066 inner = inner->next, i++, casei++) 8067 { 8068 basic_block s_entry_bb, s_exit_bb; 8069 8070 /* Skip optional reduction region. */ 8071 if (inner->type == GIMPLE_OMP_ATOMIC_LOAD) 8072 { 8073 --i; 8074 --casei; 8075 continue; 8076 } 8077 8078 s_entry_bb = inner->entry; 8079 s_exit_bb = inner->exit; 8080 8081 t = gimple_block_label (s_entry_bb); 8082 u = build_int_cst (unsigned_type_node, casei); 8083 u = build_case_label (u, NULL, t); 8084 label_vec.quick_push (u); 8085 8086 si = gsi_last_bb (s_entry_bb); 8087 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SECTION); 8088 gcc_assert (i < len || gimple_omp_section_last_p (gsi_stmt (si))); 8089 gsi_remove (&si, true); 8090 single_succ_edge (s_entry_bb)->flags = EDGE_FALLTHRU; 8091 8092 if (s_exit_bb == NULL) 8093 continue; 8094 8095 si = gsi_last_bb (s_exit_bb); 8096 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); 8097 gsi_remove (&si, true); 8098 8099 single_succ_edge (s_exit_bb)->flags = EDGE_FALLTHRU; 8100 } 8101 8102 /* Error handling code goes in DEFAULT_BB. */ 8103 t = gimple_block_label (default_bb); 8104 u = build_case_label (NULL, NULL, t); 8105 make_edge (l0_bb, default_bb, 0); 8106 add_bb_to_loop (default_bb, current_loops->tree_root); 8107 8108 stmt = gimple_build_switch (vmain, u, label_vec); 8109 gsi_insert_after (&switch_si, stmt, GSI_SAME_STMT); 8110 gsi_remove (&switch_si, true); 8111 8112 si = gsi_start_bb (default_bb); 8113 stmt = gimple_build_call (builtin_decl_explicit (BUILT_IN_TRAP), 0); 8114 gsi_insert_after (&si, stmt, GSI_CONTINUE_LINKING); 8115 8116 if (exit_reachable) 8117 { 8118 tree bfn_decl; 8119 8120 /* Code to get the next section goes in L1_BB. */ 8121 si = gsi_last_bb (l1_bb); 8122 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CONTINUE); 8123 8124 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_NEXT); 8125 stmt = gimple_build_call (bfn_decl, 0); 8126 gimple_call_set_lhs (stmt, vnext); 8127 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8128 gsi_remove (&si, true); 8129 8130 single_succ_edge (l1_bb)->flags = EDGE_FALLTHRU; 8131 } 8132 8133 /* Cleanup function replaces GIMPLE_OMP_RETURN in EXIT_BB. */ 8134 si = gsi_last_bb (l2_bb); 8135 if (gimple_omp_return_nowait_p (gsi_stmt (si))) 8136 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_NOWAIT); 8137 else if (gimple_omp_return_lhs (gsi_stmt (si))) 8138 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END_CANCEL); 8139 else 8140 t = builtin_decl_explicit (BUILT_IN_GOMP_SECTIONS_END); 8141 stmt = gimple_build_call (t, 0); 8142 if (gimple_omp_return_lhs (gsi_stmt (si))) 8143 gimple_call_set_lhs (stmt, gimple_omp_return_lhs (gsi_stmt (si))); 8144 gsi_insert_after (&si, stmt, GSI_SAME_STMT); 8145 gsi_remove (&si, true); 8146 8147 set_immediate_dominator (CDI_DOMINATORS, default_bb, l0_bb); 8148} 8149 8150 8151/* Expand code for an OpenMP single directive. We've already expanded 8152 much of the code, here we simply place the GOMP_barrier call. */ 8153 8154static void 8155expand_omp_single (struct omp_region *region) 8156{ 8157 basic_block entry_bb, exit_bb; 8158 gimple_stmt_iterator si; 8159 8160 entry_bb = region->entry; 8161 exit_bb = region->exit; 8162 8163 si = gsi_last_bb (entry_bb); 8164 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE); 8165 gsi_remove (&si, true); 8166 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8167 8168 si = gsi_last_bb (exit_bb); 8169 if (!gimple_omp_return_nowait_p (gsi_stmt (si))) 8170 { 8171 tree t = gimple_omp_return_lhs (gsi_stmt (si)); 8172 gsi_insert_after (&si, build_omp_barrier (t), GSI_SAME_STMT); 8173 } 8174 gsi_remove (&si, true); 8175 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; 8176} 8177 8178 8179/* Generic expansion for OpenMP synchronization directives: master, 8180 ordered and critical. All we need to do here is remove the entry 8181 and exit markers for REGION. */ 8182 8183static void 8184expand_omp_synch (struct omp_region *region) 8185{ 8186 basic_block entry_bb, exit_bb; 8187 gimple_stmt_iterator si; 8188 8189 entry_bb = region->entry; 8190 exit_bb = region->exit; 8191 8192 si = gsi_last_bb (entry_bb); 8193 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_SINGLE 8194 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_MASTER 8195 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TASKGROUP 8196 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ORDERED 8197 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_CRITICAL 8198 || gimple_code (gsi_stmt (si)) == GIMPLE_OMP_TEAMS); 8199 gsi_remove (&si, true); 8200 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8201 8202 if (exit_bb) 8203 { 8204 si = gsi_last_bb (exit_bb); 8205 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_RETURN); 8206 gsi_remove (&si, true); 8207 single_succ_edge (exit_bb)->flags = EDGE_FALLTHRU; 8208 } 8209} 8210 8211/* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8212 operation as a normal volatile load. */ 8213 8214static bool 8215expand_omp_atomic_load (basic_block load_bb, tree addr, 8216 tree loaded_val, int index) 8217{ 8218 enum built_in_function tmpbase; 8219 gimple_stmt_iterator gsi; 8220 basic_block store_bb; 8221 location_t loc; 8222 gimple stmt; 8223 tree decl, call, type, itype; 8224 8225 gsi = gsi_last_bb (load_bb); 8226 stmt = gsi_stmt (gsi); 8227 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); 8228 loc = gimple_location (stmt); 8229 8230 /* ??? If the target does not implement atomic_load_optab[mode], and mode 8231 is smaller than word size, then expand_atomic_load assumes that the load 8232 is atomic. We could avoid the builtin entirely in this case. */ 8233 8234 tmpbase = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); 8235 decl = builtin_decl_explicit (tmpbase); 8236 if (decl == NULL_TREE) 8237 return false; 8238 8239 type = TREE_TYPE (loaded_val); 8240 itype = TREE_TYPE (TREE_TYPE (decl)); 8241 8242 call = build_call_expr_loc (loc, decl, 2, addr, 8243 build_int_cst (NULL, 8244 gimple_omp_atomic_seq_cst_p (stmt) 8245 ? MEMMODEL_SEQ_CST 8246 : MEMMODEL_RELAXED)); 8247 if (!useless_type_conversion_p (type, itype)) 8248 call = fold_build1_loc (loc, VIEW_CONVERT_EXPR, type, call); 8249 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); 8250 8251 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8252 gsi_remove (&gsi, true); 8253 8254 store_bb = single_succ (load_bb); 8255 gsi = gsi_last_bb (store_bb); 8256 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); 8257 gsi_remove (&gsi, true); 8258 8259 if (gimple_in_ssa_p (cfun)) 8260 update_ssa (TODO_update_ssa_no_phi); 8261 8262 return true; 8263} 8264 8265/* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8266 operation as a normal volatile store. */ 8267 8268static bool 8269expand_omp_atomic_store (basic_block load_bb, tree addr, 8270 tree loaded_val, tree stored_val, int index) 8271{ 8272 enum built_in_function tmpbase; 8273 gimple_stmt_iterator gsi; 8274 basic_block store_bb = single_succ (load_bb); 8275 location_t loc; 8276 gimple stmt; 8277 tree decl, call, type, itype; 8278 machine_mode imode; 8279 bool exchange; 8280 8281 gsi = gsi_last_bb (load_bb); 8282 stmt = gsi_stmt (gsi); 8283 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_LOAD); 8284 8285 /* If the load value is needed, then this isn't a store but an exchange. */ 8286 exchange = gimple_omp_atomic_need_value_p (stmt); 8287 8288 gsi = gsi_last_bb (store_bb); 8289 stmt = gsi_stmt (gsi); 8290 gcc_assert (gimple_code (stmt) == GIMPLE_OMP_ATOMIC_STORE); 8291 loc = gimple_location (stmt); 8292 8293 /* ??? If the target does not implement atomic_store_optab[mode], and mode 8294 is smaller than word size, then expand_atomic_store assumes that the store 8295 is atomic. We could avoid the builtin entirely in this case. */ 8296 8297 tmpbase = (exchange ? BUILT_IN_ATOMIC_EXCHANGE_N : BUILT_IN_ATOMIC_STORE_N); 8298 tmpbase = (enum built_in_function) ((int) tmpbase + index + 1); 8299 decl = builtin_decl_explicit (tmpbase); 8300 if (decl == NULL_TREE) 8301 return false; 8302 8303 type = TREE_TYPE (stored_val); 8304 8305 /* Dig out the type of the function's second argument. */ 8306 itype = TREE_TYPE (decl); 8307 itype = TYPE_ARG_TYPES (itype); 8308 itype = TREE_CHAIN (itype); 8309 itype = TREE_VALUE (itype); 8310 imode = TYPE_MODE (itype); 8311 8312 if (exchange && !can_atomic_exchange_p (imode, true)) 8313 return false; 8314 8315 if (!useless_type_conversion_p (itype, type)) 8316 stored_val = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, stored_val); 8317 call = build_call_expr_loc (loc, decl, 3, addr, stored_val, 8318 build_int_cst (NULL, 8319 gimple_omp_atomic_seq_cst_p (stmt) 8320 ? MEMMODEL_SEQ_CST 8321 : MEMMODEL_RELAXED)); 8322 if (exchange) 8323 { 8324 if (!useless_type_conversion_p (type, itype)) 8325 call = build1_loc (loc, VIEW_CONVERT_EXPR, type, call); 8326 call = build2_loc (loc, MODIFY_EXPR, void_type_node, loaded_val, call); 8327 } 8328 8329 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8330 gsi_remove (&gsi, true); 8331 8332 /* Remove the GIMPLE_OMP_ATOMIC_LOAD that we verified above. */ 8333 gsi = gsi_last_bb (load_bb); 8334 gsi_remove (&gsi, true); 8335 8336 if (gimple_in_ssa_p (cfun)) 8337 update_ssa (TODO_update_ssa_no_phi); 8338 8339 return true; 8340} 8341 8342/* A subroutine of expand_omp_atomic. Attempt to implement the atomic 8343 operation as a __atomic_fetch_op builtin. INDEX is log2 of the 8344 size of the data type, and thus usable to find the index of the builtin 8345 decl. Returns false if the expression is not of the proper form. */ 8346 8347static bool 8348expand_omp_atomic_fetch_op (basic_block load_bb, 8349 tree addr, tree loaded_val, 8350 tree stored_val, int index) 8351{ 8352 enum built_in_function oldbase, newbase, tmpbase; 8353 tree decl, itype, call; 8354 tree lhs, rhs; 8355 basic_block store_bb = single_succ (load_bb); 8356 gimple_stmt_iterator gsi; 8357 gimple stmt; 8358 location_t loc; 8359 enum tree_code code; 8360 bool need_old, need_new; 8361 machine_mode imode; 8362 bool seq_cst; 8363 8364 /* We expect to find the following sequences: 8365 8366 load_bb: 8367 GIMPLE_OMP_ATOMIC_LOAD (tmp, mem) 8368 8369 store_bb: 8370 val = tmp OP something; (or: something OP tmp) 8371 GIMPLE_OMP_STORE (val) 8372 8373 ???FIXME: Allow a more flexible sequence. 8374 Perhaps use data flow to pick the statements. 8375 8376 */ 8377 8378 gsi = gsi_after_labels (store_bb); 8379 stmt = gsi_stmt (gsi); 8380 loc = gimple_location (stmt); 8381 if (!is_gimple_assign (stmt)) 8382 return false; 8383 gsi_next (&gsi); 8384 if (gimple_code (gsi_stmt (gsi)) != GIMPLE_OMP_ATOMIC_STORE) 8385 return false; 8386 need_new = gimple_omp_atomic_need_value_p (gsi_stmt (gsi)); 8387 need_old = gimple_omp_atomic_need_value_p (last_stmt (load_bb)); 8388 seq_cst = gimple_omp_atomic_seq_cst_p (last_stmt (load_bb)); 8389 gcc_checking_assert (!need_old || !need_new); 8390 8391 if (!operand_equal_p (gimple_assign_lhs (stmt), stored_val, 0)) 8392 return false; 8393 8394 /* Check for one of the supported fetch-op operations. */ 8395 code = gimple_assign_rhs_code (stmt); 8396 switch (code) 8397 { 8398 case PLUS_EXPR: 8399 case POINTER_PLUS_EXPR: 8400 oldbase = BUILT_IN_ATOMIC_FETCH_ADD_N; 8401 newbase = BUILT_IN_ATOMIC_ADD_FETCH_N; 8402 break; 8403 case MINUS_EXPR: 8404 oldbase = BUILT_IN_ATOMIC_FETCH_SUB_N; 8405 newbase = BUILT_IN_ATOMIC_SUB_FETCH_N; 8406 break; 8407 case BIT_AND_EXPR: 8408 oldbase = BUILT_IN_ATOMIC_FETCH_AND_N; 8409 newbase = BUILT_IN_ATOMIC_AND_FETCH_N; 8410 break; 8411 case BIT_IOR_EXPR: 8412 oldbase = BUILT_IN_ATOMIC_FETCH_OR_N; 8413 newbase = BUILT_IN_ATOMIC_OR_FETCH_N; 8414 break; 8415 case BIT_XOR_EXPR: 8416 oldbase = BUILT_IN_ATOMIC_FETCH_XOR_N; 8417 newbase = BUILT_IN_ATOMIC_XOR_FETCH_N; 8418 break; 8419 default: 8420 return false; 8421 } 8422 8423 /* Make sure the expression is of the proper form. */ 8424 if (operand_equal_p (gimple_assign_rhs1 (stmt), loaded_val, 0)) 8425 rhs = gimple_assign_rhs2 (stmt); 8426 else if (commutative_tree_code (gimple_assign_rhs_code (stmt)) 8427 && operand_equal_p (gimple_assign_rhs2 (stmt), loaded_val, 0)) 8428 rhs = gimple_assign_rhs1 (stmt); 8429 else 8430 return false; 8431 8432 tmpbase = ((enum built_in_function) 8433 ((need_new ? newbase : oldbase) + index + 1)); 8434 decl = builtin_decl_explicit (tmpbase); 8435 if (decl == NULL_TREE) 8436 return false; 8437 itype = TREE_TYPE (TREE_TYPE (decl)); 8438 imode = TYPE_MODE (itype); 8439 8440 /* We could test all of the various optabs involved, but the fact of the 8441 matter is that (with the exception of i486 vs i586 and xadd) all targets 8442 that support any atomic operaton optab also implements compare-and-swap. 8443 Let optabs.c take care of expanding any compare-and-swap loop. */ 8444 if (!can_compare_and_swap_p (imode, true)) 8445 return false; 8446 8447 gsi = gsi_last_bb (load_bb); 8448 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_LOAD); 8449 8450 /* OpenMP does not imply any barrier-like semantics on its atomic ops. 8451 It only requires that the operation happen atomically. Thus we can 8452 use the RELAXED memory model. */ 8453 call = build_call_expr_loc (loc, decl, 3, addr, 8454 fold_convert_loc (loc, itype, rhs), 8455 build_int_cst (NULL, 8456 seq_cst ? MEMMODEL_SEQ_CST 8457 : MEMMODEL_RELAXED)); 8458 8459 if (need_old || need_new) 8460 { 8461 lhs = need_old ? loaded_val : stored_val; 8462 call = fold_convert_loc (loc, TREE_TYPE (lhs), call); 8463 call = build2_loc (loc, MODIFY_EXPR, void_type_node, lhs, call); 8464 } 8465 else 8466 call = fold_convert_loc (loc, void_type_node, call); 8467 force_gimple_operand_gsi (&gsi, call, true, NULL_TREE, true, GSI_SAME_STMT); 8468 gsi_remove (&gsi, true); 8469 8470 gsi = gsi_last_bb (store_bb); 8471 gcc_assert (gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_ATOMIC_STORE); 8472 gsi_remove (&gsi, true); 8473 gsi = gsi_last_bb (store_bb); 8474 gsi_remove (&gsi, true); 8475 8476 if (gimple_in_ssa_p (cfun)) 8477 update_ssa (TODO_update_ssa_no_phi); 8478 8479 return true; 8480} 8481 8482/* A subroutine of expand_omp_atomic. Implement the atomic operation as: 8483 8484 oldval = *addr; 8485 repeat: 8486 newval = rhs; // with oldval replacing *addr in rhs 8487 oldval = __sync_val_compare_and_swap (addr, oldval, newval); 8488 if (oldval != newval) 8489 goto repeat; 8490 8491 INDEX is log2 of the size of the data type, and thus usable to find the 8492 index of the builtin decl. */ 8493 8494static bool 8495expand_omp_atomic_pipeline (basic_block load_bb, basic_block store_bb, 8496 tree addr, tree loaded_val, tree stored_val, 8497 int index) 8498{ 8499 tree loadedi, storedi, initial, new_storedi, old_vali; 8500 tree type, itype, cmpxchg, iaddr; 8501 gimple_stmt_iterator si; 8502 basic_block loop_header = single_succ (load_bb); 8503 gimple phi, stmt; 8504 edge e; 8505 enum built_in_function fncode; 8506 8507 /* ??? We need a non-pointer interface to __atomic_compare_exchange in 8508 order to use the RELAXED memory model effectively. */ 8509 fncode = (enum built_in_function)((int)BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_N 8510 + index + 1); 8511 cmpxchg = builtin_decl_explicit (fncode); 8512 if (cmpxchg == NULL_TREE) 8513 return false; 8514 type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); 8515 itype = TREE_TYPE (TREE_TYPE (cmpxchg)); 8516 8517 if (!can_compare_and_swap_p (TYPE_MODE (itype), true)) 8518 return false; 8519 8520 /* Load the initial value, replacing the GIMPLE_OMP_ATOMIC_LOAD. */ 8521 si = gsi_last_bb (load_bb); 8522 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); 8523 8524 /* For floating-point values, we'll need to view-convert them to integers 8525 so that we can perform the atomic compare and swap. Simplify the 8526 following code by always setting up the "i"ntegral variables. */ 8527 if (!INTEGRAL_TYPE_P (type) && !POINTER_TYPE_P (type)) 8528 { 8529 tree iaddr_val; 8530 8531 iaddr = create_tmp_reg (build_pointer_type_for_mode (itype, ptr_mode, 8532 true)); 8533 iaddr_val 8534 = force_gimple_operand_gsi (&si, 8535 fold_convert (TREE_TYPE (iaddr), addr), 8536 false, NULL_TREE, true, GSI_SAME_STMT); 8537 stmt = gimple_build_assign (iaddr, iaddr_val); 8538 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8539 loadedi = create_tmp_var (itype); 8540 if (gimple_in_ssa_p (cfun)) 8541 loadedi = make_ssa_name (loadedi); 8542 } 8543 else 8544 { 8545 iaddr = addr; 8546 loadedi = loaded_val; 8547 } 8548 8549 fncode = (enum built_in_function) (BUILT_IN_ATOMIC_LOAD_N + index + 1); 8550 tree loaddecl = builtin_decl_explicit (fncode); 8551 if (loaddecl) 8552 initial 8553 = fold_convert (TREE_TYPE (TREE_TYPE (iaddr)), 8554 build_call_expr (loaddecl, 2, iaddr, 8555 build_int_cst (NULL_TREE, 8556 MEMMODEL_RELAXED))); 8557 else 8558 initial = build2 (MEM_REF, TREE_TYPE (TREE_TYPE (iaddr)), iaddr, 8559 build_int_cst (TREE_TYPE (iaddr), 0)); 8560 8561 initial 8562 = force_gimple_operand_gsi (&si, initial, true, NULL_TREE, true, 8563 GSI_SAME_STMT); 8564 8565 /* Move the value to the LOADEDI temporary. */ 8566 if (gimple_in_ssa_p (cfun)) 8567 { 8568 gcc_assert (gimple_seq_empty_p (phi_nodes (loop_header))); 8569 phi = create_phi_node (loadedi, loop_header); 8570 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, single_succ_edge (load_bb)), 8571 initial); 8572 } 8573 else 8574 gsi_insert_before (&si, 8575 gimple_build_assign (loadedi, initial), 8576 GSI_SAME_STMT); 8577 if (loadedi != loaded_val) 8578 { 8579 gimple_stmt_iterator gsi2; 8580 tree x; 8581 8582 x = build1 (VIEW_CONVERT_EXPR, type, loadedi); 8583 gsi2 = gsi_start_bb (loop_header); 8584 if (gimple_in_ssa_p (cfun)) 8585 { 8586 gassign *stmt; 8587 x = force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, 8588 true, GSI_SAME_STMT); 8589 stmt = gimple_build_assign (loaded_val, x); 8590 gsi_insert_before (&gsi2, stmt, GSI_SAME_STMT); 8591 } 8592 else 8593 { 8594 x = build2 (MODIFY_EXPR, TREE_TYPE (loaded_val), loaded_val, x); 8595 force_gimple_operand_gsi (&gsi2, x, true, NULL_TREE, 8596 true, GSI_SAME_STMT); 8597 } 8598 } 8599 gsi_remove (&si, true); 8600 8601 si = gsi_last_bb (store_bb); 8602 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); 8603 8604 if (iaddr == addr) 8605 storedi = stored_val; 8606 else 8607 storedi = 8608 force_gimple_operand_gsi (&si, 8609 build1 (VIEW_CONVERT_EXPR, itype, 8610 stored_val), true, NULL_TREE, true, 8611 GSI_SAME_STMT); 8612 8613 /* Build the compare&swap statement. */ 8614 new_storedi = build_call_expr (cmpxchg, 3, iaddr, loadedi, storedi); 8615 new_storedi = force_gimple_operand_gsi (&si, 8616 fold_convert (TREE_TYPE (loadedi), 8617 new_storedi), 8618 true, NULL_TREE, 8619 true, GSI_SAME_STMT); 8620 8621 if (gimple_in_ssa_p (cfun)) 8622 old_vali = loadedi; 8623 else 8624 { 8625 old_vali = create_tmp_var (TREE_TYPE (loadedi)); 8626 stmt = gimple_build_assign (old_vali, loadedi); 8627 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8628 8629 stmt = gimple_build_assign (loadedi, new_storedi); 8630 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8631 } 8632 8633 /* Note that we always perform the comparison as an integer, even for 8634 floating point. This allows the atomic operation to properly 8635 succeed even with NaNs and -0.0. */ 8636 stmt = gimple_build_cond_empty 8637 (build2 (NE_EXPR, boolean_type_node, 8638 new_storedi, old_vali)); 8639 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8640 8641 /* Update cfg. */ 8642 e = single_succ_edge (store_bb); 8643 e->flags &= ~EDGE_FALLTHRU; 8644 e->flags |= EDGE_FALSE_VALUE; 8645 8646 e = make_edge (store_bb, loop_header, EDGE_TRUE_VALUE); 8647 8648 /* Copy the new value to loadedi (we already did that before the condition 8649 if we are not in SSA). */ 8650 if (gimple_in_ssa_p (cfun)) 8651 { 8652 phi = gimple_seq_first_stmt (phi_nodes (loop_header)); 8653 SET_USE (PHI_ARG_DEF_PTR_FROM_EDGE (phi, e), new_storedi); 8654 } 8655 8656 /* Remove GIMPLE_OMP_ATOMIC_STORE. */ 8657 gsi_remove (&si, true); 8658 8659 struct loop *loop = alloc_loop (); 8660 loop->header = loop_header; 8661 loop->latch = store_bb; 8662 add_loop (loop, loop_header->loop_father); 8663 8664 if (gimple_in_ssa_p (cfun)) 8665 update_ssa (TODO_update_ssa_no_phi); 8666 8667 return true; 8668} 8669 8670/* A subroutine of expand_omp_atomic. Implement the atomic operation as: 8671 8672 GOMP_atomic_start (); 8673 *addr = rhs; 8674 GOMP_atomic_end (); 8675 8676 The result is not globally atomic, but works so long as all parallel 8677 references are within #pragma omp atomic directives. According to 8678 responses received from omp@openmp.org, appears to be within spec. 8679 Which makes sense, since that's how several other compilers handle 8680 this situation as well. 8681 LOADED_VAL and ADDR are the operands of GIMPLE_OMP_ATOMIC_LOAD we're 8682 expanding. STORED_VAL is the operand of the matching 8683 GIMPLE_OMP_ATOMIC_STORE. 8684 8685 We replace 8686 GIMPLE_OMP_ATOMIC_LOAD (loaded_val, addr) with 8687 loaded_val = *addr; 8688 8689 and replace 8690 GIMPLE_OMP_ATOMIC_STORE (stored_val) with 8691 *addr = stored_val; 8692*/ 8693 8694static bool 8695expand_omp_atomic_mutex (basic_block load_bb, basic_block store_bb, 8696 tree addr, tree loaded_val, tree stored_val) 8697{ 8698 gimple_stmt_iterator si; 8699 gassign *stmt; 8700 tree t; 8701 8702 si = gsi_last_bb (load_bb); 8703 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_LOAD); 8704 8705 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_START); 8706 t = build_call_expr (t, 0); 8707 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); 8708 8709 stmt = gimple_build_assign (loaded_val, build_simple_mem_ref (addr)); 8710 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8711 gsi_remove (&si, true); 8712 8713 si = gsi_last_bb (store_bb); 8714 gcc_assert (gimple_code (gsi_stmt (si)) == GIMPLE_OMP_ATOMIC_STORE); 8715 8716 stmt = gimple_build_assign (build_simple_mem_ref (unshare_expr (addr)), 8717 stored_val); 8718 gsi_insert_before (&si, stmt, GSI_SAME_STMT); 8719 8720 t = builtin_decl_explicit (BUILT_IN_GOMP_ATOMIC_END); 8721 t = build_call_expr (t, 0); 8722 force_gimple_operand_gsi (&si, t, true, NULL_TREE, true, GSI_SAME_STMT); 8723 gsi_remove (&si, true); 8724 8725 if (gimple_in_ssa_p (cfun)) 8726 update_ssa (TODO_update_ssa_no_phi); 8727 return true; 8728} 8729 8730/* Expand an GIMPLE_OMP_ATOMIC statement. We try to expand 8731 using expand_omp_atomic_fetch_op. If it failed, we try to 8732 call expand_omp_atomic_pipeline, and if it fails too, the 8733 ultimate fallback is wrapping the operation in a mutex 8734 (expand_omp_atomic_mutex). REGION is the atomic region built 8735 by build_omp_regions_1(). */ 8736 8737static void 8738expand_omp_atomic (struct omp_region *region) 8739{ 8740 basic_block load_bb = region->entry, store_bb = region->exit; 8741 gomp_atomic_load *load = as_a <gomp_atomic_load *> (last_stmt (load_bb)); 8742 gomp_atomic_store *store = as_a <gomp_atomic_store *> (last_stmt (store_bb)); 8743 tree loaded_val = gimple_omp_atomic_load_lhs (load); 8744 tree addr = gimple_omp_atomic_load_rhs (load); 8745 tree stored_val = gimple_omp_atomic_store_val (store); 8746 tree type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (addr))); 8747 HOST_WIDE_INT index; 8748 8749 /* Make sure the type is one of the supported sizes. */ 8750 index = tree_to_uhwi (TYPE_SIZE_UNIT (type)); 8751 index = exact_log2 (index); 8752 if (index >= 0 && index <= 4) 8753 { 8754 unsigned int align = TYPE_ALIGN_UNIT (type); 8755 8756 /* __sync builtins require strict data alignment. */ 8757 if (exact_log2 (align) >= index) 8758 { 8759 /* Atomic load. */ 8760 if (loaded_val == stored_val 8761 && (GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT 8762 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) 8763 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD 8764 && expand_omp_atomic_load (load_bb, addr, loaded_val, index)) 8765 return; 8766 8767 /* Atomic store. */ 8768 if ((GET_MODE_CLASS (TYPE_MODE (type)) == MODE_INT 8769 || GET_MODE_CLASS (TYPE_MODE (type)) == MODE_FLOAT) 8770 && GET_MODE_BITSIZE (TYPE_MODE (type)) <= BITS_PER_WORD 8771 && store_bb == single_succ (load_bb) 8772 && first_stmt (store_bb) == store 8773 && expand_omp_atomic_store (load_bb, addr, loaded_val, 8774 stored_val, index)) 8775 return; 8776 8777 /* When possible, use specialized atomic update functions. */ 8778 if ((INTEGRAL_TYPE_P (type) || POINTER_TYPE_P (type)) 8779 && store_bb == single_succ (load_bb) 8780 && expand_omp_atomic_fetch_op (load_bb, addr, 8781 loaded_val, stored_val, index)) 8782 return; 8783 8784 /* If we don't have specialized __sync builtins, try and implement 8785 as a compare and swap loop. */ 8786 if (expand_omp_atomic_pipeline (load_bb, store_bb, addr, 8787 loaded_val, stored_val, index)) 8788 return; 8789 } 8790 } 8791 8792 /* The ultimate fallback is wrapping the operation in a mutex. */ 8793 expand_omp_atomic_mutex (load_bb, store_bb, addr, loaded_val, stored_val); 8794} 8795 8796 8797/* Expand the GIMPLE_OMP_TARGET starting at REGION. */ 8798 8799static void 8800expand_omp_target (struct omp_region *region) 8801{ 8802 basic_block entry_bb, exit_bb, new_bb; 8803 struct function *child_cfun; 8804 tree child_fn, block, t; 8805 gimple_stmt_iterator gsi; 8806 gomp_target *entry_stmt; 8807 gimple stmt; 8808 edge e; 8809 bool offloaded, data_region; 8810 8811 entry_stmt = as_a <gomp_target *> (last_stmt (region->entry)); 8812 new_bb = region->entry; 8813 8814 offloaded = is_gimple_omp_offloaded (entry_stmt); 8815 switch (gimple_omp_target_kind (entry_stmt)) 8816 { 8817 case GF_OMP_TARGET_KIND_REGION: 8818 case GF_OMP_TARGET_KIND_UPDATE: 8819 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 8820 case GF_OMP_TARGET_KIND_OACC_KERNELS: 8821 case GF_OMP_TARGET_KIND_OACC_UPDATE: 8822 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 8823 data_region = false; 8824 break; 8825 case GF_OMP_TARGET_KIND_DATA: 8826 case GF_OMP_TARGET_KIND_OACC_DATA: 8827 data_region = true; 8828 break; 8829 default: 8830 gcc_unreachable (); 8831 } 8832 8833 child_fn = NULL_TREE; 8834 child_cfun = NULL; 8835 if (offloaded) 8836 { 8837 child_fn = gimple_omp_target_child_fn (entry_stmt); 8838 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 8839 } 8840 8841 /* Supported by expand_omp_taskreg, but not here. */ 8842 if (child_cfun != NULL) 8843 gcc_checking_assert (!child_cfun->cfg); 8844 gcc_checking_assert (!gimple_in_ssa_p (cfun)); 8845 8846 entry_bb = region->entry; 8847 exit_bb = region->exit; 8848 8849 if (offloaded) 8850 { 8851 unsigned srcidx, dstidx, num; 8852 8853 /* If the offloading region needs data sent from the parent 8854 function, then the very first statement (except possible 8855 tree profile counter updates) of the offloading body 8856 is a copy assignment .OMP_DATA_I = &.OMP_DATA_O. Since 8857 &.OMP_DATA_O is passed as an argument to the child function, 8858 we need to replace it with the argument as seen by the child 8859 function. 8860 8861 In most cases, this will end up being the identity assignment 8862 .OMP_DATA_I = .OMP_DATA_I. However, if the offloading body had 8863 a function call that has been inlined, the original PARM_DECL 8864 .OMP_DATA_I may have been converted into a different local 8865 variable. In which case, we need to keep the assignment. */ 8866 tree data_arg = gimple_omp_target_data_arg (entry_stmt); 8867 if (data_arg) 8868 { 8869 basic_block entry_succ_bb = single_succ (entry_bb); 8870 gimple_stmt_iterator gsi; 8871 tree arg; 8872 gimple tgtcopy_stmt = NULL; 8873 tree sender = TREE_VEC_ELT (data_arg, 0); 8874 8875 for (gsi = gsi_start_bb (entry_succ_bb); ; gsi_next (&gsi)) 8876 { 8877 gcc_assert (!gsi_end_p (gsi)); 8878 stmt = gsi_stmt (gsi); 8879 if (gimple_code (stmt) != GIMPLE_ASSIGN) 8880 continue; 8881 8882 if (gimple_num_ops (stmt) == 2) 8883 { 8884 tree arg = gimple_assign_rhs1 (stmt); 8885 8886 /* We're ignoring the subcode because we're 8887 effectively doing a STRIP_NOPS. */ 8888 8889 if (TREE_CODE (arg) == ADDR_EXPR 8890 && TREE_OPERAND (arg, 0) == sender) 8891 { 8892 tgtcopy_stmt = stmt; 8893 break; 8894 } 8895 } 8896 } 8897 8898 gcc_assert (tgtcopy_stmt != NULL); 8899 arg = DECL_ARGUMENTS (child_fn); 8900 8901 gcc_assert (gimple_assign_lhs (tgtcopy_stmt) == arg); 8902 gsi_remove (&gsi, true); 8903 } 8904 8905 /* Declare local variables needed in CHILD_CFUN. */ 8906 block = DECL_INITIAL (child_fn); 8907 BLOCK_VARS (block) = vec2chain (child_cfun->local_decls); 8908 /* The gimplifier could record temporaries in the offloading block 8909 rather than in containing function's local_decls chain, 8910 which would mean cgraph missed finalizing them. Do it now. */ 8911 for (t = BLOCK_VARS (block); t; t = DECL_CHAIN (t)) 8912 if (TREE_CODE (t) == VAR_DECL 8913 && TREE_STATIC (t) 8914 && !DECL_EXTERNAL (t)) 8915 varpool_node::finalize_decl (t); 8916 DECL_SAVED_TREE (child_fn) = NULL; 8917 /* We'll create a CFG for child_fn, so no gimple body is needed. */ 8918 gimple_set_body (child_fn, NULL); 8919 TREE_USED (block) = 1; 8920 8921 /* Reset DECL_CONTEXT on function arguments. */ 8922 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 8923 DECL_CONTEXT (t) = child_fn; 8924 8925 /* Split ENTRY_BB at GIMPLE_*, 8926 so that it can be moved to the child function. */ 8927 gsi = gsi_last_bb (entry_bb); 8928 stmt = gsi_stmt (gsi); 8929 gcc_assert (stmt 8930 && gimple_code (stmt) == gimple_code (entry_stmt)); 8931 e = split_block (entry_bb, stmt); 8932 gsi_remove (&gsi, true); 8933 entry_bb = e->dest; 8934 single_succ_edge (entry_bb)->flags = EDGE_FALLTHRU; 8935 8936 /* Convert GIMPLE_OMP_RETURN into a RETURN_EXPR. */ 8937 if (exit_bb) 8938 { 8939 gsi = gsi_last_bb (exit_bb); 8940 gcc_assert (!gsi_end_p (gsi) 8941 && gimple_code (gsi_stmt (gsi)) == GIMPLE_OMP_RETURN); 8942 stmt = gimple_build_return (NULL); 8943 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); 8944 gsi_remove (&gsi, true); 8945 } 8946 8947 /* Move the offloading region into CHILD_CFUN. */ 8948 8949 block = gimple_block (entry_stmt); 8950 8951 new_bb = move_sese_region_to_fn (child_cfun, entry_bb, exit_bb, block); 8952 if (exit_bb) 8953 single_succ_edge (new_bb)->flags = EDGE_FALLTHRU; 8954 /* When the OMP expansion process cannot guarantee an up-to-date 8955 loop tree arrange for the child function to fixup loops. */ 8956 if (loops_state_satisfies_p (LOOPS_NEED_FIXUP)) 8957 child_cfun->x_current_loops->state |= LOOPS_NEED_FIXUP; 8958 8959 /* Remove non-local VAR_DECLs from child_cfun->local_decls list. */ 8960 num = vec_safe_length (child_cfun->local_decls); 8961 for (srcidx = 0, dstidx = 0; srcidx < num; srcidx++) 8962 { 8963 t = (*child_cfun->local_decls)[srcidx]; 8964 if (DECL_CONTEXT (t) == cfun->decl) 8965 continue; 8966 if (srcidx != dstidx) 8967 (*child_cfun->local_decls)[dstidx] = t; 8968 dstidx++; 8969 } 8970 if (dstidx != num) 8971 vec_safe_truncate (child_cfun->local_decls, dstidx); 8972 8973 /* Inform the callgraph about the new function. */ 8974 child_cfun->curr_properties = cfun->curr_properties; 8975 child_cfun->has_simduid_loops |= cfun->has_simduid_loops; 8976 child_cfun->has_force_vectorize_loops |= cfun->has_force_vectorize_loops; 8977 cgraph_node::add_new_function (child_fn, true); 8978 8979#ifdef ENABLE_OFFLOADING 8980 /* Add the new function to the offload table. */ 8981 vec_safe_push (offload_funcs, child_fn); 8982#endif 8983 8984 /* Fix the callgraph edges for child_cfun. Those for cfun will be 8985 fixed in a following pass. */ 8986 push_cfun (child_cfun); 8987 cgraph_edge::rebuild_edges (); 8988 8989#ifdef ENABLE_OFFLOADING 8990 /* Prevent IPA from removing child_fn as unreachable, since there are no 8991 refs from the parent function to child_fn in offload LTO mode. */ 8992 struct cgraph_node *node = cgraph_node::get (child_fn); 8993 node->mark_force_output (); 8994#endif 8995 8996 /* Some EH regions might become dead, see PR34608. If 8997 pass_cleanup_cfg isn't the first pass to happen with the 8998 new child, these dead EH edges might cause problems. 8999 Clean them up now. */ 9000 if (flag_exceptions) 9001 { 9002 basic_block bb; 9003 bool changed = false; 9004 9005 FOR_EACH_BB_FN (bb, cfun) 9006 changed |= gimple_purge_dead_eh_edges (bb); 9007 if (changed) 9008 cleanup_tree_cfg (); 9009 } 9010 pop_cfun (); 9011 } 9012 9013 /* Emit a library call to launch the offloading region, or do data 9014 transfers. */ 9015 tree t1, t2, t3, t4, device, cond, c, clauses; 9016 enum built_in_function start_ix; 9017 location_t clause_loc; 9018 9019 switch (gimple_omp_target_kind (entry_stmt)) 9020 { 9021 case GF_OMP_TARGET_KIND_REGION: 9022 start_ix = BUILT_IN_GOMP_TARGET; 9023 break; 9024 case GF_OMP_TARGET_KIND_DATA: 9025 start_ix = BUILT_IN_GOMP_TARGET_DATA; 9026 break; 9027 case GF_OMP_TARGET_KIND_UPDATE: 9028 start_ix = BUILT_IN_GOMP_TARGET_UPDATE; 9029 break; 9030 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 9031 case GF_OMP_TARGET_KIND_OACC_KERNELS: 9032 start_ix = BUILT_IN_GOACC_PARALLEL; 9033 break; 9034 case GF_OMP_TARGET_KIND_OACC_DATA: 9035 start_ix = BUILT_IN_GOACC_DATA_START; 9036 break; 9037 case GF_OMP_TARGET_KIND_OACC_UPDATE: 9038 start_ix = BUILT_IN_GOACC_UPDATE; 9039 break; 9040 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 9041 start_ix = BUILT_IN_GOACC_ENTER_EXIT_DATA; 9042 break; 9043 default: 9044 gcc_unreachable (); 9045 } 9046 9047 clauses = gimple_omp_target_clauses (entry_stmt); 9048 9049 /* By default, the value of DEVICE is GOMP_DEVICE_ICV (let runtime 9050 library choose) and there is no conditional. */ 9051 cond = NULL_TREE; 9052 device = build_int_cst (integer_type_node, GOMP_DEVICE_ICV); 9053 9054 c = find_omp_clause (clauses, OMP_CLAUSE_IF); 9055 if (c) 9056 cond = OMP_CLAUSE_IF_EXPR (c); 9057 9058 c = find_omp_clause (clauses, OMP_CLAUSE_DEVICE); 9059 if (c) 9060 { 9061 /* Even if we pass it to all library function calls, it is currently only 9062 defined/used for the OpenMP target ones. */ 9063 gcc_checking_assert (start_ix == BUILT_IN_GOMP_TARGET 9064 || start_ix == BUILT_IN_GOMP_TARGET_DATA 9065 || start_ix == BUILT_IN_GOMP_TARGET_UPDATE); 9066 9067 device = OMP_CLAUSE_DEVICE_ID (c); 9068 clause_loc = OMP_CLAUSE_LOCATION (c); 9069 } 9070 else 9071 clause_loc = gimple_location (entry_stmt); 9072 9073 /* Ensure 'device' is of the correct type. */ 9074 device = fold_convert_loc (clause_loc, integer_type_node, device); 9075 9076 /* If we found the clause 'if (cond)', build 9077 (cond ? device : GOMP_DEVICE_HOST_FALLBACK). */ 9078 if (cond) 9079 { 9080 cond = gimple_boolify (cond); 9081 9082 basic_block cond_bb, then_bb, else_bb; 9083 edge e; 9084 tree tmp_var; 9085 9086 tmp_var = create_tmp_var (TREE_TYPE (device)); 9087 if (offloaded) 9088 e = split_block (new_bb, NULL); 9089 else 9090 { 9091 gsi = gsi_last_bb (new_bb); 9092 gsi_prev (&gsi); 9093 e = split_block (new_bb, gsi_stmt (gsi)); 9094 } 9095 cond_bb = e->src; 9096 new_bb = e->dest; 9097 remove_edge (e); 9098 9099 then_bb = create_empty_bb (cond_bb); 9100 else_bb = create_empty_bb (then_bb); 9101 set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb); 9102 set_immediate_dominator (CDI_DOMINATORS, else_bb, cond_bb); 9103 9104 stmt = gimple_build_cond_empty (cond); 9105 gsi = gsi_last_bb (cond_bb); 9106 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9107 9108 gsi = gsi_start_bb (then_bb); 9109 stmt = gimple_build_assign (tmp_var, device); 9110 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9111 9112 gsi = gsi_start_bb (else_bb); 9113 stmt = gimple_build_assign (tmp_var, 9114 build_int_cst (integer_type_node, 9115 GOMP_DEVICE_HOST_FALLBACK)); 9116 gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING); 9117 9118 make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE); 9119 make_edge (cond_bb, else_bb, EDGE_FALSE_VALUE); 9120 add_bb_to_loop (then_bb, cond_bb->loop_father); 9121 add_bb_to_loop (else_bb, cond_bb->loop_father); 9122 make_edge (then_bb, new_bb, EDGE_FALLTHRU); 9123 make_edge (else_bb, new_bb, EDGE_FALLTHRU); 9124 9125 device = tmp_var; 9126 } 9127 9128 gsi = gsi_last_bb (new_bb); 9129 t = gimple_omp_target_data_arg (entry_stmt); 9130 if (t == NULL) 9131 { 9132 t1 = size_zero_node; 9133 t2 = build_zero_cst (ptr_type_node); 9134 t3 = t2; 9135 t4 = t2; 9136 } 9137 else 9138 { 9139 t1 = TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (TREE_VEC_ELT (t, 1)))); 9140 t1 = size_binop (PLUS_EXPR, t1, size_int (1)); 9141 t2 = build_fold_addr_expr (TREE_VEC_ELT (t, 0)); 9142 t3 = build_fold_addr_expr (TREE_VEC_ELT (t, 1)); 9143 t4 = build_fold_addr_expr (TREE_VEC_ELT (t, 2)); 9144 } 9145 9146 gimple g; 9147 /* The maximum number used by any start_ix, without varargs. */ 9148 auto_vec<tree, 11> args; 9149 args.quick_push (device); 9150 if (offloaded) 9151 args.quick_push (build_fold_addr_expr (child_fn)); 9152 switch (start_ix) 9153 { 9154 case BUILT_IN_GOMP_TARGET: 9155 case BUILT_IN_GOMP_TARGET_DATA: 9156 case BUILT_IN_GOMP_TARGET_UPDATE: 9157 /* This const void * is part of the current ABI, but we're not actually 9158 using it. */ 9159 args.quick_push (build_zero_cst (ptr_type_node)); 9160 break; 9161 case BUILT_IN_GOACC_DATA_START: 9162 case BUILT_IN_GOACC_ENTER_EXIT_DATA: 9163 case BUILT_IN_GOACC_PARALLEL: 9164 case BUILT_IN_GOACC_UPDATE: 9165 break; 9166 default: 9167 gcc_unreachable (); 9168 } 9169 args.quick_push (t1); 9170 args.quick_push (t2); 9171 args.quick_push (t3); 9172 args.quick_push (t4); 9173 switch (start_ix) 9174 { 9175 case BUILT_IN_GOACC_DATA_START: 9176 case BUILT_IN_GOMP_TARGET: 9177 case BUILT_IN_GOMP_TARGET_DATA: 9178 case BUILT_IN_GOMP_TARGET_UPDATE: 9179 break; 9180 case BUILT_IN_GOACC_PARALLEL: 9181 { 9182 tree t_num_gangs, t_num_workers, t_vector_length; 9183 9184 /* Default values for num_gangs, num_workers, and vector_length. */ 9185 t_num_gangs = t_num_workers = t_vector_length 9186 = fold_convert_loc (gimple_location (entry_stmt), 9187 integer_type_node, integer_one_node); 9188 /* ..., but if present, use the value specified by the respective 9189 clause, making sure that are of the correct type. */ 9190 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_GANGS); 9191 if (c) 9192 t_num_gangs = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9193 integer_type_node, 9194 OMP_CLAUSE_NUM_GANGS_EXPR (c)); 9195 c = find_omp_clause (clauses, OMP_CLAUSE_NUM_WORKERS); 9196 if (c) 9197 t_num_workers = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9198 integer_type_node, 9199 OMP_CLAUSE_NUM_WORKERS_EXPR (c)); 9200 c = find_omp_clause (clauses, OMP_CLAUSE_VECTOR_LENGTH); 9201 if (c) 9202 t_vector_length = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9203 integer_type_node, 9204 OMP_CLAUSE_VECTOR_LENGTH_EXPR (c)); 9205 args.quick_push (t_num_gangs); 9206 args.quick_push (t_num_workers); 9207 args.quick_push (t_vector_length); 9208 } 9209 /* FALLTHRU */ 9210 case BUILT_IN_GOACC_ENTER_EXIT_DATA: 9211 case BUILT_IN_GOACC_UPDATE: 9212 { 9213 tree t_async; 9214 int t_wait_idx; 9215 9216 /* Default values for t_async. */ 9217 t_async = fold_convert_loc (gimple_location (entry_stmt), 9218 integer_type_node, 9219 build_int_cst (integer_type_node, 9220 GOMP_ASYNC_SYNC)); 9221 /* ..., but if present, use the value specified by the respective 9222 clause, making sure that is of the correct type. */ 9223 c = find_omp_clause (clauses, OMP_CLAUSE_ASYNC); 9224 if (c) 9225 t_async = fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9226 integer_type_node, 9227 OMP_CLAUSE_ASYNC_EXPR (c)); 9228 9229 args.quick_push (t_async); 9230 /* Save the index, and... */ 9231 t_wait_idx = args.length (); 9232 /* ... push a default value. */ 9233 args.quick_push (fold_convert_loc (gimple_location (entry_stmt), 9234 integer_type_node, 9235 integer_zero_node)); 9236 c = find_omp_clause (clauses, OMP_CLAUSE_WAIT); 9237 if (c) 9238 { 9239 int n = 0; 9240 9241 for (; c; c = OMP_CLAUSE_CHAIN (c)) 9242 { 9243 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_WAIT) 9244 { 9245 args.safe_push (fold_convert_loc (OMP_CLAUSE_LOCATION (c), 9246 integer_type_node, 9247 OMP_CLAUSE_WAIT_EXPR (c))); 9248 n++; 9249 } 9250 } 9251 9252 /* Now that we know the number, replace the default value. */ 9253 args.ordered_remove (t_wait_idx); 9254 args.quick_insert (t_wait_idx, 9255 fold_convert_loc (gimple_location (entry_stmt), 9256 integer_type_node, 9257 build_int_cst (integer_type_node, n))); 9258 } 9259 } 9260 break; 9261 default: 9262 gcc_unreachable (); 9263 } 9264 9265 g = gimple_build_call_vec (builtin_decl_explicit (start_ix), args); 9266 gimple_set_location (g, gimple_location (entry_stmt)); 9267 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 9268 if (!offloaded) 9269 { 9270 g = gsi_stmt (gsi); 9271 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_TARGET); 9272 gsi_remove (&gsi, true); 9273 } 9274 if (data_region 9275 && region->exit) 9276 { 9277 gsi = gsi_last_bb (region->exit); 9278 g = gsi_stmt (gsi); 9279 gcc_assert (g && gimple_code (g) == GIMPLE_OMP_RETURN); 9280 gsi_remove (&gsi, true); 9281 } 9282} 9283 9284 9285/* Expand the parallel region tree rooted at REGION. Expansion 9286 proceeds in depth-first order. Innermost regions are expanded 9287 first. This way, parallel regions that require a new function to 9288 be created (e.g., GIMPLE_OMP_PARALLEL) can be expanded without having any 9289 internal dependencies in their body. */ 9290 9291static void 9292expand_omp (struct omp_region *region) 9293{ 9294 while (region) 9295 { 9296 location_t saved_location; 9297 gimple inner_stmt = NULL; 9298 9299 /* First, determine whether this is a combined parallel+workshare 9300 region. */ 9301 if (region->type == GIMPLE_OMP_PARALLEL) 9302 determine_parallel_type (region); 9303 9304 if (region->type == GIMPLE_OMP_FOR 9305 && gimple_omp_for_combined_p (last_stmt (region->entry))) 9306 inner_stmt = last_stmt (region->inner->entry); 9307 9308 if (region->inner) 9309 expand_omp (region->inner); 9310 9311 saved_location = input_location; 9312 if (gimple_has_location (last_stmt (region->entry))) 9313 input_location = gimple_location (last_stmt (region->entry)); 9314 9315 switch (region->type) 9316 { 9317 case GIMPLE_OMP_PARALLEL: 9318 case GIMPLE_OMP_TASK: 9319 expand_omp_taskreg (region); 9320 break; 9321 9322 case GIMPLE_OMP_FOR: 9323 expand_omp_for (region, inner_stmt); 9324 break; 9325 9326 case GIMPLE_OMP_SECTIONS: 9327 expand_omp_sections (region); 9328 break; 9329 9330 case GIMPLE_OMP_SECTION: 9331 /* Individual omp sections are handled together with their 9332 parent GIMPLE_OMP_SECTIONS region. */ 9333 break; 9334 9335 case GIMPLE_OMP_SINGLE: 9336 expand_omp_single (region); 9337 break; 9338 9339 case GIMPLE_OMP_MASTER: 9340 case GIMPLE_OMP_TASKGROUP: 9341 case GIMPLE_OMP_ORDERED: 9342 case GIMPLE_OMP_CRITICAL: 9343 case GIMPLE_OMP_TEAMS: 9344 expand_omp_synch (region); 9345 break; 9346 9347 case GIMPLE_OMP_ATOMIC_LOAD: 9348 expand_omp_atomic (region); 9349 break; 9350 9351 case GIMPLE_OMP_TARGET: 9352 expand_omp_target (region); 9353 break; 9354 9355 default: 9356 gcc_unreachable (); 9357 } 9358 9359 input_location = saved_location; 9360 region = region->next; 9361 } 9362} 9363 9364 9365/* Helper for build_omp_regions. Scan the dominator tree starting at 9366 block BB. PARENT is the region that contains BB. If SINGLE_TREE is 9367 true, the function ends once a single tree is built (otherwise, whole 9368 forest of OMP constructs may be built). */ 9369 9370static void 9371build_omp_regions_1 (basic_block bb, struct omp_region *parent, 9372 bool single_tree) 9373{ 9374 gimple_stmt_iterator gsi; 9375 gimple stmt; 9376 basic_block son; 9377 9378 gsi = gsi_last_bb (bb); 9379 if (!gsi_end_p (gsi) && is_gimple_omp (gsi_stmt (gsi))) 9380 { 9381 struct omp_region *region; 9382 enum gimple_code code; 9383 9384 stmt = gsi_stmt (gsi); 9385 code = gimple_code (stmt); 9386 if (code == GIMPLE_OMP_RETURN) 9387 { 9388 /* STMT is the return point out of region PARENT. Mark it 9389 as the exit point and make PARENT the immediately 9390 enclosing region. */ 9391 gcc_assert (parent); 9392 region = parent; 9393 region->exit = bb; 9394 parent = parent->outer; 9395 } 9396 else if (code == GIMPLE_OMP_ATOMIC_STORE) 9397 { 9398 /* GIMPLE_OMP_ATOMIC_STORE is analoguous to 9399 GIMPLE_OMP_RETURN, but matches with 9400 GIMPLE_OMP_ATOMIC_LOAD. */ 9401 gcc_assert (parent); 9402 gcc_assert (parent->type == GIMPLE_OMP_ATOMIC_LOAD); 9403 region = parent; 9404 region->exit = bb; 9405 parent = parent->outer; 9406 } 9407 else if (code == GIMPLE_OMP_CONTINUE) 9408 { 9409 gcc_assert (parent); 9410 parent->cont = bb; 9411 } 9412 else if (code == GIMPLE_OMP_SECTIONS_SWITCH) 9413 { 9414 /* GIMPLE_OMP_SECTIONS_SWITCH is part of 9415 GIMPLE_OMP_SECTIONS, and we do nothing for it. */ 9416 } 9417 else 9418 { 9419 region = new_omp_region (bb, code, parent); 9420 /* Otherwise... */ 9421 if (code == GIMPLE_OMP_TARGET) 9422 { 9423 switch (gimple_omp_target_kind (stmt)) 9424 { 9425 case GF_OMP_TARGET_KIND_REGION: 9426 case GF_OMP_TARGET_KIND_DATA: 9427 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 9428 case GF_OMP_TARGET_KIND_OACC_KERNELS: 9429 case GF_OMP_TARGET_KIND_OACC_DATA: 9430 break; 9431 case GF_OMP_TARGET_KIND_UPDATE: 9432 case GF_OMP_TARGET_KIND_OACC_UPDATE: 9433 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 9434 /* ..., other than for those stand-alone directives... */ 9435 region = NULL; 9436 break; 9437 default: 9438 gcc_unreachable (); 9439 } 9440 } 9441 /* ..., this directive becomes the parent for a new region. */ 9442 if (region) 9443 parent = region; 9444 } 9445 } 9446 9447 if (single_tree && !parent) 9448 return; 9449 9450 for (son = first_dom_son (CDI_DOMINATORS, bb); 9451 son; 9452 son = next_dom_son (CDI_DOMINATORS, son)) 9453 build_omp_regions_1 (son, parent, single_tree); 9454} 9455 9456/* Builds the tree of OMP regions rooted at ROOT, storing it to 9457 root_omp_region. */ 9458 9459static void 9460build_omp_regions_root (basic_block root) 9461{ 9462 gcc_assert (root_omp_region == NULL); 9463 build_omp_regions_1 (root, NULL, true); 9464 gcc_assert (root_omp_region != NULL); 9465} 9466 9467/* Expands omp construct (and its subconstructs) starting in HEAD. */ 9468 9469void 9470omp_expand_local (basic_block head) 9471{ 9472 build_omp_regions_root (head); 9473 if (dump_file && (dump_flags & TDF_DETAILS)) 9474 { 9475 fprintf (dump_file, "\nOMP region tree\n\n"); 9476 dump_omp_region (dump_file, root_omp_region, 0); 9477 fprintf (dump_file, "\n"); 9478 } 9479 9480 remove_exit_barriers (root_omp_region); 9481 expand_omp (root_omp_region); 9482 9483 free_omp_regions (); 9484} 9485 9486/* Scan the CFG and build a tree of OMP regions. Return the root of 9487 the OMP region tree. */ 9488 9489static void 9490build_omp_regions (void) 9491{ 9492 gcc_assert (root_omp_region == NULL); 9493 calculate_dominance_info (CDI_DOMINATORS); 9494 build_omp_regions_1 (ENTRY_BLOCK_PTR_FOR_FN (cfun), NULL, false); 9495} 9496 9497/* Main entry point for expanding OMP-GIMPLE into runtime calls. */ 9498 9499static unsigned int 9500execute_expand_omp (void) 9501{ 9502 build_omp_regions (); 9503 9504 if (!root_omp_region) 9505 return 0; 9506 9507 if (dump_file) 9508 { 9509 fprintf (dump_file, "\nOMP region tree\n\n"); 9510 dump_omp_region (dump_file, root_omp_region, 0); 9511 fprintf (dump_file, "\n"); 9512 } 9513 9514 remove_exit_barriers (root_omp_region); 9515 9516 expand_omp (root_omp_region); 9517 9518 cleanup_tree_cfg (); 9519 9520 free_omp_regions (); 9521 9522 return 0; 9523} 9524 9525/* OMP expansion -- the default pass, run before creation of SSA form. */ 9526 9527namespace { 9528 9529const pass_data pass_data_expand_omp = 9530{ 9531 GIMPLE_PASS, /* type */ 9532 "ompexp", /* name */ 9533 OPTGROUP_NONE, /* optinfo_flags */ 9534 TV_NONE, /* tv_id */ 9535 PROP_gimple_any, /* properties_required */ 9536 PROP_gimple_eomp, /* properties_provided */ 9537 0, /* properties_destroyed */ 9538 0, /* todo_flags_start */ 9539 0, /* todo_flags_finish */ 9540}; 9541 9542class pass_expand_omp : public gimple_opt_pass 9543{ 9544public: 9545 pass_expand_omp (gcc::context *ctxt) 9546 : gimple_opt_pass (pass_data_expand_omp, ctxt) 9547 {} 9548 9549 /* opt_pass methods: */ 9550 virtual unsigned int execute (function *) 9551 { 9552 bool gate = ((flag_cilkplus != 0 || flag_openacc != 0 || flag_openmp != 0 9553 || flag_openmp_simd != 0) 9554 && !seen_error ()); 9555 9556 /* This pass always runs, to provide PROP_gimple_eomp. 9557 But often, there is nothing to do. */ 9558 if (!gate) 9559 return 0; 9560 9561 return execute_expand_omp (); 9562 } 9563 9564}; // class pass_expand_omp 9565 9566} // anon namespace 9567 9568gimple_opt_pass * 9569make_pass_expand_omp (gcc::context *ctxt) 9570{ 9571 return new pass_expand_omp (ctxt); 9572} 9573 9574namespace { 9575 9576const pass_data pass_data_expand_omp_ssa = 9577{ 9578 GIMPLE_PASS, /* type */ 9579 "ompexpssa", /* name */ 9580 OPTGROUP_NONE, /* optinfo_flags */ 9581 TV_NONE, /* tv_id */ 9582 PROP_cfg | PROP_ssa, /* properties_required */ 9583 PROP_gimple_eomp, /* properties_provided */ 9584 0, /* properties_destroyed */ 9585 0, /* todo_flags_start */ 9586 TODO_cleanup_cfg | TODO_rebuild_alias, /* todo_flags_finish */ 9587}; 9588 9589class pass_expand_omp_ssa : public gimple_opt_pass 9590{ 9591public: 9592 pass_expand_omp_ssa (gcc::context *ctxt) 9593 : gimple_opt_pass (pass_data_expand_omp_ssa, ctxt) 9594 {} 9595 9596 /* opt_pass methods: */ 9597 virtual bool gate (function *fun) 9598 { 9599 return !(fun->curr_properties & PROP_gimple_eomp); 9600 } 9601 virtual unsigned int execute (function *) { return execute_expand_omp (); } 9602 9603}; // class pass_expand_omp_ssa 9604 9605} // anon namespace 9606 9607gimple_opt_pass * 9608make_pass_expand_omp_ssa (gcc::context *ctxt) 9609{ 9610 return new pass_expand_omp_ssa (ctxt); 9611} 9612 9613/* Routines to lower OMP directives into OMP-GIMPLE. */ 9614 9615/* Helper function to preform, potentially COMPLEX_TYPE, operation and 9616 convert it to gimple. */ 9617static void 9618oacc_gimple_assign (tree dest, tree_code op, tree src, gimple_seq *seq) 9619{ 9620 gimple stmt; 9621 9622 if (TREE_CODE (TREE_TYPE (dest)) != COMPLEX_TYPE) 9623 { 9624 stmt = gimple_build_assign (dest, op, dest, src); 9625 gimple_seq_add_stmt (seq, stmt); 9626 return; 9627 } 9628 9629 tree t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9630 tree rdest = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); 9631 gimplify_assign (t, rdest, seq); 9632 rdest = t; 9633 9634 t = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9635 tree idest = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (dest)), dest); 9636 gimplify_assign (t, idest, seq); 9637 idest = t; 9638 9639 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); 9640 tree rsrc = fold_build1 (REALPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); 9641 gimplify_assign (t, rsrc, seq); 9642 rsrc = t; 9643 9644 t = create_tmp_var (TREE_TYPE (TREE_TYPE (src))); 9645 tree isrc = fold_build1 (IMAGPART_EXPR, TREE_TYPE (TREE_TYPE (src)), src); 9646 gimplify_assign (t, isrc, seq); 9647 isrc = t; 9648 9649 tree r = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9650 tree i = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9651 tree result; 9652 9653 if (op == PLUS_EXPR) 9654 { 9655 stmt = gimple_build_assign (r, op, rdest, rsrc); 9656 gimple_seq_add_stmt (seq, stmt); 9657 9658 stmt = gimple_build_assign (i, op, idest, isrc); 9659 gimple_seq_add_stmt (seq, stmt); 9660 } 9661 else if (op == MULT_EXPR) 9662 { 9663 /* Let x = a + ib = dest, y = c + id = src. 9664 x * y = (ac - bd) + i(ad + bc) */ 9665 tree ac = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9666 tree bd = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9667 tree ad = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9668 tree bc = create_tmp_var (TREE_TYPE (TREE_TYPE (dest))); 9669 9670 stmt = gimple_build_assign (ac, MULT_EXPR, rdest, rsrc); 9671 gimple_seq_add_stmt (seq, stmt); 9672 9673 stmt = gimple_build_assign (bd, MULT_EXPR, idest, isrc); 9674 gimple_seq_add_stmt (seq, stmt); 9675 9676 stmt = gimple_build_assign (r, MINUS_EXPR, ac, bd); 9677 gimple_seq_add_stmt (seq, stmt); 9678 9679 stmt = gimple_build_assign (ad, MULT_EXPR, rdest, isrc); 9680 gimple_seq_add_stmt (seq, stmt); 9681 9682 stmt = gimple_build_assign (bd, MULT_EXPR, idest, rsrc); 9683 gimple_seq_add_stmt (seq, stmt); 9684 9685 stmt = gimple_build_assign (i, PLUS_EXPR, ad, bc); 9686 gimple_seq_add_stmt (seq, stmt); 9687 } 9688 else 9689 gcc_unreachable (); 9690 9691 result = build2 (COMPLEX_EXPR, TREE_TYPE (dest), r, i); 9692 gimplify_assign (dest, result, seq); 9693} 9694 9695/* Helper function to initialize local data for the reduction arrays. 9696 The reduction arrays need to be placed inside the calling function 9697 for accelerators, or else the host won't be able to preform the final 9698 reduction. */ 9699 9700static void 9701oacc_initialize_reduction_data (tree clauses, tree nthreads, 9702 gimple_seq *stmt_seqp, omp_context *ctx) 9703{ 9704 tree c, t, oc; 9705 gimple stmt; 9706 omp_context *octx; 9707 9708 /* Find the innermost OpenACC parallel context. */ 9709 if (gimple_code (ctx->stmt) == GIMPLE_OMP_TARGET 9710 && (gimple_omp_target_kind (ctx->stmt) 9711 == GF_OMP_TARGET_KIND_OACC_PARALLEL)) 9712 octx = ctx; 9713 else 9714 octx = ctx->outer; 9715 gcc_checking_assert (gimple_code (octx->stmt) == GIMPLE_OMP_TARGET 9716 && (gimple_omp_target_kind (octx->stmt) 9717 == GF_OMP_TARGET_KIND_OACC_PARALLEL)); 9718 9719 /* Extract the clauses. */ 9720 oc = gimple_omp_target_clauses (octx->stmt); 9721 9722 /* Find the last outer clause. */ 9723 for (; oc && OMP_CLAUSE_CHAIN (oc); oc = OMP_CLAUSE_CHAIN (oc)) 9724 ; 9725 9726 /* Allocate arrays for each reduction variable. */ 9727 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9728 { 9729 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 9730 continue; 9731 9732 tree var = OMP_CLAUSE_DECL (c); 9733 tree type = get_base_type (var); 9734 tree array = lookup_oacc_reduction (oacc_get_reduction_array_id (var), 9735 ctx); 9736 tree size, call; 9737 9738 /* Calculate size of the reduction array. */ 9739 t = create_tmp_var (TREE_TYPE (nthreads)); 9740 stmt = gimple_build_assign (t, MULT_EXPR, nthreads, 9741 fold_convert (TREE_TYPE (nthreads), 9742 TYPE_SIZE_UNIT (type))); 9743 gimple_seq_add_stmt (stmt_seqp, stmt); 9744 9745 size = create_tmp_var (sizetype); 9746 gimplify_assign (size, fold_build1 (NOP_EXPR, sizetype, t), stmt_seqp); 9747 9748 /* Now allocate memory for it. */ 9749 call = unshare_expr (builtin_decl_explicit (BUILT_IN_ALLOCA)); 9750 stmt = gimple_build_call (call, 1, size); 9751 gimple_call_set_lhs (stmt, array); 9752 gimple_seq_add_stmt (stmt_seqp, stmt); 9753 9754 /* Map this array into the accelerator. */ 9755 9756 /* Add the reduction array to the list of clauses. */ 9757 tree x = array; 9758 t = build_omp_clause (gimple_location (ctx->stmt), OMP_CLAUSE_MAP); 9759 OMP_CLAUSE_SET_MAP_KIND (t, GOMP_MAP_FORCE_FROM); 9760 OMP_CLAUSE_DECL (t) = x; 9761 OMP_CLAUSE_CHAIN (t) = NULL; 9762 if (oc) 9763 OMP_CLAUSE_CHAIN (oc) = t; 9764 else 9765 gimple_omp_target_set_clauses (as_a <gomp_target *> (octx->stmt), t); 9766 OMP_CLAUSE_SIZE (t) = size; 9767 oc = t; 9768 } 9769} 9770 9771/* Helper function to process the array of partial reductions. Nthreads 9772 indicates the number of threads. Unfortunately, GOACC_GET_NUM_THREADS 9773 cannot be used here, because nthreads on the host may be different than 9774 on the accelerator. */ 9775 9776static void 9777oacc_finalize_reduction_data (tree clauses, tree nthreads, 9778 gimple_seq *stmt_seqp, omp_context *ctx) 9779{ 9780 tree c, x, var, array, loop_header, loop_body, loop_exit, type; 9781 gimple stmt; 9782 9783 /* Create for loop. 9784 9785 let var = the original reduction variable 9786 let array = reduction variable array 9787 9788 for (i = 0; i < nthreads; i++) 9789 var op= array[i] 9790 */ 9791 9792 loop_header = create_artificial_label (UNKNOWN_LOCATION); 9793 loop_body = create_artificial_label (UNKNOWN_LOCATION); 9794 loop_exit = create_artificial_label (UNKNOWN_LOCATION); 9795 9796 /* Create and initialize an index variable. */ 9797 tree ix = create_tmp_var (sizetype); 9798 gimplify_assign (ix, fold_build1 (NOP_EXPR, sizetype, integer_zero_node), 9799 stmt_seqp); 9800 9801 /* Insert the loop header label here. */ 9802 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_header)); 9803 9804 /* Exit loop if ix >= nthreads. */ 9805 x = create_tmp_var (sizetype); 9806 gimplify_assign (x, fold_build1 (NOP_EXPR, sizetype, nthreads), stmt_seqp); 9807 stmt = gimple_build_cond (GE_EXPR, ix, x, loop_exit, loop_body); 9808 gimple_seq_add_stmt (stmt_seqp, stmt); 9809 9810 /* Insert the loop body label here. */ 9811 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_body)); 9812 9813 /* Collapse each reduction array, one element at a time. */ 9814 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9815 { 9816 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_REDUCTION) 9817 continue; 9818 9819 tree_code reduction_code = OMP_CLAUSE_REDUCTION_CODE (c); 9820 9821 /* reduction(-:var) sums up the partial results, so it acts 9822 identically to reduction(+:var). */ 9823 if (reduction_code == MINUS_EXPR) 9824 reduction_code = PLUS_EXPR; 9825 9826 /* Set up reduction variable var. */ 9827 var = OMP_CLAUSE_DECL (c); 9828 type = get_base_type (var); 9829 array = lookup_oacc_reduction (oacc_get_reduction_array_id 9830 (OMP_CLAUSE_DECL (c)), ctx); 9831 9832 /* Calculate the array offset. */ 9833 tree offset = create_tmp_var (sizetype); 9834 gimplify_assign (offset, TYPE_SIZE_UNIT (type), stmt_seqp); 9835 stmt = gimple_build_assign (offset, MULT_EXPR, offset, ix); 9836 gimple_seq_add_stmt (stmt_seqp, stmt); 9837 9838 tree ptr = create_tmp_var (TREE_TYPE (array)); 9839 stmt = gimple_build_assign (ptr, POINTER_PLUS_EXPR, array, offset); 9840 gimple_seq_add_stmt (stmt_seqp, stmt); 9841 9842 /* Extract array[ix] into mem. */ 9843 tree mem = create_tmp_var (type); 9844 gimplify_assign (mem, build_simple_mem_ref (ptr), stmt_seqp); 9845 9846 /* Find the original reduction variable. */ 9847 if (is_reference (var)) 9848 var = build_simple_mem_ref (var); 9849 9850 tree t = create_tmp_var (type); 9851 9852 x = lang_hooks.decls.omp_clause_assign_op (c, t, var); 9853 gimplify_and_add (unshare_expr(x), stmt_seqp); 9854 9855 /* var = var op mem */ 9856 switch (OMP_CLAUSE_REDUCTION_CODE (c)) 9857 { 9858 case TRUTH_ANDIF_EXPR: 9859 case TRUTH_ORIF_EXPR: 9860 t = fold_build2 (OMP_CLAUSE_REDUCTION_CODE (c), integer_type_node, 9861 t, mem); 9862 gimplify_and_add (t, stmt_seqp); 9863 break; 9864 default: 9865 /* The lhs isn't a gimple_reg when var is COMPLEX_TYPE. */ 9866 oacc_gimple_assign (t, OMP_CLAUSE_REDUCTION_CODE (c), mem, 9867 stmt_seqp); 9868 } 9869 9870 t = fold_build1 (NOP_EXPR, TREE_TYPE (var), t); 9871 x = lang_hooks.decls.omp_clause_assign_op (c, var, t); 9872 gimplify_and_add (unshare_expr(x), stmt_seqp); 9873 } 9874 9875 /* Increment the induction variable. */ 9876 tree one = fold_build1 (NOP_EXPR, sizetype, integer_one_node); 9877 stmt = gimple_build_assign (ix, PLUS_EXPR, ix, one); 9878 gimple_seq_add_stmt (stmt_seqp, stmt); 9879 9880 /* Go back to the top of the loop. */ 9881 gimple_seq_add_stmt (stmt_seqp, gimple_build_goto (loop_header)); 9882 9883 /* Place the loop exit label here. */ 9884 gimple_seq_add_stmt (stmt_seqp, gimple_build_label (loop_exit)); 9885} 9886 9887/* Scan through all of the gimple stmts searching for an OMP_FOR_EXPR, and 9888 scan that for reductions. */ 9889 9890static void 9891oacc_process_reduction_data (gimple_seq *body, gimple_seq *in_stmt_seqp, 9892 gimple_seq *out_stmt_seqp, omp_context *ctx) 9893{ 9894 gimple_stmt_iterator gsi; 9895 gimple_seq inner = NULL; 9896 9897 /* A collapse clause may have inserted a new bind block. */ 9898 gsi = gsi_start (*body); 9899 while (!gsi_end_p (gsi)) 9900 { 9901 gimple stmt = gsi_stmt (gsi); 9902 if (gbind *bind_stmt = dyn_cast <gbind *> (stmt)) 9903 { 9904 inner = gimple_bind_body (bind_stmt); 9905 body = &inner; 9906 gsi = gsi_start (*body); 9907 } 9908 else if (dyn_cast <gomp_for *> (stmt)) 9909 break; 9910 else 9911 gsi_next (&gsi); 9912 } 9913 9914 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 9915 { 9916 tree clauses, nthreads, t, c, acc_device, acc_device_host, call, 9917 enter, exit; 9918 bool reduction_found = false; 9919 9920 gimple stmt = gsi_stmt (gsi); 9921 9922 switch (gimple_code (stmt)) 9923 { 9924 case GIMPLE_OMP_FOR: 9925 clauses = gimple_omp_for_clauses (stmt); 9926 9927 /* Search for a reduction clause. */ 9928 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 9929 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_REDUCTION) 9930 { 9931 reduction_found = true; 9932 break; 9933 } 9934 9935 if (!reduction_found) 9936 break; 9937 9938 ctx = maybe_lookup_ctx (stmt); 9939 t = NULL_TREE; 9940 9941 /* Extract the number of threads. */ 9942 nthreads = create_tmp_var (sizetype); 9943 t = oacc_max_threads (ctx); 9944 gimplify_assign (nthreads, t, in_stmt_seqp); 9945 9946 /* Determine if this is kernel will be executed on the host. */ 9947 call = builtin_decl_explicit (BUILT_IN_ACC_GET_DEVICE_TYPE); 9948 acc_device = create_tmp_var (integer_type_node, ".acc_device_type"); 9949 stmt = gimple_build_call (call, 0); 9950 gimple_call_set_lhs (stmt, acc_device); 9951 gimple_seq_add_stmt (in_stmt_seqp, stmt); 9952 9953 /* Set nthreads = 1 for ACC_DEVICE_TYPE=host. */ 9954 acc_device_host = create_tmp_var (integer_type_node, 9955 ".acc_device_host"); 9956 gimplify_assign (acc_device_host, 9957 build_int_cst (integer_type_node, 9958 GOMP_DEVICE_HOST), 9959 in_stmt_seqp); 9960 9961 enter = create_artificial_label (UNKNOWN_LOCATION); 9962 exit = create_artificial_label (UNKNOWN_LOCATION); 9963 9964 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, 9965 enter, exit); 9966 gimple_seq_add_stmt (in_stmt_seqp, stmt); 9967 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); 9968 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, 9969 integer_one_node), 9970 in_stmt_seqp); 9971 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); 9972 9973 /* Also, set nthreads = 1 for ACC_DEVICE_TYPE=host_nonshm. */ 9974 gimplify_assign (acc_device_host, 9975 build_int_cst (integer_type_node, 9976 GOMP_DEVICE_HOST_NONSHM), 9977 in_stmt_seqp); 9978 9979 enter = create_artificial_label (UNKNOWN_LOCATION); 9980 exit = create_artificial_label (UNKNOWN_LOCATION); 9981 9982 stmt = gimple_build_cond (EQ_EXPR, acc_device, acc_device_host, 9983 enter, exit); 9984 gimple_seq_add_stmt (in_stmt_seqp, stmt); 9985 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (enter)); 9986 gimplify_assign (nthreads, fold_build1 (NOP_EXPR, sizetype, 9987 integer_one_node), 9988 in_stmt_seqp); 9989 gimple_seq_add_stmt (in_stmt_seqp, gimple_build_label (exit)); 9990 9991 oacc_initialize_reduction_data (clauses, nthreads, in_stmt_seqp, 9992 ctx); 9993 oacc_finalize_reduction_data (clauses, nthreads, out_stmt_seqp, ctx); 9994 break; 9995 default: 9996 // Scan for other directives which support reduction here. 9997 break; 9998 } 9999 } 10000} 10001 10002/* If ctx is a worksharing context inside of a cancellable parallel 10003 region and it isn't nowait, add lhs to its GIMPLE_OMP_RETURN 10004 and conditional branch to parallel's cancel_label to handle 10005 cancellation in the implicit barrier. */ 10006 10007static void 10008maybe_add_implicit_barrier_cancel (omp_context *ctx, gimple_seq *body) 10009{ 10010 gimple omp_return = gimple_seq_last_stmt (*body); 10011 gcc_assert (gimple_code (omp_return) == GIMPLE_OMP_RETURN); 10012 if (gimple_omp_return_nowait_p (omp_return)) 10013 return; 10014 if (ctx->outer 10015 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_PARALLEL 10016 && ctx->outer->cancellable) 10017 { 10018 tree fndecl = builtin_decl_explicit (BUILT_IN_GOMP_CANCEL); 10019 tree c_bool_type = TREE_TYPE (TREE_TYPE (fndecl)); 10020 tree lhs = create_tmp_var (c_bool_type); 10021 gimple_omp_return_set_lhs (omp_return, lhs); 10022 tree fallthru_label = create_artificial_label (UNKNOWN_LOCATION); 10023 gimple g = gimple_build_cond (NE_EXPR, lhs, 10024 fold_convert (c_bool_type, 10025 boolean_false_node), 10026 ctx->outer->cancel_label, fallthru_label); 10027 gimple_seq_add_stmt (body, g); 10028 gimple_seq_add_stmt (body, gimple_build_label (fallthru_label)); 10029 } 10030} 10031 10032/* Lower the OpenMP sections directive in the current statement in GSI_P. 10033 CTX is the enclosing OMP context for the current statement. */ 10034 10035static void 10036lower_omp_sections (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10037{ 10038 tree block, control; 10039 gimple_stmt_iterator tgsi; 10040 gomp_sections *stmt; 10041 gimple t; 10042 gbind *new_stmt, *bind; 10043 gimple_seq ilist, dlist, olist, new_body; 10044 10045 stmt = as_a <gomp_sections *> (gsi_stmt (*gsi_p)); 10046 10047 push_gimplify_context (); 10048 10049 dlist = NULL; 10050 ilist = NULL; 10051 lower_rec_input_clauses (gimple_omp_sections_clauses (stmt), 10052 &ilist, &dlist, ctx, NULL); 10053 10054 new_body = gimple_omp_body (stmt); 10055 gimple_omp_set_body (stmt, NULL); 10056 tgsi = gsi_start (new_body); 10057 for (; !gsi_end_p (tgsi); gsi_next (&tgsi)) 10058 { 10059 omp_context *sctx; 10060 gimple sec_start; 10061 10062 sec_start = gsi_stmt (tgsi); 10063 sctx = maybe_lookup_ctx (sec_start); 10064 gcc_assert (sctx); 10065 10066 lower_omp (gimple_omp_body_ptr (sec_start), sctx); 10067 gsi_insert_seq_after (&tgsi, gimple_omp_body (sec_start), 10068 GSI_CONTINUE_LINKING); 10069 gimple_omp_set_body (sec_start, NULL); 10070 10071 if (gsi_one_before_end_p (tgsi)) 10072 { 10073 gimple_seq l = NULL; 10074 lower_lastprivate_clauses (gimple_omp_sections_clauses (stmt), NULL, 10075 &l, ctx); 10076 gsi_insert_seq_after (&tgsi, l, GSI_CONTINUE_LINKING); 10077 gimple_omp_section_set_last (sec_start); 10078 } 10079 10080 gsi_insert_after (&tgsi, gimple_build_omp_return (false), 10081 GSI_CONTINUE_LINKING); 10082 } 10083 10084 block = make_node (BLOCK); 10085 bind = gimple_build_bind (NULL, new_body, block); 10086 10087 olist = NULL; 10088 lower_reduction_clauses (gimple_omp_sections_clauses (stmt), &olist, ctx); 10089 10090 block = make_node (BLOCK); 10091 new_stmt = gimple_build_bind (NULL, NULL, block); 10092 gsi_replace (gsi_p, new_stmt, true); 10093 10094 pop_gimplify_context (new_stmt); 10095 gimple_bind_append_vars (new_stmt, ctx->block_vars); 10096 BLOCK_VARS (block) = gimple_bind_vars (bind); 10097 if (BLOCK_VARS (block)) 10098 TREE_USED (block) = 1; 10099 10100 new_body = NULL; 10101 gimple_seq_add_seq (&new_body, ilist); 10102 gimple_seq_add_stmt (&new_body, stmt); 10103 gimple_seq_add_stmt (&new_body, gimple_build_omp_sections_switch ()); 10104 gimple_seq_add_stmt (&new_body, bind); 10105 10106 control = create_tmp_var (unsigned_type_node, ".section"); 10107 t = gimple_build_omp_continue (control, control); 10108 gimple_omp_sections_set_control (stmt, control); 10109 gimple_seq_add_stmt (&new_body, t); 10110 10111 gimple_seq_add_seq (&new_body, olist); 10112 if (ctx->cancellable) 10113 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); 10114 gimple_seq_add_seq (&new_body, dlist); 10115 10116 new_body = maybe_catch_exception (new_body); 10117 10118 t = gimple_build_omp_return 10119 (!!find_omp_clause (gimple_omp_sections_clauses (stmt), 10120 OMP_CLAUSE_NOWAIT)); 10121 gimple_seq_add_stmt (&new_body, t); 10122 maybe_add_implicit_barrier_cancel (ctx, &new_body); 10123 10124 gimple_bind_set_body (new_stmt, new_body); 10125} 10126 10127 10128/* A subroutine of lower_omp_single. Expand the simple form of 10129 a GIMPLE_OMP_SINGLE, without a copyprivate clause: 10130 10131 if (GOMP_single_start ()) 10132 BODY; 10133 [ GOMP_barrier (); ] -> unless 'nowait' is present. 10134 10135 FIXME. It may be better to delay expanding the logic of this until 10136 pass_expand_omp. The expanded logic may make the job more difficult 10137 to a synchronization analysis pass. */ 10138 10139static void 10140lower_omp_single_simple (gomp_single *single_stmt, gimple_seq *pre_p) 10141{ 10142 location_t loc = gimple_location (single_stmt); 10143 tree tlabel = create_artificial_label (loc); 10144 tree flabel = create_artificial_label (loc); 10145 gimple call, cond; 10146 tree lhs, decl; 10147 10148 decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_START); 10149 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (decl))); 10150 call = gimple_build_call (decl, 0); 10151 gimple_call_set_lhs (call, lhs); 10152 gimple_seq_add_stmt (pre_p, call); 10153 10154 cond = gimple_build_cond (EQ_EXPR, lhs, 10155 fold_convert_loc (loc, TREE_TYPE (lhs), 10156 boolean_true_node), 10157 tlabel, flabel); 10158 gimple_seq_add_stmt (pre_p, cond); 10159 gimple_seq_add_stmt (pre_p, gimple_build_label (tlabel)); 10160 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); 10161 gimple_seq_add_stmt (pre_p, gimple_build_label (flabel)); 10162} 10163 10164 10165/* A subroutine of lower_omp_single. Expand the simple form of 10166 a GIMPLE_OMP_SINGLE, with a copyprivate clause: 10167 10168 #pragma omp single copyprivate (a, b, c) 10169 10170 Create a new structure to hold copies of 'a', 'b' and 'c' and emit: 10171 10172 { 10173 if ((copyout_p = GOMP_single_copy_start ()) == NULL) 10174 { 10175 BODY; 10176 copyout.a = a; 10177 copyout.b = b; 10178 copyout.c = c; 10179 GOMP_single_copy_end (©out); 10180 } 10181 else 10182 { 10183 a = copyout_p->a; 10184 b = copyout_p->b; 10185 c = copyout_p->c; 10186 } 10187 GOMP_barrier (); 10188 } 10189 10190 FIXME. It may be better to delay expanding the logic of this until 10191 pass_expand_omp. The expanded logic may make the job more difficult 10192 to a synchronization analysis pass. */ 10193 10194static void 10195lower_omp_single_copy (gomp_single *single_stmt, gimple_seq *pre_p, 10196 omp_context *ctx) 10197{ 10198 tree ptr_type, t, l0, l1, l2, bfn_decl; 10199 gimple_seq copyin_seq; 10200 location_t loc = gimple_location (single_stmt); 10201 10202 ctx->sender_decl = create_tmp_var (ctx->record_type, ".omp_copy_o"); 10203 10204 ptr_type = build_pointer_type (ctx->record_type); 10205 ctx->receiver_decl = create_tmp_var (ptr_type, ".omp_copy_i"); 10206 10207 l0 = create_artificial_label (loc); 10208 l1 = create_artificial_label (loc); 10209 l2 = create_artificial_label (loc); 10210 10211 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_START); 10212 t = build_call_expr_loc (loc, bfn_decl, 0); 10213 t = fold_convert_loc (loc, ptr_type, t); 10214 gimplify_assign (ctx->receiver_decl, t, pre_p); 10215 10216 t = build2 (EQ_EXPR, boolean_type_node, ctx->receiver_decl, 10217 build_int_cst (ptr_type, 0)); 10218 t = build3 (COND_EXPR, void_type_node, t, 10219 build_and_jump (&l0), build_and_jump (&l1)); 10220 gimplify_and_add (t, pre_p); 10221 10222 gimple_seq_add_stmt (pre_p, gimple_build_label (l0)); 10223 10224 gimple_seq_add_seq (pre_p, gimple_omp_body (single_stmt)); 10225 10226 copyin_seq = NULL; 10227 lower_copyprivate_clauses (gimple_omp_single_clauses (single_stmt), pre_p, 10228 ©in_seq, ctx); 10229 10230 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 10231 bfn_decl = builtin_decl_explicit (BUILT_IN_GOMP_SINGLE_COPY_END); 10232 t = build_call_expr_loc (loc, bfn_decl, 1, t); 10233 gimplify_and_add (t, pre_p); 10234 10235 t = build_and_jump (&l2); 10236 gimplify_and_add (t, pre_p); 10237 10238 gimple_seq_add_stmt (pre_p, gimple_build_label (l1)); 10239 10240 gimple_seq_add_seq (pre_p, copyin_seq); 10241 10242 gimple_seq_add_stmt (pre_p, gimple_build_label (l2)); 10243} 10244 10245 10246/* Expand code for an OpenMP single directive. */ 10247 10248static void 10249lower_omp_single (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10250{ 10251 tree block; 10252 gimple t; 10253 gomp_single *single_stmt = as_a <gomp_single *> (gsi_stmt (*gsi_p)); 10254 gbind *bind; 10255 gimple_seq bind_body, bind_body_tail = NULL, dlist; 10256 10257 push_gimplify_context (); 10258 10259 block = make_node (BLOCK); 10260 bind = gimple_build_bind (NULL, NULL, block); 10261 gsi_replace (gsi_p, bind, true); 10262 bind_body = NULL; 10263 dlist = NULL; 10264 lower_rec_input_clauses (gimple_omp_single_clauses (single_stmt), 10265 &bind_body, &dlist, ctx, NULL); 10266 lower_omp (gimple_omp_body_ptr (single_stmt), ctx); 10267 10268 gimple_seq_add_stmt (&bind_body, single_stmt); 10269 10270 if (ctx->record_type) 10271 lower_omp_single_copy (single_stmt, &bind_body, ctx); 10272 else 10273 lower_omp_single_simple (single_stmt, &bind_body); 10274 10275 gimple_omp_set_body (single_stmt, NULL); 10276 10277 gimple_seq_add_seq (&bind_body, dlist); 10278 10279 bind_body = maybe_catch_exception (bind_body); 10280 10281 t = gimple_build_omp_return 10282 (!!find_omp_clause (gimple_omp_single_clauses (single_stmt), 10283 OMP_CLAUSE_NOWAIT)); 10284 gimple_seq_add_stmt (&bind_body_tail, t); 10285 maybe_add_implicit_barrier_cancel (ctx, &bind_body_tail); 10286 if (ctx->record_type) 10287 { 10288 gimple_stmt_iterator gsi = gsi_start (bind_body_tail); 10289 tree clobber = build_constructor (ctx->record_type, NULL); 10290 TREE_THIS_VOLATILE (clobber) = 1; 10291 gsi_insert_after (&gsi, gimple_build_assign (ctx->sender_decl, 10292 clobber), GSI_SAME_STMT); 10293 } 10294 gimple_seq_add_seq (&bind_body, bind_body_tail); 10295 gimple_bind_set_body (bind, bind_body); 10296 10297 pop_gimplify_context (bind); 10298 10299 gimple_bind_append_vars (bind, ctx->block_vars); 10300 BLOCK_VARS (block) = ctx->block_vars; 10301 if (BLOCK_VARS (block)) 10302 TREE_USED (block) = 1; 10303} 10304 10305 10306/* Expand code for an OpenMP master directive. */ 10307 10308static void 10309lower_omp_master (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10310{ 10311 tree block, lab = NULL, x, bfn_decl; 10312 gimple stmt = gsi_stmt (*gsi_p); 10313 gbind *bind; 10314 location_t loc = gimple_location (stmt); 10315 gimple_seq tseq; 10316 10317 push_gimplify_context (); 10318 10319 block = make_node (BLOCK); 10320 bind = gimple_build_bind (NULL, NULL, block); 10321 gsi_replace (gsi_p, bind, true); 10322 gimple_bind_add_stmt (bind, stmt); 10323 10324 bfn_decl = builtin_decl_explicit (BUILT_IN_OMP_GET_THREAD_NUM); 10325 x = build_call_expr_loc (loc, bfn_decl, 0); 10326 x = build2 (EQ_EXPR, boolean_type_node, x, integer_zero_node); 10327 x = build3 (COND_EXPR, void_type_node, x, NULL, build_and_jump (&lab)); 10328 tseq = NULL; 10329 gimplify_and_add (x, &tseq); 10330 gimple_bind_add_seq (bind, tseq); 10331 10332 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10333 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10334 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10335 gimple_omp_set_body (stmt, NULL); 10336 10337 gimple_bind_add_stmt (bind, gimple_build_label (lab)); 10338 10339 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10340 10341 pop_gimplify_context (bind); 10342 10343 gimple_bind_append_vars (bind, ctx->block_vars); 10344 BLOCK_VARS (block) = ctx->block_vars; 10345} 10346 10347 10348/* Expand code for an OpenMP taskgroup directive. */ 10349 10350static void 10351lower_omp_taskgroup (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10352{ 10353 gimple stmt = gsi_stmt (*gsi_p); 10354 gcall *x; 10355 gbind *bind; 10356 tree block = make_node (BLOCK); 10357 10358 bind = gimple_build_bind (NULL, NULL, block); 10359 gsi_replace (gsi_p, bind, true); 10360 gimple_bind_add_stmt (bind, stmt); 10361 10362 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_TASKGROUP_START), 10363 0); 10364 gimple_bind_add_stmt (bind, x); 10365 10366 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10367 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10368 gimple_omp_set_body (stmt, NULL); 10369 10370 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10371 10372 gimple_bind_append_vars (bind, ctx->block_vars); 10373 BLOCK_VARS (block) = ctx->block_vars; 10374} 10375 10376 10377/* Expand code for an OpenMP ordered directive. */ 10378 10379static void 10380lower_omp_ordered (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10381{ 10382 tree block; 10383 gimple stmt = gsi_stmt (*gsi_p); 10384 gcall *x; 10385 gbind *bind; 10386 10387 push_gimplify_context (); 10388 10389 block = make_node (BLOCK); 10390 bind = gimple_build_bind (NULL, NULL, block); 10391 gsi_replace (gsi_p, bind, true); 10392 gimple_bind_add_stmt (bind, stmt); 10393 10394 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_START), 10395 0); 10396 gimple_bind_add_stmt (bind, x); 10397 10398 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10399 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10400 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10401 gimple_omp_set_body (stmt, NULL); 10402 10403 x = gimple_build_call (builtin_decl_explicit (BUILT_IN_GOMP_ORDERED_END), 0); 10404 gimple_bind_add_stmt (bind, x); 10405 10406 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10407 10408 pop_gimplify_context (bind); 10409 10410 gimple_bind_append_vars (bind, ctx->block_vars); 10411 BLOCK_VARS (block) = gimple_bind_vars (bind); 10412} 10413 10414 10415/* Gimplify a GIMPLE_OMP_CRITICAL statement. This is a relatively simple 10416 substitution of a couple of function calls. But in the NAMED case, 10417 requires that languages coordinate a symbol name. It is therefore 10418 best put here in common code. */ 10419 10420static GTY(()) hash_map<tree, tree> *critical_name_mutexes; 10421 10422static void 10423lower_omp_critical (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10424{ 10425 tree block; 10426 tree name, lock, unlock; 10427 gomp_critical *stmt = as_a <gomp_critical *> (gsi_stmt (*gsi_p)); 10428 gbind *bind; 10429 location_t loc = gimple_location (stmt); 10430 gimple_seq tbody; 10431 10432 name = gimple_omp_critical_name (stmt); 10433 if (name) 10434 { 10435 tree decl; 10436 10437 if (!critical_name_mutexes) 10438 critical_name_mutexes = hash_map<tree, tree>::create_ggc (10); 10439 10440 tree *n = critical_name_mutexes->get (name); 10441 if (n == NULL) 10442 { 10443 char *new_str; 10444 10445 decl = create_tmp_var_raw (ptr_type_node); 10446 10447 new_str = ACONCAT ((".gomp_critical_user_", 10448 IDENTIFIER_POINTER (name), NULL)); 10449 DECL_NAME (decl) = get_identifier (new_str); 10450 TREE_PUBLIC (decl) = 1; 10451 TREE_STATIC (decl) = 1; 10452 DECL_COMMON (decl) = 1; 10453 DECL_ARTIFICIAL (decl) = 1; 10454 DECL_IGNORED_P (decl) = 1; 10455 10456 varpool_node::finalize_decl (decl); 10457 10458 critical_name_mutexes->put (name, decl); 10459 } 10460 else 10461 decl = *n; 10462 10463 /* If '#pragma omp critical' is inside offloaded region or 10464 inside function marked as offloadable, the symbol must be 10465 marked as offloadable too. */ 10466 omp_context *octx; 10467 if (cgraph_node::get (current_function_decl)->offloadable) 10468 varpool_node::get_create (decl)->offloadable = 1; 10469 else 10470 for (octx = ctx->outer; octx; octx = octx->outer) 10471 if (is_gimple_omp_offloaded (octx->stmt)) 10472 { 10473 varpool_node::get_create (decl)->offloadable = 1; 10474 break; 10475 } 10476 10477 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_START); 10478 lock = build_call_expr_loc (loc, lock, 1, build_fold_addr_expr_loc (loc, decl)); 10479 10480 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_NAME_END); 10481 unlock = build_call_expr_loc (loc, unlock, 1, 10482 build_fold_addr_expr_loc (loc, decl)); 10483 } 10484 else 10485 { 10486 lock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_START); 10487 lock = build_call_expr_loc (loc, lock, 0); 10488 10489 unlock = builtin_decl_explicit (BUILT_IN_GOMP_CRITICAL_END); 10490 unlock = build_call_expr_loc (loc, unlock, 0); 10491 } 10492 10493 push_gimplify_context (); 10494 10495 block = make_node (BLOCK); 10496 bind = gimple_build_bind (NULL, NULL, block); 10497 gsi_replace (gsi_p, bind, true); 10498 gimple_bind_add_stmt (bind, stmt); 10499 10500 tbody = gimple_bind_body (bind); 10501 gimplify_and_add (lock, &tbody); 10502 gimple_bind_set_body (bind, tbody); 10503 10504 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10505 gimple_omp_set_body (stmt, maybe_catch_exception (gimple_omp_body (stmt))); 10506 gimple_bind_add_seq (bind, gimple_omp_body (stmt)); 10507 gimple_omp_set_body (stmt, NULL); 10508 10509 tbody = gimple_bind_body (bind); 10510 gimplify_and_add (unlock, &tbody); 10511 gimple_bind_set_body (bind, tbody); 10512 10513 gimple_bind_add_stmt (bind, gimple_build_omp_return (true)); 10514 10515 pop_gimplify_context (bind); 10516 gimple_bind_append_vars (bind, ctx->block_vars); 10517 BLOCK_VARS (block) = gimple_bind_vars (bind); 10518} 10519 10520 10521/* A subroutine of lower_omp_for. Generate code to emit the predicate 10522 for a lastprivate clause. Given a loop control predicate of (V 10523 cond N2), we gate the clause on (!(V cond N2)). The lowered form 10524 is appended to *DLIST, iterator initialization is appended to 10525 *BODY_P. */ 10526 10527static void 10528lower_omp_for_lastprivate (struct omp_for_data *fd, gimple_seq *body_p, 10529 gimple_seq *dlist, struct omp_context *ctx) 10530{ 10531 tree clauses, cond, vinit; 10532 enum tree_code cond_code; 10533 gimple_seq stmts; 10534 10535 cond_code = fd->loop.cond_code; 10536 cond_code = cond_code == LT_EXPR ? GE_EXPR : LE_EXPR; 10537 10538 /* When possible, use a strict equality expression. This can let VRP 10539 type optimizations deduce the value and remove a copy. */ 10540 if (tree_fits_shwi_p (fd->loop.step)) 10541 { 10542 HOST_WIDE_INT step = tree_to_shwi (fd->loop.step); 10543 if (step == 1 || step == -1) 10544 cond_code = EQ_EXPR; 10545 } 10546 10547 tree n2 = fd->loop.n2; 10548 if (fd->collapse > 1 10549 && TREE_CODE (n2) != INTEGER_CST 10550 && gimple_omp_for_combined_into_p (fd->for_stmt) 10551 && gimple_code (ctx->outer->stmt) == GIMPLE_OMP_FOR) 10552 { 10553 gomp_for *gfor = as_a <gomp_for *> (ctx->outer->stmt); 10554 if (gimple_omp_for_kind (gfor) == GF_OMP_FOR_KIND_FOR) 10555 { 10556 struct omp_for_data outer_fd; 10557 extract_omp_for_data (gfor, &outer_fd, NULL); 10558 n2 = fold_convert (TREE_TYPE (n2), outer_fd.loop.n2); 10559 } 10560 } 10561 cond = build2 (cond_code, boolean_type_node, fd->loop.v, n2); 10562 10563 clauses = gimple_omp_for_clauses (fd->for_stmt); 10564 stmts = NULL; 10565 lower_lastprivate_clauses (clauses, cond, &stmts, ctx); 10566 if (!gimple_seq_empty_p (stmts)) 10567 { 10568 gimple_seq_add_seq (&stmts, *dlist); 10569 *dlist = stmts; 10570 10571 /* Optimize: v = 0; is usually cheaper than v = some_other_constant. */ 10572 vinit = fd->loop.n1; 10573 if (cond_code == EQ_EXPR 10574 && tree_fits_shwi_p (fd->loop.n2) 10575 && ! integer_zerop (fd->loop.n2)) 10576 vinit = build_int_cst (TREE_TYPE (fd->loop.v), 0); 10577 else 10578 vinit = unshare_expr (vinit); 10579 10580 /* Initialize the iterator variable, so that threads that don't execute 10581 any iterations don't execute the lastprivate clauses by accident. */ 10582 gimplify_assign (fd->loop.v, vinit, body_p); 10583 } 10584} 10585 10586 10587/* Lower code for an OMP loop directive. */ 10588 10589static void 10590lower_omp_for (gimple_stmt_iterator *gsi_p, omp_context *ctx) 10591{ 10592 tree *rhs_p, block; 10593 struct omp_for_data fd, *fdp = NULL; 10594 gomp_for *stmt = as_a <gomp_for *> (gsi_stmt (*gsi_p)); 10595 gbind *new_stmt; 10596 gimple_seq omp_for_body, body, dlist; 10597 size_t i; 10598 10599 push_gimplify_context (); 10600 10601 lower_omp (gimple_omp_for_pre_body_ptr (stmt), ctx); 10602 10603 block = make_node (BLOCK); 10604 new_stmt = gimple_build_bind (NULL, NULL, block); 10605 /* Replace at gsi right away, so that 'stmt' is no member 10606 of a sequence anymore as we're going to add to to a different 10607 one below. */ 10608 gsi_replace (gsi_p, new_stmt, true); 10609 10610 /* Move declaration of temporaries in the loop body before we make 10611 it go away. */ 10612 omp_for_body = gimple_omp_body (stmt); 10613 if (!gimple_seq_empty_p (omp_for_body) 10614 && gimple_code (gimple_seq_first_stmt (omp_for_body)) == GIMPLE_BIND) 10615 { 10616 gbind *inner_bind 10617 = as_a <gbind *> (gimple_seq_first_stmt (omp_for_body)); 10618 tree vars = gimple_bind_vars (inner_bind); 10619 gimple_bind_append_vars (new_stmt, vars); 10620 /* bind_vars/BLOCK_VARS are being moved to new_stmt/block, don't 10621 keep them on the inner_bind and it's block. */ 10622 gimple_bind_set_vars (inner_bind, NULL_TREE); 10623 if (gimple_bind_block (inner_bind)) 10624 BLOCK_VARS (gimple_bind_block (inner_bind)) = NULL_TREE; 10625 } 10626 10627 if (gimple_omp_for_combined_into_p (stmt)) 10628 { 10629 extract_omp_for_data (stmt, &fd, NULL); 10630 fdp = &fd; 10631 10632 /* We need two temporaries with fd.loop.v type (istart/iend) 10633 and then (fd.collapse - 1) temporaries with the same 10634 type for count2 ... countN-1 vars if not constant. */ 10635 size_t count = 2; 10636 tree type = fd.iter_type; 10637 if (fd.collapse > 1 10638 && TREE_CODE (fd.loop.n2) != INTEGER_CST) 10639 count += fd.collapse - 1; 10640 bool parallel_for = gimple_omp_for_kind (stmt) == GF_OMP_FOR_KIND_FOR; 10641 tree outerc = NULL, *pc = gimple_omp_for_clauses_ptr (stmt); 10642 tree clauses = *pc; 10643 if (parallel_for) 10644 outerc 10645 = find_omp_clause (gimple_omp_parallel_clauses (ctx->outer->stmt), 10646 OMP_CLAUSE__LOOPTEMP_); 10647 for (i = 0; i < count; i++) 10648 { 10649 tree temp; 10650 if (parallel_for) 10651 { 10652 gcc_assert (outerc); 10653 temp = lookup_decl (OMP_CLAUSE_DECL (outerc), ctx->outer); 10654 outerc = find_omp_clause (OMP_CLAUSE_CHAIN (outerc), 10655 OMP_CLAUSE__LOOPTEMP_); 10656 } 10657 else 10658 { 10659 temp = create_tmp_var (type); 10660 insert_decl_map (&ctx->outer->cb, temp, temp); 10661 } 10662 *pc = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE__LOOPTEMP_); 10663 OMP_CLAUSE_DECL (*pc) = temp; 10664 pc = &OMP_CLAUSE_CHAIN (*pc); 10665 } 10666 *pc = clauses; 10667 } 10668 10669 /* The pre-body and input clauses go before the lowered GIMPLE_OMP_FOR. */ 10670 dlist = NULL; 10671 body = NULL; 10672 lower_rec_input_clauses (gimple_omp_for_clauses (stmt), &body, &dlist, ctx, 10673 fdp); 10674 gimple_seq_add_seq (&body, gimple_omp_for_pre_body (stmt)); 10675 10676 lower_omp (gimple_omp_body_ptr (stmt), ctx); 10677 10678 /* Lower the header expressions. At this point, we can assume that 10679 the header is of the form: 10680 10681 #pragma omp for (V = VAL1; V {<|>|<=|>=} VAL2; V = V [+-] VAL3) 10682 10683 We just need to make sure that VAL1, VAL2 and VAL3 are lowered 10684 using the .omp_data_s mapping, if needed. */ 10685 for (i = 0; i < gimple_omp_for_collapse (stmt); i++) 10686 { 10687 rhs_p = gimple_omp_for_initial_ptr (stmt, i); 10688 if (!is_gimple_min_invariant (*rhs_p)) 10689 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10690 10691 rhs_p = gimple_omp_for_final_ptr (stmt, i); 10692 if (!is_gimple_min_invariant (*rhs_p)) 10693 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10694 10695 rhs_p = &TREE_OPERAND (gimple_omp_for_incr (stmt, i), 1); 10696 if (!is_gimple_min_invariant (*rhs_p)) 10697 *rhs_p = get_formal_tmp_var (*rhs_p, &body); 10698 } 10699 10700 /* Once lowered, extract the bounds and clauses. */ 10701 extract_omp_for_data (stmt, &fd, NULL); 10702 10703 lower_omp_for_lastprivate (&fd, &body, &dlist, ctx); 10704 10705 gimple_seq_add_stmt (&body, stmt); 10706 gimple_seq_add_seq (&body, gimple_omp_body (stmt)); 10707 10708 gimple_seq_add_stmt (&body, gimple_build_omp_continue (fd.loop.v, 10709 fd.loop.v)); 10710 10711 /* After the loop, add exit clauses. */ 10712 lower_reduction_clauses (gimple_omp_for_clauses (stmt), &body, ctx); 10713 10714 if (ctx->cancellable) 10715 gimple_seq_add_stmt (&body, gimple_build_label (ctx->cancel_label)); 10716 10717 gimple_seq_add_seq (&body, dlist); 10718 10719 body = maybe_catch_exception (body); 10720 10721 /* Region exit marker goes at the end of the loop body. */ 10722 gimple_seq_add_stmt (&body, gimple_build_omp_return (fd.have_nowait)); 10723 maybe_add_implicit_barrier_cancel (ctx, &body); 10724 pop_gimplify_context (new_stmt); 10725 10726 gimple_bind_append_vars (new_stmt, ctx->block_vars); 10727 BLOCK_VARS (block) = gimple_bind_vars (new_stmt); 10728 if (BLOCK_VARS (block)) 10729 TREE_USED (block) = 1; 10730 10731 gimple_bind_set_body (new_stmt, body); 10732 gimple_omp_set_body (stmt, NULL); 10733 gimple_omp_for_set_pre_body (stmt, NULL); 10734} 10735 10736/* Callback for walk_stmts. Check if the current statement only contains 10737 GIMPLE_OMP_FOR or GIMPLE_OMP_SECTIONS. */ 10738 10739static tree 10740check_combined_parallel (gimple_stmt_iterator *gsi_p, 10741 bool *handled_ops_p, 10742 struct walk_stmt_info *wi) 10743{ 10744 int *info = (int *) wi->info; 10745 gimple stmt = gsi_stmt (*gsi_p); 10746 10747 *handled_ops_p = true; 10748 switch (gimple_code (stmt)) 10749 { 10750 WALK_SUBSTMTS; 10751 10752 case GIMPLE_OMP_FOR: 10753 case GIMPLE_OMP_SECTIONS: 10754 *info = *info == 0 ? 1 : -1; 10755 break; 10756 default: 10757 *info = -1; 10758 break; 10759 } 10760 return NULL; 10761} 10762 10763struct omp_taskcopy_context 10764{ 10765 /* This field must be at the beginning, as we do "inheritance": Some 10766 callback functions for tree-inline.c (e.g., omp_copy_decl) 10767 receive a copy_body_data pointer that is up-casted to an 10768 omp_context pointer. */ 10769 copy_body_data cb; 10770 omp_context *ctx; 10771}; 10772 10773static tree 10774task_copyfn_copy_decl (tree var, copy_body_data *cb) 10775{ 10776 struct omp_taskcopy_context *tcctx = (struct omp_taskcopy_context *) cb; 10777 10778 if (splay_tree_lookup (tcctx->ctx->sfield_map, (splay_tree_key) var)) 10779 return create_tmp_var (TREE_TYPE (var)); 10780 10781 return var; 10782} 10783 10784static tree 10785task_copyfn_remap_type (struct omp_taskcopy_context *tcctx, tree orig_type) 10786{ 10787 tree name, new_fields = NULL, type, f; 10788 10789 type = lang_hooks.types.make_type (RECORD_TYPE); 10790 name = DECL_NAME (TYPE_NAME (orig_type)); 10791 name = build_decl (gimple_location (tcctx->ctx->stmt), 10792 TYPE_DECL, name, type); 10793 TYPE_NAME (type) = name; 10794 10795 for (f = TYPE_FIELDS (orig_type); f ; f = TREE_CHAIN (f)) 10796 { 10797 tree new_f = copy_node (f); 10798 DECL_CONTEXT (new_f) = type; 10799 TREE_TYPE (new_f) = remap_type (TREE_TYPE (f), &tcctx->cb); 10800 TREE_CHAIN (new_f) = new_fields; 10801 walk_tree (&DECL_SIZE (new_f), copy_tree_body_r, &tcctx->cb, NULL); 10802 walk_tree (&DECL_SIZE_UNIT (new_f), copy_tree_body_r, &tcctx->cb, NULL); 10803 walk_tree (&DECL_FIELD_OFFSET (new_f), copy_tree_body_r, 10804 &tcctx->cb, NULL); 10805 new_fields = new_f; 10806 tcctx->cb.decl_map->put (f, new_f); 10807 } 10808 TYPE_FIELDS (type) = nreverse (new_fields); 10809 layout_type (type); 10810 return type; 10811} 10812 10813/* Create task copyfn. */ 10814 10815static void 10816create_task_copyfn (gomp_task *task_stmt, omp_context *ctx) 10817{ 10818 struct function *child_cfun; 10819 tree child_fn, t, c, src, dst, f, sf, arg, sarg, decl; 10820 tree record_type, srecord_type, bind, list; 10821 bool record_needs_remap = false, srecord_needs_remap = false; 10822 splay_tree_node n; 10823 struct omp_taskcopy_context tcctx; 10824 location_t loc = gimple_location (task_stmt); 10825 10826 child_fn = gimple_omp_task_copy_fn (task_stmt); 10827 child_cfun = DECL_STRUCT_FUNCTION (child_fn); 10828 gcc_assert (child_cfun->cfg == NULL); 10829 DECL_SAVED_TREE (child_fn) = alloc_stmt_list (); 10830 10831 /* Reset DECL_CONTEXT on function arguments. */ 10832 for (t = DECL_ARGUMENTS (child_fn); t; t = DECL_CHAIN (t)) 10833 DECL_CONTEXT (t) = child_fn; 10834 10835 /* Populate the function. */ 10836 push_gimplify_context (); 10837 push_cfun (child_cfun); 10838 10839 bind = build3 (BIND_EXPR, void_type_node, NULL, NULL, NULL); 10840 TREE_SIDE_EFFECTS (bind) = 1; 10841 list = NULL; 10842 DECL_SAVED_TREE (child_fn) = bind; 10843 DECL_SOURCE_LOCATION (child_fn) = gimple_location (task_stmt); 10844 10845 /* Remap src and dst argument types if needed. */ 10846 record_type = ctx->record_type; 10847 srecord_type = ctx->srecord_type; 10848 for (f = TYPE_FIELDS (record_type); f ; f = DECL_CHAIN (f)) 10849 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 10850 { 10851 record_needs_remap = true; 10852 break; 10853 } 10854 for (f = TYPE_FIELDS (srecord_type); f ; f = DECL_CHAIN (f)) 10855 if (variably_modified_type_p (TREE_TYPE (f), ctx->cb.src_fn)) 10856 { 10857 srecord_needs_remap = true; 10858 break; 10859 } 10860 10861 if (record_needs_remap || srecord_needs_remap) 10862 { 10863 memset (&tcctx, '\0', sizeof (tcctx)); 10864 tcctx.cb.src_fn = ctx->cb.src_fn; 10865 tcctx.cb.dst_fn = child_fn; 10866 tcctx.cb.src_node = cgraph_node::get (tcctx.cb.src_fn); 10867 gcc_checking_assert (tcctx.cb.src_node); 10868 tcctx.cb.dst_node = tcctx.cb.src_node; 10869 tcctx.cb.src_cfun = ctx->cb.src_cfun; 10870 tcctx.cb.copy_decl = task_copyfn_copy_decl; 10871 tcctx.cb.eh_lp_nr = 0; 10872 tcctx.cb.transform_call_graph_edges = CB_CGE_MOVE; 10873 tcctx.cb.decl_map = new hash_map<tree, tree>; 10874 tcctx.ctx = ctx; 10875 10876 if (record_needs_remap) 10877 record_type = task_copyfn_remap_type (&tcctx, record_type); 10878 if (srecord_needs_remap) 10879 srecord_type = task_copyfn_remap_type (&tcctx, srecord_type); 10880 } 10881 else 10882 tcctx.cb.decl_map = NULL; 10883 10884 arg = DECL_ARGUMENTS (child_fn); 10885 TREE_TYPE (arg) = build_pointer_type (record_type); 10886 sarg = DECL_CHAIN (arg); 10887 TREE_TYPE (sarg) = build_pointer_type (srecord_type); 10888 10889 /* First pass: initialize temporaries used in record_type and srecord_type 10890 sizes and field offsets. */ 10891 if (tcctx.cb.decl_map) 10892 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 10893 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) 10894 { 10895 tree *p; 10896 10897 decl = OMP_CLAUSE_DECL (c); 10898 p = tcctx.cb.decl_map->get (decl); 10899 if (p == NULL) 10900 continue; 10901 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10902 sf = (tree) n->value; 10903 sf = *tcctx.cb.decl_map->get (sf); 10904 src = build_simple_mem_ref_loc (loc, sarg); 10905 src = omp_build_component_ref (src, sf); 10906 t = build2 (MODIFY_EXPR, TREE_TYPE (*p), *p, src); 10907 append_to_statement_list (t, &list); 10908 } 10909 10910 /* Second pass: copy shared var pointers and copy construct non-VLA 10911 firstprivate vars. */ 10912 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 10913 switch (OMP_CLAUSE_CODE (c)) 10914 { 10915 case OMP_CLAUSE_SHARED: 10916 decl = OMP_CLAUSE_DECL (c); 10917 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10918 if (n == NULL) 10919 break; 10920 f = (tree) n->value; 10921 if (tcctx.cb.decl_map) 10922 f = *tcctx.cb.decl_map->get (f); 10923 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10924 sf = (tree) n->value; 10925 if (tcctx.cb.decl_map) 10926 sf = *tcctx.cb.decl_map->get (sf); 10927 src = build_simple_mem_ref_loc (loc, sarg); 10928 src = omp_build_component_ref (src, sf); 10929 dst = build_simple_mem_ref_loc (loc, arg); 10930 dst = omp_build_component_ref (dst, f); 10931 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); 10932 append_to_statement_list (t, &list); 10933 break; 10934 case OMP_CLAUSE_FIRSTPRIVATE: 10935 decl = OMP_CLAUSE_DECL (c); 10936 if (is_variable_sized (decl)) 10937 break; 10938 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10939 if (n == NULL) 10940 break; 10941 f = (tree) n->value; 10942 if (tcctx.cb.decl_map) 10943 f = *tcctx.cb.decl_map->get (f); 10944 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10945 if (n != NULL) 10946 { 10947 sf = (tree) n->value; 10948 if (tcctx.cb.decl_map) 10949 sf = *tcctx.cb.decl_map->get (sf); 10950 src = build_simple_mem_ref_loc (loc, sarg); 10951 src = omp_build_component_ref (src, sf); 10952 if (use_pointer_for_field (decl, NULL) || is_reference (decl)) 10953 src = build_simple_mem_ref_loc (loc, src); 10954 } 10955 else 10956 src = decl; 10957 dst = build_simple_mem_ref_loc (loc, arg); 10958 dst = omp_build_component_ref (dst, f); 10959 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); 10960 append_to_statement_list (t, &list); 10961 break; 10962 case OMP_CLAUSE_PRIVATE: 10963 if (! OMP_CLAUSE_PRIVATE_OUTER_REF (c)) 10964 break; 10965 decl = OMP_CLAUSE_DECL (c); 10966 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 10967 f = (tree) n->value; 10968 if (tcctx.cb.decl_map) 10969 f = *tcctx.cb.decl_map->get (f); 10970 n = splay_tree_lookup (ctx->sfield_map, (splay_tree_key) decl); 10971 if (n != NULL) 10972 { 10973 sf = (tree) n->value; 10974 if (tcctx.cb.decl_map) 10975 sf = *tcctx.cb.decl_map->get (sf); 10976 src = build_simple_mem_ref_loc (loc, sarg); 10977 src = omp_build_component_ref (src, sf); 10978 if (use_pointer_for_field (decl, NULL)) 10979 src = build_simple_mem_ref_loc (loc, src); 10980 } 10981 else 10982 src = decl; 10983 dst = build_simple_mem_ref_loc (loc, arg); 10984 dst = omp_build_component_ref (dst, f); 10985 t = build2 (MODIFY_EXPR, TREE_TYPE (dst), dst, src); 10986 append_to_statement_list (t, &list); 10987 break; 10988 default: 10989 break; 10990 } 10991 10992 /* Last pass: handle VLA firstprivates. */ 10993 if (tcctx.cb.decl_map) 10994 for (c = gimple_omp_task_clauses (task_stmt); c; c = OMP_CLAUSE_CHAIN (c)) 10995 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_FIRSTPRIVATE) 10996 { 10997 tree ind, ptr, df; 10998 10999 decl = OMP_CLAUSE_DECL (c); 11000 if (!is_variable_sized (decl)) 11001 continue; 11002 n = splay_tree_lookup (ctx->field_map, (splay_tree_key) decl); 11003 if (n == NULL) 11004 continue; 11005 f = (tree) n->value; 11006 f = *tcctx.cb.decl_map->get (f); 11007 gcc_assert (DECL_HAS_VALUE_EXPR_P (decl)); 11008 ind = DECL_VALUE_EXPR (decl); 11009 gcc_assert (TREE_CODE (ind) == INDIRECT_REF); 11010 gcc_assert (DECL_P (TREE_OPERAND (ind, 0))); 11011 n = splay_tree_lookup (ctx->sfield_map, 11012 (splay_tree_key) TREE_OPERAND (ind, 0)); 11013 sf = (tree) n->value; 11014 sf = *tcctx.cb.decl_map->get (sf); 11015 src = build_simple_mem_ref_loc (loc, sarg); 11016 src = omp_build_component_ref (src, sf); 11017 src = build_simple_mem_ref_loc (loc, src); 11018 dst = build_simple_mem_ref_loc (loc, arg); 11019 dst = omp_build_component_ref (dst, f); 11020 t = lang_hooks.decls.omp_clause_copy_ctor (c, dst, src); 11021 append_to_statement_list (t, &list); 11022 n = splay_tree_lookup (ctx->field_map, 11023 (splay_tree_key) TREE_OPERAND (ind, 0)); 11024 df = (tree) n->value; 11025 df = *tcctx.cb.decl_map->get (df); 11026 ptr = build_simple_mem_ref_loc (loc, arg); 11027 ptr = omp_build_component_ref (ptr, df); 11028 t = build2 (MODIFY_EXPR, TREE_TYPE (ptr), ptr, 11029 build_fold_addr_expr_loc (loc, dst)); 11030 append_to_statement_list (t, &list); 11031 } 11032 11033 t = build1 (RETURN_EXPR, void_type_node, NULL); 11034 append_to_statement_list (t, &list); 11035 11036 if (tcctx.cb.decl_map) 11037 delete tcctx.cb.decl_map; 11038 pop_gimplify_context (NULL); 11039 BIND_EXPR_BODY (bind) = list; 11040 pop_cfun (); 11041} 11042 11043static void 11044lower_depend_clauses (gimple stmt, gimple_seq *iseq, gimple_seq *oseq) 11045{ 11046 tree c, clauses; 11047 gimple g; 11048 size_t n_in = 0, n_out = 0, idx = 2, i; 11049 11050 clauses = find_omp_clause (gimple_omp_task_clauses (stmt), 11051 OMP_CLAUSE_DEPEND); 11052 gcc_assert (clauses); 11053 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 11054 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND) 11055 switch (OMP_CLAUSE_DEPEND_KIND (c)) 11056 { 11057 case OMP_CLAUSE_DEPEND_IN: 11058 n_in++; 11059 break; 11060 case OMP_CLAUSE_DEPEND_OUT: 11061 case OMP_CLAUSE_DEPEND_INOUT: 11062 n_out++; 11063 break; 11064 default: 11065 gcc_unreachable (); 11066 } 11067 tree type = build_array_type_nelts (ptr_type_node, n_in + n_out + 2); 11068 tree array = create_tmp_var (type); 11069 tree r = build4 (ARRAY_REF, ptr_type_node, array, size_int (0), NULL_TREE, 11070 NULL_TREE); 11071 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_in + n_out)); 11072 gimple_seq_add_stmt (iseq, g); 11073 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (1), NULL_TREE, 11074 NULL_TREE); 11075 g = gimple_build_assign (r, build_int_cst (ptr_type_node, n_out)); 11076 gimple_seq_add_stmt (iseq, g); 11077 for (i = 0; i < 2; i++) 11078 { 11079 if ((i ? n_in : n_out) == 0) 11080 continue; 11081 for (c = clauses; c; c = OMP_CLAUSE_CHAIN (c)) 11082 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_DEPEND 11083 && ((OMP_CLAUSE_DEPEND_KIND (c) != OMP_CLAUSE_DEPEND_IN) ^ i)) 11084 { 11085 tree t = OMP_CLAUSE_DECL (c); 11086 t = fold_convert (ptr_type_node, t); 11087 gimplify_expr (&t, iseq, NULL, is_gimple_val, fb_rvalue); 11088 r = build4 (ARRAY_REF, ptr_type_node, array, size_int (idx++), 11089 NULL_TREE, NULL_TREE); 11090 g = gimple_build_assign (r, t); 11091 gimple_seq_add_stmt (iseq, g); 11092 } 11093 } 11094 tree *p = gimple_omp_task_clauses_ptr (stmt); 11095 c = build_omp_clause (UNKNOWN_LOCATION, OMP_CLAUSE_DEPEND); 11096 OMP_CLAUSE_DECL (c) = build_fold_addr_expr (array); 11097 OMP_CLAUSE_CHAIN (c) = *p; 11098 *p = c; 11099 tree clobber = build_constructor (type, NULL); 11100 TREE_THIS_VOLATILE (clobber) = 1; 11101 g = gimple_build_assign (array, clobber); 11102 gimple_seq_add_stmt (oseq, g); 11103} 11104 11105/* Lower the OpenMP parallel or task directive in the current statement 11106 in GSI_P. CTX holds context information for the directive. */ 11107 11108static void 11109lower_omp_taskreg (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11110{ 11111 tree clauses; 11112 tree child_fn, t; 11113 gimple stmt = gsi_stmt (*gsi_p); 11114 gbind *par_bind, *bind, *dep_bind = NULL; 11115 gimple_seq par_body, olist, ilist, par_olist, par_rlist, par_ilist, new_body; 11116 location_t loc = gimple_location (stmt); 11117 11118 clauses = gimple_omp_taskreg_clauses (stmt); 11119 par_bind 11120 = as_a <gbind *> (gimple_seq_first_stmt (gimple_omp_body (stmt))); 11121 par_body = gimple_bind_body (par_bind); 11122 child_fn = ctx->cb.dst_fn; 11123 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL 11124 && !gimple_omp_parallel_combined_p (stmt)) 11125 { 11126 struct walk_stmt_info wi; 11127 int ws_num = 0; 11128 11129 memset (&wi, 0, sizeof (wi)); 11130 wi.info = &ws_num; 11131 wi.val_only = true; 11132 walk_gimple_seq (par_body, check_combined_parallel, NULL, &wi); 11133 if (ws_num == 1) 11134 gimple_omp_parallel_set_combined_p (stmt, true); 11135 } 11136 gimple_seq dep_ilist = NULL; 11137 gimple_seq dep_olist = NULL; 11138 if (gimple_code (stmt) == GIMPLE_OMP_TASK 11139 && find_omp_clause (clauses, OMP_CLAUSE_DEPEND)) 11140 { 11141 push_gimplify_context (); 11142 dep_bind = gimple_build_bind (NULL, NULL, make_node (BLOCK)); 11143 lower_depend_clauses (stmt, &dep_ilist, &dep_olist); 11144 } 11145 11146 if (ctx->srecord_type) 11147 create_task_copyfn (as_a <gomp_task *> (stmt), ctx); 11148 11149 push_gimplify_context (); 11150 11151 par_olist = NULL; 11152 par_ilist = NULL; 11153 par_rlist = NULL; 11154 lower_rec_input_clauses (clauses, &par_ilist, &par_olist, ctx, NULL); 11155 lower_omp (&par_body, ctx); 11156 if (gimple_code (stmt) == GIMPLE_OMP_PARALLEL) 11157 lower_reduction_clauses (clauses, &par_rlist, ctx); 11158 11159 /* Declare all the variables created by mapping and the variables 11160 declared in the scope of the parallel body. */ 11161 record_vars_into (ctx->block_vars, child_fn); 11162 record_vars_into (gimple_bind_vars (par_bind), child_fn); 11163 11164 if (ctx->record_type) 11165 { 11166 ctx->sender_decl 11167 = create_tmp_var (ctx->srecord_type ? ctx->srecord_type 11168 : ctx->record_type, ".omp_data_o"); 11169 DECL_NAMELESS (ctx->sender_decl) = 1; 11170 TREE_ADDRESSABLE (ctx->sender_decl) = 1; 11171 gimple_omp_taskreg_set_data_arg (stmt, ctx->sender_decl); 11172 } 11173 11174 olist = NULL; 11175 ilist = NULL; 11176 lower_send_clauses (clauses, &ilist, &olist, ctx); 11177 lower_send_shared_vars (&ilist, &olist, ctx); 11178 11179 if (ctx->record_type) 11180 { 11181 tree clobber = build_constructor (TREE_TYPE (ctx->sender_decl), NULL); 11182 TREE_THIS_VOLATILE (clobber) = 1; 11183 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, 11184 clobber)); 11185 } 11186 11187 /* Once all the expansions are done, sequence all the different 11188 fragments inside gimple_omp_body. */ 11189 11190 new_body = NULL; 11191 11192 if (ctx->record_type) 11193 { 11194 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 11195 /* fixup_child_record_type might have changed receiver_decl's type. */ 11196 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); 11197 gimple_seq_add_stmt (&new_body, 11198 gimple_build_assign (ctx->receiver_decl, t)); 11199 } 11200 11201 gimple_seq_add_seq (&new_body, par_ilist); 11202 gimple_seq_add_seq (&new_body, par_body); 11203 gimple_seq_add_seq (&new_body, par_rlist); 11204 if (ctx->cancellable) 11205 gimple_seq_add_stmt (&new_body, gimple_build_label (ctx->cancel_label)); 11206 gimple_seq_add_seq (&new_body, par_olist); 11207 new_body = maybe_catch_exception (new_body); 11208 if (gimple_code (stmt) == GIMPLE_OMP_TASK) 11209 gimple_seq_add_stmt (&new_body, 11210 gimple_build_omp_continue (integer_zero_node, 11211 integer_zero_node)); 11212 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); 11213 gimple_omp_set_body (stmt, new_body); 11214 11215 bind = gimple_build_bind (NULL, NULL, gimple_bind_block (par_bind)); 11216 gsi_replace (gsi_p, dep_bind ? dep_bind : bind, true); 11217 gimple_bind_add_seq (bind, ilist); 11218 gimple_bind_add_stmt (bind, stmt); 11219 gimple_bind_add_seq (bind, olist); 11220 11221 pop_gimplify_context (NULL); 11222 11223 if (dep_bind) 11224 { 11225 gimple_bind_add_seq (dep_bind, dep_ilist); 11226 gimple_bind_add_stmt (dep_bind, bind); 11227 gimple_bind_add_seq (dep_bind, dep_olist); 11228 pop_gimplify_context (dep_bind); 11229 } 11230} 11231 11232/* Lower the GIMPLE_OMP_TARGET in the current statement 11233 in GSI_P. CTX holds context information for the directive. */ 11234 11235static void 11236lower_omp_target (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11237{ 11238 tree clauses; 11239 tree child_fn, t, c; 11240 gomp_target *stmt = as_a <gomp_target *> (gsi_stmt (*gsi_p)); 11241 gbind *tgt_bind, *bind; 11242 gimple_seq tgt_body, olist, ilist, orlist, irlist, new_body; 11243 location_t loc = gimple_location (stmt); 11244 bool offloaded, data_region; 11245 unsigned int map_cnt = 0; 11246 11247 offloaded = is_gimple_omp_offloaded (stmt); 11248 switch (gimple_omp_target_kind (stmt)) 11249 { 11250 case GF_OMP_TARGET_KIND_REGION: 11251 case GF_OMP_TARGET_KIND_UPDATE: 11252 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 11253 case GF_OMP_TARGET_KIND_OACC_KERNELS: 11254 case GF_OMP_TARGET_KIND_OACC_UPDATE: 11255 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 11256 data_region = false; 11257 break; 11258 case GF_OMP_TARGET_KIND_DATA: 11259 case GF_OMP_TARGET_KIND_OACC_DATA: 11260 data_region = true; 11261 break; 11262 default: 11263 gcc_unreachable (); 11264 } 11265 11266 clauses = gimple_omp_target_clauses (stmt); 11267 11268 tgt_bind = NULL; 11269 tgt_body = NULL; 11270 if (offloaded) 11271 { 11272 tgt_bind = gimple_seq_first_stmt_as_a_bind (gimple_omp_body (stmt)); 11273 tgt_body = gimple_bind_body (tgt_bind); 11274 } 11275 else if (data_region) 11276 tgt_body = gimple_omp_body (stmt); 11277 child_fn = ctx->cb.dst_fn; 11278 11279 push_gimplify_context (); 11280 11281 irlist = NULL; 11282 orlist = NULL; 11283 if (offloaded 11284 && is_gimple_omp_oacc (stmt)) 11285 oacc_process_reduction_data (&tgt_body, &irlist, &orlist, ctx); 11286 11287 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 11288 switch (OMP_CLAUSE_CODE (c)) 11289 { 11290 tree var, x; 11291 11292 default: 11293 break; 11294 case OMP_CLAUSE_MAP: 11295#ifdef ENABLE_CHECKING 11296 /* First check what we're prepared to handle in the following. */ 11297 switch (OMP_CLAUSE_MAP_KIND (c)) 11298 { 11299 case GOMP_MAP_ALLOC: 11300 case GOMP_MAP_TO: 11301 case GOMP_MAP_FROM: 11302 case GOMP_MAP_TOFROM: 11303 case GOMP_MAP_POINTER: 11304 case GOMP_MAP_TO_PSET: 11305 break; 11306 case GOMP_MAP_FORCE_ALLOC: 11307 case GOMP_MAP_FORCE_TO: 11308 case GOMP_MAP_FORCE_FROM: 11309 case GOMP_MAP_FORCE_TOFROM: 11310 case GOMP_MAP_FORCE_PRESENT: 11311 case GOMP_MAP_FORCE_DEALLOC: 11312 case GOMP_MAP_FORCE_DEVICEPTR: 11313 gcc_assert (is_gimple_omp_oacc (stmt)); 11314 break; 11315 default: 11316 gcc_unreachable (); 11317 } 11318#endif 11319 /* FALLTHRU */ 11320 case OMP_CLAUSE_TO: 11321 case OMP_CLAUSE_FROM: 11322 var = OMP_CLAUSE_DECL (c); 11323 if (!DECL_P (var)) 11324 { 11325 if (OMP_CLAUSE_CODE (c) != OMP_CLAUSE_MAP 11326 || !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 11327 map_cnt++; 11328 continue; 11329 } 11330 11331 if (DECL_SIZE (var) 11332 && TREE_CODE (DECL_SIZE (var)) != INTEGER_CST) 11333 { 11334 tree var2 = DECL_VALUE_EXPR (var); 11335 gcc_assert (TREE_CODE (var2) == INDIRECT_REF); 11336 var2 = TREE_OPERAND (var2, 0); 11337 gcc_assert (DECL_P (var2)); 11338 var = var2; 11339 } 11340 11341 if (!maybe_lookup_field (var, ctx)) 11342 continue; 11343 11344 if (offloaded) 11345 { 11346 x = build_receiver_ref (var, true, ctx); 11347 tree new_var = lookup_decl (var, ctx); 11348 if (OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 11349 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 11350 && TREE_CODE (TREE_TYPE (var)) == ARRAY_TYPE) 11351 x = build_simple_mem_ref (x); 11352 SET_DECL_VALUE_EXPR (new_var, x); 11353 DECL_HAS_VALUE_EXPR_P (new_var) = 1; 11354 } 11355 map_cnt++; 11356 } 11357 11358 if (offloaded) 11359 { 11360 target_nesting_level++; 11361 lower_omp (&tgt_body, ctx); 11362 target_nesting_level--; 11363 } 11364 else if (data_region) 11365 lower_omp (&tgt_body, ctx); 11366 11367 if (offloaded) 11368 { 11369 /* Declare all the variables created by mapping and the variables 11370 declared in the scope of the target body. */ 11371 record_vars_into (ctx->block_vars, child_fn); 11372 record_vars_into (gimple_bind_vars (tgt_bind), child_fn); 11373 } 11374 11375 olist = NULL; 11376 ilist = NULL; 11377 if (ctx->record_type) 11378 { 11379 ctx->sender_decl 11380 = create_tmp_var (ctx->record_type, ".omp_data_arr"); 11381 DECL_NAMELESS (ctx->sender_decl) = 1; 11382 TREE_ADDRESSABLE (ctx->sender_decl) = 1; 11383 t = make_tree_vec (3); 11384 TREE_VEC_ELT (t, 0) = ctx->sender_decl; 11385 TREE_VEC_ELT (t, 1) 11386 = create_tmp_var (build_array_type_nelts (size_type_node, map_cnt), 11387 ".omp_data_sizes"); 11388 DECL_NAMELESS (TREE_VEC_ELT (t, 1)) = 1; 11389 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 1)) = 1; 11390 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 1; 11391 tree tkind_type; 11392 int talign_shift; 11393 if (is_gimple_omp_oacc (stmt)) 11394 { 11395 tkind_type = short_unsigned_type_node; 11396 talign_shift = 8; 11397 } 11398 else 11399 { 11400 tkind_type = unsigned_char_type_node; 11401 talign_shift = 3; 11402 } 11403 TREE_VEC_ELT (t, 2) 11404 = create_tmp_var (build_array_type_nelts (tkind_type, map_cnt), 11405 ".omp_data_kinds"); 11406 DECL_NAMELESS (TREE_VEC_ELT (t, 2)) = 1; 11407 TREE_ADDRESSABLE (TREE_VEC_ELT (t, 2)) = 1; 11408 TREE_STATIC (TREE_VEC_ELT (t, 2)) = 1; 11409 gimple_omp_target_set_data_arg (stmt, t); 11410 11411 vec<constructor_elt, va_gc> *vsize; 11412 vec<constructor_elt, va_gc> *vkind; 11413 vec_alloc (vsize, map_cnt); 11414 vec_alloc (vkind, map_cnt); 11415 unsigned int map_idx = 0; 11416 11417 for (c = clauses; c ; c = OMP_CLAUSE_CHAIN (c)) 11418 switch (OMP_CLAUSE_CODE (c)) 11419 { 11420 tree ovar, nc; 11421 11422 default: 11423 break; 11424 case OMP_CLAUSE_MAP: 11425 case OMP_CLAUSE_TO: 11426 case OMP_CLAUSE_FROM: 11427 nc = c; 11428 ovar = OMP_CLAUSE_DECL (c); 11429 if (!DECL_P (ovar)) 11430 { 11431 if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 11432 && OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c)) 11433 { 11434 gcc_checking_assert (OMP_CLAUSE_DECL (OMP_CLAUSE_CHAIN (c)) 11435 == get_base_address (ovar)); 11436 nc = OMP_CLAUSE_CHAIN (c); 11437 ovar = OMP_CLAUSE_DECL (nc); 11438 } 11439 else 11440 { 11441 tree x = build_sender_ref (ovar, ctx); 11442 tree v 11443 = build_fold_addr_expr_with_type (ovar, ptr_type_node); 11444 gimplify_assign (x, v, &ilist); 11445 nc = NULL_TREE; 11446 } 11447 } 11448 else 11449 { 11450 if (DECL_SIZE (ovar) 11451 && TREE_CODE (DECL_SIZE (ovar)) != INTEGER_CST) 11452 { 11453 tree ovar2 = DECL_VALUE_EXPR (ovar); 11454 gcc_assert (TREE_CODE (ovar2) == INDIRECT_REF); 11455 ovar2 = TREE_OPERAND (ovar2, 0); 11456 gcc_assert (DECL_P (ovar2)); 11457 ovar = ovar2; 11458 } 11459 if (!maybe_lookup_field (ovar, ctx)) 11460 continue; 11461 } 11462 11463 unsigned int talign = TYPE_ALIGN_UNIT (TREE_TYPE (ovar)); 11464 if (DECL_P (ovar) && DECL_ALIGN_UNIT (ovar) > talign) 11465 talign = DECL_ALIGN_UNIT (ovar); 11466 if (nc) 11467 { 11468 tree var = lookup_decl_in_outer_ctx (ovar, ctx); 11469 tree x = build_sender_ref (ovar, ctx); 11470 if (maybe_lookup_oacc_reduction (var, ctx)) 11471 { 11472 gcc_checking_assert (offloaded 11473 && is_gimple_omp_oacc (stmt)); 11474 gimplify_assign (x, var, &ilist); 11475 } 11476 else if (OMP_CLAUSE_CODE (c) == OMP_CLAUSE_MAP 11477 && OMP_CLAUSE_MAP_KIND (c) == GOMP_MAP_POINTER 11478 && !OMP_CLAUSE_MAP_ZERO_BIAS_ARRAY_SECTION (c) 11479 && TREE_CODE (TREE_TYPE (ovar)) == ARRAY_TYPE) 11480 { 11481 gcc_assert (offloaded); 11482 tree avar 11483 = create_tmp_var (TREE_TYPE (TREE_TYPE (x))); 11484 mark_addressable (avar); 11485 gimplify_assign (avar, build_fold_addr_expr (var), &ilist); 11486 talign = DECL_ALIGN_UNIT (avar); 11487 avar = build_fold_addr_expr (avar); 11488 gimplify_assign (x, avar, &ilist); 11489 } 11490 else if (is_gimple_reg (var)) 11491 { 11492 gcc_assert (offloaded); 11493 tree avar = create_tmp_var (TREE_TYPE (var)); 11494 mark_addressable (avar); 11495 enum gomp_map_kind map_kind = OMP_CLAUSE_MAP_KIND (c); 11496 if (GOMP_MAP_COPY_TO_P (map_kind) 11497 || map_kind == GOMP_MAP_POINTER 11498 || map_kind == GOMP_MAP_TO_PSET 11499 || map_kind == GOMP_MAP_FORCE_DEVICEPTR) 11500 gimplify_assign (avar, var, &ilist); 11501 avar = build_fold_addr_expr (avar); 11502 gimplify_assign (x, avar, &ilist); 11503 if ((GOMP_MAP_COPY_FROM_P (map_kind) 11504 || map_kind == GOMP_MAP_FORCE_DEVICEPTR) 11505 && !TYPE_READONLY (TREE_TYPE (var))) 11506 { 11507 x = build_sender_ref (ovar, ctx); 11508 x = build_simple_mem_ref (x); 11509 gimplify_assign (var, x, &olist); 11510 } 11511 } 11512 else 11513 { 11514 var = build_fold_addr_expr (var); 11515 gimplify_assign (x, var, &ilist); 11516 } 11517 } 11518 tree s = OMP_CLAUSE_SIZE (c); 11519 if (s == NULL_TREE) 11520 s = TYPE_SIZE_UNIT (TREE_TYPE (ovar)); 11521 s = fold_convert (size_type_node, s); 11522 tree purpose = size_int (map_idx++); 11523 CONSTRUCTOR_APPEND_ELT (vsize, purpose, s); 11524 if (TREE_CODE (s) != INTEGER_CST) 11525 TREE_STATIC (TREE_VEC_ELT (t, 1)) = 0; 11526 11527 unsigned HOST_WIDE_INT tkind; 11528 switch (OMP_CLAUSE_CODE (c)) 11529 { 11530 case OMP_CLAUSE_MAP: 11531 tkind = OMP_CLAUSE_MAP_KIND (c); 11532 break; 11533 case OMP_CLAUSE_TO: 11534 tkind = GOMP_MAP_TO; 11535 break; 11536 case OMP_CLAUSE_FROM: 11537 tkind = GOMP_MAP_FROM; 11538 break; 11539 default: 11540 gcc_unreachable (); 11541 } 11542 gcc_checking_assert (tkind 11543 < (HOST_WIDE_INT_C (1U) << talign_shift)); 11544 talign = ceil_log2 (talign); 11545 tkind |= talign << talign_shift; 11546 gcc_checking_assert (tkind 11547 <= tree_to_uhwi (TYPE_MAX_VALUE (tkind_type))); 11548 CONSTRUCTOR_APPEND_ELT (vkind, purpose, 11549 build_int_cstu (tkind_type, tkind)); 11550 if (nc && nc != c) 11551 c = nc; 11552 } 11553 11554 gcc_assert (map_idx == map_cnt); 11555 11556 DECL_INITIAL (TREE_VEC_ELT (t, 1)) 11557 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), vsize); 11558 DECL_INITIAL (TREE_VEC_ELT (t, 2)) 11559 = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 2)), vkind); 11560 if (!TREE_STATIC (TREE_VEC_ELT (t, 1))) 11561 { 11562 gimple_seq initlist = NULL; 11563 force_gimple_operand (build1 (DECL_EXPR, void_type_node, 11564 TREE_VEC_ELT (t, 1)), 11565 &initlist, true, NULL_TREE); 11566 gimple_seq_add_seq (&ilist, initlist); 11567 11568 tree clobber = build_constructor (TREE_TYPE (TREE_VEC_ELT (t, 1)), 11569 NULL); 11570 TREE_THIS_VOLATILE (clobber) = 1; 11571 gimple_seq_add_stmt (&olist, 11572 gimple_build_assign (TREE_VEC_ELT (t, 1), 11573 clobber)); 11574 } 11575 11576 tree clobber = build_constructor (ctx->record_type, NULL); 11577 TREE_THIS_VOLATILE (clobber) = 1; 11578 gimple_seq_add_stmt (&olist, gimple_build_assign (ctx->sender_decl, 11579 clobber)); 11580 } 11581 11582 /* Once all the expansions are done, sequence all the different 11583 fragments inside gimple_omp_body. */ 11584 11585 new_body = NULL; 11586 11587 if (offloaded 11588 && ctx->record_type) 11589 { 11590 t = build_fold_addr_expr_loc (loc, ctx->sender_decl); 11591 /* fixup_child_record_type might have changed receiver_decl's type. */ 11592 t = fold_convert_loc (loc, TREE_TYPE (ctx->receiver_decl), t); 11593 gimple_seq_add_stmt (&new_body, 11594 gimple_build_assign (ctx->receiver_decl, t)); 11595 } 11596 11597 if (offloaded) 11598 { 11599 gimple_seq_add_seq (&new_body, tgt_body); 11600 new_body = maybe_catch_exception (new_body); 11601 } 11602 else if (data_region) 11603 new_body = tgt_body; 11604 if (offloaded || data_region) 11605 { 11606 gimple_seq_add_stmt (&new_body, gimple_build_omp_return (false)); 11607 gimple_omp_set_body (stmt, new_body); 11608 } 11609 11610 bind = gimple_build_bind (NULL, NULL, 11611 tgt_bind ? gimple_bind_block (tgt_bind) 11612 : NULL_TREE); 11613 gsi_replace (gsi_p, bind, true); 11614 gimple_bind_add_seq (bind, irlist); 11615 gimple_bind_add_seq (bind, ilist); 11616 gimple_bind_add_stmt (bind, stmt); 11617 gimple_bind_add_seq (bind, olist); 11618 gimple_bind_add_seq (bind, orlist); 11619 11620 pop_gimplify_context (NULL); 11621} 11622 11623/* Expand code for an OpenMP teams directive. */ 11624 11625static void 11626lower_omp_teams (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11627{ 11628 gomp_teams *teams_stmt = as_a <gomp_teams *> (gsi_stmt (*gsi_p)); 11629 push_gimplify_context (); 11630 11631 tree block = make_node (BLOCK); 11632 gbind *bind = gimple_build_bind (NULL, NULL, block); 11633 gsi_replace (gsi_p, bind, true); 11634 gimple_seq bind_body = NULL; 11635 gimple_seq dlist = NULL; 11636 gimple_seq olist = NULL; 11637 11638 tree num_teams = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), 11639 OMP_CLAUSE_NUM_TEAMS); 11640 if (num_teams == NULL_TREE) 11641 num_teams = build_int_cst (unsigned_type_node, 0); 11642 else 11643 { 11644 num_teams = OMP_CLAUSE_NUM_TEAMS_EXPR (num_teams); 11645 num_teams = fold_convert (unsigned_type_node, num_teams); 11646 gimplify_expr (&num_teams, &bind_body, NULL, is_gimple_val, fb_rvalue); 11647 } 11648 tree thread_limit = find_omp_clause (gimple_omp_teams_clauses (teams_stmt), 11649 OMP_CLAUSE_THREAD_LIMIT); 11650 if (thread_limit == NULL_TREE) 11651 thread_limit = build_int_cst (unsigned_type_node, 0); 11652 else 11653 { 11654 thread_limit = OMP_CLAUSE_THREAD_LIMIT_EXPR (thread_limit); 11655 thread_limit = fold_convert (unsigned_type_node, thread_limit); 11656 gimplify_expr (&thread_limit, &bind_body, NULL, is_gimple_val, 11657 fb_rvalue); 11658 } 11659 11660 lower_rec_input_clauses (gimple_omp_teams_clauses (teams_stmt), 11661 &bind_body, &dlist, ctx, NULL); 11662 lower_omp (gimple_omp_body_ptr (teams_stmt), ctx); 11663 lower_reduction_clauses (gimple_omp_teams_clauses (teams_stmt), &olist, ctx); 11664 gimple_seq_add_stmt (&bind_body, teams_stmt); 11665 11666 location_t loc = gimple_location (teams_stmt); 11667 tree decl = builtin_decl_explicit (BUILT_IN_GOMP_TEAMS); 11668 gimple call = gimple_build_call (decl, 2, num_teams, thread_limit); 11669 gimple_set_location (call, loc); 11670 gimple_seq_add_stmt (&bind_body, call); 11671 11672 gimple_seq_add_seq (&bind_body, gimple_omp_body (teams_stmt)); 11673 gimple_omp_set_body (teams_stmt, NULL); 11674 gimple_seq_add_seq (&bind_body, olist); 11675 gimple_seq_add_seq (&bind_body, dlist); 11676 gimple_seq_add_stmt (&bind_body, gimple_build_omp_return (true)); 11677 gimple_bind_set_body (bind, bind_body); 11678 11679 pop_gimplify_context (bind); 11680 11681 gimple_bind_append_vars (bind, ctx->block_vars); 11682 BLOCK_VARS (block) = ctx->block_vars; 11683 if (BLOCK_VARS (block)) 11684 TREE_USED (block) = 1; 11685} 11686 11687 11688/* Callback for lower_omp_1. Return non-NULL if *tp needs to be 11689 regimplified. If DATA is non-NULL, lower_omp_1 is outside 11690 of OMP context, but with task_shared_vars set. */ 11691 11692static tree 11693lower_omp_regimplify_p (tree *tp, int *walk_subtrees, 11694 void *data) 11695{ 11696 tree t = *tp; 11697 11698 /* Any variable with DECL_VALUE_EXPR needs to be regimplified. */ 11699 if (TREE_CODE (t) == VAR_DECL && data == NULL && DECL_HAS_VALUE_EXPR_P (t)) 11700 return t; 11701 11702 if (task_shared_vars 11703 && DECL_P (t) 11704 && bitmap_bit_p (task_shared_vars, DECL_UID (t))) 11705 return t; 11706 11707 /* If a global variable has been privatized, TREE_CONSTANT on 11708 ADDR_EXPR might be wrong. */ 11709 if (data == NULL && TREE_CODE (t) == ADDR_EXPR) 11710 recompute_tree_invariant_for_addr_expr (t); 11711 11712 *walk_subtrees = !TYPE_P (t) && !DECL_P (t); 11713 return NULL_TREE; 11714} 11715 11716static void 11717lower_omp_1 (gimple_stmt_iterator *gsi_p, omp_context *ctx) 11718{ 11719 gimple stmt = gsi_stmt (*gsi_p); 11720 struct walk_stmt_info wi; 11721 gcall *call_stmt; 11722 11723 if (gimple_has_location (stmt)) 11724 input_location = gimple_location (stmt); 11725 11726 if (task_shared_vars) 11727 memset (&wi, '\0', sizeof (wi)); 11728 11729 /* If we have issued syntax errors, avoid doing any heavy lifting. 11730 Just replace the OMP directives with a NOP to avoid 11731 confusing RTL expansion. */ 11732 if (seen_error () && is_gimple_omp (stmt)) 11733 { 11734 gsi_replace (gsi_p, gimple_build_nop (), true); 11735 return; 11736 } 11737 11738 switch (gimple_code (stmt)) 11739 { 11740 case GIMPLE_COND: 11741 { 11742 gcond *cond_stmt = as_a <gcond *> (stmt); 11743 if ((ctx || task_shared_vars) 11744 && (walk_tree (gimple_cond_lhs_ptr (cond_stmt), 11745 lower_omp_regimplify_p, 11746 ctx ? NULL : &wi, NULL) 11747 || walk_tree (gimple_cond_rhs_ptr (cond_stmt), 11748 lower_omp_regimplify_p, 11749 ctx ? NULL : &wi, NULL))) 11750 gimple_regimplify_operands (cond_stmt, gsi_p); 11751 } 11752 break; 11753 case GIMPLE_CATCH: 11754 lower_omp (gimple_catch_handler_ptr (as_a <gcatch *> (stmt)), ctx); 11755 break; 11756 case GIMPLE_EH_FILTER: 11757 lower_omp (gimple_eh_filter_failure_ptr (stmt), ctx); 11758 break; 11759 case GIMPLE_TRY: 11760 lower_omp (gimple_try_eval_ptr (stmt), ctx); 11761 lower_omp (gimple_try_cleanup_ptr (stmt), ctx); 11762 break; 11763 case GIMPLE_TRANSACTION: 11764 lower_omp (gimple_transaction_body_ptr ( 11765 as_a <gtransaction *> (stmt)), 11766 ctx); 11767 break; 11768 case GIMPLE_BIND: 11769 lower_omp (gimple_bind_body_ptr (as_a <gbind *> (stmt)), ctx); 11770 break; 11771 case GIMPLE_OMP_PARALLEL: 11772 case GIMPLE_OMP_TASK: 11773 ctx = maybe_lookup_ctx (stmt); 11774 gcc_assert (ctx); 11775 if (ctx->cancellable) 11776 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11777 lower_omp_taskreg (gsi_p, ctx); 11778 break; 11779 case GIMPLE_OMP_FOR: 11780 ctx = maybe_lookup_ctx (stmt); 11781 gcc_assert (ctx); 11782 if (ctx->cancellable) 11783 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11784 lower_omp_for (gsi_p, ctx); 11785 break; 11786 case GIMPLE_OMP_SECTIONS: 11787 ctx = maybe_lookup_ctx (stmt); 11788 gcc_assert (ctx); 11789 if (ctx->cancellable) 11790 ctx->cancel_label = create_artificial_label (UNKNOWN_LOCATION); 11791 lower_omp_sections (gsi_p, ctx); 11792 break; 11793 case GIMPLE_OMP_SINGLE: 11794 ctx = maybe_lookup_ctx (stmt); 11795 gcc_assert (ctx); 11796 lower_omp_single (gsi_p, ctx); 11797 break; 11798 case GIMPLE_OMP_MASTER: 11799 ctx = maybe_lookup_ctx (stmt); 11800 gcc_assert (ctx); 11801 lower_omp_master (gsi_p, ctx); 11802 break; 11803 case GIMPLE_OMP_TASKGROUP: 11804 ctx = maybe_lookup_ctx (stmt); 11805 gcc_assert (ctx); 11806 lower_omp_taskgroup (gsi_p, ctx); 11807 break; 11808 case GIMPLE_OMP_ORDERED: 11809 ctx = maybe_lookup_ctx (stmt); 11810 gcc_assert (ctx); 11811 lower_omp_ordered (gsi_p, ctx); 11812 break; 11813 case GIMPLE_OMP_CRITICAL: 11814 ctx = maybe_lookup_ctx (stmt); 11815 gcc_assert (ctx); 11816 lower_omp_critical (gsi_p, ctx); 11817 break; 11818 case GIMPLE_OMP_ATOMIC_LOAD: 11819 if ((ctx || task_shared_vars) 11820 && walk_tree (gimple_omp_atomic_load_rhs_ptr ( 11821 as_a <gomp_atomic_load *> (stmt)), 11822 lower_omp_regimplify_p, ctx ? NULL : &wi, NULL)) 11823 gimple_regimplify_operands (stmt, gsi_p); 11824 break; 11825 case GIMPLE_OMP_TARGET: 11826 ctx = maybe_lookup_ctx (stmt); 11827 gcc_assert (ctx); 11828 lower_omp_target (gsi_p, ctx); 11829 break; 11830 case GIMPLE_OMP_TEAMS: 11831 ctx = maybe_lookup_ctx (stmt); 11832 gcc_assert (ctx); 11833 lower_omp_teams (gsi_p, ctx); 11834 break; 11835 case GIMPLE_CALL: 11836 tree fndecl; 11837 call_stmt = as_a <gcall *> (stmt); 11838 fndecl = gimple_call_fndecl (call_stmt); 11839 if (fndecl 11840 && DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL) 11841 switch (DECL_FUNCTION_CODE (fndecl)) 11842 { 11843 case BUILT_IN_GOMP_BARRIER: 11844 if (ctx == NULL) 11845 break; 11846 /* FALLTHRU */ 11847 case BUILT_IN_GOMP_CANCEL: 11848 case BUILT_IN_GOMP_CANCELLATION_POINT: 11849 omp_context *cctx; 11850 cctx = ctx; 11851 if (gimple_code (cctx->stmt) == GIMPLE_OMP_SECTION) 11852 cctx = cctx->outer; 11853 gcc_assert (gimple_call_lhs (call_stmt) == NULL_TREE); 11854 if (!cctx->cancellable) 11855 { 11856 if (DECL_FUNCTION_CODE (fndecl) 11857 == BUILT_IN_GOMP_CANCELLATION_POINT) 11858 { 11859 stmt = gimple_build_nop (); 11860 gsi_replace (gsi_p, stmt, false); 11861 } 11862 break; 11863 } 11864 if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_GOMP_BARRIER) 11865 { 11866 fndecl = builtin_decl_explicit (BUILT_IN_GOMP_BARRIER_CANCEL); 11867 gimple_call_set_fndecl (call_stmt, fndecl); 11868 gimple_call_set_fntype (call_stmt, TREE_TYPE (fndecl)); 11869 } 11870 tree lhs; 11871 lhs = create_tmp_var (TREE_TYPE (TREE_TYPE (fndecl))); 11872 gimple_call_set_lhs (call_stmt, lhs); 11873 tree fallthru_label; 11874 fallthru_label = create_artificial_label (UNKNOWN_LOCATION); 11875 gimple g; 11876 g = gimple_build_label (fallthru_label); 11877 gsi_insert_after (gsi_p, g, GSI_SAME_STMT); 11878 g = gimple_build_cond (NE_EXPR, lhs, 11879 fold_convert (TREE_TYPE (lhs), 11880 boolean_false_node), 11881 cctx->cancel_label, fallthru_label); 11882 gsi_insert_after (gsi_p, g, GSI_SAME_STMT); 11883 break; 11884 default: 11885 break; 11886 } 11887 /* FALLTHRU */ 11888 default: 11889 if ((ctx || task_shared_vars) 11890 && walk_gimple_op (stmt, lower_omp_regimplify_p, 11891 ctx ? NULL : &wi)) 11892 { 11893 /* Just remove clobbers, this should happen only if we have 11894 "privatized" local addressable variables in SIMD regions, 11895 the clobber isn't needed in that case and gimplifying address 11896 of the ARRAY_REF into a pointer and creating MEM_REF based 11897 clobber would create worse code than we get with the clobber 11898 dropped. */ 11899 if (gimple_clobber_p (stmt)) 11900 { 11901 gsi_replace (gsi_p, gimple_build_nop (), true); 11902 break; 11903 } 11904 gimple_regimplify_operands (stmt, gsi_p); 11905 } 11906 break; 11907 } 11908} 11909 11910static void 11911lower_omp (gimple_seq *body, omp_context *ctx) 11912{ 11913 location_t saved_location = input_location; 11914 gimple_stmt_iterator gsi; 11915 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 11916 lower_omp_1 (&gsi, ctx); 11917 /* During gimplification, we haven't folded statments inside offloading 11918 or taskreg regions (gimplify.c:maybe_fold_stmt); do that now. */ 11919 if (target_nesting_level || taskreg_nesting_level) 11920 for (gsi = gsi_start (*body); !gsi_end_p (gsi); gsi_next (&gsi)) 11921 fold_stmt (&gsi); 11922 input_location = saved_location; 11923} 11924 11925/* Main entry point. */ 11926 11927static unsigned int 11928execute_lower_omp (void) 11929{ 11930 gimple_seq body; 11931 int i; 11932 omp_context *ctx; 11933 11934 /* This pass always runs, to provide PROP_gimple_lomp. 11935 But often, there is nothing to do. */ 11936 if (flag_cilkplus == 0 && flag_openacc == 0 && flag_openmp == 0 11937 && flag_openmp_simd == 0) 11938 return 0; 11939 11940 all_contexts = splay_tree_new (splay_tree_compare_pointers, 0, 11941 delete_omp_context); 11942 11943 body = gimple_body (current_function_decl); 11944 scan_omp (&body, NULL); 11945 gcc_assert (taskreg_nesting_level == 0); 11946 FOR_EACH_VEC_ELT (taskreg_contexts, i, ctx) 11947 finish_taskreg_scan (ctx); 11948 taskreg_contexts.release (); 11949 11950 if (all_contexts->root) 11951 { 11952 if (task_shared_vars) 11953 push_gimplify_context (); 11954 lower_omp (&body, NULL); 11955 if (task_shared_vars) 11956 pop_gimplify_context (NULL); 11957 } 11958 11959 if (all_contexts) 11960 { 11961 splay_tree_delete (all_contexts); 11962 all_contexts = NULL; 11963 } 11964 BITMAP_FREE (task_shared_vars); 11965 return 0; 11966} 11967 11968namespace { 11969 11970const pass_data pass_data_lower_omp = 11971{ 11972 GIMPLE_PASS, /* type */ 11973 "omplower", /* name */ 11974 OPTGROUP_NONE, /* optinfo_flags */ 11975 TV_NONE, /* tv_id */ 11976 PROP_gimple_any, /* properties_required */ 11977 PROP_gimple_lomp, /* properties_provided */ 11978 0, /* properties_destroyed */ 11979 0, /* todo_flags_start */ 11980 0, /* todo_flags_finish */ 11981}; 11982 11983class pass_lower_omp : public gimple_opt_pass 11984{ 11985public: 11986 pass_lower_omp (gcc::context *ctxt) 11987 : gimple_opt_pass (pass_data_lower_omp, ctxt) 11988 {} 11989 11990 /* opt_pass methods: */ 11991 virtual unsigned int execute (function *) { return execute_lower_omp (); } 11992 11993}; // class pass_lower_omp 11994 11995} // anon namespace 11996 11997gimple_opt_pass * 11998make_pass_lower_omp (gcc::context *ctxt) 11999{ 12000 return new pass_lower_omp (ctxt); 12001} 12002 12003/* The following is a utility to diagnose structured block violations. 12004 It is not part of the "omplower" pass, as that's invoked too late. It 12005 should be invoked by the respective front ends after gimplification. */ 12006 12007static splay_tree all_labels; 12008 12009/* Check for mismatched contexts and generate an error if needed. Return 12010 true if an error is detected. */ 12011 12012static bool 12013diagnose_sb_0 (gimple_stmt_iterator *gsi_p, 12014 gimple branch_ctx, gimple label_ctx) 12015{ 12016 gcc_checking_assert (!branch_ctx || is_gimple_omp (branch_ctx)); 12017 gcc_checking_assert (!label_ctx || is_gimple_omp (label_ctx)); 12018 12019 if (label_ctx == branch_ctx) 12020 return false; 12021 12022 const char* kind = NULL; 12023 12024 if (flag_cilkplus) 12025 { 12026 if ((branch_ctx 12027 && gimple_code (branch_ctx) == GIMPLE_OMP_FOR 12028 && gimple_omp_for_kind (branch_ctx) == GF_OMP_FOR_KIND_CILKSIMD) 12029 || (label_ctx 12030 && gimple_code (label_ctx) == GIMPLE_OMP_FOR 12031 && gimple_omp_for_kind (label_ctx) == GF_OMP_FOR_KIND_CILKSIMD)) 12032 kind = "Cilk Plus"; 12033 } 12034 if (flag_openacc) 12035 { 12036 if ((branch_ctx && is_gimple_omp_oacc (branch_ctx)) 12037 || (label_ctx && is_gimple_omp_oacc (label_ctx))) 12038 { 12039 gcc_checking_assert (kind == NULL); 12040 kind = "OpenACC"; 12041 } 12042 } 12043 if (kind == NULL) 12044 { 12045 gcc_checking_assert (flag_openmp); 12046 kind = "OpenMP"; 12047 } 12048 12049 /* 12050 Previously we kept track of the label's entire context in diagnose_sb_[12] 12051 so we could traverse it and issue a correct "exit" or "enter" error 12052 message upon a structured block violation. 12053 12054 We built the context by building a list with tree_cons'ing, but there is 12055 no easy counterpart in gimple tuples. It seems like far too much work 12056 for issuing exit/enter error messages. If someone really misses the 12057 distinct error message... patches welcome. 12058 */ 12059 12060#if 0 12061 /* Try to avoid confusing the user by producing and error message 12062 with correct "exit" or "enter" verbiage. We prefer "exit" 12063 unless we can show that LABEL_CTX is nested within BRANCH_CTX. */ 12064 if (branch_ctx == NULL) 12065 exit_p = false; 12066 else 12067 { 12068 while (label_ctx) 12069 { 12070 if (TREE_VALUE (label_ctx) == branch_ctx) 12071 { 12072 exit_p = false; 12073 break; 12074 } 12075 label_ctx = TREE_CHAIN (label_ctx); 12076 } 12077 } 12078 12079 if (exit_p) 12080 error ("invalid exit from %s structured block", kind); 12081 else 12082 error ("invalid entry to %s structured block", kind); 12083#endif 12084 12085 /* If it's obvious we have an invalid entry, be specific about the error. */ 12086 if (branch_ctx == NULL) 12087 error ("invalid entry to %s structured block", kind); 12088 else 12089 { 12090 /* Otherwise, be vague and lazy, but efficient. */ 12091 error ("invalid branch to/from %s structured block", kind); 12092 } 12093 12094 gsi_replace (gsi_p, gimple_build_nop (), false); 12095 return true; 12096} 12097 12098/* Pass 1: Create a minimal tree of structured blocks, and record 12099 where each label is found. */ 12100 12101static tree 12102diagnose_sb_1 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, 12103 struct walk_stmt_info *wi) 12104{ 12105 gimple context = (gimple) wi->info; 12106 gimple inner_context; 12107 gimple stmt = gsi_stmt (*gsi_p); 12108 12109 *handled_ops_p = true; 12110 12111 switch (gimple_code (stmt)) 12112 { 12113 WALK_SUBSTMTS; 12114 12115 case GIMPLE_OMP_PARALLEL: 12116 case GIMPLE_OMP_TASK: 12117 case GIMPLE_OMP_SECTIONS: 12118 case GIMPLE_OMP_SINGLE: 12119 case GIMPLE_OMP_SECTION: 12120 case GIMPLE_OMP_MASTER: 12121 case GIMPLE_OMP_ORDERED: 12122 case GIMPLE_OMP_CRITICAL: 12123 case GIMPLE_OMP_TARGET: 12124 case GIMPLE_OMP_TEAMS: 12125 case GIMPLE_OMP_TASKGROUP: 12126 /* The minimal context here is just the current OMP construct. */ 12127 inner_context = stmt; 12128 wi->info = inner_context; 12129 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); 12130 wi->info = context; 12131 break; 12132 12133 case GIMPLE_OMP_FOR: 12134 inner_context = stmt; 12135 wi->info = inner_context; 12136 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to 12137 walk them. */ 12138 walk_gimple_seq (gimple_omp_for_pre_body (stmt), 12139 diagnose_sb_1, NULL, wi); 12140 walk_gimple_seq (gimple_omp_body (stmt), diagnose_sb_1, NULL, wi); 12141 wi->info = context; 12142 break; 12143 12144 case GIMPLE_LABEL: 12145 splay_tree_insert (all_labels, 12146 (splay_tree_key) gimple_label_label ( 12147 as_a <glabel *> (stmt)), 12148 (splay_tree_value) context); 12149 break; 12150 12151 default: 12152 break; 12153 } 12154 12155 return NULL_TREE; 12156} 12157 12158/* Pass 2: Check each branch and see if its context differs from that of 12159 the destination label's context. */ 12160 12161static tree 12162diagnose_sb_2 (gimple_stmt_iterator *gsi_p, bool *handled_ops_p, 12163 struct walk_stmt_info *wi) 12164{ 12165 gimple context = (gimple) wi->info; 12166 splay_tree_node n; 12167 gimple stmt = gsi_stmt (*gsi_p); 12168 12169 *handled_ops_p = true; 12170 12171 switch (gimple_code (stmt)) 12172 { 12173 WALK_SUBSTMTS; 12174 12175 case GIMPLE_OMP_PARALLEL: 12176 case GIMPLE_OMP_TASK: 12177 case GIMPLE_OMP_SECTIONS: 12178 case GIMPLE_OMP_SINGLE: 12179 case GIMPLE_OMP_SECTION: 12180 case GIMPLE_OMP_MASTER: 12181 case GIMPLE_OMP_ORDERED: 12182 case GIMPLE_OMP_CRITICAL: 12183 case GIMPLE_OMP_TARGET: 12184 case GIMPLE_OMP_TEAMS: 12185 case GIMPLE_OMP_TASKGROUP: 12186 wi->info = stmt; 12187 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); 12188 wi->info = context; 12189 break; 12190 12191 case GIMPLE_OMP_FOR: 12192 wi->info = stmt; 12193 /* gimple_omp_for_{index,initial,final} are all DECLs; no need to 12194 walk them. */ 12195 walk_gimple_seq_mod (gimple_omp_for_pre_body_ptr (stmt), 12196 diagnose_sb_2, NULL, wi); 12197 walk_gimple_seq_mod (gimple_omp_body_ptr (stmt), diagnose_sb_2, NULL, wi); 12198 wi->info = context; 12199 break; 12200 12201 case GIMPLE_COND: 12202 { 12203 gcond *cond_stmt = as_a <gcond *> (stmt); 12204 tree lab = gimple_cond_true_label (cond_stmt); 12205 if (lab) 12206 { 12207 n = splay_tree_lookup (all_labels, 12208 (splay_tree_key) lab); 12209 diagnose_sb_0 (gsi_p, context, 12210 n ? (gimple) n->value : NULL); 12211 } 12212 lab = gimple_cond_false_label (cond_stmt); 12213 if (lab) 12214 { 12215 n = splay_tree_lookup (all_labels, 12216 (splay_tree_key) lab); 12217 diagnose_sb_0 (gsi_p, context, 12218 n ? (gimple) n->value : NULL); 12219 } 12220 } 12221 break; 12222 12223 case GIMPLE_GOTO: 12224 { 12225 tree lab = gimple_goto_dest (stmt); 12226 if (TREE_CODE (lab) != LABEL_DECL) 12227 break; 12228 12229 n = splay_tree_lookup (all_labels, (splay_tree_key) lab); 12230 diagnose_sb_0 (gsi_p, context, n ? (gimple) n->value : NULL); 12231 } 12232 break; 12233 12234 case GIMPLE_SWITCH: 12235 { 12236 gswitch *switch_stmt = as_a <gswitch *> (stmt); 12237 unsigned int i; 12238 for (i = 0; i < gimple_switch_num_labels (switch_stmt); ++i) 12239 { 12240 tree lab = CASE_LABEL (gimple_switch_label (switch_stmt, i)); 12241 n = splay_tree_lookup (all_labels, (splay_tree_key) lab); 12242 if (n && diagnose_sb_0 (gsi_p, context, (gimple) n->value)) 12243 break; 12244 } 12245 } 12246 break; 12247 12248 case GIMPLE_RETURN: 12249 diagnose_sb_0 (gsi_p, context, NULL); 12250 break; 12251 12252 default: 12253 break; 12254 } 12255 12256 return NULL_TREE; 12257} 12258 12259/* Called from tree-cfg.c::make_edges to create cfg edges for all relevant 12260 GIMPLE_* codes. */ 12261bool 12262make_gimple_omp_edges (basic_block bb, struct omp_region **region, 12263 int *region_idx) 12264{ 12265 gimple last = last_stmt (bb); 12266 enum gimple_code code = gimple_code (last); 12267 struct omp_region *cur_region = *region; 12268 bool fallthru = false; 12269 12270 switch (code) 12271 { 12272 case GIMPLE_OMP_PARALLEL: 12273 case GIMPLE_OMP_TASK: 12274 case GIMPLE_OMP_FOR: 12275 case GIMPLE_OMP_SINGLE: 12276 case GIMPLE_OMP_TEAMS: 12277 case GIMPLE_OMP_MASTER: 12278 case GIMPLE_OMP_TASKGROUP: 12279 case GIMPLE_OMP_ORDERED: 12280 case GIMPLE_OMP_CRITICAL: 12281 case GIMPLE_OMP_SECTION: 12282 cur_region = new_omp_region (bb, code, cur_region); 12283 fallthru = true; 12284 break; 12285 12286 case GIMPLE_OMP_TARGET: 12287 cur_region = new_omp_region (bb, code, cur_region); 12288 fallthru = true; 12289 switch (gimple_omp_target_kind (last)) 12290 { 12291 case GF_OMP_TARGET_KIND_REGION: 12292 case GF_OMP_TARGET_KIND_DATA: 12293 case GF_OMP_TARGET_KIND_OACC_PARALLEL: 12294 case GF_OMP_TARGET_KIND_OACC_KERNELS: 12295 case GF_OMP_TARGET_KIND_OACC_DATA: 12296 break; 12297 case GF_OMP_TARGET_KIND_UPDATE: 12298 case GF_OMP_TARGET_KIND_OACC_UPDATE: 12299 case GF_OMP_TARGET_KIND_OACC_ENTER_EXIT_DATA: 12300 cur_region = cur_region->outer; 12301 break; 12302 default: 12303 gcc_unreachable (); 12304 } 12305 break; 12306 12307 case GIMPLE_OMP_SECTIONS: 12308 cur_region = new_omp_region (bb, code, cur_region); 12309 fallthru = true; 12310 break; 12311 12312 case GIMPLE_OMP_SECTIONS_SWITCH: 12313 fallthru = false; 12314 break; 12315 12316 case GIMPLE_OMP_ATOMIC_LOAD: 12317 case GIMPLE_OMP_ATOMIC_STORE: 12318 fallthru = true; 12319 break; 12320 12321 case GIMPLE_OMP_RETURN: 12322 /* In the case of a GIMPLE_OMP_SECTION, the edge will go 12323 somewhere other than the next block. This will be 12324 created later. */ 12325 cur_region->exit = bb; 12326 if (cur_region->type == GIMPLE_OMP_TASK) 12327 /* Add an edge corresponding to not scheduling the task 12328 immediately. */ 12329 make_edge (cur_region->entry, bb, EDGE_ABNORMAL); 12330 fallthru = cur_region->type != GIMPLE_OMP_SECTION; 12331 cur_region = cur_region->outer; 12332 break; 12333 12334 case GIMPLE_OMP_CONTINUE: 12335 cur_region->cont = bb; 12336 switch (cur_region->type) 12337 { 12338 case GIMPLE_OMP_FOR: 12339 /* Mark all GIMPLE_OMP_FOR and GIMPLE_OMP_CONTINUE 12340 succs edges as abnormal to prevent splitting 12341 them. */ 12342 single_succ_edge (cur_region->entry)->flags |= EDGE_ABNORMAL; 12343 /* Make the loopback edge. */ 12344 make_edge (bb, single_succ (cur_region->entry), 12345 EDGE_ABNORMAL); 12346 12347 /* Create an edge from GIMPLE_OMP_FOR to exit, which 12348 corresponds to the case that the body of the loop 12349 is not executed at all. */ 12350 make_edge (cur_region->entry, bb->next_bb, EDGE_ABNORMAL); 12351 make_edge (bb, bb->next_bb, EDGE_FALLTHRU | EDGE_ABNORMAL); 12352 fallthru = false; 12353 break; 12354 12355 case GIMPLE_OMP_SECTIONS: 12356 /* Wire up the edges into and out of the nested sections. */ 12357 { 12358 basic_block switch_bb = single_succ (cur_region->entry); 12359 12360 struct omp_region *i; 12361 for (i = cur_region->inner; i ; i = i->next) 12362 { 12363 gcc_assert (i->type == GIMPLE_OMP_SECTION); 12364 make_edge (switch_bb, i->entry, 0); 12365 make_edge (i->exit, bb, EDGE_FALLTHRU); 12366 } 12367 12368 /* Make the loopback edge to the block with 12369 GIMPLE_OMP_SECTIONS_SWITCH. */ 12370 make_edge (bb, switch_bb, 0); 12371 12372 /* Make the edge from the switch to exit. */ 12373 make_edge (switch_bb, bb->next_bb, 0); 12374 fallthru = false; 12375 } 12376 break; 12377 12378 case GIMPLE_OMP_TASK: 12379 fallthru = true; 12380 break; 12381 12382 default: 12383 gcc_unreachable (); 12384 } 12385 break; 12386 12387 default: 12388 gcc_unreachable (); 12389 } 12390 12391 if (*region != cur_region) 12392 { 12393 *region = cur_region; 12394 if (cur_region) 12395 *region_idx = cur_region->entry->index; 12396 else 12397 *region_idx = 0; 12398 } 12399 12400 return fallthru; 12401} 12402 12403static unsigned int 12404diagnose_omp_structured_block_errors (void) 12405{ 12406 struct walk_stmt_info wi; 12407 gimple_seq body = gimple_body (current_function_decl); 12408 12409 all_labels = splay_tree_new (splay_tree_compare_pointers, 0, 0); 12410 12411 memset (&wi, 0, sizeof (wi)); 12412 walk_gimple_seq (body, diagnose_sb_1, NULL, &wi); 12413 12414 memset (&wi, 0, sizeof (wi)); 12415 wi.want_locations = true; 12416 walk_gimple_seq_mod (&body, diagnose_sb_2, NULL, &wi); 12417 12418 gimple_set_body (current_function_decl, body); 12419 12420 splay_tree_delete (all_labels); 12421 all_labels = NULL; 12422 12423 return 0; 12424} 12425 12426namespace { 12427 12428const pass_data pass_data_diagnose_omp_blocks = 12429{ 12430 GIMPLE_PASS, /* type */ 12431 "*diagnose_omp_blocks", /* name */ 12432 OPTGROUP_NONE, /* optinfo_flags */ 12433 TV_NONE, /* tv_id */ 12434 PROP_gimple_any, /* properties_required */ 12435 0, /* properties_provided */ 12436 0, /* properties_destroyed */ 12437 0, /* todo_flags_start */ 12438 0, /* todo_flags_finish */ 12439}; 12440 12441class pass_diagnose_omp_blocks : public gimple_opt_pass 12442{ 12443public: 12444 pass_diagnose_omp_blocks (gcc::context *ctxt) 12445 : gimple_opt_pass (pass_data_diagnose_omp_blocks, ctxt) 12446 {} 12447 12448 /* opt_pass methods: */ 12449 virtual bool gate (function *) 12450 { 12451 return flag_cilkplus || flag_openacc || flag_openmp; 12452 } 12453 virtual unsigned int execute (function *) 12454 { 12455 return diagnose_omp_structured_block_errors (); 12456 } 12457 12458}; // class pass_diagnose_omp_blocks 12459 12460} // anon namespace 12461 12462gimple_opt_pass * 12463make_pass_diagnose_omp_blocks (gcc::context *ctxt) 12464{ 12465 return new pass_diagnose_omp_blocks (ctxt); 12466} 12467 12468/* SIMD clone supporting code. */ 12469 12470/* Allocate a fresh `simd_clone' and return it. NARGS is the number 12471 of arguments to reserve space for. */ 12472 12473static struct cgraph_simd_clone * 12474simd_clone_struct_alloc (int nargs) 12475{ 12476 struct cgraph_simd_clone *clone_info; 12477 size_t len = (sizeof (struct cgraph_simd_clone) 12478 + nargs * sizeof (struct cgraph_simd_clone_arg)); 12479 clone_info = (struct cgraph_simd_clone *) 12480 ggc_internal_cleared_alloc (len); 12481 return clone_info; 12482} 12483 12484/* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */ 12485 12486static inline void 12487simd_clone_struct_copy (struct cgraph_simd_clone *to, 12488 struct cgraph_simd_clone *from) 12489{ 12490 memcpy (to, from, (sizeof (struct cgraph_simd_clone) 12491 + ((from->nargs - from->inbranch) 12492 * sizeof (struct cgraph_simd_clone_arg)))); 12493} 12494 12495/* Return vector of parameter types of function FNDECL. This uses 12496 TYPE_ARG_TYPES if available, otherwise falls back to types of 12497 DECL_ARGUMENTS types. */ 12498 12499vec<tree> 12500simd_clone_vector_of_formal_parm_types (tree fndecl) 12501{ 12502 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl))) 12503 return ipa_get_vector_of_formal_parm_types (TREE_TYPE (fndecl)); 12504 vec<tree> args = ipa_get_vector_of_formal_parms (fndecl); 12505 unsigned int i; 12506 tree arg; 12507 FOR_EACH_VEC_ELT (args, i, arg) 12508 args[i] = TREE_TYPE (args[i]); 12509 return args; 12510} 12511 12512/* Given a simd function in NODE, extract the simd specific 12513 information from the OMP clauses passed in CLAUSES, and return 12514 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED 12515 is set to TRUE if the `inbranch' or `notinbranch' clause specified, 12516 otherwise set to FALSE. */ 12517 12518static struct cgraph_simd_clone * 12519simd_clone_clauses_extract (struct cgraph_node *node, tree clauses, 12520 bool *inbranch_specified) 12521{ 12522 vec<tree> args = simd_clone_vector_of_formal_parm_types (node->decl); 12523 tree t; 12524 int n; 12525 *inbranch_specified = false; 12526 12527 n = args.length (); 12528 if (n > 0 && args.last () == void_type_node) 12529 n--; 12530 12531 /* To distinguish from an OpenMP simd clone, Cilk Plus functions to 12532 be cloned have a distinctive artificial label in addition to "omp 12533 declare simd". */ 12534 bool cilk_clone 12535 = (flag_cilkplus 12536 && lookup_attribute ("cilk simd function", 12537 DECL_ATTRIBUTES (node->decl))); 12538 12539 /* Allocate one more than needed just in case this is an in-branch 12540 clone which will require a mask argument. */ 12541 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1); 12542 clone_info->nargs = n; 12543 clone_info->cilk_elemental = cilk_clone; 12544 12545 if (!clauses) 12546 { 12547 args.release (); 12548 return clone_info; 12549 } 12550 clauses = TREE_VALUE (clauses); 12551 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE) 12552 return clone_info; 12553 12554 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t)) 12555 { 12556 switch (OMP_CLAUSE_CODE (t)) 12557 { 12558 case OMP_CLAUSE_INBRANCH: 12559 clone_info->inbranch = 1; 12560 *inbranch_specified = true; 12561 break; 12562 case OMP_CLAUSE_NOTINBRANCH: 12563 clone_info->inbranch = 0; 12564 *inbranch_specified = true; 12565 break; 12566 case OMP_CLAUSE_SIMDLEN: 12567 clone_info->simdlen 12568 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t)); 12569 break; 12570 case OMP_CLAUSE_LINEAR: 12571 { 12572 tree decl = OMP_CLAUSE_DECL (t); 12573 tree step = OMP_CLAUSE_LINEAR_STEP (t); 12574 int argno = TREE_INT_CST_LOW (decl); 12575 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t)) 12576 { 12577 clone_info->args[argno].arg_type 12578 = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP; 12579 clone_info->args[argno].linear_step = tree_to_shwi (step); 12580 gcc_assert (clone_info->args[argno].linear_step >= 0 12581 && clone_info->args[argno].linear_step < n); 12582 } 12583 else 12584 { 12585 if (POINTER_TYPE_P (args[argno])) 12586 step = fold_convert (ssizetype, step); 12587 if (!tree_fits_shwi_p (step)) 12588 { 12589 warning_at (OMP_CLAUSE_LOCATION (t), 0, 12590 "ignoring large linear step"); 12591 args.release (); 12592 return NULL; 12593 } 12594 else if (integer_zerop (step)) 12595 { 12596 warning_at (OMP_CLAUSE_LOCATION (t), 0, 12597 "ignoring zero linear step"); 12598 args.release (); 12599 return NULL; 12600 } 12601 else 12602 { 12603 clone_info->args[argno].arg_type 12604 = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP; 12605 clone_info->args[argno].linear_step = tree_to_shwi (step); 12606 } 12607 } 12608 break; 12609 } 12610 case OMP_CLAUSE_UNIFORM: 12611 { 12612 tree decl = OMP_CLAUSE_DECL (t); 12613 int argno = tree_to_uhwi (decl); 12614 clone_info->args[argno].arg_type 12615 = SIMD_CLONE_ARG_TYPE_UNIFORM; 12616 break; 12617 } 12618 case OMP_CLAUSE_ALIGNED: 12619 { 12620 tree decl = OMP_CLAUSE_DECL (t); 12621 int argno = tree_to_uhwi (decl); 12622 clone_info->args[argno].alignment 12623 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t)); 12624 break; 12625 } 12626 default: 12627 break; 12628 } 12629 } 12630 args.release (); 12631 return clone_info; 12632} 12633 12634/* Given a SIMD clone in NODE, calculate the characteristic data 12635 type and return the coresponding type. The characteristic data 12636 type is computed as described in the Intel Vector ABI. */ 12637 12638static tree 12639simd_clone_compute_base_data_type (struct cgraph_node *node, 12640 struct cgraph_simd_clone *clone_info) 12641{ 12642 tree type = integer_type_node; 12643 tree fndecl = node->decl; 12644 12645 /* a) For non-void function, the characteristic data type is the 12646 return type. */ 12647 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE) 12648 type = TREE_TYPE (TREE_TYPE (fndecl)); 12649 12650 /* b) If the function has any non-uniform, non-linear parameters, 12651 then the characteristic data type is the type of the first 12652 such parameter. */ 12653 else 12654 { 12655 vec<tree> map = simd_clone_vector_of_formal_parm_types (fndecl); 12656 for (unsigned int i = 0; i < clone_info->nargs; ++i) 12657 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) 12658 { 12659 type = map[i]; 12660 break; 12661 } 12662 map.release (); 12663 } 12664 12665 /* c) If the characteristic data type determined by a) or b) above 12666 is struct, union, or class type which is pass-by-value (except 12667 for the type that maps to the built-in complex data type), the 12668 characteristic data type is int. */ 12669 if (RECORD_OR_UNION_TYPE_P (type) 12670 && !aggregate_value_p (type, NULL) 12671 && TREE_CODE (type) != COMPLEX_TYPE) 12672 return integer_type_node; 12673 12674 /* d) If none of the above three classes is applicable, the 12675 characteristic data type is int. */ 12676 12677 return type; 12678 12679 /* e) For Intel Xeon Phi native and offload compilation, if the 12680 resulting characteristic data type is 8-bit or 16-bit integer 12681 data type, the characteristic data type is int. */ 12682 /* Well, we don't handle Xeon Phi yet. */ 12683} 12684 12685static tree 12686simd_clone_mangle (struct cgraph_node *node, 12687 struct cgraph_simd_clone *clone_info) 12688{ 12689 char vecsize_mangle = clone_info->vecsize_mangle; 12690 char mask = clone_info->inbranch ? 'M' : 'N'; 12691 unsigned int simdlen = clone_info->simdlen; 12692 unsigned int n; 12693 pretty_printer pp; 12694 12695 gcc_assert (vecsize_mangle && simdlen); 12696 12697 pp_string (&pp, "_ZGV"); 12698 pp_character (&pp, vecsize_mangle); 12699 pp_character (&pp, mask); 12700 pp_decimal_int (&pp, simdlen); 12701 12702 for (n = 0; n < clone_info->nargs; ++n) 12703 { 12704 struct cgraph_simd_clone_arg arg = clone_info->args[n]; 12705 12706 if (arg.arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM) 12707 pp_character (&pp, 'u'); 12708 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 12709 { 12710 gcc_assert (arg.linear_step != 0); 12711 pp_character (&pp, 'l'); 12712 if (arg.linear_step > 1) 12713 pp_unsigned_wide_integer (&pp, arg.linear_step); 12714 else if (arg.linear_step < 0) 12715 { 12716 pp_character (&pp, 'n'); 12717 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT) 12718 arg.linear_step)); 12719 } 12720 } 12721 else if (arg.arg_type == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP) 12722 { 12723 pp_character (&pp, 's'); 12724 pp_unsigned_wide_integer (&pp, arg.linear_step); 12725 } 12726 else 12727 pp_character (&pp, 'v'); 12728 if (arg.alignment) 12729 { 12730 pp_character (&pp, 'a'); 12731 pp_decimal_int (&pp, arg.alignment); 12732 } 12733 } 12734 12735 pp_underscore (&pp); 12736 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); 12737 if (*str == '*') 12738 ++str; 12739 pp_string (&pp, str); 12740 str = pp_formatted_text (&pp); 12741 12742 /* If there already is a SIMD clone with the same mangled name, don't 12743 add another one. This can happen e.g. for 12744 #pragma omp declare simd 12745 #pragma omp declare simd simdlen(8) 12746 int foo (int, int); 12747 if the simdlen is assumed to be 8 for the first one, etc. */ 12748 for (struct cgraph_node *clone = node->simd_clones; clone; 12749 clone = clone->simdclone->next_clone) 12750 if (strcmp (IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (clone->decl)), 12751 str) == 0) 12752 return NULL_TREE; 12753 12754 return get_identifier (str); 12755} 12756 12757/* Create a simd clone of OLD_NODE and return it. */ 12758 12759static struct cgraph_node * 12760simd_clone_create (struct cgraph_node *old_node) 12761{ 12762 struct cgraph_node *new_node; 12763 if (old_node->definition) 12764 { 12765 if (!old_node->has_gimple_body_p ()) 12766 return NULL; 12767 old_node->get_body (); 12768 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL, 12769 false, NULL, NULL, 12770 "simdclone"); 12771 } 12772 else 12773 { 12774 tree old_decl = old_node->decl; 12775 tree new_decl = copy_node (old_node->decl); 12776 DECL_NAME (new_decl) = clone_function_name (old_decl, "simdclone"); 12777 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 12778 SET_DECL_RTL (new_decl, NULL); 12779 DECL_STATIC_CONSTRUCTOR (new_decl) = 0; 12780 DECL_STATIC_DESTRUCTOR (new_decl) = 0; 12781 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 12782 if (old_node->in_other_partition) 12783 new_node->in_other_partition = 1; 12784 } 12785 if (new_node == NULL) 12786 return new_node; 12787 12788 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl); 12789 12790 /* The function cgraph_function_versioning () will force the new 12791 symbol local. Undo this, and inherit external visability from 12792 the old node. */ 12793 new_node->local.local = old_node->local.local; 12794 new_node->externally_visible = old_node->externally_visible; 12795 12796 return new_node; 12797} 12798 12799/* Adjust the return type of the given function to its appropriate 12800 vector counterpart. Returns a simd array to be used throughout the 12801 function as a return value. */ 12802 12803static tree 12804simd_clone_adjust_return_type (struct cgraph_node *node) 12805{ 12806 tree fndecl = node->decl; 12807 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl)); 12808 unsigned int veclen; 12809 tree t; 12810 12811 /* Adjust the function return type. */ 12812 if (orig_rettype == void_type_node) 12813 return NULL_TREE; 12814 TREE_TYPE (fndecl) = build_distinct_type_copy (TREE_TYPE (fndecl)); 12815 t = TREE_TYPE (TREE_TYPE (fndecl)); 12816 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t)) 12817 veclen = node->simdclone->vecsize_int; 12818 else 12819 veclen = node->simdclone->vecsize_float; 12820 veclen /= GET_MODE_BITSIZE (TYPE_MODE (t)); 12821 if (veclen > node->simdclone->simdlen) 12822 veclen = node->simdclone->simdlen; 12823 if (POINTER_TYPE_P (t)) 12824 t = pointer_sized_int_node; 12825 if (veclen == node->simdclone->simdlen) 12826 t = build_vector_type (t, node->simdclone->simdlen); 12827 else 12828 { 12829 t = build_vector_type (t, veclen); 12830 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen); 12831 } 12832 TREE_TYPE (TREE_TYPE (fndecl)) = t; 12833 if (!node->definition) 12834 return NULL_TREE; 12835 12836 t = DECL_RESULT (fndecl); 12837 /* Adjust the DECL_RESULT. */ 12838 gcc_assert (TREE_TYPE (t) != void_type_node); 12839 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl)); 12840 relayout_decl (t); 12841 12842 tree atype = build_array_type_nelts (orig_rettype, 12843 node->simdclone->simdlen); 12844 if (veclen != node->simdclone->simdlen) 12845 return build1 (VIEW_CONVERT_EXPR, atype, t); 12846 12847 /* Set up a SIMD array to use as the return value. */ 12848 tree retval = create_tmp_var_raw (atype, "retval"); 12849 gimple_add_tmp_var (retval); 12850 return retval; 12851} 12852 12853/* Each vector argument has a corresponding array to be used locally 12854 as part of the eventual loop. Create such temporary array and 12855 return it. 12856 12857 PREFIX is the prefix to be used for the temporary. 12858 12859 TYPE is the inner element type. 12860 12861 SIMDLEN is the number of elements. */ 12862 12863static tree 12864create_tmp_simd_array (const char *prefix, tree type, int simdlen) 12865{ 12866 tree atype = build_array_type_nelts (type, simdlen); 12867 tree avar = create_tmp_var_raw (atype, prefix); 12868 gimple_add_tmp_var (avar); 12869 return avar; 12870} 12871 12872/* Modify the function argument types to their corresponding vector 12873 counterparts if appropriate. Also, create one array for each simd 12874 argument to be used locally when using the function arguments as 12875 part of the loop. 12876 12877 NODE is the function whose arguments are to be adjusted. 12878 12879 Returns an adjustment vector that will be filled describing how the 12880 argument types will be adjusted. */ 12881 12882static ipa_parm_adjustment_vec 12883simd_clone_adjust_argument_types (struct cgraph_node *node) 12884{ 12885 vec<tree> args; 12886 ipa_parm_adjustment_vec adjustments; 12887 12888 if (node->definition) 12889 args = ipa_get_vector_of_formal_parms (node->decl); 12890 else 12891 args = simd_clone_vector_of_formal_parm_types (node->decl); 12892 adjustments.create (args.length ()); 12893 unsigned i, j, veclen; 12894 struct ipa_parm_adjustment adj; 12895 for (i = 0; i < node->simdclone->nargs; ++i) 12896 { 12897 memset (&adj, 0, sizeof (adj)); 12898 tree parm = args[i]; 12899 tree parm_type = node->definition ? TREE_TYPE (parm) : parm; 12900 adj.base_index = i; 12901 adj.base = parm; 12902 12903 node->simdclone->args[i].orig_arg = node->definition ? parm : NULL_TREE; 12904 node->simdclone->args[i].orig_type = parm_type; 12905 12906 if (node->simdclone->args[i].arg_type != SIMD_CLONE_ARG_TYPE_VECTOR) 12907 { 12908 /* No adjustment necessary for scalar arguments. */ 12909 adj.op = IPA_PARM_OP_COPY; 12910 } 12911 else 12912 { 12913 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type)) 12914 veclen = node->simdclone->vecsize_int; 12915 else 12916 veclen = node->simdclone->vecsize_float; 12917 veclen /= GET_MODE_BITSIZE (TYPE_MODE (parm_type)); 12918 if (veclen > node->simdclone->simdlen) 12919 veclen = node->simdclone->simdlen; 12920 adj.arg_prefix = "simd"; 12921 if (POINTER_TYPE_P (parm_type)) 12922 adj.type = build_vector_type (pointer_sized_int_node, veclen); 12923 else 12924 adj.type = build_vector_type (parm_type, veclen); 12925 node->simdclone->args[i].vector_type = adj.type; 12926 for (j = veclen; j < node->simdclone->simdlen; j += veclen) 12927 { 12928 adjustments.safe_push (adj); 12929 if (j == veclen) 12930 { 12931 memset (&adj, 0, sizeof (adj)); 12932 adj.op = IPA_PARM_OP_NEW; 12933 adj.arg_prefix = "simd"; 12934 adj.base_index = i; 12935 adj.type = node->simdclone->args[i].vector_type; 12936 } 12937 } 12938 12939 if (node->definition) 12940 node->simdclone->args[i].simd_array 12941 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)), 12942 parm_type, node->simdclone->simdlen); 12943 } 12944 adjustments.safe_push (adj); 12945 } 12946 12947 if (node->simdclone->inbranch) 12948 { 12949 tree base_type 12950 = simd_clone_compute_base_data_type (node->simdclone->origin, 12951 node->simdclone); 12952 12953 memset (&adj, 0, sizeof (adj)); 12954 adj.op = IPA_PARM_OP_NEW; 12955 adj.arg_prefix = "mask"; 12956 12957 adj.base_index = i; 12958 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type)) 12959 veclen = node->simdclone->vecsize_int; 12960 else 12961 veclen = node->simdclone->vecsize_float; 12962 veclen /= GET_MODE_BITSIZE (TYPE_MODE (base_type)); 12963 if (veclen > node->simdclone->simdlen) 12964 veclen = node->simdclone->simdlen; 12965 if (POINTER_TYPE_P (base_type)) 12966 adj.type = build_vector_type (pointer_sized_int_node, veclen); 12967 else 12968 adj.type = build_vector_type (base_type, veclen); 12969 adjustments.safe_push (adj); 12970 12971 for (j = veclen; j < node->simdclone->simdlen; j += veclen) 12972 adjustments.safe_push (adj); 12973 12974 /* We have previously allocated one extra entry for the mask. Use 12975 it and fill it. */ 12976 struct cgraph_simd_clone *sc = node->simdclone; 12977 sc->nargs++; 12978 if (node->definition) 12979 { 12980 sc->args[i].orig_arg 12981 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type); 12982 sc->args[i].simd_array 12983 = create_tmp_simd_array ("mask", base_type, sc->simdlen); 12984 } 12985 sc->args[i].orig_type = base_type; 12986 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK; 12987 } 12988 12989 if (node->definition) 12990 ipa_modify_formal_parameters (node->decl, adjustments); 12991 else 12992 { 12993 tree new_arg_types = NULL_TREE, new_reversed; 12994 bool last_parm_void = false; 12995 if (args.length () > 0 && args.last () == void_type_node) 12996 last_parm_void = true; 12997 12998 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl))); 12999 j = adjustments.length (); 13000 for (i = 0; i < j; i++) 13001 { 13002 struct ipa_parm_adjustment *adj = &adjustments[i]; 13003 tree ptype; 13004 if (adj->op == IPA_PARM_OP_COPY) 13005 ptype = args[adj->base_index]; 13006 else 13007 ptype = adj->type; 13008 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types); 13009 } 13010 new_reversed = nreverse (new_arg_types); 13011 if (last_parm_void) 13012 { 13013 if (new_reversed) 13014 TREE_CHAIN (new_arg_types) = void_list_node; 13015 else 13016 new_reversed = void_list_node; 13017 } 13018 13019 tree new_type = build_distinct_type_copy (TREE_TYPE (node->decl)); 13020 TYPE_ARG_TYPES (new_type) = new_reversed; 13021 TREE_TYPE (node->decl) = new_type; 13022 13023 adjustments.release (); 13024 } 13025 args.release (); 13026 return adjustments; 13027} 13028 13029/* Initialize and copy the function arguments in NODE to their 13030 corresponding local simd arrays. Returns a fresh gimple_seq with 13031 the instruction sequence generated. */ 13032 13033static gimple_seq 13034simd_clone_init_simd_arrays (struct cgraph_node *node, 13035 ipa_parm_adjustment_vec adjustments) 13036{ 13037 gimple_seq seq = NULL; 13038 unsigned i = 0, j = 0, k; 13039 13040 for (tree arg = DECL_ARGUMENTS (node->decl); 13041 arg; 13042 arg = DECL_CHAIN (arg), i++, j++) 13043 { 13044 if (adjustments[j].op == IPA_PARM_OP_COPY) 13045 continue; 13046 13047 node->simdclone->args[i].vector_arg = arg; 13048 13049 tree array = node->simdclone->args[i].simd_array; 13050 if (TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)) == node->simdclone->simdlen) 13051 { 13052 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 13053 tree ptr = build_fold_addr_expr (array); 13054 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 13055 build_int_cst (ptype, 0)); 13056 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 13057 gimplify_and_add (t, &seq); 13058 } 13059 else 13060 { 13061 unsigned int simdlen = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg)); 13062 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 13063 for (k = 0; k < node->simdclone->simdlen; k += simdlen) 13064 { 13065 tree ptr = build_fold_addr_expr (array); 13066 int elemsize; 13067 if (k) 13068 { 13069 arg = DECL_CHAIN (arg); 13070 j++; 13071 } 13072 elemsize 13073 = GET_MODE_SIZE (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg)))); 13074 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 13075 build_int_cst (ptype, k * elemsize)); 13076 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 13077 gimplify_and_add (t, &seq); 13078 } 13079 } 13080 } 13081 return seq; 13082} 13083 13084/* Callback info for ipa_simd_modify_stmt_ops below. */ 13085 13086struct modify_stmt_info { 13087 ipa_parm_adjustment_vec adjustments; 13088 gimple stmt; 13089 /* True if the parent statement was modified by 13090 ipa_simd_modify_stmt_ops. */ 13091 bool modified; 13092}; 13093 13094/* Callback for walk_gimple_op. 13095 13096 Adjust operands from a given statement as specified in the 13097 adjustments vector in the callback data. */ 13098 13099static tree 13100ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data) 13101{ 13102 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 13103 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info; 13104 tree *orig_tp = tp; 13105 if (TREE_CODE (*tp) == ADDR_EXPR) 13106 tp = &TREE_OPERAND (*tp, 0); 13107 struct ipa_parm_adjustment *cand = NULL; 13108 if (TREE_CODE (*tp) == PARM_DECL) 13109 cand = ipa_get_adjustment_candidate (&tp, NULL, info->adjustments, true); 13110 else 13111 { 13112 if (TYPE_P (*tp)) 13113 *walk_subtrees = 0; 13114 } 13115 13116 tree repl = NULL_TREE; 13117 if (cand) 13118 repl = unshare_expr (cand->new_decl); 13119 else 13120 { 13121 if (tp != orig_tp) 13122 { 13123 *walk_subtrees = 0; 13124 bool modified = info->modified; 13125 info->modified = false; 13126 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset); 13127 if (!info->modified) 13128 { 13129 info->modified = modified; 13130 return NULL_TREE; 13131 } 13132 info->modified = modified; 13133 repl = *tp; 13134 } 13135 else 13136 return NULL_TREE; 13137 } 13138 13139 if (tp != orig_tp) 13140 { 13141 repl = build_fold_addr_expr (repl); 13142 gimple stmt; 13143 if (is_gimple_debug (info->stmt)) 13144 { 13145 tree vexpr = make_node (DEBUG_EXPR_DECL); 13146 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL); 13147 DECL_ARTIFICIAL (vexpr) = 1; 13148 TREE_TYPE (vexpr) = TREE_TYPE (repl); 13149 DECL_MODE (vexpr) = TYPE_MODE (TREE_TYPE (repl)); 13150 repl = vexpr; 13151 } 13152 else 13153 { 13154 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl); 13155 repl = gimple_assign_lhs (stmt); 13156 } 13157 gimple_stmt_iterator gsi = gsi_for_stmt (info->stmt); 13158 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 13159 *orig_tp = repl; 13160 } 13161 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl))) 13162 { 13163 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl); 13164 *tp = vce; 13165 } 13166 else 13167 *tp = repl; 13168 13169 info->modified = true; 13170 return NULL_TREE; 13171} 13172 13173/* Traverse the function body and perform all modifications as 13174 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be 13175 modified such that the replacement/reduction value will now be an 13176 offset into the corresponding simd_array. 13177 13178 This function will replace all function argument uses with their 13179 corresponding simd array elements, and ajust the return values 13180 accordingly. */ 13181 13182static void 13183ipa_simd_modify_function_body (struct cgraph_node *node, 13184 ipa_parm_adjustment_vec adjustments, 13185 tree retval_array, tree iter) 13186{ 13187 basic_block bb; 13188 unsigned int i, j, l; 13189 13190 /* Re-use the adjustments array, but this time use it to replace 13191 every function argument use to an offset into the corresponding 13192 simd_array. */ 13193 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j) 13194 { 13195 if (!node->simdclone->args[i].vector_arg) 13196 continue; 13197 13198 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg); 13199 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg); 13200 adjustments[j].new_decl 13201 = build4 (ARRAY_REF, 13202 basetype, 13203 node->simdclone->args[i].simd_array, 13204 iter, 13205 NULL_TREE, NULL_TREE); 13206 if (adjustments[j].op == IPA_PARM_OP_NONE 13207 && TYPE_VECTOR_SUBPARTS (vectype) < node->simdclone->simdlen) 13208 j += node->simdclone->simdlen / TYPE_VECTOR_SUBPARTS (vectype) - 1; 13209 } 13210 13211 l = adjustments.length (); 13212 for (i = 1; i < num_ssa_names; i++) 13213 { 13214 tree name = ssa_name (i); 13215 if (name 13216 && SSA_NAME_VAR (name) 13217 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL) 13218 { 13219 for (j = 0; j < l; j++) 13220 if (SSA_NAME_VAR (name) == adjustments[j].base 13221 && adjustments[j].new_decl) 13222 { 13223 tree base_var; 13224 if (adjustments[j].new_ssa_base == NULL_TREE) 13225 { 13226 base_var 13227 = copy_var_decl (adjustments[j].base, 13228 DECL_NAME (adjustments[j].base), 13229 TREE_TYPE (adjustments[j].base)); 13230 adjustments[j].new_ssa_base = base_var; 13231 } 13232 else 13233 base_var = adjustments[j].new_ssa_base; 13234 if (SSA_NAME_IS_DEFAULT_DEF (name)) 13235 { 13236 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13237 gimple_stmt_iterator gsi = gsi_after_labels (bb); 13238 tree new_decl = unshare_expr (adjustments[j].new_decl); 13239 set_ssa_default_def (cfun, adjustments[j].base, NULL_TREE); 13240 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 13241 SSA_NAME_IS_DEFAULT_DEF (name) = 0; 13242 gimple stmt = gimple_build_assign (name, new_decl); 13243 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 13244 } 13245 else 13246 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 13247 } 13248 } 13249 } 13250 13251 struct modify_stmt_info info; 13252 info.adjustments = adjustments; 13253 13254 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) 13255 { 13256 gimple_stmt_iterator gsi; 13257 13258 gsi = gsi_start_bb (bb); 13259 while (!gsi_end_p (gsi)) 13260 { 13261 gimple stmt = gsi_stmt (gsi); 13262 info.stmt = stmt; 13263 struct walk_stmt_info wi; 13264 13265 memset (&wi, 0, sizeof (wi)); 13266 info.modified = false; 13267 wi.info = &info; 13268 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi); 13269 13270 if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) 13271 { 13272 tree retval = gimple_return_retval (return_stmt); 13273 if (!retval) 13274 { 13275 gsi_remove (&gsi, true); 13276 continue; 13277 } 13278 13279 /* Replace `return foo' with `retval_array[iter] = foo'. */ 13280 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval), 13281 retval_array, iter, NULL, NULL); 13282 stmt = gimple_build_assign (ref, retval); 13283 gsi_replace (&gsi, stmt, true); 13284 info.modified = true; 13285 } 13286 13287 if (info.modified) 13288 { 13289 update_stmt (stmt); 13290 if (maybe_clean_eh_stmt (stmt)) 13291 gimple_purge_dead_eh_edges (gimple_bb (stmt)); 13292 } 13293 gsi_next (&gsi); 13294 } 13295 } 13296} 13297 13298/* Adjust the argument types in NODE to their appropriate vector 13299 counterparts. */ 13300 13301static void 13302simd_clone_adjust (struct cgraph_node *node) 13303{ 13304 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 13305 13306 targetm.simd_clone.adjust (node); 13307 13308 tree retval = simd_clone_adjust_return_type (node); 13309 ipa_parm_adjustment_vec adjustments 13310 = simd_clone_adjust_argument_types (node); 13311 13312 push_gimplify_context (); 13313 13314 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments); 13315 13316 /* Adjust all uses of vector arguments accordingly. Adjust all 13317 return values accordingly. */ 13318 tree iter = create_tmp_var (unsigned_type_node, "iter"); 13319 tree iter1 = make_ssa_name (iter); 13320 tree iter2 = make_ssa_name (iter); 13321 ipa_simd_modify_function_body (node, adjustments, retval, iter1); 13322 13323 /* Initialize the iteration variable. */ 13324 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13325 basic_block body_bb = split_block_after_labels (entry_bb)->dest; 13326 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 13327 /* Insert the SIMD array and iv initialization at function 13328 entry. */ 13329 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT); 13330 13331 pop_gimplify_context (NULL); 13332 13333 /* Create a new BB right before the original exit BB, to hold the 13334 iteration increment and the condition/branch. */ 13335 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src; 13336 basic_block incr_bb = create_empty_bb (orig_exit); 13337 add_bb_to_loop (incr_bb, body_bb->loop_father); 13338 /* The succ of orig_exit was EXIT_BLOCK_PTR_FOR_FN (cfun), with an empty 13339 flag. Set it now to be a FALLTHRU_EDGE. */ 13340 gcc_assert (EDGE_COUNT (orig_exit->succs) == 1); 13341 EDGE_SUCC (orig_exit, 0)->flags |= EDGE_FALLTHRU; 13342 for (unsigned i = 0; 13343 i < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds); ++i) 13344 { 13345 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), i); 13346 redirect_edge_succ (e, incr_bb); 13347 } 13348 edge e = make_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 13349 e->probability = REG_BR_PROB_BASE; 13350 gsi = gsi_last_bb (incr_bb); 13351 gimple g = gimple_build_assign (iter2, PLUS_EXPR, iter1, 13352 build_int_cst (unsigned_type_node, 1)); 13353 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13354 13355 /* Mostly annotate the loop for the vectorizer (the rest is done below). */ 13356 struct loop *loop = alloc_loop (); 13357 cfun->has_force_vectorize_loops = true; 13358 loop->safelen = node->simdclone->simdlen; 13359 loop->force_vectorize = true; 13360 loop->header = body_bb; 13361 13362 /* Branch around the body if the mask applies. */ 13363 if (node->simdclone->inbranch) 13364 { 13365 gimple_stmt_iterator gsi = gsi_last_bb (loop->header); 13366 tree mask_array 13367 = node->simdclone->args[node->simdclone->nargs - 1].simd_array; 13368 tree mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 13369 tree aref = build4 (ARRAY_REF, 13370 TREE_TYPE (TREE_TYPE (mask_array)), 13371 mask_array, iter1, 13372 NULL, NULL); 13373 g = gimple_build_assign (mask, aref); 13374 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13375 int bitsize = GET_MODE_BITSIZE (TYPE_MODE (TREE_TYPE (aref))); 13376 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref))) 13377 { 13378 aref = build1 (VIEW_CONVERT_EXPR, 13379 build_nonstandard_integer_type (bitsize, 0), mask); 13380 mask = make_ssa_name (TREE_TYPE (aref)); 13381 g = gimple_build_assign (mask, aref); 13382 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13383 } 13384 13385 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)), 13386 NULL, NULL); 13387 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13388 make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE); 13389 FALLTHRU_EDGE (loop->header)->flags = EDGE_FALSE_VALUE; 13390 } 13391 13392 /* Generate the condition. */ 13393 g = gimple_build_cond (LT_EXPR, 13394 iter2, 13395 build_int_cst (unsigned_type_node, 13396 node->simdclone->simdlen), 13397 NULL, NULL); 13398 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13399 e = split_block (incr_bb, gsi_stmt (gsi)); 13400 basic_block latch_bb = e->dest; 13401 basic_block new_exit_bb; 13402 new_exit_bb = split_block (latch_bb, NULL)->dest; 13403 loop->latch = latch_bb; 13404 13405 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); 13406 13407 make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); 13408 /* The successor of incr_bb is already pointing to latch_bb; just 13409 change the flags. 13410 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ 13411 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE; 13412 13413 gphi *phi = create_phi_node (iter1, body_bb); 13414 edge preheader_edge = find_edge (entry_bb, body_bb); 13415 edge latch_edge = single_succ_edge (latch_bb); 13416 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge, 13417 UNKNOWN_LOCATION); 13418 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 13419 13420 /* Generate the new return. */ 13421 gsi = gsi_last_bb (new_exit_bb); 13422 if (retval 13423 && TREE_CODE (retval) == VIEW_CONVERT_EXPR 13424 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL) 13425 retval = TREE_OPERAND (retval, 0); 13426 else if (retval) 13427 { 13428 retval = build1 (VIEW_CONVERT_EXPR, 13429 TREE_TYPE (TREE_TYPE (node->decl)), 13430 retval); 13431 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL, 13432 false, GSI_CONTINUE_LINKING); 13433 } 13434 g = gimple_build_return (retval); 13435 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 13436 13437 /* Handle aligned clauses by replacing default defs of the aligned 13438 uniform args with __builtin_assume_aligned (arg_N(D), alignment) 13439 lhs. Handle linear by adding PHIs. */ 13440 for (unsigned i = 0; i < node->simdclone->nargs; i++) 13441 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 13442 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg) 13443 || !is_gimple_reg_type 13444 (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 13445 { 13446 tree orig_arg = node->simdclone->args[i].orig_arg; 13447 if (is_gimple_reg_type (TREE_TYPE (orig_arg))) 13448 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 13449 else 13450 { 13451 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg)); 13452 gimple_add_tmp_var (iter1); 13453 } 13454 gsi = gsi_after_labels (entry_bb); 13455 g = gimple_build_assign (iter1, orig_arg); 13456 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13457 gsi = gsi_after_labels (body_bb); 13458 g = gimple_build_assign (orig_arg, iter1); 13459 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13460 } 13461 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 13462 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg) 13463 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 13464 == REFERENCE_TYPE 13465 && TREE_ADDRESSABLE 13466 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 13467 { 13468 tree orig_arg = node->simdclone->args[i].orig_arg; 13469 tree def = ssa_default_def (cfun, orig_arg); 13470 if (def && !has_zero_uses (def)) 13471 { 13472 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg))); 13473 gimple_add_tmp_var (iter1); 13474 gsi = gsi_after_labels (entry_bb); 13475 g = gimple_build_assign (iter1, build_simple_mem_ref (def)); 13476 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13477 gsi = gsi_after_labels (body_bb); 13478 g = gimple_build_assign (build_simple_mem_ref (def), iter1); 13479 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13480 } 13481 } 13482 else if (node->simdclone->args[i].alignment 13483 && node->simdclone->args[i].arg_type 13484 == SIMD_CLONE_ARG_TYPE_UNIFORM 13485 && (node->simdclone->args[i].alignment 13486 & (node->simdclone->args[i].alignment - 1)) == 0 13487 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 13488 == POINTER_TYPE) 13489 { 13490 unsigned int alignment = node->simdclone->args[i].alignment; 13491 tree orig_arg = node->simdclone->args[i].orig_arg; 13492 tree def = ssa_default_def (cfun, orig_arg); 13493 if (def && !has_zero_uses (def)) 13494 { 13495 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 13496 gimple_seq seq = NULL; 13497 bool need_cvt = false; 13498 gcall *call 13499 = gimple_build_call (fn, 2, def, size_int (alignment)); 13500 g = call; 13501 if (!useless_type_conversion_p (TREE_TYPE (orig_arg), 13502 ptr_type_node)) 13503 need_cvt = true; 13504 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg); 13505 gimple_call_set_lhs (g, t); 13506 gimple_seq_add_stmt_without_update (&seq, g); 13507 if (need_cvt) 13508 { 13509 t = make_ssa_name (orig_arg); 13510 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g)); 13511 gimple_seq_add_stmt_without_update (&seq, g); 13512 } 13513 gsi_insert_seq_on_edge_immediate 13514 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); 13515 13516 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 13517 int freq = compute_call_stmt_bb_frequency (current_function_decl, 13518 entry_bb); 13519 node->create_edge (cgraph_node::get_create (fn), 13520 call, entry_bb->count, freq); 13521 13522 imm_use_iterator iter; 13523 use_operand_p use_p; 13524 gimple use_stmt; 13525 tree repl = gimple_get_lhs (g); 13526 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 13527 if (is_gimple_debug (use_stmt) || use_stmt == call) 13528 continue; 13529 else 13530 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 13531 SET_USE (use_p, repl); 13532 } 13533 } 13534 else if (node->simdclone->args[i].arg_type 13535 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 13536 { 13537 tree orig_arg = node->simdclone->args[i].orig_arg; 13538 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13539 || POINTER_TYPE_P (TREE_TYPE (orig_arg))); 13540 tree def = NULL_TREE; 13541 if (TREE_ADDRESSABLE (orig_arg)) 13542 { 13543 def = make_ssa_name (TREE_TYPE (orig_arg)); 13544 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 13545 iter2 = make_ssa_name (TREE_TYPE (orig_arg)); 13546 gsi = gsi_after_labels (entry_bb); 13547 g = gimple_build_assign (def, orig_arg); 13548 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13549 } 13550 else 13551 { 13552 def = ssa_default_def (cfun, orig_arg); 13553 if (!def || has_zero_uses (def)) 13554 def = NULL_TREE; 13555 else 13556 { 13557 iter1 = make_ssa_name (orig_arg); 13558 iter2 = make_ssa_name (orig_arg); 13559 } 13560 } 13561 if (def) 13562 { 13563 phi = create_phi_node (iter1, body_bb); 13564 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION); 13565 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 13566 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13567 ? PLUS_EXPR : POINTER_PLUS_EXPR; 13568 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 13569 ? TREE_TYPE (orig_arg) : sizetype; 13570 tree addcst 13571 = build_int_cst (addtype, node->simdclone->args[i].linear_step); 13572 g = gimple_build_assign (iter2, code, iter1, addcst); 13573 gsi = gsi_last_bb (incr_bb); 13574 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 13575 13576 imm_use_iterator iter; 13577 use_operand_p use_p; 13578 gimple use_stmt; 13579 if (TREE_ADDRESSABLE (orig_arg)) 13580 { 13581 gsi = gsi_after_labels (body_bb); 13582 g = gimple_build_assign (orig_arg, iter1); 13583 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 13584 } 13585 else 13586 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 13587 if (use_stmt == phi) 13588 continue; 13589 else 13590 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 13591 SET_USE (use_p, iter1); 13592 } 13593 } 13594 13595 calculate_dominance_info (CDI_DOMINATORS); 13596 add_loop (loop, loop->header->loop_father); 13597 update_ssa (TODO_update_ssa); 13598 13599 pop_cfun (); 13600} 13601 13602/* If the function in NODE is tagged as an elemental SIMD function, 13603 create the appropriate SIMD clones. */ 13604 13605static void 13606expand_simd_clones (struct cgraph_node *node) 13607{ 13608 tree attr = lookup_attribute ("omp declare simd", 13609 DECL_ATTRIBUTES (node->decl)); 13610 if (attr == NULL_TREE 13611 || node->global.inlined_to 13612 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) 13613 return; 13614 13615 /* Ignore 13616 #pragma omp declare simd 13617 extern int foo (); 13618 in C, there we don't know the argument types at all. */ 13619 if (!node->definition 13620 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE) 13621 return; 13622 13623 /* Call this before creating clone_info, as it might ggc_collect. */ 13624 if (node->definition && node->has_gimple_body_p ()) 13625 node->get_body (); 13626 13627 do 13628 { 13629 /* Start with parsing the "omp declare simd" attribute(s). */ 13630 bool inbranch_clause_specified; 13631 struct cgraph_simd_clone *clone_info 13632 = simd_clone_clauses_extract (node, TREE_VALUE (attr), 13633 &inbranch_clause_specified); 13634 if (clone_info == NULL) 13635 continue; 13636 13637 int orig_simdlen = clone_info->simdlen; 13638 tree base_type = simd_clone_compute_base_data_type (node, clone_info); 13639 /* The target can return 0 (no simd clones should be created), 13640 1 (just one ISA of simd clones should be created) or higher 13641 count of ISA variants. In that case, clone_info is initialized 13642 for the first ISA variant. */ 13643 int count 13644 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info, 13645 base_type, 0); 13646 if (count == 0) 13647 continue; 13648 13649 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED, 13650 also create one inbranch and one !inbranch clone of it. */ 13651 for (int i = 0; i < count * 2; i++) 13652 { 13653 struct cgraph_simd_clone *clone = clone_info; 13654 if (inbranch_clause_specified && (i & 1) != 0) 13655 continue; 13656 13657 if (i != 0) 13658 { 13659 clone = simd_clone_struct_alloc (clone_info->nargs 13660 + ((i & 1) != 0)); 13661 simd_clone_struct_copy (clone, clone_info); 13662 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen 13663 and simd_clone_adjust_argument_types did to the first 13664 clone's info. */ 13665 clone->nargs -= clone_info->inbranch; 13666 clone->simdlen = orig_simdlen; 13667 /* And call the target hook again to get the right ISA. */ 13668 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone, 13669 base_type, 13670 i / 2); 13671 if ((i & 1) != 0) 13672 clone->inbranch = 1; 13673 } 13674 13675 /* simd_clone_mangle might fail if such a clone has been created 13676 already. */ 13677 tree id = simd_clone_mangle (node, clone); 13678 if (id == NULL_TREE) 13679 continue; 13680 13681 /* Only when we are sure we want to create the clone actually 13682 clone the function (or definitions) or create another 13683 extern FUNCTION_DECL (for prototypes without definitions). */ 13684 struct cgraph_node *n = simd_clone_create (node); 13685 if (n == NULL) 13686 continue; 13687 13688 n->simdclone = clone; 13689 clone->origin = node; 13690 clone->next_clone = NULL; 13691 if (node->simd_clones == NULL) 13692 { 13693 clone->prev_clone = n; 13694 node->simd_clones = n; 13695 } 13696 else 13697 { 13698 clone->prev_clone = node->simd_clones->simdclone->prev_clone; 13699 clone->prev_clone->simdclone->next_clone = n; 13700 node->simd_clones->simdclone->prev_clone = n; 13701 } 13702 symtab->change_decl_assembler_name (n->decl, id); 13703 /* And finally adjust the return type, parameters and for 13704 definitions also function body. */ 13705 if (node->definition) 13706 simd_clone_adjust (n); 13707 else 13708 { 13709 simd_clone_adjust_return_type (n); 13710 simd_clone_adjust_argument_types (n); 13711 } 13712 } 13713 } 13714 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr)))); 13715} 13716 13717/* Entry point for IPA simd clone creation pass. */ 13718 13719static unsigned int 13720ipa_omp_simd_clone (void) 13721{ 13722 struct cgraph_node *node; 13723 FOR_EACH_FUNCTION (node) 13724 expand_simd_clones (node); 13725 return 0; 13726} 13727 13728namespace { 13729 13730const pass_data pass_data_omp_simd_clone = 13731{ 13732 SIMPLE_IPA_PASS, /* type */ 13733 "simdclone", /* name */ 13734 OPTGROUP_NONE, /* optinfo_flags */ 13735 TV_NONE, /* tv_id */ 13736 ( PROP_ssa | PROP_cfg ), /* properties_required */ 13737 0, /* properties_provided */ 13738 0, /* properties_destroyed */ 13739 0, /* todo_flags_start */ 13740 0, /* todo_flags_finish */ 13741}; 13742 13743class pass_omp_simd_clone : public simple_ipa_opt_pass 13744{ 13745public: 13746 pass_omp_simd_clone(gcc::context *ctxt) 13747 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt) 13748 {} 13749 13750 /* opt_pass methods: */ 13751 virtual bool gate (function *); 13752 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); } 13753}; 13754 13755bool 13756pass_omp_simd_clone::gate (function *) 13757{ 13758 return ((flag_openmp || flag_openmp_simd 13759 || flag_cilkplus 13760 || (in_lto_p && !flag_wpa)) 13761 && (targetm.simd_clone.compute_vecsize_and_simdlen != NULL)); 13762} 13763 13764} // anon namespace 13765 13766simple_ipa_opt_pass * 13767make_pass_omp_simd_clone (gcc::context *ctxt) 13768{ 13769 return new pass_omp_simd_clone (ctxt); 13770} 13771 13772/* Helper function for omp_finish_file routine. Takes decls from V_DECLS and 13773 adds their addresses and sizes to constructor-vector V_CTOR. */ 13774static void 13775add_decls_addresses_to_decl_constructor (vec<tree, va_gc> *v_decls, 13776 vec<constructor_elt, va_gc> *v_ctor) 13777{ 13778 unsigned len = vec_safe_length (v_decls); 13779 for (unsigned i = 0; i < len; i++) 13780 { 13781 tree it = (*v_decls)[i]; 13782 bool is_function = TREE_CODE (it) != VAR_DECL; 13783 13784 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, build_fold_addr_expr (it)); 13785 if (!is_function) 13786 CONSTRUCTOR_APPEND_ELT (v_ctor, NULL_TREE, 13787 fold_convert (const_ptr_type_node, 13788 DECL_SIZE_UNIT (it))); 13789 } 13790} 13791 13792/* Create new symbols containing (address, size) pairs for global variables, 13793 marked with "omp declare target" attribute, as well as addresses for the 13794 functions, which are outlined offloading regions. */ 13795void 13796omp_finish_file (void) 13797{ 13798 unsigned num_funcs = vec_safe_length (offload_funcs); 13799 unsigned num_vars = vec_safe_length (offload_vars); 13800 13801 if (num_funcs == 0 && num_vars == 0) 13802 return; 13803 13804 if (targetm_common.have_named_sections) 13805 { 13806 vec<constructor_elt, va_gc> *v_f, *v_v; 13807 vec_alloc (v_f, num_funcs); 13808 vec_alloc (v_v, num_vars * 2); 13809 13810 add_decls_addresses_to_decl_constructor (offload_funcs, v_f); 13811 add_decls_addresses_to_decl_constructor (offload_vars, v_v); 13812 13813 tree vars_decl_type = build_array_type_nelts (pointer_sized_int_node, 13814 num_vars * 2); 13815 tree funcs_decl_type = build_array_type_nelts (pointer_sized_int_node, 13816 num_funcs); 13817 TYPE_ALIGN (vars_decl_type) = TYPE_ALIGN (pointer_sized_int_node); 13818 TYPE_ALIGN (funcs_decl_type) = TYPE_ALIGN (pointer_sized_int_node); 13819 tree ctor_v = build_constructor (vars_decl_type, v_v); 13820 tree ctor_f = build_constructor (funcs_decl_type, v_f); 13821 TREE_CONSTANT (ctor_v) = TREE_CONSTANT (ctor_f) = 1; 13822 TREE_STATIC (ctor_v) = TREE_STATIC (ctor_f) = 1; 13823 tree funcs_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, 13824 get_identifier (".offload_func_table"), 13825 funcs_decl_type); 13826 tree vars_decl = build_decl (UNKNOWN_LOCATION, VAR_DECL, 13827 get_identifier (".offload_var_table"), 13828 vars_decl_type); 13829 TREE_STATIC (funcs_decl) = TREE_STATIC (vars_decl) = 1; 13830 /* Do not align tables more than TYPE_ALIGN (pointer_sized_int_node), 13831 otherwise a joint table in a binary will contain padding between 13832 tables from multiple object files. */ 13833 DECL_USER_ALIGN (funcs_decl) = DECL_USER_ALIGN (vars_decl) = 1; 13834 DECL_ALIGN (funcs_decl) = TYPE_ALIGN (funcs_decl_type); 13835 DECL_ALIGN (vars_decl) = TYPE_ALIGN (vars_decl_type); 13836 DECL_INITIAL (funcs_decl) = ctor_f; 13837 DECL_INITIAL (vars_decl) = ctor_v; 13838 set_decl_section_name (funcs_decl, OFFLOAD_FUNC_TABLE_SECTION_NAME); 13839 set_decl_section_name (vars_decl, OFFLOAD_VAR_TABLE_SECTION_NAME); 13840 13841 varpool_node::finalize_decl (vars_decl); 13842 varpool_node::finalize_decl (funcs_decl); 13843 } 13844 else 13845 { 13846 for (unsigned i = 0; i < num_funcs; i++) 13847 { 13848 tree it = (*offload_funcs)[i]; 13849 targetm.record_offload_symbol (it); 13850 } 13851 for (unsigned i = 0; i < num_vars; i++) 13852 { 13853 tree it = (*offload_vars)[i]; 13854 targetm.record_offload_symbol (it); 13855 } 13856 } 13857} 13858 13859#include "gt-omp-low.h" 13860