1/* OMP constructs' SIMD clone supporting code. 2 3Copyright (C) 2005-2020 Free Software Foundation, Inc. 4 5This file is part of GCC. 6 7GCC is free software; you can redistribute it and/or modify it under 8the terms of the GNU General Public License as published by the Free 9Software Foundation; either version 3, or (at your option) any later 10version. 11 12GCC is distributed in the hope that it will be useful, but WITHOUT ANY 13WARRANTY; without even the implied warranty of MERCHANTABILITY or 14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 15for more details. 16 17You should have received a copy of the GNU General Public License 18along with GCC; see the file COPYING3. If not see 19<http://www.gnu.org/licenses/>. */ 20 21#include "config.h" 22#include "system.h" 23#include "coretypes.h" 24#include "backend.h" 25#include "target.h" 26#include "tree.h" 27#include "gimple.h" 28#include "cfghooks.h" 29#include "alloc-pool.h" 30#include "tree-pass.h" 31#include "ssa.h" 32#include "cgraph.h" 33#include "pretty-print.h" 34#include "diagnostic-core.h" 35#include "fold-const.h" 36#include "stor-layout.h" 37#include "cfganal.h" 38#include "gimplify.h" 39#include "gimple-iterator.h" 40#include "gimplify-me.h" 41#include "gimple-walk.h" 42#include "langhooks.h" 43#include "tree-cfg.h" 44#include "tree-into-ssa.h" 45#include "tree-dfa.h" 46#include "cfgloop.h" 47#include "symbol-summary.h" 48#include "ipa-param-manipulation.h" 49#include "tree-eh.h" 50#include "varasm.h" 51#include "stringpool.h" 52#include "attribs.h" 53#include "omp-simd-clone.h" 54 55/* Return the number of elements in vector type VECTYPE, which is associated 56 with a SIMD clone. At present these always have a constant length. */ 57 58static unsigned HOST_WIDE_INT 59simd_clone_subparts (tree vectype) 60{ 61 return TYPE_VECTOR_SUBPARTS (vectype).to_constant (); 62} 63 64/* Allocate a fresh `simd_clone' and return it. NARGS is the number 65 of arguments to reserve space for. */ 66 67static struct cgraph_simd_clone * 68simd_clone_struct_alloc (int nargs) 69{ 70 struct cgraph_simd_clone *clone_info; 71 size_t len = (sizeof (struct cgraph_simd_clone) 72 + nargs * sizeof (struct cgraph_simd_clone_arg)); 73 clone_info = (struct cgraph_simd_clone *) 74 ggc_internal_cleared_alloc (len); 75 return clone_info; 76} 77 78/* Make a copy of the `struct cgraph_simd_clone' in FROM to TO. */ 79 80static inline void 81simd_clone_struct_copy (struct cgraph_simd_clone *to, 82 struct cgraph_simd_clone *from) 83{ 84 memcpy (to, from, (sizeof (struct cgraph_simd_clone) 85 + ((from->nargs - from->inbranch) 86 * sizeof (struct cgraph_simd_clone_arg)))); 87} 88 89/* Fill an empty vector ARGS with parameter types of function FNDECL. This 90 uses TYPE_ARG_TYPES if available, otherwise falls back to types of 91 DECL_ARGUMENTS types. */ 92 93static void 94simd_clone_vector_of_formal_parm_types (vec<tree> *args, tree fndecl) 95{ 96 if (TYPE_ARG_TYPES (TREE_TYPE (fndecl))) 97 { 98 push_function_arg_types (args, TREE_TYPE (fndecl)); 99 return; 100 } 101 push_function_arg_decls (args, fndecl); 102 unsigned int i; 103 tree arg; 104 FOR_EACH_VEC_ELT (*args, i, arg) 105 (*args)[i] = TREE_TYPE ((*args)[i]); 106} 107 108/* Given a simd function in NODE, extract the simd specific 109 information from the OMP clauses passed in CLAUSES, and return 110 the struct cgraph_simd_clone * if it should be cloned. *INBRANCH_SPECIFIED 111 is set to TRUE if the `inbranch' or `notinbranch' clause specified, 112 otherwise set to FALSE. */ 113 114static struct cgraph_simd_clone * 115simd_clone_clauses_extract (struct cgraph_node *node, tree clauses, 116 bool *inbranch_specified) 117{ 118 auto_vec<tree> args; 119 simd_clone_vector_of_formal_parm_types (&args, node->decl); 120 tree t; 121 int n; 122 *inbranch_specified = false; 123 124 n = args.length (); 125 if (n > 0 && args.last () == void_type_node) 126 n--; 127 128 /* Allocate one more than needed just in case this is an in-branch 129 clone which will require a mask argument. */ 130 struct cgraph_simd_clone *clone_info = simd_clone_struct_alloc (n + 1); 131 clone_info->nargs = n; 132 133 if (!clauses) 134 goto out; 135 136 clauses = TREE_VALUE (clauses); 137 if (!clauses || TREE_CODE (clauses) != OMP_CLAUSE) 138 goto out; 139 140 for (t = clauses; t; t = OMP_CLAUSE_CHAIN (t)) 141 { 142 switch (OMP_CLAUSE_CODE (t)) 143 { 144 case OMP_CLAUSE_INBRANCH: 145 clone_info->inbranch = 1; 146 *inbranch_specified = true; 147 break; 148 case OMP_CLAUSE_NOTINBRANCH: 149 clone_info->inbranch = 0; 150 *inbranch_specified = true; 151 break; 152 case OMP_CLAUSE_SIMDLEN: 153 clone_info->simdlen 154 = TREE_INT_CST_LOW (OMP_CLAUSE_SIMDLEN_EXPR (t)); 155 break; 156 case OMP_CLAUSE_LINEAR: 157 { 158 tree decl = OMP_CLAUSE_DECL (t); 159 tree step = OMP_CLAUSE_LINEAR_STEP (t); 160 int argno = TREE_INT_CST_LOW (decl); 161 if (OMP_CLAUSE_LINEAR_VARIABLE_STRIDE (t)) 162 { 163 enum cgraph_simd_clone_arg_type arg_type; 164 if (TREE_CODE (args[argno]) == REFERENCE_TYPE) 165 switch (OMP_CLAUSE_LINEAR_KIND (t)) 166 { 167 case OMP_CLAUSE_LINEAR_REF: 168 arg_type 169 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP; 170 break; 171 case OMP_CLAUSE_LINEAR_UVAL: 172 arg_type 173 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP; 174 break; 175 case OMP_CLAUSE_LINEAR_VAL: 176 case OMP_CLAUSE_LINEAR_DEFAULT: 177 arg_type 178 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP; 179 break; 180 default: 181 gcc_unreachable (); 182 } 183 else 184 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP; 185 clone_info->args[argno].arg_type = arg_type; 186 clone_info->args[argno].linear_step = tree_to_shwi (step); 187 gcc_assert (clone_info->args[argno].linear_step >= 0 188 && clone_info->args[argno].linear_step < n); 189 } 190 else 191 { 192 if (POINTER_TYPE_P (args[argno])) 193 step = fold_convert (ssizetype, step); 194 if (!tree_fits_shwi_p (step)) 195 { 196 warning_at (OMP_CLAUSE_LOCATION (t), 0, 197 "ignoring large linear step"); 198 return NULL; 199 } 200 else if (integer_zerop (step)) 201 { 202 warning_at (OMP_CLAUSE_LOCATION (t), 0, 203 "ignoring zero linear step"); 204 return NULL; 205 } 206 else 207 { 208 enum cgraph_simd_clone_arg_type arg_type; 209 if (TREE_CODE (args[argno]) == REFERENCE_TYPE) 210 switch (OMP_CLAUSE_LINEAR_KIND (t)) 211 { 212 case OMP_CLAUSE_LINEAR_REF: 213 arg_type 214 = SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP; 215 break; 216 case OMP_CLAUSE_LINEAR_UVAL: 217 arg_type 218 = SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP; 219 break; 220 case OMP_CLAUSE_LINEAR_VAL: 221 case OMP_CLAUSE_LINEAR_DEFAULT: 222 arg_type 223 = SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP; 224 break; 225 default: 226 gcc_unreachable (); 227 } 228 else 229 arg_type = SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP; 230 clone_info->args[argno].arg_type = arg_type; 231 clone_info->args[argno].linear_step = tree_to_shwi (step); 232 } 233 } 234 break; 235 } 236 case OMP_CLAUSE_UNIFORM: 237 { 238 tree decl = OMP_CLAUSE_DECL (t); 239 int argno = tree_to_uhwi (decl); 240 clone_info->args[argno].arg_type 241 = SIMD_CLONE_ARG_TYPE_UNIFORM; 242 break; 243 } 244 case OMP_CLAUSE_ALIGNED: 245 { 246 /* Ignore aligned (x) for declare simd, for the ABI we really 247 need an alignment specified. */ 248 if (OMP_CLAUSE_ALIGNED_ALIGNMENT (t) == NULL_TREE) 249 break; 250 tree decl = OMP_CLAUSE_DECL (t); 251 int argno = tree_to_uhwi (decl); 252 clone_info->args[argno].alignment 253 = TREE_INT_CST_LOW (OMP_CLAUSE_ALIGNED_ALIGNMENT (t)); 254 break; 255 } 256 default: 257 break; 258 } 259 } 260 261 out: 262 if (TYPE_ATOMIC (TREE_TYPE (TREE_TYPE (node->decl)))) 263 { 264 warning_at (DECL_SOURCE_LOCATION (node->decl), 0, 265 "ignoring %<#pragma omp declare simd%> on function " 266 "with %<_Atomic%> qualified return type"); 267 return NULL; 268 } 269 270 for (unsigned int argno = 0; argno < clone_info->nargs; argno++) 271 if (TYPE_ATOMIC (args[argno]) 272 && clone_info->args[argno].arg_type != SIMD_CLONE_ARG_TYPE_UNIFORM) 273 { 274 warning_at (DECL_SOURCE_LOCATION (node->decl), 0, 275 "ignoring %<#pragma omp declare simd%> on function " 276 "with %<_Atomic%> qualified non-%<uniform%> argument"); 277 args.release (); 278 return NULL; 279 } 280 281 return clone_info; 282} 283 284/* Given a SIMD clone in NODE, calculate the characteristic data 285 type and return the coresponding type. The characteristic data 286 type is computed as described in the Intel Vector ABI. */ 287 288static tree 289simd_clone_compute_base_data_type (struct cgraph_node *node, 290 struct cgraph_simd_clone *clone_info) 291{ 292 tree type = integer_type_node; 293 tree fndecl = node->decl; 294 295 /* a) For non-void function, the characteristic data type is the 296 return type. */ 297 if (TREE_CODE (TREE_TYPE (TREE_TYPE (fndecl))) != VOID_TYPE) 298 type = TREE_TYPE (TREE_TYPE (fndecl)); 299 300 /* b) If the function has any non-uniform, non-linear parameters, 301 then the characteristic data type is the type of the first 302 such parameter. */ 303 else 304 { 305 auto_vec<tree> map; 306 simd_clone_vector_of_formal_parm_types (&map, fndecl); 307 for (unsigned int i = 0; i < clone_info->nargs; ++i) 308 if (clone_info->args[i].arg_type == SIMD_CLONE_ARG_TYPE_VECTOR) 309 { 310 type = map[i]; 311 break; 312 } 313 } 314 315 /* c) If the characteristic data type determined by a) or b) above 316 is struct, union, or class type which is pass-by-value (except 317 for the type that maps to the built-in complex data type), the 318 characteristic data type is int. */ 319 if (RECORD_OR_UNION_TYPE_P (type) 320 && !aggregate_value_p (type, NULL) 321 && TREE_CODE (type) != COMPLEX_TYPE) 322 return integer_type_node; 323 324 /* d) If none of the above three classes is applicable, the 325 characteristic data type is int. */ 326 327 return type; 328 329 /* e) For Intel Xeon Phi native and offload compilation, if the 330 resulting characteristic data type is 8-bit or 16-bit integer 331 data type, the characteristic data type is int. */ 332 /* Well, we don't handle Xeon Phi yet. */ 333} 334 335static tree 336simd_clone_mangle (struct cgraph_node *node, 337 struct cgraph_simd_clone *clone_info) 338{ 339 char vecsize_mangle = clone_info->vecsize_mangle; 340 char mask = clone_info->inbranch ? 'M' : 'N'; 341 unsigned int simdlen = clone_info->simdlen; 342 unsigned int n; 343 pretty_printer pp; 344 345 gcc_assert (vecsize_mangle && simdlen); 346 347 pp_string (&pp, "_ZGV"); 348 pp_character (&pp, vecsize_mangle); 349 pp_character (&pp, mask); 350 pp_decimal_int (&pp, simdlen); 351 352 for (n = 0; n < clone_info->nargs; ++n) 353 { 354 struct cgraph_simd_clone_arg arg = clone_info->args[n]; 355 356 switch (arg.arg_type) 357 { 358 case SIMD_CLONE_ARG_TYPE_UNIFORM: 359 pp_character (&pp, 'u'); 360 break; 361 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: 362 pp_character (&pp, 'l'); 363 goto mangle_linear; 364 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP: 365 pp_character (&pp, 'R'); 366 goto mangle_linear; 367 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 368 pp_character (&pp, 'L'); 369 goto mangle_linear; 370 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 371 pp_character (&pp, 'U'); 372 goto mangle_linear; 373 mangle_linear: 374 gcc_assert (arg.linear_step != 0); 375 if (arg.linear_step > 1) 376 pp_unsigned_wide_integer (&pp, arg.linear_step); 377 else if (arg.linear_step < 0) 378 { 379 pp_character (&pp, 'n'); 380 pp_unsigned_wide_integer (&pp, (-(unsigned HOST_WIDE_INT) 381 arg.linear_step)); 382 } 383 break; 384 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP: 385 pp_string (&pp, "ls"); 386 pp_unsigned_wide_integer (&pp, arg.linear_step); 387 break; 388 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP: 389 pp_string (&pp, "Rs"); 390 pp_unsigned_wide_integer (&pp, arg.linear_step); 391 break; 392 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 393 pp_string (&pp, "Ls"); 394 pp_unsigned_wide_integer (&pp, arg.linear_step); 395 break; 396 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 397 pp_string (&pp, "Us"); 398 pp_unsigned_wide_integer (&pp, arg.linear_step); 399 break; 400 default: 401 pp_character (&pp, 'v'); 402 } 403 if (arg.alignment) 404 { 405 pp_character (&pp, 'a'); 406 pp_decimal_int (&pp, arg.alignment); 407 } 408 } 409 410 pp_underscore (&pp); 411 const char *str = IDENTIFIER_POINTER (DECL_ASSEMBLER_NAME (node->decl)); 412 if (*str == '*') 413 ++str; 414 pp_string (&pp, str); 415 str = pp_formatted_text (&pp); 416 417 /* If there already is a SIMD clone with the same mangled name, don't 418 add another one. This can happen e.g. for 419 #pragma omp declare simd 420 #pragma omp declare simd simdlen(8) 421 int foo (int, int); 422 if the simdlen is assumed to be 8 for the first one, etc. */ 423 for (struct cgraph_node *clone = node->simd_clones; clone; 424 clone = clone->simdclone->next_clone) 425 if (id_equal (DECL_ASSEMBLER_NAME (clone->decl), str)) 426 return NULL_TREE; 427 428 return get_identifier (str); 429} 430 431/* Create a simd clone of OLD_NODE and return it. */ 432 433static struct cgraph_node * 434simd_clone_create (struct cgraph_node *old_node) 435{ 436 struct cgraph_node *new_node; 437 if (old_node->definition) 438 { 439 if (!old_node->has_gimple_body_p ()) 440 return NULL; 441 old_node->get_body (); 442 new_node = old_node->create_version_clone_with_body (vNULL, NULL, NULL, 443 NULL, NULL, 444 "simdclone"); 445 } 446 else 447 { 448 tree old_decl = old_node->decl; 449 tree new_decl = copy_node (old_node->decl); 450 DECL_NAME (new_decl) = clone_function_name_numbered (old_decl, 451 "simdclone"); 452 SET_DECL_ASSEMBLER_NAME (new_decl, DECL_NAME (new_decl)); 453 SET_DECL_RTL (new_decl, NULL); 454 DECL_STATIC_CONSTRUCTOR (new_decl) = 0; 455 DECL_STATIC_DESTRUCTOR (new_decl) = 0; 456 new_node = old_node->create_version_clone (new_decl, vNULL, NULL); 457 if (old_node->in_other_partition) 458 new_node->in_other_partition = 1; 459 } 460 if (new_node == NULL) 461 return new_node; 462 463 set_decl_built_in_function (new_node->decl, NOT_BUILT_IN, 0); 464 TREE_PUBLIC (new_node->decl) = TREE_PUBLIC (old_node->decl); 465 DECL_COMDAT (new_node->decl) = DECL_COMDAT (old_node->decl); 466 DECL_WEAK (new_node->decl) = DECL_WEAK (old_node->decl); 467 DECL_EXTERNAL (new_node->decl) = DECL_EXTERNAL (old_node->decl); 468 DECL_VISIBILITY_SPECIFIED (new_node->decl) 469 = DECL_VISIBILITY_SPECIFIED (old_node->decl); 470 DECL_VISIBILITY (new_node->decl) = DECL_VISIBILITY (old_node->decl); 471 DECL_DLLIMPORT_P (new_node->decl) = DECL_DLLIMPORT_P (old_node->decl); 472 if (DECL_ONE_ONLY (old_node->decl)) 473 make_decl_one_only (new_node->decl, DECL_ASSEMBLER_NAME (new_node->decl)); 474 475 /* The method cgraph_version_clone_with_body () will force the new 476 symbol local. Undo this, and inherit external visibility from 477 the old node. */ 478 new_node->local = old_node->local; 479 new_node->externally_visible = old_node->externally_visible; 480 481 return new_node; 482} 483 484/* Adjust the return type of the given function to its appropriate 485 vector counterpart. Returns a simd array to be used throughout the 486 function as a return value. */ 487 488static tree 489simd_clone_adjust_return_type (struct cgraph_node *node) 490{ 491 tree fndecl = node->decl; 492 tree orig_rettype = TREE_TYPE (TREE_TYPE (fndecl)); 493 unsigned int veclen; 494 tree t; 495 496 /* Adjust the function return type. */ 497 if (orig_rettype == void_type_node) 498 return NULL_TREE; 499 t = TREE_TYPE (TREE_TYPE (fndecl)); 500 if (INTEGRAL_TYPE_P (t) || POINTER_TYPE_P (t)) 501 veclen = node->simdclone->vecsize_int; 502 else 503 veclen = node->simdclone->vecsize_float; 504 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (t)); 505 if (veclen > node->simdclone->simdlen) 506 veclen = node->simdclone->simdlen; 507 if (POINTER_TYPE_P (t)) 508 t = pointer_sized_int_node; 509 if (veclen == node->simdclone->simdlen) 510 t = build_vector_type (t, node->simdclone->simdlen); 511 else 512 { 513 t = build_vector_type (t, veclen); 514 t = build_array_type_nelts (t, node->simdclone->simdlen / veclen); 515 } 516 TREE_TYPE (TREE_TYPE (fndecl)) = t; 517 if (!node->definition) 518 return NULL_TREE; 519 520 t = DECL_RESULT (fndecl); 521 /* Adjust the DECL_RESULT. */ 522 gcc_assert (TREE_TYPE (t) != void_type_node); 523 TREE_TYPE (t) = TREE_TYPE (TREE_TYPE (fndecl)); 524 relayout_decl (t); 525 526 tree atype = build_array_type_nelts (orig_rettype, 527 node->simdclone->simdlen); 528 if (veclen != node->simdclone->simdlen) 529 return build1 (VIEW_CONVERT_EXPR, atype, t); 530 531 /* Set up a SIMD array to use as the return value. */ 532 tree retval = create_tmp_var_raw (atype, "retval"); 533 gimple_add_tmp_var (retval); 534 return retval; 535} 536 537/* Each vector argument has a corresponding array to be used locally 538 as part of the eventual loop. Create such temporary array and 539 return it. 540 541 PREFIX is the prefix to be used for the temporary. 542 543 TYPE is the inner element type. 544 545 SIMDLEN is the number of elements. */ 546 547static tree 548create_tmp_simd_array (const char *prefix, tree type, int simdlen) 549{ 550 tree atype = build_array_type_nelts (type, simdlen); 551 tree avar = create_tmp_var_raw (atype, prefix); 552 gimple_add_tmp_var (avar); 553 return avar; 554} 555 556/* Modify the function argument types to their corresponding vector 557 counterparts if appropriate. Also, create one array for each simd 558 argument to be used locally when using the function arguments as 559 part of the loop. 560 561 NODE is the function whose arguments are to be adjusted. 562 563 If NODE does not represent function definition, returns NULL. Otherwise 564 returns an adjustment class that will be filled describing how the argument 565 declarations will be remapped. New arguments which are not to be remapped 566 are marked with USER_FLAG. */ 567 568static ipa_param_body_adjustments * 569simd_clone_adjust_argument_types (struct cgraph_node *node) 570{ 571 auto_vec<tree> args; 572 573 if (node->definition) 574 push_function_arg_decls (&args, node->decl); 575 else 576 simd_clone_vector_of_formal_parm_types (&args, node->decl); 577 struct cgraph_simd_clone *sc = node->simdclone; 578 vec<ipa_adjusted_param, va_gc> *new_params = NULL; 579 vec_safe_reserve (new_params, sc->nargs); 580 unsigned i, j, veclen; 581 582 for (i = 0; i < sc->nargs; ++i) 583 { 584 ipa_adjusted_param adj; 585 memset (&adj, 0, sizeof (adj)); 586 tree parm = args[i]; 587 tree parm_type = node->definition ? TREE_TYPE (parm) : parm; 588 adj.base_index = i; 589 adj.prev_clone_index = i; 590 591 sc->args[i].orig_arg = node->definition ? parm : NULL_TREE; 592 sc->args[i].orig_type = parm_type; 593 594 switch (sc->args[i].arg_type) 595 { 596 default: 597 /* No adjustment necessary for scalar arguments. */ 598 adj.op = IPA_PARAM_OP_COPY; 599 break; 600 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 601 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 602 if (node->definition) 603 sc->args[i].simd_array 604 = create_tmp_simd_array (IDENTIFIER_POINTER (DECL_NAME (parm)), 605 TREE_TYPE (parm_type), 606 sc->simdlen); 607 adj.op = IPA_PARAM_OP_COPY; 608 break; 609 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 610 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 611 case SIMD_CLONE_ARG_TYPE_VECTOR: 612 if (INTEGRAL_TYPE_P (parm_type) || POINTER_TYPE_P (parm_type)) 613 veclen = sc->vecsize_int; 614 else 615 veclen = sc->vecsize_float; 616 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (parm_type)); 617 if (veclen > sc->simdlen) 618 veclen = sc->simdlen; 619 adj.op = IPA_PARAM_OP_NEW; 620 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD; 621 if (POINTER_TYPE_P (parm_type)) 622 adj.type = build_vector_type (pointer_sized_int_node, veclen); 623 else 624 adj.type = build_vector_type (parm_type, veclen); 625 sc->args[i].vector_type = adj.type; 626 for (j = veclen; j < sc->simdlen; j += veclen) 627 { 628 vec_safe_push (new_params, adj); 629 if (j == veclen) 630 { 631 memset (&adj, 0, sizeof (adj)); 632 adj.op = IPA_PARAM_OP_NEW; 633 adj.user_flag = 1; 634 adj.param_prefix_index = IPA_PARAM_PREFIX_SIMD; 635 adj.base_index = i; 636 adj.prev_clone_index = i; 637 adj.type = sc->args[i].vector_type; 638 } 639 } 640 641 if (node->definition) 642 sc->args[i].simd_array 643 = create_tmp_simd_array (DECL_NAME (parm) 644 ? IDENTIFIER_POINTER (DECL_NAME (parm)) 645 : NULL, parm_type, sc->simdlen); 646 } 647 vec_safe_push (new_params, adj); 648 } 649 650 if (sc->inbranch) 651 { 652 tree base_type = simd_clone_compute_base_data_type (sc->origin, sc); 653 ipa_adjusted_param adj; 654 memset (&adj, 0, sizeof (adj)); 655 adj.op = IPA_PARAM_OP_NEW; 656 adj.user_flag = 1; 657 adj.param_prefix_index = IPA_PARAM_PREFIX_MASK; 658 659 adj.base_index = i; 660 adj.prev_clone_index = i; 661 if (INTEGRAL_TYPE_P (base_type) || POINTER_TYPE_P (base_type)) 662 veclen = sc->vecsize_int; 663 else 664 veclen = sc->vecsize_float; 665 veclen /= GET_MODE_BITSIZE (SCALAR_TYPE_MODE (base_type)); 666 if (veclen > sc->simdlen) 667 veclen = sc->simdlen; 668 if (sc->mask_mode != VOIDmode) 669 adj.type 670 = lang_hooks.types.type_for_mode (sc->mask_mode, 1); 671 else if (POINTER_TYPE_P (base_type)) 672 adj.type = build_vector_type (pointer_sized_int_node, veclen); 673 else 674 adj.type = build_vector_type (base_type, veclen); 675 vec_safe_push (new_params, adj); 676 677 for (j = veclen; j < sc->simdlen; j += veclen) 678 vec_safe_push (new_params, adj); 679 680 /* We have previously allocated one extra entry for the mask. Use 681 it and fill it. */ 682 sc->nargs++; 683 if (sc->mask_mode != VOIDmode) 684 base_type = boolean_type_node; 685 if (node->definition) 686 { 687 sc->args[i].orig_arg 688 = build_decl (UNKNOWN_LOCATION, PARM_DECL, NULL, base_type); 689 if (sc->mask_mode == VOIDmode) 690 sc->args[i].simd_array 691 = create_tmp_simd_array ("mask", base_type, sc->simdlen); 692 else if (veclen < sc->simdlen) 693 sc->args[i].simd_array 694 = create_tmp_simd_array ("mask", adj.type, sc->simdlen / veclen); 695 else 696 sc->args[i].simd_array = NULL_TREE; 697 } 698 sc->args[i].orig_type = base_type; 699 sc->args[i].arg_type = SIMD_CLONE_ARG_TYPE_MASK; 700 } 701 702 if (node->definition) 703 { 704 ipa_param_body_adjustments *adjustments 705 = new ipa_param_body_adjustments (new_params, node->decl); 706 707 adjustments->modify_formal_parameters (); 708 return adjustments; 709 } 710 else 711 { 712 tree new_arg_types = NULL_TREE, new_reversed; 713 bool last_parm_void = false; 714 if (args.length () > 0 && args.last () == void_type_node) 715 last_parm_void = true; 716 717 gcc_assert (TYPE_ARG_TYPES (TREE_TYPE (node->decl))); 718 j = vec_safe_length (new_params); 719 for (i = 0; i < j; i++) 720 { 721 struct ipa_adjusted_param *adj = &(*new_params)[i]; 722 tree ptype; 723 if (adj->op == IPA_PARAM_OP_COPY) 724 ptype = args[adj->base_index]; 725 else 726 ptype = adj->type; 727 new_arg_types = tree_cons (NULL_TREE, ptype, new_arg_types); 728 } 729 new_reversed = nreverse (new_arg_types); 730 if (last_parm_void) 731 { 732 if (new_reversed) 733 TREE_CHAIN (new_arg_types) = void_list_node; 734 else 735 new_reversed = void_list_node; 736 } 737 TYPE_ARG_TYPES (TREE_TYPE (node->decl)) = new_reversed; 738 return NULL; 739 } 740} 741 742/* Initialize and copy the function arguments in NODE to their 743 corresponding local simd arrays. Returns a fresh gimple_seq with 744 the instruction sequence generated. */ 745 746static gimple_seq 747simd_clone_init_simd_arrays (struct cgraph_node *node, 748 ipa_param_body_adjustments *adjustments) 749{ 750 gimple_seq seq = NULL; 751 unsigned i = 0, j = 0, k; 752 753 for (tree arg = DECL_ARGUMENTS (node->decl); 754 arg; 755 arg = DECL_CHAIN (arg), i++, j++) 756 { 757 if ((*adjustments->m_adj_params)[j].op == IPA_PARAM_OP_COPY 758 || POINTER_TYPE_P (TREE_TYPE (arg))) 759 continue; 760 761 node->simdclone->args[i].vector_arg = arg; 762 763 tree array = node->simdclone->args[i].simd_array; 764 if (node->simdclone->mask_mode != VOIDmode 765 && node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_MASK) 766 { 767 if (array == NULL_TREE) 768 continue; 769 unsigned int l 770 = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (TREE_TYPE (array)))); 771 for (k = 0; k <= l; k++) 772 { 773 if (k) 774 { 775 arg = DECL_CHAIN (arg); 776 j++; 777 } 778 tree t = build4 (ARRAY_REF, TREE_TYPE (TREE_TYPE (array)), 779 array, size_int (k), NULL, NULL); 780 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 781 gimplify_and_add (t, &seq); 782 } 783 continue; 784 } 785 if (simd_clone_subparts (TREE_TYPE (arg)) == node->simdclone->simdlen) 786 { 787 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 788 tree ptr = build_fold_addr_expr (array); 789 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 790 build_int_cst (ptype, 0)); 791 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 792 gimplify_and_add (t, &seq); 793 } 794 else 795 { 796 unsigned int simdlen = simd_clone_subparts (TREE_TYPE (arg)); 797 tree ptype = build_pointer_type (TREE_TYPE (TREE_TYPE (array))); 798 for (k = 0; k < node->simdclone->simdlen; k += simdlen) 799 { 800 tree ptr = build_fold_addr_expr (array); 801 int elemsize; 802 if (k) 803 { 804 arg = DECL_CHAIN (arg); 805 j++; 806 } 807 tree elemtype = TREE_TYPE (TREE_TYPE (arg)); 808 elemsize = GET_MODE_SIZE (SCALAR_TYPE_MODE (elemtype)); 809 tree t = build2 (MEM_REF, TREE_TYPE (arg), ptr, 810 build_int_cst (ptype, k * elemsize)); 811 t = build2 (MODIFY_EXPR, TREE_TYPE (t), t, arg); 812 gimplify_and_add (t, &seq); 813 } 814 } 815 } 816 return seq; 817} 818 819/* Callback info for ipa_simd_modify_stmt_ops below. */ 820 821struct modify_stmt_info { 822 ipa_param_body_adjustments *adjustments; 823 gimple *stmt; 824 gimple *after_stmt; 825 /* True if the parent statement was modified by 826 ipa_simd_modify_stmt_ops. */ 827 bool modified; 828}; 829 830/* Callback for walk_gimple_op. 831 832 Adjust operands from a given statement as specified in the 833 adjustments vector in the callback data. */ 834 835static tree 836ipa_simd_modify_stmt_ops (tree *tp, int *walk_subtrees, void *data) 837{ 838 struct walk_stmt_info *wi = (struct walk_stmt_info *) data; 839 struct modify_stmt_info *info = (struct modify_stmt_info *) wi->info; 840 tree *orig_tp = tp; 841 if (TREE_CODE (*tp) == ADDR_EXPR) 842 tp = &TREE_OPERAND (*tp, 0); 843 844 if (TREE_CODE (*tp) == BIT_FIELD_REF 845 || TREE_CODE (*tp) == IMAGPART_EXPR 846 || TREE_CODE (*tp) == REALPART_EXPR) 847 tp = &TREE_OPERAND (*tp, 0); 848 849 tree repl = NULL_TREE; 850 ipa_param_body_replacement *pbr = NULL; 851 852 if (TREE_CODE (*tp) == PARM_DECL) 853 { 854 pbr = info->adjustments->get_expr_replacement (*tp, true); 855 if (pbr) 856 repl = pbr->repl; 857 } 858 else if (TYPE_P (*tp)) 859 *walk_subtrees = 0; 860 861 if (repl) 862 repl = unshare_expr (repl); 863 else 864 { 865 if (tp != orig_tp) 866 { 867 *walk_subtrees = 0; 868 bool modified = info->modified; 869 info->modified = false; 870 walk_tree (tp, ipa_simd_modify_stmt_ops, wi, wi->pset); 871 if (!info->modified) 872 { 873 info->modified = modified; 874 return NULL_TREE; 875 } 876 info->modified = modified; 877 repl = *tp; 878 } 879 else 880 return NULL_TREE; 881 } 882 883 if (tp != orig_tp) 884 { 885 if (gimple_code (info->stmt) == GIMPLE_PHI 886 && pbr 887 && TREE_CODE (*orig_tp) == ADDR_EXPR 888 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL 889 && pbr->dummy) 890 { 891 gcc_assert (TREE_CODE (pbr->dummy) == SSA_NAME); 892 *orig_tp = pbr->dummy; 893 info->modified = true; 894 return NULL_TREE; 895 } 896 897 repl = build_fold_addr_expr (repl); 898 gimple *stmt; 899 if (is_gimple_debug (info->stmt)) 900 { 901 tree vexpr = make_node (DEBUG_EXPR_DECL); 902 stmt = gimple_build_debug_source_bind (vexpr, repl, NULL); 903 DECL_ARTIFICIAL (vexpr) = 1; 904 TREE_TYPE (vexpr) = TREE_TYPE (repl); 905 SET_DECL_MODE (vexpr, TYPE_MODE (TREE_TYPE (repl))); 906 repl = vexpr; 907 } 908 else 909 { 910 stmt = gimple_build_assign (make_ssa_name (TREE_TYPE (repl)), repl); 911 repl = gimple_assign_lhs (stmt); 912 } 913 gimple_stmt_iterator gsi; 914 if (gimple_code (info->stmt) == GIMPLE_PHI) 915 { 916 if (info->after_stmt) 917 gsi = gsi_for_stmt (info->after_stmt); 918 else 919 gsi = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))); 920 /* Cache SSA_NAME for next time. */ 921 if (pbr 922 && TREE_CODE (*orig_tp) == ADDR_EXPR 923 && TREE_CODE (TREE_OPERAND (*orig_tp, 0)) == PARM_DECL) 924 { 925 gcc_assert (!pbr->dummy); 926 pbr->dummy = repl; 927 } 928 } 929 else 930 gsi = gsi_for_stmt (info->stmt); 931 if (info->after_stmt) 932 gsi_insert_after (&gsi, stmt, GSI_SAME_STMT); 933 else 934 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 935 if (gimple_code (info->stmt) == GIMPLE_PHI) 936 info->after_stmt = stmt; 937 *orig_tp = repl; 938 } 939 else if (!useless_type_conversion_p (TREE_TYPE (*tp), TREE_TYPE (repl))) 940 { 941 tree vce = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (*tp), repl); 942 *tp = vce; 943 } 944 else 945 *tp = repl; 946 947 info->modified = true; 948 return NULL_TREE; 949} 950 951/* Traverse the function body and perform all modifications as 952 described in ADJUSTMENTS. At function return, ADJUSTMENTS will be 953 modified such that the replacement/reduction value will now be an 954 offset into the corresponding simd_array. 955 956 This function will replace all function argument uses with their 957 corresponding simd array elements, and ajust the return values 958 accordingly. */ 959 960static void 961ipa_simd_modify_function_body (struct cgraph_node *node, 962 ipa_param_body_adjustments *adjustments, 963 tree retval_array, tree iter) 964{ 965 basic_block bb; 966 unsigned int i, j; 967 968 969 /* Register replacements for every function argument use to an offset into 970 the corresponding simd_array. */ 971 for (i = 0, j = 0; i < node->simdclone->nargs; ++i, ++j) 972 { 973 if (!node->simdclone->args[i].vector_arg 974 || (*adjustments->m_adj_params)[j].user_flag) 975 continue; 976 977 tree basetype = TREE_TYPE (node->simdclone->args[i].orig_arg); 978 tree vectype = TREE_TYPE (node->simdclone->args[i].vector_arg); 979 tree r = build4 (ARRAY_REF, basetype, node->simdclone->args[i].simd_array, 980 iter, NULL_TREE, NULL_TREE); 981 adjustments->register_replacement (&(*adjustments->m_adj_params)[j], r); 982 983 if (simd_clone_subparts (vectype) < node->simdclone->simdlen) 984 j += node->simdclone->simdlen / simd_clone_subparts (vectype) - 1; 985 } 986 987 tree name; 988 FOR_EACH_SSA_NAME (i, name, cfun) 989 { 990 tree base_var; 991 if (SSA_NAME_VAR (name) 992 && TREE_CODE (SSA_NAME_VAR (name)) == PARM_DECL 993 && (base_var 994 = adjustments->get_replacement_ssa_base (SSA_NAME_VAR (name)))) 995 { 996 if (SSA_NAME_IS_DEFAULT_DEF (name)) 997 { 998 tree old_decl = SSA_NAME_VAR (name); 999 bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 1000 gimple_stmt_iterator gsi = gsi_after_labels (bb); 1001 tree repl = adjustments->lookup_replacement (old_decl, 0); 1002 gcc_checking_assert (repl); 1003 repl = unshare_expr (repl); 1004 set_ssa_default_def (cfun, old_decl, NULL_TREE); 1005 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 1006 SSA_NAME_IS_DEFAULT_DEF (name) = 0; 1007 gimple *stmt = gimple_build_assign (name, repl); 1008 gsi_insert_before (&gsi, stmt, GSI_SAME_STMT); 1009 } 1010 else 1011 SET_SSA_NAME_VAR_OR_IDENTIFIER (name, base_var); 1012 } 1013 } 1014 1015 struct modify_stmt_info info; 1016 info.adjustments = adjustments; 1017 1018 FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl)) 1019 { 1020 gimple_stmt_iterator gsi; 1021 1022 for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi)) 1023 { 1024 gphi *phi = as_a <gphi *> (gsi_stmt (gsi)); 1025 int i, n = gimple_phi_num_args (phi); 1026 info.stmt = phi; 1027 info.after_stmt = NULL; 1028 struct walk_stmt_info wi; 1029 memset (&wi, 0, sizeof (wi)); 1030 info.modified = false; 1031 wi.info = &info; 1032 for (i = 0; i < n; ++i) 1033 { 1034 int walk_subtrees = 1; 1035 tree arg = gimple_phi_arg_def (phi, i); 1036 tree op = arg; 1037 ipa_simd_modify_stmt_ops (&op, &walk_subtrees, &wi); 1038 if (op != arg) 1039 { 1040 SET_PHI_ARG_DEF (phi, i, op); 1041 gcc_assert (TREE_CODE (op) == SSA_NAME); 1042 if (gimple_phi_arg_edge (phi, i)->flags & EDGE_ABNORMAL) 1043 SSA_NAME_OCCURS_IN_ABNORMAL_PHI (op) = 1; 1044 } 1045 } 1046 } 1047 1048 gsi = gsi_start_bb (bb); 1049 while (!gsi_end_p (gsi)) 1050 { 1051 gimple *stmt = gsi_stmt (gsi); 1052 info.stmt = stmt; 1053 info.after_stmt = NULL; 1054 struct walk_stmt_info wi; 1055 1056 memset (&wi, 0, sizeof (wi)); 1057 info.modified = false; 1058 wi.info = &info; 1059 walk_gimple_op (stmt, ipa_simd_modify_stmt_ops, &wi); 1060 1061 if (greturn *return_stmt = dyn_cast <greturn *> (stmt)) 1062 { 1063 tree retval = gimple_return_retval (return_stmt); 1064 edge e = find_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun)); 1065 e->flags |= EDGE_FALLTHRU; 1066 if (!retval) 1067 { 1068 gsi_remove (&gsi, true); 1069 continue; 1070 } 1071 1072 /* Replace `return foo' with `retval_array[iter] = foo'. */ 1073 tree ref = build4 (ARRAY_REF, TREE_TYPE (retval), 1074 retval_array, iter, NULL, NULL); 1075 stmt = gimple_build_assign (ref, retval); 1076 gsi_replace (&gsi, stmt, true); 1077 info.modified = true; 1078 } 1079 1080 if (info.modified) 1081 { 1082 update_stmt (stmt); 1083 /* If the above changed the var of a debug bind into something 1084 different, remove the debug stmt. We could also for all the 1085 replaced parameters add VAR_DECLs for debug info purposes, 1086 add debug stmts for those to be the simd array accesses and 1087 replace debug stmt var operand with that var. Debugging of 1088 vectorized loops doesn't work too well, so don't bother for 1089 now. */ 1090 if ((gimple_debug_bind_p (stmt) 1091 && !DECL_P (gimple_debug_bind_get_var (stmt))) 1092 || (gimple_debug_source_bind_p (stmt) 1093 && !DECL_P (gimple_debug_source_bind_get_var (stmt)))) 1094 { 1095 gsi_remove (&gsi, true); 1096 continue; 1097 } 1098 if (maybe_clean_eh_stmt (stmt)) 1099 gimple_purge_dead_eh_edges (gimple_bb (stmt)); 1100 } 1101 gsi_next (&gsi); 1102 } 1103 } 1104} 1105 1106/* Helper function of simd_clone_adjust, return linear step addend 1107 of Ith argument. */ 1108 1109static tree 1110simd_clone_linear_addend (struct cgraph_node *node, unsigned int i, 1111 tree addtype, basic_block entry_bb) 1112{ 1113 tree ptype = NULL_TREE; 1114 switch (node->simdclone->args[i].arg_type) 1115 { 1116 case SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP: 1117 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP: 1118 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_CONSTANT_STEP: 1119 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP: 1120 return build_int_cst (addtype, node->simdclone->args[i].linear_step); 1121 case SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP: 1122 case SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP: 1123 ptype = TREE_TYPE (node->simdclone->args[i].orig_arg); 1124 break; 1125 case SIMD_CLONE_ARG_TYPE_LINEAR_VAL_VARIABLE_STEP: 1126 case SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP: 1127 ptype = TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)); 1128 break; 1129 default: 1130 gcc_unreachable (); 1131 } 1132 1133 unsigned int idx = node->simdclone->args[i].linear_step; 1134 tree arg = node->simdclone->args[idx].orig_arg; 1135 gcc_assert (is_gimple_reg_type (TREE_TYPE (arg))); 1136 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 1137 gimple *g; 1138 tree ret; 1139 if (is_gimple_reg (arg)) 1140 ret = get_or_create_ssa_default_def (cfun, arg); 1141 else 1142 { 1143 g = gimple_build_assign (make_ssa_name (TREE_TYPE (arg)), arg); 1144 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1145 ret = gimple_assign_lhs (g); 1146 } 1147 if (TREE_CODE (TREE_TYPE (arg)) == REFERENCE_TYPE) 1148 { 1149 g = gimple_build_assign (make_ssa_name (TREE_TYPE (TREE_TYPE (arg))), 1150 build_simple_mem_ref (ret)); 1151 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1152 ret = gimple_assign_lhs (g); 1153 } 1154 if (!useless_type_conversion_p (addtype, TREE_TYPE (ret))) 1155 { 1156 g = gimple_build_assign (make_ssa_name (addtype), NOP_EXPR, ret); 1157 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1158 ret = gimple_assign_lhs (g); 1159 } 1160 if (POINTER_TYPE_P (ptype)) 1161 { 1162 tree size = TYPE_SIZE_UNIT (TREE_TYPE (ptype)); 1163 if (size && TREE_CODE (size) == INTEGER_CST) 1164 { 1165 g = gimple_build_assign (make_ssa_name (addtype), MULT_EXPR, 1166 ret, fold_convert (addtype, size)); 1167 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1168 ret = gimple_assign_lhs (g); 1169 } 1170 } 1171 return ret; 1172} 1173 1174/* Adjust the argument types in NODE to their appropriate vector 1175 counterparts. */ 1176 1177static void 1178simd_clone_adjust (struct cgraph_node *node) 1179{ 1180 push_cfun (DECL_STRUCT_FUNCTION (node->decl)); 1181 1182 TREE_TYPE (node->decl) = build_distinct_type_copy (TREE_TYPE (node->decl)); 1183 targetm.simd_clone.adjust (node); 1184 1185 tree retval = simd_clone_adjust_return_type (node); 1186 ipa_param_body_adjustments *adjustments 1187 = simd_clone_adjust_argument_types (node); 1188 gcc_assert (adjustments); 1189 1190 push_gimplify_context (); 1191 1192 gimple_seq seq = simd_clone_init_simd_arrays (node, adjustments); 1193 1194 /* Adjust all uses of vector arguments accordingly. Adjust all 1195 return values accordingly. */ 1196 tree iter = create_tmp_var (unsigned_type_node, "iter"); 1197 tree iter1 = make_ssa_name (iter); 1198 tree iter2 = NULL_TREE; 1199 ipa_simd_modify_function_body (node, adjustments, retval, iter1); 1200 delete adjustments; 1201 1202 /* Initialize the iteration variable. */ 1203 basic_block entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 1204 basic_block body_bb = split_block_after_labels (entry_bb)->dest; 1205 gimple_stmt_iterator gsi = gsi_after_labels (entry_bb); 1206 /* Insert the SIMD array and iv initialization at function 1207 entry. */ 1208 gsi_insert_seq_before (&gsi, seq, GSI_NEW_STMT); 1209 1210 pop_gimplify_context (NULL); 1211 1212 gimple *g; 1213 basic_block incr_bb = NULL; 1214 class loop *loop = NULL; 1215 1216 /* Create a new BB right before the original exit BB, to hold the 1217 iteration increment and the condition/branch. */ 1218 if (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)) 1219 { 1220 basic_block orig_exit = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0)->src; 1221 incr_bb = create_empty_bb (orig_exit); 1222 incr_bb->count = profile_count::zero (); 1223 add_bb_to_loop (incr_bb, body_bb->loop_father); 1224 while (EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)) 1225 { 1226 edge e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 1227 redirect_edge_succ (e, incr_bb); 1228 incr_bb->count += e->count (); 1229 } 1230 } 1231 else if (node->simdclone->inbranch) 1232 { 1233 incr_bb = create_empty_bb (entry_bb); 1234 incr_bb->count = profile_count::zero (); 1235 add_bb_to_loop (incr_bb, body_bb->loop_father); 1236 } 1237 1238 if (incr_bb) 1239 { 1240 make_single_succ_edge (incr_bb, EXIT_BLOCK_PTR_FOR_FN (cfun), 0); 1241 gsi = gsi_last_bb (incr_bb); 1242 iter2 = make_ssa_name (iter); 1243 g = gimple_build_assign (iter2, PLUS_EXPR, iter1, 1244 build_int_cst (unsigned_type_node, 1)); 1245 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1246 1247 /* Mostly annotate the loop for the vectorizer (the rest is done 1248 below). */ 1249 loop = alloc_loop (); 1250 cfun->has_force_vectorize_loops = true; 1251 loop->safelen = node->simdclone->simdlen; 1252 loop->force_vectorize = true; 1253 loop->header = body_bb; 1254 } 1255 1256 /* Branch around the body if the mask applies. */ 1257 if (node->simdclone->inbranch) 1258 { 1259 gsi = gsi_last_bb (loop->header); 1260 tree mask_array 1261 = node->simdclone->args[node->simdclone->nargs - 1].simd_array; 1262 tree mask; 1263 if (node->simdclone->mask_mode != VOIDmode) 1264 { 1265 tree shift_cnt; 1266 if (mask_array == NULL_TREE) 1267 { 1268 tree arg = node->simdclone->args[node->simdclone->nargs 1269 - 1].vector_arg; 1270 mask = get_or_create_ssa_default_def (cfun, arg); 1271 shift_cnt = iter1; 1272 } 1273 else 1274 { 1275 tree maskt = TREE_TYPE (mask_array); 1276 int c = tree_to_uhwi (TYPE_MAX_VALUE (TYPE_DOMAIN (maskt))); 1277 c = node->simdclone->simdlen / (c + 1); 1278 int s = exact_log2 (c); 1279 gcc_assert (s > 0); 1280 c--; 1281 tree idx = make_ssa_name (TREE_TYPE (iter1)); 1282 g = gimple_build_assign (idx, RSHIFT_EXPR, iter1, 1283 build_int_cst (NULL_TREE, s)); 1284 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1285 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 1286 tree aref = build4 (ARRAY_REF, 1287 TREE_TYPE (TREE_TYPE (mask_array)), 1288 mask_array, idx, NULL, NULL); 1289 g = gimple_build_assign (mask, aref); 1290 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1291 shift_cnt = make_ssa_name (TREE_TYPE (iter1)); 1292 g = gimple_build_assign (shift_cnt, BIT_AND_EXPR, iter1, 1293 build_int_cst (TREE_TYPE (iter1), c)); 1294 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1295 } 1296 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)), 1297 RSHIFT_EXPR, mask, shift_cnt); 1298 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1299 mask = gimple_assign_lhs (g); 1300 g = gimple_build_assign (make_ssa_name (TREE_TYPE (mask)), 1301 BIT_AND_EXPR, mask, 1302 build_int_cst (TREE_TYPE (mask), 1)); 1303 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1304 mask = gimple_assign_lhs (g); 1305 } 1306 else 1307 { 1308 mask = make_ssa_name (TREE_TYPE (TREE_TYPE (mask_array))); 1309 tree aref = build4 (ARRAY_REF, 1310 TREE_TYPE (TREE_TYPE (mask_array)), 1311 mask_array, iter1, NULL, NULL); 1312 g = gimple_build_assign (mask, aref); 1313 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1314 int bitsize = GET_MODE_BITSIZE (SCALAR_TYPE_MODE (TREE_TYPE (aref))); 1315 if (!INTEGRAL_TYPE_P (TREE_TYPE (aref))) 1316 { 1317 aref = build1 (VIEW_CONVERT_EXPR, 1318 build_nonstandard_integer_type (bitsize, 0), 1319 mask); 1320 mask = make_ssa_name (TREE_TYPE (aref)); 1321 g = gimple_build_assign (mask, aref); 1322 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1323 } 1324 } 1325 1326 g = gimple_build_cond (EQ_EXPR, mask, build_zero_cst (TREE_TYPE (mask)), 1327 NULL, NULL); 1328 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1329 edge e = make_edge (loop->header, incr_bb, EDGE_TRUE_VALUE); 1330 e->probability = profile_probability::unlikely ().guessed (); 1331 incr_bb->count += e->count (); 1332 edge fallthru = FALLTHRU_EDGE (loop->header); 1333 fallthru->flags = EDGE_FALSE_VALUE; 1334 fallthru->probability = profile_probability::likely ().guessed (); 1335 } 1336 1337 basic_block latch_bb = NULL; 1338 basic_block new_exit_bb = NULL; 1339 1340 /* Generate the condition. */ 1341 if (incr_bb) 1342 { 1343 gsi = gsi_last_bb (incr_bb); 1344 g = gimple_build_cond (LT_EXPR, iter2, 1345 build_int_cst (unsigned_type_node, 1346 node->simdclone->simdlen), 1347 NULL, NULL); 1348 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1349 edge e = split_block (incr_bb, gsi_stmt (gsi)); 1350 latch_bb = e->dest; 1351 new_exit_bb = split_block_after_labels (latch_bb)->dest; 1352 loop->latch = latch_bb; 1353 1354 redirect_edge_succ (FALLTHRU_EDGE (latch_bb), body_bb); 1355 1356 edge new_e = make_edge (incr_bb, new_exit_bb, EDGE_FALSE_VALUE); 1357 1358 /* FIXME: Do we need to distribute probabilities for the conditional? */ 1359 new_e->probability = profile_probability::guessed_never (); 1360 /* The successor of incr_bb is already pointing to latch_bb; just 1361 change the flags. 1362 make_edge (incr_bb, latch_bb, EDGE_TRUE_VALUE); */ 1363 FALLTHRU_EDGE (incr_bb)->flags = EDGE_TRUE_VALUE; 1364 } 1365 1366 gphi *phi = create_phi_node (iter1, body_bb); 1367 edge preheader_edge = find_edge (entry_bb, body_bb); 1368 edge latch_edge = NULL; 1369 add_phi_arg (phi, build_zero_cst (unsigned_type_node), preheader_edge, 1370 UNKNOWN_LOCATION); 1371 if (incr_bb) 1372 { 1373 latch_edge = single_succ_edge (latch_bb); 1374 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1375 1376 /* Generate the new return. */ 1377 gsi = gsi_last_bb (new_exit_bb); 1378 if (retval 1379 && TREE_CODE (retval) == VIEW_CONVERT_EXPR 1380 && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL) 1381 retval = TREE_OPERAND (retval, 0); 1382 else if (retval) 1383 { 1384 retval = build1 (VIEW_CONVERT_EXPR, 1385 TREE_TYPE (TREE_TYPE (node->decl)), 1386 retval); 1387 retval = force_gimple_operand_gsi (&gsi, retval, true, NULL, 1388 false, GSI_CONTINUE_LINKING); 1389 } 1390 g = gimple_build_return (retval); 1391 gsi_insert_after (&gsi, g, GSI_CONTINUE_LINKING); 1392 } 1393 1394 /* Handle aligned clauses by replacing default defs of the aligned 1395 uniform args with __builtin_assume_aligned (arg_N(D), alignment) 1396 lhs. Handle linear by adding PHIs. */ 1397 for (unsigned i = 0; i < node->simdclone->nargs; i++) 1398 if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 1399 && (TREE_ADDRESSABLE (node->simdclone->args[i].orig_arg) 1400 || !is_gimple_reg_type 1401 (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 1402 { 1403 tree orig_arg = node->simdclone->args[i].orig_arg; 1404 if (is_gimple_reg_type (TREE_TYPE (orig_arg))) 1405 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 1406 else 1407 { 1408 iter1 = create_tmp_var_raw (TREE_TYPE (orig_arg)); 1409 gimple_add_tmp_var (iter1); 1410 } 1411 gsi = gsi_after_labels (entry_bb); 1412 g = gimple_build_assign (iter1, orig_arg); 1413 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1414 gsi = gsi_after_labels (body_bb); 1415 g = gimple_build_assign (orig_arg, iter1); 1416 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1417 } 1418 else if (node->simdclone->args[i].arg_type == SIMD_CLONE_ARG_TYPE_UNIFORM 1419 && DECL_BY_REFERENCE (node->simdclone->args[i].orig_arg) 1420 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 1421 == REFERENCE_TYPE 1422 && TREE_ADDRESSABLE 1423 (TREE_TYPE (TREE_TYPE (node->simdclone->args[i].orig_arg)))) 1424 { 1425 tree orig_arg = node->simdclone->args[i].orig_arg; 1426 tree def = ssa_default_def (cfun, orig_arg); 1427 if (def && !has_zero_uses (def)) 1428 { 1429 iter1 = create_tmp_var_raw (TREE_TYPE (TREE_TYPE (orig_arg))); 1430 gimple_add_tmp_var (iter1); 1431 gsi = gsi_after_labels (entry_bb); 1432 g = gimple_build_assign (iter1, build_simple_mem_ref (def)); 1433 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1434 gsi = gsi_after_labels (body_bb); 1435 g = gimple_build_assign (build_simple_mem_ref (def), iter1); 1436 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1437 } 1438 } 1439 else if (node->simdclone->args[i].alignment 1440 && node->simdclone->args[i].arg_type 1441 == SIMD_CLONE_ARG_TYPE_UNIFORM 1442 && (node->simdclone->args[i].alignment 1443 & (node->simdclone->args[i].alignment - 1)) == 0 1444 && TREE_CODE (TREE_TYPE (node->simdclone->args[i].orig_arg)) 1445 == POINTER_TYPE) 1446 { 1447 unsigned int alignment = node->simdclone->args[i].alignment; 1448 tree orig_arg = node->simdclone->args[i].orig_arg; 1449 tree def = ssa_default_def (cfun, orig_arg); 1450 if (def && !has_zero_uses (def)) 1451 { 1452 tree fn = builtin_decl_explicit (BUILT_IN_ASSUME_ALIGNED); 1453 gimple_seq seq = NULL; 1454 bool need_cvt = false; 1455 gcall *call 1456 = gimple_build_call (fn, 2, def, size_int (alignment)); 1457 g = call; 1458 if (!useless_type_conversion_p (TREE_TYPE (orig_arg), 1459 ptr_type_node)) 1460 need_cvt = true; 1461 tree t = make_ssa_name (need_cvt ? ptr_type_node : orig_arg); 1462 gimple_call_set_lhs (g, t); 1463 gimple_seq_add_stmt_without_update (&seq, g); 1464 if (need_cvt) 1465 { 1466 t = make_ssa_name (orig_arg); 1467 g = gimple_build_assign (t, NOP_EXPR, gimple_call_lhs (g)); 1468 gimple_seq_add_stmt_without_update (&seq, g); 1469 } 1470 gsi_insert_seq_on_edge_immediate 1471 (single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun)), seq); 1472 1473 entry_bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)); 1474 node->create_edge (cgraph_node::get_create (fn), 1475 call, entry_bb->count); 1476 1477 imm_use_iterator iter; 1478 use_operand_p use_p; 1479 gimple *use_stmt; 1480 tree repl = gimple_get_lhs (g); 1481 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1482 if (is_gimple_debug (use_stmt) || use_stmt == call) 1483 continue; 1484 else 1485 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1486 SET_USE (use_p, repl); 1487 } 1488 } 1489 else if ((node->simdclone->args[i].arg_type 1490 == SIMD_CLONE_ARG_TYPE_LINEAR_CONSTANT_STEP) 1491 || (node->simdclone->args[i].arg_type 1492 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_CONSTANT_STEP) 1493 || (node->simdclone->args[i].arg_type 1494 == SIMD_CLONE_ARG_TYPE_LINEAR_VARIABLE_STEP) 1495 || (node->simdclone->args[i].arg_type 1496 == SIMD_CLONE_ARG_TYPE_LINEAR_REF_VARIABLE_STEP)) 1497 { 1498 tree orig_arg = node->simdclone->args[i].orig_arg; 1499 gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1500 || POINTER_TYPE_P (TREE_TYPE (orig_arg))); 1501 tree def = NULL_TREE; 1502 if (TREE_ADDRESSABLE (orig_arg)) 1503 { 1504 def = make_ssa_name (TREE_TYPE (orig_arg)); 1505 iter1 = make_ssa_name (TREE_TYPE (orig_arg)); 1506 if (incr_bb) 1507 iter2 = make_ssa_name (TREE_TYPE (orig_arg)); 1508 gsi = gsi_after_labels (entry_bb); 1509 g = gimple_build_assign (def, orig_arg); 1510 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1511 } 1512 else 1513 { 1514 def = ssa_default_def (cfun, orig_arg); 1515 if (!def || has_zero_uses (def)) 1516 def = NULL_TREE; 1517 else 1518 { 1519 iter1 = make_ssa_name (orig_arg); 1520 if (incr_bb) 1521 iter2 = make_ssa_name (orig_arg); 1522 } 1523 } 1524 if (def) 1525 { 1526 phi = create_phi_node (iter1, body_bb); 1527 add_phi_arg (phi, def, preheader_edge, UNKNOWN_LOCATION); 1528 if (incr_bb) 1529 { 1530 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1531 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1532 ? PLUS_EXPR : POINTER_PLUS_EXPR; 1533 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (orig_arg)) 1534 ? TREE_TYPE (orig_arg) : sizetype; 1535 tree addcst = simd_clone_linear_addend (node, i, addtype, 1536 entry_bb); 1537 gsi = gsi_last_bb (incr_bb); 1538 g = gimple_build_assign (iter2, code, iter1, addcst); 1539 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1540 } 1541 1542 imm_use_iterator iter; 1543 use_operand_p use_p; 1544 gimple *use_stmt; 1545 if (TREE_ADDRESSABLE (orig_arg)) 1546 { 1547 gsi = gsi_after_labels (body_bb); 1548 g = gimple_build_assign (orig_arg, iter1); 1549 gsi_insert_before (&gsi, g, GSI_NEW_STMT); 1550 } 1551 else 1552 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1553 if (use_stmt == phi) 1554 continue; 1555 else 1556 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1557 SET_USE (use_p, iter1); 1558 } 1559 } 1560 else if (node->simdclone->args[i].arg_type 1561 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_CONSTANT_STEP 1562 || (node->simdclone->args[i].arg_type 1563 == SIMD_CLONE_ARG_TYPE_LINEAR_UVAL_VARIABLE_STEP)) 1564 { 1565 tree orig_arg = node->simdclone->args[i].orig_arg; 1566 tree def = ssa_default_def (cfun, orig_arg); 1567 gcc_assert (!TREE_ADDRESSABLE (orig_arg) 1568 && TREE_CODE (TREE_TYPE (orig_arg)) == REFERENCE_TYPE); 1569 if (def && !has_zero_uses (def)) 1570 { 1571 tree rtype = TREE_TYPE (TREE_TYPE (orig_arg)); 1572 iter1 = make_ssa_name (orig_arg); 1573 if (incr_bb) 1574 iter2 = make_ssa_name (orig_arg); 1575 tree iter3 = make_ssa_name (rtype); 1576 tree iter4 = make_ssa_name (rtype); 1577 tree iter5 = incr_bb ? make_ssa_name (rtype) : NULL_TREE; 1578 gsi = gsi_after_labels (entry_bb); 1579 gimple *load 1580 = gimple_build_assign (iter3, build_simple_mem_ref (def)); 1581 gsi_insert_before (&gsi, load, GSI_NEW_STMT); 1582 1583 tree array = node->simdclone->args[i].simd_array; 1584 TREE_ADDRESSABLE (array) = 1; 1585 tree ptr = build_fold_addr_expr (array); 1586 phi = create_phi_node (iter1, body_bb); 1587 add_phi_arg (phi, ptr, preheader_edge, UNKNOWN_LOCATION); 1588 if (incr_bb) 1589 { 1590 add_phi_arg (phi, iter2, latch_edge, UNKNOWN_LOCATION); 1591 g = gimple_build_assign (iter2, POINTER_PLUS_EXPR, iter1, 1592 TYPE_SIZE_UNIT (TREE_TYPE (iter3))); 1593 gsi = gsi_last_bb (incr_bb); 1594 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1595 } 1596 1597 phi = create_phi_node (iter4, body_bb); 1598 add_phi_arg (phi, iter3, preheader_edge, UNKNOWN_LOCATION); 1599 if (incr_bb) 1600 { 1601 add_phi_arg (phi, iter5, latch_edge, UNKNOWN_LOCATION); 1602 enum tree_code code = INTEGRAL_TYPE_P (TREE_TYPE (iter3)) 1603 ? PLUS_EXPR : POINTER_PLUS_EXPR; 1604 tree addtype = INTEGRAL_TYPE_P (TREE_TYPE (iter3)) 1605 ? TREE_TYPE (iter3) : sizetype; 1606 tree addcst = simd_clone_linear_addend (node, i, addtype, 1607 entry_bb); 1608 g = gimple_build_assign (iter5, code, iter4, addcst); 1609 gsi = gsi_last_bb (incr_bb); 1610 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1611 } 1612 1613 g = gimple_build_assign (build_simple_mem_ref (iter1), iter4); 1614 gsi = gsi_after_labels (body_bb); 1615 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1616 1617 imm_use_iterator iter; 1618 use_operand_p use_p; 1619 gimple *use_stmt; 1620 FOR_EACH_IMM_USE_STMT (use_stmt, iter, def) 1621 if (use_stmt == load) 1622 continue; 1623 else 1624 FOR_EACH_IMM_USE_ON_STMT (use_p, iter) 1625 SET_USE (use_p, iter1); 1626 1627 if (!TYPE_READONLY (rtype) && incr_bb) 1628 { 1629 tree v = make_ssa_name (rtype); 1630 tree aref = build4 (ARRAY_REF, rtype, array, 1631 size_zero_node, NULL_TREE, 1632 NULL_TREE); 1633 gsi = gsi_after_labels (new_exit_bb); 1634 g = gimple_build_assign (v, aref); 1635 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1636 g = gimple_build_assign (build_simple_mem_ref (def), v); 1637 gsi_insert_before (&gsi, g, GSI_SAME_STMT); 1638 } 1639 } 1640 } 1641 1642 calculate_dominance_info (CDI_DOMINATORS); 1643 if (loop) 1644 add_loop (loop, loop->header->loop_father); 1645 update_ssa (TODO_update_ssa); 1646 1647 pop_cfun (); 1648} 1649 1650/* If the function in NODE is tagged as an elemental SIMD function, 1651 create the appropriate SIMD clones. */ 1652 1653void 1654expand_simd_clones (struct cgraph_node *node) 1655{ 1656 tree attr = lookup_attribute ("omp declare simd", 1657 DECL_ATTRIBUTES (node->decl)); 1658 if (attr == NULL_TREE 1659 || node->inlined_to 1660 || lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl))) 1661 return; 1662 1663 /* Ignore 1664 #pragma omp declare simd 1665 extern int foo (); 1666 in C, there we don't know the argument types at all. */ 1667 if (!node->definition 1668 && TYPE_ARG_TYPES (TREE_TYPE (node->decl)) == NULL_TREE) 1669 return; 1670 1671 /* Call this before creating clone_info, as it might ggc_collect. */ 1672 if (node->definition && node->has_gimple_body_p ()) 1673 node->get_body (); 1674 1675 do 1676 { 1677 /* Start with parsing the "omp declare simd" attribute(s). */ 1678 bool inbranch_clause_specified; 1679 struct cgraph_simd_clone *clone_info 1680 = simd_clone_clauses_extract (node, TREE_VALUE (attr), 1681 &inbranch_clause_specified); 1682 if (clone_info == NULL) 1683 continue; 1684 1685 int orig_simdlen = clone_info->simdlen; 1686 tree base_type = simd_clone_compute_base_data_type (node, clone_info); 1687 /* The target can return 0 (no simd clones should be created), 1688 1 (just one ISA of simd clones should be created) or higher 1689 count of ISA variants. In that case, clone_info is initialized 1690 for the first ISA variant. */ 1691 int count 1692 = targetm.simd_clone.compute_vecsize_and_simdlen (node, clone_info, 1693 base_type, 0); 1694 if (count == 0) 1695 continue; 1696 1697 /* Loop over all COUNT ISA variants, and if !INBRANCH_CLAUSE_SPECIFIED, 1698 also create one inbranch and one !inbranch clone of it. */ 1699 for (int i = 0; i < count * 2; i++) 1700 { 1701 struct cgraph_simd_clone *clone = clone_info; 1702 if (inbranch_clause_specified && (i & 1) != 0) 1703 continue; 1704 1705 if (i != 0) 1706 { 1707 clone = simd_clone_struct_alloc (clone_info->nargs 1708 + ((i & 1) != 0)); 1709 simd_clone_struct_copy (clone, clone_info); 1710 /* Undo changes targetm.simd_clone.compute_vecsize_and_simdlen 1711 and simd_clone_adjust_argument_types did to the first 1712 clone's info. */ 1713 clone->nargs -= clone_info->inbranch; 1714 clone->simdlen = orig_simdlen; 1715 /* And call the target hook again to get the right ISA. */ 1716 targetm.simd_clone.compute_vecsize_and_simdlen (node, clone, 1717 base_type, 1718 i / 2); 1719 if ((i & 1) != 0) 1720 clone->inbranch = 1; 1721 } 1722 1723 /* simd_clone_mangle might fail if such a clone has been created 1724 already. */ 1725 tree id = simd_clone_mangle (node, clone); 1726 if (id == NULL_TREE) 1727 { 1728 if (i == 0) 1729 clone->nargs += clone->inbranch; 1730 continue; 1731 } 1732 1733 /* Only when we are sure we want to create the clone actually 1734 clone the function (or definitions) or create another 1735 extern FUNCTION_DECL (for prototypes without definitions). */ 1736 struct cgraph_node *n = simd_clone_create (node); 1737 if (n == NULL) 1738 { 1739 if (i == 0) 1740 clone->nargs += clone->inbranch; 1741 continue; 1742 } 1743 1744 n->simdclone = clone; 1745 clone->origin = node; 1746 clone->next_clone = NULL; 1747 if (node->simd_clones == NULL) 1748 { 1749 clone->prev_clone = n; 1750 node->simd_clones = n; 1751 } 1752 else 1753 { 1754 clone->prev_clone = node->simd_clones->simdclone->prev_clone; 1755 clone->prev_clone->simdclone->next_clone = n; 1756 node->simd_clones->simdclone->prev_clone = n; 1757 } 1758 symtab->change_decl_assembler_name (n->decl, id); 1759 /* And finally adjust the return type, parameters and for 1760 definitions also function body. */ 1761 if (node->definition) 1762 simd_clone_adjust (n); 1763 else 1764 { 1765 TREE_TYPE (n->decl) 1766 = build_distinct_type_copy (TREE_TYPE (n->decl)); 1767 targetm.simd_clone.adjust (n); 1768 simd_clone_adjust_return_type (n); 1769 simd_clone_adjust_argument_types (n); 1770 } 1771 } 1772 } 1773 while ((attr = lookup_attribute ("omp declare simd", TREE_CHAIN (attr)))); 1774} 1775 1776/* Entry point for IPA simd clone creation pass. */ 1777 1778static unsigned int 1779ipa_omp_simd_clone (void) 1780{ 1781 struct cgraph_node *node; 1782 FOR_EACH_FUNCTION (node) 1783 expand_simd_clones (node); 1784 return 0; 1785} 1786 1787namespace { 1788 1789const pass_data pass_data_omp_simd_clone = 1790{ 1791 SIMPLE_IPA_PASS, /* type */ 1792 "simdclone", /* name */ 1793 OPTGROUP_OMP, /* optinfo_flags */ 1794 TV_NONE, /* tv_id */ 1795 ( PROP_ssa | PROP_cfg ), /* properties_required */ 1796 0, /* properties_provided */ 1797 0, /* properties_destroyed */ 1798 0, /* todo_flags_start */ 1799 0, /* todo_flags_finish */ 1800}; 1801 1802class pass_omp_simd_clone : public simple_ipa_opt_pass 1803{ 1804public: 1805 pass_omp_simd_clone(gcc::context *ctxt) 1806 : simple_ipa_opt_pass(pass_data_omp_simd_clone, ctxt) 1807 {} 1808 1809 /* opt_pass methods: */ 1810 virtual bool gate (function *); 1811 virtual unsigned int execute (function *) { return ipa_omp_simd_clone (); } 1812}; 1813 1814bool 1815pass_omp_simd_clone::gate (function *) 1816{ 1817 return targetm.simd_clone.compute_vecsize_and_simdlen != NULL; 1818} 1819 1820} // anon namespace 1821 1822simple_ipa_opt_pass * 1823make_pass_omp_simd_clone (gcc::context *ctxt) 1824{ 1825 return new pass_omp_simd_clone (ctxt); 1826} 1827