1/* 2 * kmp_gsupport.cpp 3 */ 4 5//===----------------------------------------------------------------------===// 6// 7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8// See https://llvm.org/LICENSE.txt for license information. 9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10// 11//===----------------------------------------------------------------------===// 12 13#include "kmp.h" 14#include "kmp_atomic.h" 15 16#if OMPT_SUPPORT 17#include "ompt-specific.h" 18#endif 19 20#ifdef __cplusplus 21extern "C" { 22#endif // __cplusplus 23 24#define MKLOC(loc, routine) \ 25 static ident_t loc = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"}; 26 27#include "kmp_ftn_os.h" 28 29void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) { 30 int gtid = __kmp_entry_gtid(); 31 MKLOC(loc, "GOMP_barrier"); 32 KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid)); 33#if OMPT_SUPPORT && OMPT_OPTIONAL 34 ompt_frame_t *ompt_frame; 35 if (ompt_enabled.enabled) { 36 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 37 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 38 OMPT_STORE_RETURN_ADDRESS(gtid); 39 } 40#endif 41 __kmpc_barrier(&loc, gtid); 42#if OMPT_SUPPORT && OMPT_OPTIONAL 43 if (ompt_enabled.enabled) { 44 ompt_frame->enter_frame = ompt_data_none; 45 } 46#endif 47} 48 49// Mutual exclusion 50 51// The symbol that icc/ifort generates for unnamed for unnamed critical sections 52// - .gomp_critical_user_ - is defined using .comm in any objects reference it. 53// We can't reference it directly here in C code, as the symbol contains a ".". 54// 55// The RTL contains an assembly language definition of .gomp_critical_user_ 56// with another symbol __kmp_unnamed_critical_addr initialized with it's 57// address. 58extern kmp_critical_name *__kmp_unnamed_critical_addr; 59 60void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_START)(void) { 61 int gtid = __kmp_entry_gtid(); 62 MKLOC(loc, "GOMP_critical_start"); 63 KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid)); 64#if OMPT_SUPPORT && OMPT_OPTIONAL 65 OMPT_STORE_RETURN_ADDRESS(gtid); 66#endif 67 __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr); 68} 69 70void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_END)(void) { 71 int gtid = __kmp_get_gtid(); 72 MKLOC(loc, "GOMP_critical_end"); 73 KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid)); 74#if OMPT_SUPPORT && OMPT_OPTIONAL 75 OMPT_STORE_RETURN_ADDRESS(gtid); 76#endif 77 __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr); 78} 79 80void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr) { 81 int gtid = __kmp_entry_gtid(); 82 MKLOC(loc, "GOMP_critical_name_start"); 83 KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid)); 84 __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr); 85} 86 87void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr) { 88 int gtid = __kmp_get_gtid(); 89 MKLOC(loc, "GOMP_critical_name_end"); 90 KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid)); 91 __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr); 92} 93 94// The Gnu codegen tries to use locked operations to perform atomic updates 95// inline. If it can't, then it calls GOMP_atomic_start() before performing 96// the update and GOMP_atomic_end() afterward, regardless of the data type. 97void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_START)(void) { 98 int gtid = __kmp_entry_gtid(); 99 KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid)); 100 101#if OMPT_SUPPORT 102 __ompt_thread_assign_wait_id(0); 103#endif 104 105 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid); 106} 107 108void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_END)(void) { 109 int gtid = __kmp_get_gtid(); 110 KA_TRACE(20, ("GOMP_atomic_end: T#%d\n", gtid)); 111 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid); 112} 113 114int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) { 115 int gtid = __kmp_entry_gtid(); 116 MKLOC(loc, "GOMP_single_start"); 117 KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid)); 118 119 if (!TCR_4(__kmp_init_parallel)) 120 __kmp_parallel_initialize(); 121 __kmp_resume_if_soft_paused(); 122 123 // 3rd parameter == FALSE prevents kmp_enter_single from pushing a 124 // workshare when USE_CHECKS is defined. We need to avoid the push, 125 // as there is no corresponding GOMP_single_end() call. 126 kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE); 127 128#if OMPT_SUPPORT && OMPT_OPTIONAL 129 kmp_info_t *this_thr = __kmp_threads[gtid]; 130 kmp_team_t *team = this_thr->th.th_team; 131 int tid = __kmp_tid_from_gtid(gtid); 132 133 if (ompt_enabled.enabled) { 134 if (rc) { 135 if (ompt_enabled.ompt_callback_work) { 136 ompt_callbacks.ompt_callback(ompt_callback_work)( 137 ompt_work_single_executor, ompt_scope_begin, 138 &(team->t.ompt_team_info.parallel_data), 139 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 140 1, OMPT_GET_RETURN_ADDRESS(0)); 141 } 142 } else { 143 if (ompt_enabled.ompt_callback_work) { 144 ompt_callbacks.ompt_callback(ompt_callback_work)( 145 ompt_work_single_other, ompt_scope_begin, 146 &(team->t.ompt_team_info.parallel_data), 147 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 148 1, OMPT_GET_RETURN_ADDRESS(0)); 149 ompt_callbacks.ompt_callback(ompt_callback_work)( 150 ompt_work_single_other, ompt_scope_end, 151 &(team->t.ompt_team_info.parallel_data), 152 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 153 1, OMPT_GET_RETURN_ADDRESS(0)); 154 } 155 } 156 } 157#endif 158 159 return rc; 160} 161 162void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) { 163 void *retval; 164 int gtid = __kmp_entry_gtid(); 165 MKLOC(loc, "GOMP_single_copy_start"); 166 KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid)); 167 168 if (!TCR_4(__kmp_init_parallel)) 169 __kmp_parallel_initialize(); 170 __kmp_resume_if_soft_paused(); 171 172 // If this is the first thread to enter, return NULL. The generated code will 173 // then call GOMP_single_copy_end() for this thread only, with the 174 // copyprivate data pointer as an argument. 175 if (__kmp_enter_single(gtid, &loc, FALSE)) 176 return NULL; 177 178// Wait for the first thread to set the copyprivate data pointer, 179// and for all other threads to reach this point. 180 181#if OMPT_SUPPORT && OMPT_OPTIONAL 182 ompt_frame_t *ompt_frame; 183 if (ompt_enabled.enabled) { 184 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 185 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 186 OMPT_STORE_RETURN_ADDRESS(gtid); 187 } 188#endif 189 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 190 191 // Retrieve the value of the copyprivate data point, and wait for all 192 // threads to do likewise, then return. 193 retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data; 194#if OMPT_SUPPORT && OMPT_OPTIONAL 195 if (ompt_enabled.enabled) { 196 OMPT_STORE_RETURN_ADDRESS(gtid); 197 } 198#endif 199 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 200#if OMPT_SUPPORT && OMPT_OPTIONAL 201 if (ompt_enabled.enabled) { 202 ompt_frame->enter_frame = ompt_data_none; 203 } 204#endif 205 return retval; 206} 207 208void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data) { 209 int gtid = __kmp_get_gtid(); 210 KA_TRACE(20, ("GOMP_single_copy_end: T#%d\n", gtid)); 211 212 // Set the copyprivate data pointer fo the team, then hit the barrier so that 213 // the other threads will continue on and read it. Hit another barrier before 214 // continuing, so that the know that the copyprivate data pointer has been 215 // propagated to all threads before trying to reuse the t_copypriv_data field. 216 __kmp_team_from_gtid(gtid)->t.t_copypriv_data = data; 217#if OMPT_SUPPORT && OMPT_OPTIONAL 218 ompt_frame_t *ompt_frame; 219 if (ompt_enabled.enabled) { 220 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 221 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 222 OMPT_STORE_RETURN_ADDRESS(gtid); 223 } 224#endif 225 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 226#if OMPT_SUPPORT && OMPT_OPTIONAL 227 if (ompt_enabled.enabled) { 228 OMPT_STORE_RETURN_ADDRESS(gtid); 229 } 230#endif 231 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 232#if OMPT_SUPPORT && OMPT_OPTIONAL 233 if (ompt_enabled.enabled) { 234 ompt_frame->enter_frame = ompt_data_none; 235 } 236#endif 237} 238 239void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_START)(void) { 240 int gtid = __kmp_entry_gtid(); 241 MKLOC(loc, "GOMP_ordered_start"); 242 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 243#if OMPT_SUPPORT && OMPT_OPTIONAL 244 OMPT_STORE_RETURN_ADDRESS(gtid); 245#endif 246 __kmpc_ordered(&loc, gtid); 247} 248 249void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) { 250 int gtid = __kmp_get_gtid(); 251 MKLOC(loc, "GOMP_ordered_end"); 252 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 253#if OMPT_SUPPORT && OMPT_OPTIONAL 254 OMPT_STORE_RETURN_ADDRESS(gtid); 255#endif 256 __kmpc_end_ordered(&loc, gtid); 257} 258 259// Dispatch macro defs 260// 261// They come in two flavors: 64-bit unsigned, and either 32-bit signed 262// (IA-32 architecture) or 64-bit signed (Intel(R) 64). 263 264#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS 265#define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4 266#define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4 267#define KMP_DISPATCH_NEXT __kmpc_dispatch_next_4 268#else 269#define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_8 270#define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_8 271#define KMP_DISPATCH_NEXT __kmpc_dispatch_next_8 272#endif /* KMP_ARCH_X86 */ 273 274#define KMP_DISPATCH_INIT_ULL __kmp_aux_dispatch_init_8u 275#define KMP_DISPATCH_FINI_CHUNK_ULL __kmp_aux_dispatch_fini_chunk_8u 276#define KMP_DISPATCH_NEXT_ULL __kmpc_dispatch_next_8u 277 278// The parallel contruct 279 280#ifndef KMP_DEBUG 281static 282#endif /* KMP_DEBUG */ 283 void 284 __kmp_GOMP_microtask_wrapper(int *gtid, int *npr, void (*task)(void *), 285 void *data) { 286#if OMPT_SUPPORT 287 kmp_info_t *thr; 288 ompt_frame_t *ompt_frame; 289 ompt_state_t enclosing_state; 290 291 if (ompt_enabled.enabled) { 292 // get pointer to thread data structure 293 thr = __kmp_threads[*gtid]; 294 295 // save enclosing task state; set current state for task 296 enclosing_state = thr->th.ompt_thread_info.state; 297 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 298 299 // set task frame 300 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 301 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 302 } 303#endif 304 305 task(data); 306 307#if OMPT_SUPPORT 308 if (ompt_enabled.enabled) { 309 // clear task frame 310 ompt_frame->exit_frame = ompt_data_none; 311 312 // restore enclosing state 313 thr->th.ompt_thread_info.state = enclosing_state; 314 } 315#endif 316} 317 318#ifndef KMP_DEBUG 319static 320#endif /* KMP_DEBUG */ 321 void 322 __kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr, 323 void (*task)(void *), void *data, 324 unsigned num_threads, ident_t *loc, 325 enum sched_type schedule, long start, 326 long end, long incr, 327 long chunk_size) { 328 // Intialize the loop worksharing construct. 329 330 KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size, 331 schedule != kmp_sch_static); 332 333#if OMPT_SUPPORT 334 kmp_info_t *thr; 335 ompt_frame_t *ompt_frame; 336 ompt_state_t enclosing_state; 337 338 if (ompt_enabled.enabled) { 339 thr = __kmp_threads[*gtid]; 340 // save enclosing task state; set current state for task 341 enclosing_state = thr->th.ompt_thread_info.state; 342 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 343 344 // set task frame 345 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 346 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 347 } 348#endif 349 350 // Now invoke the microtask. 351 task(data); 352 353#if OMPT_SUPPORT 354 if (ompt_enabled.enabled) { 355 // clear task frame 356 ompt_frame->exit_frame = ompt_data_none; 357 358 // reset enclosing state 359 thr->th.ompt_thread_info.state = enclosing_state; 360 } 361#endif 362} 363 364#ifndef KMP_DEBUG 365static 366#endif /* KMP_DEBUG */ 367 void 368 __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), 369 microtask_t wrapper, int argc, ...) { 370 int rc; 371 kmp_info_t *thr = __kmp_threads[gtid]; 372 kmp_team_t *team = thr->th.th_team; 373 int tid = __kmp_tid_from_gtid(gtid); 374 375 va_list ap; 376 va_start(ap, argc); 377 378 rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc, wrapper, 379 __kmp_invoke_task_func, kmp_va_addr_of(ap)); 380 381 va_end(ap); 382 383 if (rc) { 384 __kmp_run_before_invoked_task(gtid, tid, thr, team); 385 } 386 387#if OMPT_SUPPORT 388 int ompt_team_size; 389 if (ompt_enabled.enabled) { 390 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 391 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 392 393 // implicit task callback 394 if (ompt_enabled.ompt_callback_implicit_task) { 395 ompt_team_size = __kmp_team_from_gtid(gtid)->t.t_nproc; 396 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 397 ompt_scope_begin, &(team_info->parallel_data), 398 &(task_info->task_data), ompt_team_size, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 399 task_info->thread_num = __kmp_tid_from_gtid(gtid); 400 } 401 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 402 } 403#endif 404} 405 406static void __kmp_GOMP_serialized_parallel(ident_t *loc, kmp_int32 gtid, 407 void (*task)(void *)) { 408#if OMPT_SUPPORT 409 OMPT_STORE_RETURN_ADDRESS(gtid); 410#endif 411 __kmp_serialized_parallel(loc, gtid); 412} 413 414void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), 415 void *data, 416 unsigned num_threads) { 417 int gtid = __kmp_entry_gtid(); 418 419#if OMPT_SUPPORT 420 ompt_frame_t *parent_frame, *frame; 421 422 if (ompt_enabled.enabled) { 423 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 424 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 425 OMPT_STORE_RETURN_ADDRESS(gtid); 426 } 427#endif 428 429 MKLOC(loc, "GOMP_parallel_start"); 430 KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid)); 431 432 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 433 if (num_threads != 0) { 434 __kmp_push_num_threads(&loc, gtid, num_threads); 435 } 436 __kmp_GOMP_fork_call(&loc, gtid, task, 437 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 438 data); 439 } else { 440 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 441 } 442 443#if OMPT_SUPPORT 444 if (ompt_enabled.enabled) { 445 __ompt_get_task_info_internal(0, NULL, NULL, &frame, NULL, NULL); 446 frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 447 } 448#endif 449} 450 451void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(void) { 452 int gtid = __kmp_get_gtid(); 453 kmp_info_t *thr; 454 455 thr = __kmp_threads[gtid]; 456 457 MKLOC(loc, "GOMP_parallel_end"); 458 KA_TRACE(20, ("GOMP_parallel_end: T#%d\n", gtid)); 459 460 if (!thr->th.th_team->t.t_serialized) { 461 __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr, 462 thr->th.th_team); 463 464#if OMPT_SUPPORT 465 if (ompt_enabled.enabled) { 466 // Implicit task is finished here, in the barrier we might schedule 467 // deferred tasks, 468 // these don't see the implicit task on the stack 469 OMPT_CUR_TASK_INFO(thr)->frame.exit_frame = ompt_data_none; 470 } 471#endif 472 473 __kmp_join_call(&loc, gtid 474#if OMPT_SUPPORT 475 , 476 fork_context_gnu 477#endif 478 ); 479 } else { 480 __kmpc_end_serialized_parallel(&loc, gtid); 481 } 482} 483 484// Loop worksharing constructs 485 486// The Gnu codegen passes in an exclusive upper bound for the overall range, 487// but the libguide dispatch code expects an inclusive upper bound, hence the 488// "end - incr" 5th argument to KMP_DISPATCH_INIT (and the " ub - str" 11th 489// argument to __kmp_GOMP_fork_call). 490// 491// Conversely, KMP_DISPATCH_NEXT returns and inclusive upper bound in *p_ub, 492// but the Gnu codegen expects an exclusive upper bound, so the adjustment 493// "*p_ub += stride" compensates for the discrepancy. 494// 495// Correction: the gnu codegen always adjusts the upper bound by +-1, not the 496// stride value. We adjust the dispatch parameters accordingly (by +-1), but 497// we still adjust p_ub by the actual stride value. 498// 499// The "runtime" versions do not take a chunk_sz parameter. 500// 501// The profile lib cannot support construct checking of unordered loops that 502// are predetermined by the compiler to be statically scheduled, as the gcc 503// codegen will not always emit calls to GOMP_loop_static_next() to get the 504// next iteration. Instead, it emits inline code to call omp_get_thread_num() 505// num and calculate the iteration space using the result. It doesn't do this 506// with ordered static loop, so they can be checked. 507 508#if OMPT_SUPPORT 509#define IF_OMPT_SUPPORT(code) code 510#else 511#define IF_OMPT_SUPPORT(code) 512#endif 513 514#define LOOP_START(func, schedule) \ 515 int func(long lb, long ub, long str, long chunk_sz, long *p_lb, \ 516 long *p_ub) { \ 517 int status; \ 518 long stride; \ 519 int gtid = __kmp_entry_gtid(); \ 520 MKLOC(loc, KMP_STR(func)); \ 521 KA_TRACE( \ 522 20, \ 523 (KMP_STR( \ 524 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 525 gtid, lb, ub, str, chunk_sz)); \ 526 \ 527 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 528 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 529 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 530 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 531 (schedule) != kmp_sch_static); \ 532 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 533 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 534 (kmp_int *)p_ub, (kmp_int *)&stride); \ 535 if (status) { \ 536 KMP_DEBUG_ASSERT(stride == str); \ 537 *p_ub += (str > 0) ? 1 : -1; \ 538 } \ 539 } else { \ 540 status = 0; \ 541 } \ 542 \ 543 KA_TRACE( \ 544 20, \ 545 (KMP_STR( \ 546 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 547 gtid, *p_lb, *p_ub, status)); \ 548 return status; \ 549 } 550 551#define LOOP_RUNTIME_START(func, schedule) \ 552 int func(long lb, long ub, long str, long *p_lb, long *p_ub) { \ 553 int status; \ 554 long stride; \ 555 long chunk_sz = 0; \ 556 int gtid = __kmp_entry_gtid(); \ 557 MKLOC(loc, KMP_STR(func)); \ 558 KA_TRACE( \ 559 20, \ 560 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 561 gtid, lb, ub, str, chunk_sz)); \ 562 \ 563 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 564 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 565 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 566 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \ 567 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 568 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 569 (kmp_int *)p_ub, (kmp_int *)&stride); \ 570 if (status) { \ 571 KMP_DEBUG_ASSERT(stride == str); \ 572 *p_ub += (str > 0) ? 1 : -1; \ 573 } \ 574 } else { \ 575 status = 0; \ 576 } \ 577 \ 578 KA_TRACE( \ 579 20, \ 580 (KMP_STR( \ 581 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 582 gtid, *p_lb, *p_ub, status)); \ 583 return status; \ 584 } 585 586#define KMP_DOACROSS_FINI(status, gtid) \ 587 if (!status && __kmp_threads[gtid]->th.th_dispatch->th_doacross_flags) { \ 588 __kmpc_doacross_fini(NULL, gtid); \ 589 } 590 591#define LOOP_NEXT(func, fini_code) \ 592 int func(long *p_lb, long *p_ub) { \ 593 int status; \ 594 long stride; \ 595 int gtid = __kmp_get_gtid(); \ 596 MKLOC(loc, KMP_STR(func)); \ 597 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 598 \ 599 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 600 fini_code status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 601 (kmp_int *)p_ub, (kmp_int *)&stride); \ 602 if (status) { \ 603 *p_ub += (stride > 0) ? 1 : -1; \ 604 } \ 605 KMP_DOACROSS_FINI(status, gtid) \ 606 \ 607 KA_TRACE( \ 608 20, \ 609 (KMP_STR(func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, stride 0x%lx, " \ 610 "returning %d\n", \ 611 gtid, *p_lb, *p_ub, stride, status)); \ 612 return status; \ 613 } 614 615LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_START), kmp_sch_static) 616LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT), {}) 617LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START), 618 kmp_sch_dynamic_chunked) 619LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START), 620 kmp_sch_dynamic_chunked) 621LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT), {}) 622LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT), {}) 623LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_START), 624 kmp_sch_guided_chunked) 625LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START), 626 kmp_sch_guided_chunked) 627LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT), {}) 628LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT), {}) 629LOOP_RUNTIME_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_START), 630 kmp_sch_runtime) 631LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {}) 632 633LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START), 634 kmp_ord_static) 635LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT), 636 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 637LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START), 638 kmp_ord_dynamic_chunked) 639LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT), 640 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 641LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START), 642 kmp_ord_guided_chunked) 643LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT), 644 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 645LOOP_RUNTIME_START( 646 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START), 647 kmp_ord_runtime) 648LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT), 649 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 650 651#define LOOP_DOACROSS_START(func, schedule) \ 652 bool func(unsigned ncounts, long *counts, long chunk_sz, long *p_lb, \ 653 long *p_ub) { \ 654 int status; \ 655 long stride, lb, ub, str; \ 656 int gtid = __kmp_entry_gtid(); \ 657 struct kmp_dim *dims = \ 658 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 659 MKLOC(loc, KMP_STR(func)); \ 660 for (unsigned i = 0; i < ncounts; ++i) { \ 661 dims[i].lo = 0; \ 662 dims[i].up = counts[i] - 1; \ 663 dims[i].st = 1; \ 664 } \ 665 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 666 lb = 0; \ 667 ub = counts[0]; \ 668 str = 1; \ 669 KA_TRACE(20, (KMP_STR(func) ": T#%d, ncounts %u, lb 0x%lx, ub 0x%lx, str " \ 670 "0x%lx, chunk_sz " \ 671 "0x%lx\n", \ 672 gtid, ncounts, lb, ub, str, chunk_sz)); \ 673 \ 674 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 675 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 676 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 677 (schedule) != kmp_sch_static); \ 678 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 679 (kmp_int *)p_ub, (kmp_int *)&stride); \ 680 if (status) { \ 681 KMP_DEBUG_ASSERT(stride == str); \ 682 *p_ub += (str > 0) ? 1 : -1; \ 683 } \ 684 } else { \ 685 status = 0; \ 686 } \ 687 KMP_DOACROSS_FINI(status, gtid); \ 688 \ 689 KA_TRACE( \ 690 20, \ 691 (KMP_STR( \ 692 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 693 gtid, *p_lb, *p_ub, status)); \ 694 __kmp_free(dims); \ 695 return status; \ 696 } 697 698#define LOOP_DOACROSS_RUNTIME_START(func, schedule) \ 699 int func(unsigned ncounts, long *counts, long *p_lb, long *p_ub) { \ 700 int status; \ 701 long stride, lb, ub, str; \ 702 long chunk_sz = 0; \ 703 int gtid = __kmp_entry_gtid(); \ 704 struct kmp_dim *dims = \ 705 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 706 MKLOC(loc, KMP_STR(func)); \ 707 for (unsigned i = 0; i < ncounts; ++i) { \ 708 dims[i].lo = 0; \ 709 dims[i].up = counts[i] - 1; \ 710 dims[i].st = 1; \ 711 } \ 712 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 713 lb = 0; \ 714 ub = counts[0]; \ 715 str = 1; \ 716 KA_TRACE( \ 717 20, \ 718 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 719 gtid, lb, ub, str, chunk_sz)); \ 720 \ 721 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 722 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 723 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \ 724 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 725 (kmp_int *)p_ub, (kmp_int *)&stride); \ 726 if (status) { \ 727 KMP_DEBUG_ASSERT(stride == str); \ 728 *p_ub += (str > 0) ? 1 : -1; \ 729 } \ 730 } else { \ 731 status = 0; \ 732 } \ 733 KMP_DOACROSS_FINI(status, gtid); \ 734 \ 735 KA_TRACE( \ 736 20, \ 737 (KMP_STR( \ 738 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 739 gtid, *p_lb, *p_ub, status)); \ 740 __kmp_free(dims); \ 741 return status; \ 742 } 743 744LOOP_DOACROSS_START( 745 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START), 746 kmp_sch_static) 747LOOP_DOACROSS_START( 748 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START), 749 kmp_sch_dynamic_chunked) 750LOOP_DOACROSS_START( 751 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START), 752 kmp_sch_guided_chunked) 753LOOP_DOACROSS_RUNTIME_START( 754 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START), 755 kmp_sch_runtime) 756 757void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END)(void) { 758 int gtid = __kmp_get_gtid(); 759 KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid)) 760 761#if OMPT_SUPPORT && OMPT_OPTIONAL 762 ompt_frame_t *ompt_frame; 763 if (ompt_enabled.enabled) { 764 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 765 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 766 OMPT_STORE_RETURN_ADDRESS(gtid); 767 } 768#endif 769 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 770#if OMPT_SUPPORT && OMPT_OPTIONAL 771 if (ompt_enabled.enabled) { 772 ompt_frame->enter_frame = ompt_data_none; 773 } 774#endif 775 776 KA_TRACE(20, ("GOMP_loop_end exit: T#%d\n", gtid)) 777} 778 779void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_NOWAIT)(void) { 780 KA_TRACE(20, ("GOMP_loop_end_nowait: T#%d\n", __kmp_get_gtid())) 781} 782 783// Unsigned long long loop worksharing constructs 784// 785// These are new with gcc 4.4 786 787#define LOOP_START_ULL(func, schedule) \ 788 int func(int up, unsigned long long lb, unsigned long long ub, \ 789 unsigned long long str, unsigned long long chunk_sz, \ 790 unsigned long long *p_lb, unsigned long long *p_ub) { \ 791 int status; \ 792 long long str2 = up ? ((long long)str) : -((long long)str); \ 793 long long stride; \ 794 int gtid = __kmp_entry_gtid(); \ 795 MKLOC(loc, KMP_STR(func)); \ 796 \ 797 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 798 "0x%llx, chunk_sz 0x%llx\n", \ 799 gtid, up, lb, ub, str, chunk_sz)); \ 800 \ 801 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 802 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 803 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 804 (schedule) != kmp_sch_static); \ 805 status = \ 806 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 807 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 808 if (status) { \ 809 KMP_DEBUG_ASSERT(stride == str2); \ 810 *p_ub += (str > 0) ? 1 : -1; \ 811 } \ 812 } else { \ 813 status = 0; \ 814 } \ 815 \ 816 KA_TRACE( \ 817 20, \ 818 (KMP_STR( \ 819 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 820 gtid, *p_lb, *p_ub, status)); \ 821 return status; \ 822 } 823 824#define LOOP_RUNTIME_START_ULL(func, schedule) \ 825 int func(int up, unsigned long long lb, unsigned long long ub, \ 826 unsigned long long str, unsigned long long *p_lb, \ 827 unsigned long long *p_ub) { \ 828 int status; \ 829 long long str2 = up ? ((long long)str) : -((long long)str); \ 830 unsigned long long stride; \ 831 unsigned long long chunk_sz = 0; \ 832 int gtid = __kmp_entry_gtid(); \ 833 MKLOC(loc, KMP_STR(func)); \ 834 \ 835 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 836 "0x%llx, chunk_sz 0x%llx\n", \ 837 gtid, up, lb, ub, str, chunk_sz)); \ 838 \ 839 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 840 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 841 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 842 TRUE); \ 843 status = \ 844 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 845 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 846 if (status) { \ 847 KMP_DEBUG_ASSERT((long long)stride == str2); \ 848 *p_ub += (str > 0) ? 1 : -1; \ 849 } \ 850 } else { \ 851 status = 0; \ 852 } \ 853 \ 854 KA_TRACE( \ 855 20, \ 856 (KMP_STR( \ 857 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 858 gtid, *p_lb, *p_ub, status)); \ 859 return status; \ 860 } 861 862#define LOOP_NEXT_ULL(func, fini_code) \ 863 int func(unsigned long long *p_lb, unsigned long long *p_ub) { \ 864 int status; \ 865 long long stride; \ 866 int gtid = __kmp_get_gtid(); \ 867 MKLOC(loc, KMP_STR(func)); \ 868 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 869 \ 870 fini_code status = \ 871 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 872 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 873 if (status) { \ 874 *p_ub += (stride > 0) ? 1 : -1; \ 875 } \ 876 \ 877 KA_TRACE( \ 878 20, \ 879 (KMP_STR( \ 880 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, stride 0x%llx, " \ 881 "returning %d\n", \ 882 gtid, *p_lb, *p_ub, stride, status)); \ 883 return status; \ 884 } 885 886LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START), 887 kmp_sch_static) 888LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT), {}) 889LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START), 890 kmp_sch_dynamic_chunked) 891LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT), {}) 892LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START), 893 kmp_sch_guided_chunked) 894LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT), {}) 895LOOP_START_ULL( 896 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START), 897 kmp_sch_dynamic_chunked) 898LOOP_NEXT_ULL( 899 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT), {}) 900LOOP_START_ULL( 901 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START), 902 kmp_sch_guided_chunked) 903LOOP_NEXT_ULL( 904 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT), {}) 905LOOP_RUNTIME_START_ULL( 906 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime) 907LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {}) 908 909LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START), 910 kmp_ord_static) 911LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT), 912 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 913LOOP_START_ULL( 914 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START), 915 kmp_ord_dynamic_chunked) 916LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT), 917 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 918LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START), 919 kmp_ord_guided_chunked) 920LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT), 921 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 922LOOP_RUNTIME_START_ULL( 923 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START), 924 kmp_ord_runtime) 925LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), 926 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 927 928#define LOOP_DOACROSS_START_ULL(func, schedule) \ 929 int func(unsigned ncounts, unsigned long long *counts, \ 930 unsigned long long chunk_sz, unsigned long long *p_lb, \ 931 unsigned long long *p_ub) { \ 932 int status; \ 933 long long stride, str, lb, ub; \ 934 int gtid = __kmp_entry_gtid(); \ 935 struct kmp_dim *dims = \ 936 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 937 MKLOC(loc, KMP_STR(func)); \ 938 for (unsigned i = 0; i < ncounts; ++i) { \ 939 dims[i].lo = 0; \ 940 dims[i].up = counts[i] - 1; \ 941 dims[i].st = 1; \ 942 } \ 943 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 944 lb = 0; \ 945 ub = counts[0]; \ 946 str = 1; \ 947 \ 948 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 949 "0x%llx, chunk_sz 0x%llx\n", \ 950 gtid, lb, ub, str, chunk_sz)); \ 951 \ 952 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 953 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 954 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 955 (schedule) != kmp_sch_static); \ 956 status = \ 957 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 958 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 959 if (status) { \ 960 KMP_DEBUG_ASSERT(stride == str); \ 961 *p_ub += (str > 0) ? 1 : -1; \ 962 } \ 963 } else { \ 964 status = 0; \ 965 } \ 966 KMP_DOACROSS_FINI(status, gtid); \ 967 \ 968 KA_TRACE( \ 969 20, \ 970 (KMP_STR( \ 971 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 972 gtid, *p_lb, *p_ub, status)); \ 973 __kmp_free(dims); \ 974 return status; \ 975 } 976 977#define LOOP_DOACROSS_RUNTIME_START_ULL(func, schedule) \ 978 int func(unsigned ncounts, unsigned long long *counts, \ 979 unsigned long long *p_lb, unsigned long long *p_ub) { \ 980 int status; \ 981 unsigned long long stride, str, lb, ub; \ 982 unsigned long long chunk_sz = 0; \ 983 int gtid = __kmp_entry_gtid(); \ 984 struct kmp_dim *dims = \ 985 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 986 MKLOC(loc, KMP_STR(func)); \ 987 for (unsigned i = 0; i < ncounts; ++i) { \ 988 dims[i].lo = 0; \ 989 dims[i].up = counts[i] - 1; \ 990 dims[i].st = 1; \ 991 } \ 992 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 993 lb = 0; \ 994 ub = counts[0]; \ 995 str = 1; \ 996 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 997 "0x%llx, chunk_sz 0x%llx\n", \ 998 gtid, lb, ub, str, chunk_sz)); \ 999 \ 1000 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 1001 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 1002 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1003 TRUE); \ 1004 status = \ 1005 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 1006 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 1007 if (status) { \ 1008 KMP_DEBUG_ASSERT(stride == str); \ 1009 *p_ub += (str > 0) ? 1 : -1; \ 1010 } \ 1011 } else { \ 1012 status = 0; \ 1013 } \ 1014 KMP_DOACROSS_FINI(status, gtid); \ 1015 \ 1016 KA_TRACE( \ 1017 20, \ 1018 (KMP_STR( \ 1019 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 1020 gtid, *p_lb, *p_ub, status)); \ 1021 __kmp_free(dims); \ 1022 return status; \ 1023 } 1024 1025LOOP_DOACROSS_START_ULL( 1026 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START), 1027 kmp_sch_static) 1028LOOP_DOACROSS_START_ULL( 1029 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START), 1030 kmp_sch_dynamic_chunked) 1031LOOP_DOACROSS_START_ULL( 1032 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START), 1033 kmp_sch_guided_chunked) 1034LOOP_DOACROSS_RUNTIME_START_ULL( 1035 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START), 1036 kmp_sch_runtime) 1037 1038// Combined parallel / loop worksharing constructs 1039// 1040// There are no ull versions (yet). 1041 1042#define PARALLEL_LOOP_START(func, schedule, ompt_pre, ompt_post) \ 1043 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1044 long ub, long str, long chunk_sz) { \ 1045 int gtid = __kmp_entry_gtid(); \ 1046 MKLOC(loc, KMP_STR(func)); \ 1047 KA_TRACE( \ 1048 20, \ 1049 (KMP_STR( \ 1050 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1051 gtid, lb, ub, str, chunk_sz)); \ 1052 \ 1053 ompt_pre(); \ 1054 \ 1055 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \ 1056 if (num_threads != 0) { \ 1057 __kmp_push_num_threads(&loc, gtid, num_threads); \ 1058 } \ 1059 __kmp_GOMP_fork_call(&loc, gtid, task, \ 1060 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1061 9, task, data, num_threads, &loc, (schedule), lb, \ 1062 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1063 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid)); \ 1064 } else { \ 1065 __kmp_GOMP_serialized_parallel(&loc, gtid, task); \ 1066 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid)); \ 1067 } \ 1068 \ 1069 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1070 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1071 (schedule) != kmp_sch_static); \ 1072 \ 1073 ompt_post(); \ 1074 \ 1075 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1076 } 1077 1078#if OMPT_SUPPORT && OMPT_OPTIONAL 1079 1080#define OMPT_LOOP_PRE() \ 1081 ompt_frame_t *parent_frame; \ 1082 if (ompt_enabled.enabled) { \ 1083 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); \ 1084 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); \ 1085 OMPT_STORE_RETURN_ADDRESS(gtid); \ 1086 } 1087 1088#define OMPT_LOOP_POST() \ 1089 if (ompt_enabled.enabled) { \ 1090 parent_frame->enter_frame = ompt_data_none; \ 1091 } 1092 1093#else 1094 1095#define OMPT_LOOP_PRE() 1096 1097#define OMPT_LOOP_POST() 1098 1099#endif 1100 1101PARALLEL_LOOP_START( 1102 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START), 1103 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1104PARALLEL_LOOP_START( 1105 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START), 1106 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1107PARALLEL_LOOP_START( 1108 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START), 1109 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1110PARALLEL_LOOP_START( 1111 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START), 1112 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1113 1114// Tasking constructs 1115 1116void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, 1117 void (*copy_func)(void *, void *), 1118 long arg_size, long arg_align, 1119 bool if_cond, unsigned gomp_flags, 1120 void **depend) { 1121 MKLOC(loc, "GOMP_task"); 1122 int gtid = __kmp_entry_gtid(); 1123 kmp_int32 flags = 0; 1124 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1125 1126 KA_TRACE(20, ("GOMP_task: T#%d\n", gtid)); 1127 1128 // The low-order bit is the "untied" flag 1129 if (!(gomp_flags & 1)) { 1130 input_flags->tiedness = 1; 1131 } 1132 // The second low-order bit is the "final" flag 1133 if (gomp_flags & 2) { 1134 input_flags->final = 1; 1135 } 1136 input_flags->native = 1; 1137 // __kmp_task_alloc() sets up all other flags 1138 1139 if (!if_cond) { 1140 arg_size = 0; 1141 } 1142 1143 kmp_task_t *task = __kmp_task_alloc( 1144 &loc, gtid, input_flags, sizeof(kmp_task_t), 1145 arg_size ? arg_size + arg_align - 1 : 0, (kmp_routine_entry_t)func); 1146 1147 if (arg_size > 0) { 1148 if (arg_align > 0) { 1149 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1150 arg_align * arg_align); 1151 } 1152 // else error?? 1153 1154 if (copy_func) { 1155 (*copy_func)(task->shareds, data); 1156 } else { 1157 KMP_MEMCPY(task->shareds, data, arg_size); 1158 } 1159 } 1160 1161#if OMPT_SUPPORT 1162 kmp_taskdata_t *current_task; 1163 if (ompt_enabled.enabled) { 1164 OMPT_STORE_RETURN_ADDRESS(gtid); 1165 current_task = __kmp_threads[gtid]->th.th_current_task; 1166 current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1167 } 1168#endif 1169 1170 if (if_cond) { 1171 if (gomp_flags & 8) { 1172 KMP_ASSERT(depend); 1173 const size_t ndeps = (kmp_intptr_t)depend[0]; 1174 const size_t nout = (kmp_intptr_t)depend[1]; 1175 kmp_depend_info_t dep_list[ndeps]; 1176 1177 for (size_t i = 0U; i < ndeps; i++) { 1178 dep_list[i].base_addr = (kmp_intptr_t)depend[2U + i]; 1179 dep_list[i].len = 0U; 1180 dep_list[i].flags.in = 1; 1181 dep_list[i].flags.out = (i < nout); 1182 } 1183 __kmpc_omp_task_with_deps(&loc, gtid, task, ndeps, dep_list, 0, NULL); 1184 } else { 1185 __kmpc_omp_task(&loc, gtid, task); 1186 } 1187 } else { 1188#if OMPT_SUPPORT 1189 ompt_thread_info_t oldInfo; 1190 kmp_info_t *thread; 1191 kmp_taskdata_t *taskdata; 1192 if (ompt_enabled.enabled) { 1193 // Store the threads states and restore them after the task 1194 thread = __kmp_threads[gtid]; 1195 taskdata = KMP_TASK_TO_TASKDATA(task); 1196 oldInfo = thread->th.ompt_thread_info; 1197 thread->th.ompt_thread_info.wait_id = 0; 1198 thread->th.ompt_thread_info.state = ompt_state_work_parallel; 1199 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1200 OMPT_STORE_RETURN_ADDRESS(gtid); 1201 } 1202#endif 1203 1204 __kmpc_omp_task_begin_if0(&loc, gtid, task); 1205 func(data); 1206 __kmpc_omp_task_complete_if0(&loc, gtid, task); 1207 1208#if OMPT_SUPPORT 1209 if (ompt_enabled.enabled) { 1210 thread->th.ompt_thread_info = oldInfo; 1211 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1212 } 1213#endif 1214 } 1215#if OMPT_SUPPORT 1216 if (ompt_enabled.enabled) { 1217 current_task->ompt_task_info.frame.enter_frame = ompt_data_none; 1218 } 1219#endif 1220 1221 KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid)); 1222} 1223 1224void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) { 1225 MKLOC(loc, "GOMP_taskwait"); 1226 int gtid = __kmp_entry_gtid(); 1227 1228#if OMPT_SUPPORT 1229 if (ompt_enabled.enabled) 1230 OMPT_STORE_RETURN_ADDRESS(gtid); 1231#endif 1232 1233 KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid)); 1234 1235 __kmpc_omp_taskwait(&loc, gtid); 1236 1237 KA_TRACE(20, ("GOMP_taskwait exit: T#%d\n", gtid)); 1238} 1239 1240// Sections worksharing constructs 1241// 1242// For the sections construct, we initialize a dynamically scheduled loop 1243// worksharing construct with lb 1 and stride 1, and use the iteration #'s 1244// that its returns as sections ids. 1245// 1246// There are no special entry points for ordered sections, so we always use 1247// the dynamically scheduled workshare, even if the sections aren't ordered. 1248 1249unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_START)(unsigned count) { 1250 int status; 1251 kmp_int lb, ub, stride; 1252 int gtid = __kmp_entry_gtid(); 1253 MKLOC(loc, "GOMP_sections_start"); 1254 KA_TRACE(20, ("GOMP_sections_start: T#%d\n", gtid)); 1255 1256 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1257 1258 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1259 if (status) { 1260 KMP_DEBUG_ASSERT(stride == 1); 1261 KMP_DEBUG_ASSERT(lb > 0); 1262 KMP_ASSERT(lb == ub); 1263 } else { 1264 lb = 0; 1265 } 1266 1267 KA_TRACE(20, ("GOMP_sections_start exit: T#%d returning %u\n", gtid, 1268 (unsigned)lb)); 1269 return (unsigned)lb; 1270} 1271 1272unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_NEXT)(void) { 1273 int status; 1274 kmp_int lb, ub, stride; 1275 int gtid = __kmp_get_gtid(); 1276 MKLOC(loc, "GOMP_sections_next"); 1277 KA_TRACE(20, ("GOMP_sections_next: T#%d\n", gtid)); 1278 1279#if OMPT_SUPPORT 1280 OMPT_STORE_RETURN_ADDRESS(gtid); 1281#endif 1282 1283 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1284 if (status) { 1285 KMP_DEBUG_ASSERT(stride == 1); 1286 KMP_DEBUG_ASSERT(lb > 0); 1287 KMP_ASSERT(lb == ub); 1288 } else { 1289 lb = 0; 1290 } 1291 1292 KA_TRACE( 1293 20, ("GOMP_sections_next exit: T#%d returning %u\n", gtid, (unsigned)lb)); 1294 return (unsigned)lb; 1295} 1296 1297void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)( 1298 void (*task)(void *), void *data, unsigned num_threads, unsigned count) { 1299 int gtid = __kmp_entry_gtid(); 1300 1301#if OMPT_SUPPORT 1302 ompt_frame_t *parent_frame; 1303 1304 if (ompt_enabled.enabled) { 1305 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 1306 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1307 OMPT_STORE_RETURN_ADDRESS(gtid); 1308 } 1309#endif 1310 1311 MKLOC(loc, "GOMP_parallel_sections_start"); 1312 KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid)); 1313 1314 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1315 if (num_threads != 0) { 1316 __kmp_push_num_threads(&loc, gtid, num_threads); 1317 } 1318 __kmp_GOMP_fork_call(&loc, gtid, task, 1319 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1320 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1321 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1322 } else { 1323 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1324 } 1325 1326#if OMPT_SUPPORT 1327 if (ompt_enabled.enabled) { 1328 parent_frame->enter_frame = ompt_data_none; 1329 } 1330#endif 1331 1332 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1333 1334 KA_TRACE(20, ("GOMP_parallel_sections_start exit: T#%d\n", gtid)); 1335} 1336 1337void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END)(void) { 1338 int gtid = __kmp_get_gtid(); 1339 KA_TRACE(20, ("GOMP_sections_end: T#%d\n", gtid)) 1340 1341#if OMPT_SUPPORT 1342 ompt_frame_t *ompt_frame; 1343 if (ompt_enabled.enabled) { 1344 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 1345 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1346 OMPT_STORE_RETURN_ADDRESS(gtid); 1347 } 1348#endif 1349 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1350#if OMPT_SUPPORT 1351 if (ompt_enabled.enabled) { 1352 ompt_frame->enter_frame = ompt_data_none; 1353 } 1354#endif 1355 1356 KA_TRACE(20, ("GOMP_sections_end exit: T#%d\n", gtid)) 1357} 1358 1359void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT)(void) { 1360 KA_TRACE(20, ("GOMP_sections_end_nowait: T#%d\n", __kmp_get_gtid())) 1361} 1362 1363// libgomp has an empty function for GOMP_taskyield as of 2013-10-10 1364void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKYIELD)(void) { 1365 KA_TRACE(20, ("GOMP_taskyield: T#%d\n", __kmp_get_gtid())) 1366 return; 1367} 1368 1369void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), 1370 void *data, 1371 unsigned num_threads, 1372 unsigned int flags) { 1373 int gtid = __kmp_entry_gtid(); 1374 MKLOC(loc, "GOMP_parallel"); 1375 KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid)); 1376 1377#if OMPT_SUPPORT 1378 ompt_task_info_t *parent_task_info, *task_info; 1379 if (ompt_enabled.enabled) { 1380 parent_task_info = __ompt_get_task_info_object(0); 1381 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1382 OMPT_STORE_RETURN_ADDRESS(gtid); 1383 } 1384#endif 1385 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1386 if (num_threads != 0) { 1387 __kmp_push_num_threads(&loc, gtid, num_threads); 1388 } 1389 if (flags != 0) { 1390 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); 1391 } 1392 __kmp_GOMP_fork_call(&loc, gtid, task, 1393 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 1394 data); 1395 } else { 1396 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1397 } 1398#if OMPT_SUPPORT 1399 if (ompt_enabled.enabled) { 1400 task_info = __ompt_get_task_info_object(0); 1401 task_info->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1402 } 1403#endif 1404 task(data); 1405#if OMPT_SUPPORT 1406 if (ompt_enabled.enabled) { 1407 OMPT_STORE_RETURN_ADDRESS(gtid); 1408 } 1409#endif 1410 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1411#if OMPT_SUPPORT 1412 if (ompt_enabled.enabled) { 1413 task_info->frame.exit_frame = ompt_data_none; 1414 parent_task_info->frame.enter_frame = ompt_data_none; 1415 } 1416#endif 1417} 1418 1419void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task)(void *), 1420 void *data, 1421 unsigned num_threads, 1422 unsigned count, 1423 unsigned flags) { 1424 int gtid = __kmp_entry_gtid(); 1425 MKLOC(loc, "GOMP_parallel_sections"); 1426 KA_TRACE(20, ("GOMP_parallel_sections: T#%d\n", gtid)); 1427 1428#if OMPT_SUPPORT 1429 OMPT_STORE_RETURN_ADDRESS(gtid); 1430#endif 1431 1432 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1433 if (num_threads != 0) { 1434 __kmp_push_num_threads(&loc, gtid, num_threads); 1435 } 1436 if (flags != 0) { 1437 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); 1438 } 1439 __kmp_GOMP_fork_call(&loc, gtid, task, 1440 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1441 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1442 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1443 } else { 1444 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1445 } 1446 1447#if OMPT_SUPPORT 1448 OMPT_STORE_RETURN_ADDRESS(gtid); 1449#endif 1450 1451 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1452 1453 task(data); 1454 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1455 KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid)); 1456} 1457 1458#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post) \ 1459 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1460 long ub, long str, long chunk_sz, unsigned flags) { \ 1461 int gtid = __kmp_entry_gtid(); \ 1462 MKLOC(loc, KMP_STR(func)); \ 1463 KA_TRACE( \ 1464 20, \ 1465 (KMP_STR( \ 1466 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1467 gtid, lb, ub, str, chunk_sz)); \ 1468 \ 1469 ompt_pre(); \ 1470 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \ 1471 if (num_threads != 0) { \ 1472 __kmp_push_num_threads(&loc, gtid, num_threads); \ 1473 } \ 1474 if (flags != 0) { \ 1475 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); \ 1476 } \ 1477 __kmp_GOMP_fork_call(&loc, gtid, task, \ 1478 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1479 9, task, data, num_threads, &loc, (schedule), lb, \ 1480 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1481 } else { \ 1482 __kmp_GOMP_serialized_parallel(&loc, gtid, task); \ 1483 } \ 1484 \ 1485 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 1486 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1487 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1488 (schedule) != kmp_sch_static); \ 1489 task(data); \ 1490 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); \ 1491 ompt_post(); \ 1492 \ 1493 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1494 } 1495 1496PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), 1497 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1498PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), 1499 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1500PARALLEL_LOOP( 1501 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED), 1502 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1503PARALLEL_LOOP( 1504 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC), 1505 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1506PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), 1507 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1508PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), 1509 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1510 1511void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) { 1512 int gtid = __kmp_entry_gtid(); 1513 MKLOC(loc, "GOMP_taskgroup_start"); 1514 KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid)); 1515 1516#if OMPT_SUPPORT 1517 if (ompt_enabled.enabled) 1518 OMPT_STORE_RETURN_ADDRESS(gtid); 1519#endif 1520 1521 __kmpc_taskgroup(&loc, gtid); 1522 1523 return; 1524} 1525 1526void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_END)(void) { 1527 int gtid = __kmp_get_gtid(); 1528 MKLOC(loc, "GOMP_taskgroup_end"); 1529 KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid)); 1530 1531#if OMPT_SUPPORT 1532 if (ompt_enabled.enabled) 1533 OMPT_STORE_RETURN_ADDRESS(gtid); 1534#endif 1535 1536 __kmpc_end_taskgroup(&loc, gtid); 1537 1538 return; 1539} 1540 1541static kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) { 1542 kmp_int32 cncl_kind = 0; 1543 switch (gomp_kind) { 1544 case 1: 1545 cncl_kind = cancel_parallel; 1546 break; 1547 case 2: 1548 cncl_kind = cancel_loop; 1549 break; 1550 case 4: 1551 cncl_kind = cancel_sections; 1552 break; 1553 case 8: 1554 cncl_kind = cancel_taskgroup; 1555 break; 1556 } 1557 return cncl_kind; 1558} 1559 1560// Return true if cancellation should take place, false otherwise 1561bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) { 1562 int gtid = __kmp_get_gtid(); 1563 MKLOC(loc, "GOMP_cancellation_point"); 1564 KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which)); 1565 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1566 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1567} 1568 1569// Return true if cancellation should take place, false otherwise 1570bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) { 1571 int gtid = __kmp_get_gtid(); 1572 MKLOC(loc, "GOMP_cancel"); 1573 KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which, 1574 (int)do_cancel)); 1575 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1576 1577 if (do_cancel == FALSE) { 1578 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1579 } else { 1580 return __kmpc_cancel(&loc, gtid, cncl_kind); 1581 } 1582} 1583 1584// Return true if cancellation should take place, false otherwise 1585bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) { 1586 int gtid = __kmp_get_gtid(); 1587 KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid)); 1588 return __kmp_barrier_gomp_cancel(gtid); 1589} 1590 1591// Return true if cancellation should take place, false otherwise 1592bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) { 1593 int gtid = __kmp_get_gtid(); 1594 KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid)); 1595 return __kmp_barrier_gomp_cancel(gtid); 1596} 1597 1598// Return true if cancellation should take place, false otherwise 1599bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void) { 1600 int gtid = __kmp_get_gtid(); 1601 KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid)); 1602 return __kmp_barrier_gomp_cancel(gtid); 1603} 1604 1605// All target functions are empty as of 2014-05-29 1606void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn)(void *), 1607 const void *openmp_target, 1608 size_t mapnum, void **hostaddrs, 1609 size_t *sizes, 1610 unsigned char *kinds) { 1611 return; 1612} 1613 1614void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_DATA)( 1615 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1616 size_t *sizes, unsigned char *kinds) { 1617 return; 1618} 1619 1620void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_END_DATA)(void) { return; } 1621 1622void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_UPDATE)( 1623 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1624 size_t *sizes, unsigned char *kinds) { 1625 return; 1626} 1627 1628void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS)(unsigned int num_teams, 1629 unsigned int thread_limit) { 1630 return; 1631} 1632 1633// Task duplication function which copies src to dest (both are 1634// preallocated task structures) 1635static void __kmp_gomp_task_dup(kmp_task_t *dest, kmp_task_t *src, 1636 kmp_int32 last_private) { 1637 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(src); 1638 if (taskdata->td_copy_func) { 1639 (taskdata->td_copy_func)(dest->shareds, src->shareds); 1640 } 1641} 1642 1643#ifdef __cplusplus 1644} // extern "C" 1645#endif 1646 1647template <typename T> 1648void __GOMP_taskloop(void (*func)(void *), void *data, 1649 void (*copy_func)(void *, void *), long arg_size, 1650 long arg_align, unsigned gomp_flags, 1651 unsigned long num_tasks, int priority, T start, T end, 1652 T step) { 1653 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 1654 MKLOC(loc, "GOMP_taskloop"); 1655 int sched; 1656 T *loop_bounds; 1657 int gtid = __kmp_entry_gtid(); 1658 kmp_int32 flags = 0; 1659 int if_val = gomp_flags & (1u << 10); 1660 int nogroup = gomp_flags & (1u << 11); 1661 int up = gomp_flags & (1u << 8); 1662 p_task_dup_t task_dup = NULL; 1663 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1664#ifdef KMP_DEBUG 1665 { 1666 char *buff; 1667 buff = __kmp_str_format( 1668 "GOMP_taskloop: T#%%d: func:%%p data:%%p copy_func:%%p " 1669 "arg_size:%%ld arg_align:%%ld gomp_flags:0x%%x num_tasks:%%lu " 1670 "priority:%%d start:%%%s end:%%%s step:%%%s\n", 1671 traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec); 1672 KA_TRACE(20, (buff, gtid, func, data, copy_func, arg_size, arg_align, 1673 gomp_flags, num_tasks, priority, start, end, step)); 1674 __kmp_str_free(&buff); 1675 } 1676#endif 1677 KMP_ASSERT((size_t)arg_size >= 2 * sizeof(T)); 1678 KMP_ASSERT(arg_align > 0); 1679 // The low-order bit is the "untied" flag 1680 if (!(gomp_flags & 1)) { 1681 input_flags->tiedness = 1; 1682 } 1683 // The second low-order bit is the "final" flag 1684 if (gomp_flags & 2) { 1685 input_flags->final = 1; 1686 } 1687 // Negative step flag 1688 if (!up) { 1689 // If step is flagged as negative, but isn't properly sign extended 1690 // Then manually sign extend it. Could be a short, int, char embedded 1691 // in a long. So cannot assume any cast. 1692 if (step > 0) { 1693 for (int i = sizeof(T) * CHAR_BIT - 1; i >= 0L; --i) { 1694 // break at the first 1 bit 1695 if (step & ((T)1 << i)) 1696 break; 1697 step |= ((T)1 << i); 1698 } 1699 } 1700 } 1701 input_flags->native = 1; 1702 // Figure out if none/grainsize/num_tasks clause specified 1703 if (num_tasks > 0) { 1704 if (gomp_flags & (1u << 9)) 1705 sched = 1; // grainsize specified 1706 else 1707 sched = 2; // num_tasks specified 1708 // neither grainsize nor num_tasks specified 1709 } else { 1710 sched = 0; 1711 } 1712 1713 // __kmp_task_alloc() sets up all other flags 1714 kmp_task_t *task = 1715 __kmp_task_alloc(&loc, gtid, input_flags, sizeof(kmp_task_t), 1716 arg_size + arg_align - 1, (kmp_routine_entry_t)func); 1717 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1718 taskdata->td_copy_func = copy_func; 1719 taskdata->td_size_loop_bounds = sizeof(T); 1720 1721 // re-align shareds if needed and setup firstprivate copy constructors 1722 // through the task_dup mechanism 1723 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1724 arg_align * arg_align); 1725 if (copy_func) { 1726 task_dup = __kmp_gomp_task_dup; 1727 } 1728 KMP_MEMCPY(task->shareds, data, arg_size); 1729 1730 loop_bounds = (T *)task->shareds; 1731 loop_bounds[0] = start; 1732 loop_bounds[1] = end + (up ? -1 : 1); 1733 __kmpc_taskloop(&loc, gtid, task, if_val, (kmp_uint64 *)&(loop_bounds[0]), 1734 (kmp_uint64 *)&(loop_bounds[1]), (kmp_int64)step, nogroup, 1735 sched, (kmp_uint64)num_tasks, (void *)task_dup); 1736} 1737 1738// 4 byte version of GOMP_doacross_post 1739// This verison needs to create a temporary array which converts 4 byte 1740// integers into 8 byte integers 1741template <typename T, bool need_conversion = (sizeof(long) == 4)> 1742void __kmp_GOMP_doacross_post(T *count); 1743 1744template <> void __kmp_GOMP_doacross_post<long, true>(long *count) { 1745 int gtid = __kmp_entry_gtid(); 1746 kmp_info_t *th = __kmp_threads[gtid]; 1747 MKLOC(loc, "GOMP_doacross_post"); 1748 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1749 kmp_int64 *vec = 1750 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1751 for (kmp_int64 i = 0; i < num_dims; ++i) { 1752 vec[i] = (kmp_int64)count[i]; 1753 } 1754 __kmpc_doacross_post(&loc, gtid, vec); 1755 __kmp_thread_free(th, vec); 1756} 1757 1758// 8 byte versions of GOMP_doacross_post 1759// This version can just pass in the count array directly instead of creating 1760// a temporary array 1761template <> void __kmp_GOMP_doacross_post<long, false>(long *count) { 1762 int gtid = __kmp_entry_gtid(); 1763 MKLOC(loc, "GOMP_doacross_post"); 1764 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1765} 1766 1767template <typename T> void __kmp_GOMP_doacross_wait(T first, va_list args) { 1768 int gtid = __kmp_entry_gtid(); 1769 kmp_info_t *th = __kmp_threads[gtid]; 1770 MKLOC(loc, "GOMP_doacross_wait"); 1771 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1772 kmp_int64 *vec = 1773 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1774 vec[0] = (kmp_int64)first; 1775 for (kmp_int64 i = 1; i < num_dims; ++i) { 1776 T item = va_arg(args, T); 1777 vec[i] = (kmp_int64)item; 1778 } 1779 __kmpc_doacross_wait(&loc, gtid, vec); 1780 __kmp_thread_free(th, vec); 1781 return; 1782} 1783 1784#ifdef __cplusplus 1785extern "C" { 1786#endif // __cplusplus 1787 1788void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP)( 1789 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1790 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1791 int priority, long start, long end, long step) { 1792 __GOMP_taskloop<long>(func, data, copy_func, arg_size, arg_align, gomp_flags, 1793 num_tasks, priority, start, end, step); 1794} 1795 1796void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP_ULL)( 1797 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1798 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1799 int priority, unsigned long long start, unsigned long long end, 1800 unsigned long long step) { 1801 __GOMP_taskloop<unsigned long long>(func, data, copy_func, arg_size, 1802 arg_align, gomp_flags, num_tasks, 1803 priority, start, end, step); 1804} 1805 1806void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_POST)(long *count) { 1807 __kmp_GOMP_doacross_post(count); 1808} 1809 1810void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_WAIT)(long first, ...) { 1811 va_list args; 1812 va_start(args, first); 1813 __kmp_GOMP_doacross_wait<long>(first, args); 1814 va_end(args); 1815} 1816 1817void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_POST)( 1818 unsigned long long *count) { 1819 int gtid = __kmp_entry_gtid(); 1820 MKLOC(loc, "GOMP_doacross_ull_post"); 1821 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1822} 1823 1824void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT)( 1825 unsigned long long first, ...) { 1826 va_list args; 1827 va_start(args, first); 1828 __kmp_GOMP_doacross_wait<unsigned long long>(first, args); 1829 va_end(args); 1830} 1831 1832/* The following sections of code create aliases for the GOMP_* functions, then 1833 create versioned symbols using the assembler directive .symver. This is only 1834 pertinent for ELF .so library. The KMP_VERSION_SYMBOL macro is defined in 1835 kmp_os.h */ 1836 1837#ifdef KMP_USE_VERSION_SYMBOLS 1838// GOMP_1.0 versioned symbols 1839KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0"); 1840KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_START, 10, "GOMP_1.0"); 1841KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER, 10, "GOMP_1.0"); 1842KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_END, 10, "GOMP_1.0"); 1843KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10, "GOMP_1.0"); 1844KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10, "GOMP_1.0"); 1845KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_START, 10, "GOMP_1.0"); 1846KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1847KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10, "GOMP_1.0"); 1848KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END, 10, "GOMP_1.0"); 1849KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10, "GOMP_1.0"); 1850KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10, "GOMP_1.0"); 1851KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10, "GOMP_1.0"); 1852KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1853KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10, 1854 "GOMP_1.0"); 1855KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10, "GOMP_1.0"); 1856KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10, "GOMP_1.0"); 1857KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10, "GOMP_1.0"); 1858KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10, 1859 "GOMP_1.0"); 1860KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10, "GOMP_1.0"); 1861KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10, "GOMP_1.0"); 1862KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10, "GOMP_1.0"); 1863KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10, "GOMP_1.0"); 1864KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10, "GOMP_1.0"); 1865KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10, "GOMP_1.0"); 1866KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_END, 10, "GOMP_1.0"); 1867KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_START, 10, "GOMP_1.0"); 1868KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_END, 10, "GOMP_1.0"); 1869KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10, 1870 "GOMP_1.0"); 1871KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10, 1872 "GOMP_1.0"); 1873KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10, 1874 "GOMP_1.0"); 1875KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10, 1876 "GOMP_1.0"); 1877KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10, "GOMP_1.0"); 1878KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_START, 10, "GOMP_1.0"); 1879KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END, 10, "GOMP_1.0"); 1880KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10, "GOMP_1.0"); 1881KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10, "GOMP_1.0"); 1882KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_START, 10, "GOMP_1.0"); 1883KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10, "GOMP_1.0"); 1884KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10, "GOMP_1.0"); 1885KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_START, 10, "GOMP_1.0"); 1886 1887// GOMP_2.0 versioned symbols 1888KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASK, 20, "GOMP_2.0"); 1889KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT, 20, "GOMP_2.0"); 1890KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20, "GOMP_2.0"); 1891KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20, "GOMP_2.0"); 1892KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20, "GOMP_2.0"); 1893KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20, "GOMP_2.0"); 1894KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20, 1895 "GOMP_2.0"); 1896KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20, 1897 "GOMP_2.0"); 1898KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20, 1899 "GOMP_2.0"); 1900KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20, 1901 "GOMP_2.0"); 1902KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20, 1903 "GOMP_2.0"); 1904KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20, 1905 "GOMP_2.0"); 1906KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20, 1907 "GOMP_2.0"); 1908KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20, 1909 "GOMP_2.0"); 1910KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20, "GOMP_2.0"); 1911KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20, "GOMP_2.0"); 1912KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20, "GOMP_2.0"); 1913KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20, "GOMP_2.0"); 1914 1915// GOMP_3.0 versioned symbols 1916KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKYIELD, 30, "GOMP_3.0"); 1917 1918// GOMP_4.0 versioned symbols 1919KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL, 40, "GOMP_4.0"); 1920KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40, "GOMP_4.0"); 1921KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40, "GOMP_4.0"); 1922KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40, "GOMP_4.0"); 1923KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40, "GOMP_4.0"); 1924KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40, "GOMP_4.0"); 1925KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_START, 40, "GOMP_4.0"); 1926KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_END, 40, "GOMP_4.0"); 1927KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40, "GOMP_4.0"); 1928KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCEL, 40, "GOMP_4.0"); 1929KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40, "GOMP_4.0"); 1930KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40, "GOMP_4.0"); 1931KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40, "GOMP_4.0"); 1932KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET, 40, "GOMP_4.0"); 1933KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_DATA, 40, "GOMP_4.0"); 1934KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_END_DATA, 40, "GOMP_4.0"); 1935KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_UPDATE, 40, "GOMP_4.0"); 1936KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS, 40, "GOMP_4.0"); 1937 1938// GOMP_4.5 versioned symbols 1939KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP, 45, "GOMP_4.5"); 1940KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP_ULL, 45, "GOMP_4.5"); 1941KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_POST, 45, "GOMP_4.5"); 1942KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_WAIT, 45, "GOMP_4.5"); 1943KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START, 45, 1944 "GOMP_4.5"); 1945KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START, 45, 1946 "GOMP_4.5"); 1947KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START, 45, 1948 "GOMP_4.5"); 1949KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START, 45, 1950 "GOMP_4.5"); 1951KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_POST, 45, "GOMP_4.5"); 1952KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT, 45, "GOMP_4.5"); 1953KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START, 45, 1954 "GOMP_4.5"); 1955KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START, 45, 1956 "GOMP_4.5"); 1957KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START, 45, 1958 "GOMP_4.5"); 1959KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START, 45, 1960 "GOMP_4.5"); 1961KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START, 45, 1962 "GOMP_4.5"); 1963KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT, 45, 1964 "GOMP_4.5"); 1965KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START, 45, 1966 "GOMP_4.5"); 1967KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT, 45, 1968 "GOMP_4.5"); 1969KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START, 45, 1970 "GOMP_4.5"); 1971KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT, 45, 1972 "GOMP_4.5"); 1973KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START, 45, 1974 "GOMP_4.5"); 1975KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT, 45, 1976 "GOMP_4.5"); 1977KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC, 45, 1978 "GOMP_4.5"); 1979KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED, 45, 1980 "GOMP_4.5"); 1981 1982#endif // KMP_USE_VERSION_SYMBOLS 1983 1984#ifdef __cplusplus 1985} // extern "C" 1986#endif // __cplusplus 1987