1/* 2 * kmp_gsupport.cpp 3 */ 4 5//===----------------------------------------------------------------------===// 6// 7// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 8// See https://llvm.org/LICENSE.txt for license information. 9// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 10// 11//===----------------------------------------------------------------------===// 12 13#include "kmp.h" 14#include "kmp_atomic.h" 15 16#if OMPT_SUPPORT 17#include "ompt-specific.h" 18#endif 19 20#ifdef __cplusplus 21extern "C" { 22#endif // __cplusplus 23 24#define MKLOC(loc, routine) \ 25 static ident_t loc = {0, KMP_IDENT_KMPC, 0, 0, ";unknown;unknown;0;0;;"}; 26 27#include "kmp_ftn_os.h" 28 29void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER)(void) { 30 int gtid = __kmp_entry_gtid(); 31 MKLOC(loc, "GOMP_barrier"); 32 KA_TRACE(20, ("GOMP_barrier: T#%d\n", gtid)); 33#if OMPT_SUPPORT && OMPT_OPTIONAL 34 ompt_frame_t *ompt_frame; 35 if (ompt_enabled.enabled) { 36 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 37 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 38 OMPT_STORE_RETURN_ADDRESS(gtid); 39 } 40#endif 41 __kmpc_barrier(&loc, gtid); 42#if OMPT_SUPPORT && OMPT_OPTIONAL 43 if (ompt_enabled.enabled) { 44 ompt_frame->enter_frame = ompt_data_none; 45 } 46#endif 47} 48 49// Mutual exclusion 50 51// The symbol that icc/ifort generates for unnamed for unnamed critical sections 52// - .gomp_critical_user_ - is defined using .comm in any objects reference it. 53// We can't reference it directly here in C code, as the symbol contains a ".". 54// 55// The RTL contains an assembly language definition of .gomp_critical_user_ 56// with another symbol __kmp_unnamed_critical_addr initialized with it's 57// address. 58extern kmp_critical_name *__kmp_unnamed_critical_addr; 59 60void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_START)(void) { 61 int gtid = __kmp_entry_gtid(); 62 MKLOC(loc, "GOMP_critical_start"); 63 KA_TRACE(20, ("GOMP_critical_start: T#%d\n", gtid)); 64#if OMPT_SUPPORT && OMPT_OPTIONAL 65 OMPT_STORE_RETURN_ADDRESS(gtid); 66#endif 67 __kmpc_critical(&loc, gtid, __kmp_unnamed_critical_addr); 68} 69 70void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_END)(void) { 71 int gtid = __kmp_get_gtid(); 72 MKLOC(loc, "GOMP_critical_end"); 73 KA_TRACE(20, ("GOMP_critical_end: T#%d\n", gtid)); 74#if OMPT_SUPPORT && OMPT_OPTIONAL 75 OMPT_STORE_RETURN_ADDRESS(gtid); 76#endif 77 __kmpc_end_critical(&loc, gtid, __kmp_unnamed_critical_addr); 78} 79 80void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_START)(void **pptr) { 81 int gtid = __kmp_entry_gtid(); 82 MKLOC(loc, "GOMP_critical_name_start"); 83 KA_TRACE(20, ("GOMP_critical_name_start: T#%d\n", gtid)); 84 __kmpc_critical(&loc, gtid, (kmp_critical_name *)pptr); 85} 86 87void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CRITICAL_NAME_END)(void **pptr) { 88 int gtid = __kmp_get_gtid(); 89 MKLOC(loc, "GOMP_critical_name_end"); 90 KA_TRACE(20, ("GOMP_critical_name_end: T#%d\n", gtid)); 91 __kmpc_end_critical(&loc, gtid, (kmp_critical_name *)pptr); 92} 93 94// The Gnu codegen tries to use locked operations to perform atomic updates 95// inline. If it can't, then it calls GOMP_atomic_start() before performing 96// the update and GOMP_atomic_end() afterward, regardless of the data type. 97void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_START)(void) { 98 int gtid = __kmp_entry_gtid(); 99 KA_TRACE(20, ("GOMP_atomic_start: T#%d\n", gtid)); 100 101#if OMPT_SUPPORT 102 __ompt_thread_assign_wait_id(0); 103#endif 104 105 __kmp_acquire_atomic_lock(&__kmp_atomic_lock, gtid); 106} 107 108void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ATOMIC_END)(void) { 109 int gtid = __kmp_get_gtid(); 110 KA_TRACE(20, ("GOMP_atomic_end: T#%d\n", gtid)); 111 __kmp_release_atomic_lock(&__kmp_atomic_lock, gtid); 112} 113 114int KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_START)(void) { 115 int gtid = __kmp_entry_gtid(); 116 MKLOC(loc, "GOMP_single_start"); 117 KA_TRACE(20, ("GOMP_single_start: T#%d\n", gtid)); 118 119 if (!TCR_4(__kmp_init_parallel)) 120 __kmp_parallel_initialize(); 121 __kmp_resume_if_soft_paused(); 122 123 // 3rd parameter == FALSE prevents kmp_enter_single from pushing a 124 // workshare when USE_CHECKS is defined. We need to avoid the push, 125 // as there is no corresponding GOMP_single_end() call. 126 kmp_int32 rc = __kmp_enter_single(gtid, &loc, FALSE); 127 128#if OMPT_SUPPORT && OMPT_OPTIONAL 129 kmp_info_t *this_thr = __kmp_threads[gtid]; 130 kmp_team_t *team = this_thr->th.th_team; 131 int tid = __kmp_tid_from_gtid(gtid); 132 133 if (ompt_enabled.enabled) { 134 if (rc) { 135 if (ompt_enabled.ompt_callback_work) { 136 ompt_callbacks.ompt_callback(ompt_callback_work)( 137 ompt_work_single_executor, ompt_scope_begin, 138 &(team->t.ompt_team_info.parallel_data), 139 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 140 1, OMPT_GET_RETURN_ADDRESS(0)); 141 } 142 } else { 143 if (ompt_enabled.ompt_callback_work) { 144 ompt_callbacks.ompt_callback(ompt_callback_work)( 145 ompt_work_single_other, ompt_scope_begin, 146 &(team->t.ompt_team_info.parallel_data), 147 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 148 1, OMPT_GET_RETURN_ADDRESS(0)); 149 ompt_callbacks.ompt_callback(ompt_callback_work)( 150 ompt_work_single_other, ompt_scope_end, 151 &(team->t.ompt_team_info.parallel_data), 152 &(team->t.t_implicit_task_taskdata[tid].ompt_task_info.task_data), 153 1, OMPT_GET_RETURN_ADDRESS(0)); 154 } 155 } 156 } 157#endif 158 159 return rc; 160} 161 162void *KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_START)(void) { 163 void *retval; 164 int gtid = __kmp_entry_gtid(); 165 MKLOC(loc, "GOMP_single_copy_start"); 166 KA_TRACE(20, ("GOMP_single_copy_start: T#%d\n", gtid)); 167 168 if (!TCR_4(__kmp_init_parallel)) 169 __kmp_parallel_initialize(); 170 __kmp_resume_if_soft_paused(); 171 172 // If this is the first thread to enter, return NULL. The generated code will 173 // then call GOMP_single_copy_end() for this thread only, with the 174 // copyprivate data pointer as an argument. 175 if (__kmp_enter_single(gtid, &loc, FALSE)) 176 return NULL; 177 178// Wait for the first thread to set the copyprivate data pointer, 179// and for all other threads to reach this point. 180 181#if OMPT_SUPPORT && OMPT_OPTIONAL 182 ompt_frame_t *ompt_frame; 183 if (ompt_enabled.enabled) { 184 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 185 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 186 OMPT_STORE_RETURN_ADDRESS(gtid); 187 } 188#endif 189 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 190 191 // Retrieve the value of the copyprivate data point, and wait for all 192 // threads to do likewise, then return. 193 retval = __kmp_team_from_gtid(gtid)->t.t_copypriv_data; 194#if OMPT_SUPPORT && OMPT_OPTIONAL 195 if (ompt_enabled.enabled) { 196 OMPT_STORE_RETURN_ADDRESS(gtid); 197 } 198#endif 199 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 200#if OMPT_SUPPORT && OMPT_OPTIONAL 201 if (ompt_enabled.enabled) { 202 ompt_frame->enter_frame = ompt_data_none; 203 } 204#endif 205 return retval; 206} 207 208void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SINGLE_COPY_END)(void *data) { 209 int gtid = __kmp_get_gtid(); 210 KA_TRACE(20, ("GOMP_single_copy_end: T#%d\n", gtid)); 211 212 // Set the copyprivate data pointer fo the team, then hit the barrier so that 213 // the other threads will continue on and read it. Hit another barrier before 214 // continuing, so that the know that the copyprivate data pointer has been 215 // propagated to all threads before trying to reuse the t_copypriv_data field. 216 __kmp_team_from_gtid(gtid)->t.t_copypriv_data = data; 217#if OMPT_SUPPORT && OMPT_OPTIONAL 218 ompt_frame_t *ompt_frame; 219 if (ompt_enabled.enabled) { 220 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 221 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 222 OMPT_STORE_RETURN_ADDRESS(gtid); 223 } 224#endif 225 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 226#if OMPT_SUPPORT && OMPT_OPTIONAL 227 if (ompt_enabled.enabled) { 228 OMPT_STORE_RETURN_ADDRESS(gtid); 229 } 230#endif 231 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 232#if OMPT_SUPPORT && OMPT_OPTIONAL 233 if (ompt_enabled.enabled) { 234 ompt_frame->enter_frame = ompt_data_none; 235 } 236#endif 237} 238 239void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_START)(void) { 240 int gtid = __kmp_entry_gtid(); 241 MKLOC(loc, "GOMP_ordered_start"); 242 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 243#if OMPT_SUPPORT && OMPT_OPTIONAL 244 OMPT_STORE_RETURN_ADDRESS(gtid); 245#endif 246 __kmpc_ordered(&loc, gtid); 247} 248 249void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_ORDERED_END)(void) { 250 int gtid = __kmp_get_gtid(); 251 MKLOC(loc, "GOMP_ordered_end"); 252 KA_TRACE(20, ("GOMP_ordered_start: T#%d\n", gtid)); 253#if OMPT_SUPPORT && OMPT_OPTIONAL 254 OMPT_STORE_RETURN_ADDRESS(gtid); 255#endif 256 __kmpc_end_ordered(&loc, gtid); 257} 258 259// Dispatch macro defs 260// 261// They come in two flavors: 64-bit unsigned, and either 32-bit signed 262// (IA-32 architecture) or 64-bit signed (Intel(R) 64). 263 264#if KMP_ARCH_X86 || KMP_ARCH_ARM || KMP_ARCH_MIPS 265#define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_4 266#define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_4 267#define KMP_DISPATCH_NEXT __kmpc_dispatch_next_4 268#else 269#define KMP_DISPATCH_INIT __kmp_aux_dispatch_init_8 270#define KMP_DISPATCH_FINI_CHUNK __kmp_aux_dispatch_fini_chunk_8 271#define KMP_DISPATCH_NEXT __kmpc_dispatch_next_8 272#endif /* KMP_ARCH_X86 */ 273 274#define KMP_DISPATCH_INIT_ULL __kmp_aux_dispatch_init_8u 275#define KMP_DISPATCH_FINI_CHUNK_ULL __kmp_aux_dispatch_fini_chunk_8u 276#define KMP_DISPATCH_NEXT_ULL __kmpc_dispatch_next_8u 277 278// The parallel construct 279 280#ifndef KMP_DEBUG 281static 282#endif /* KMP_DEBUG */ 283 void 284 __kmp_GOMP_microtask_wrapper(int *gtid, int *npr, void (*task)(void *), 285 void *data) { 286#if OMPT_SUPPORT 287 kmp_info_t *thr; 288 ompt_frame_t *ompt_frame; 289 ompt_state_t enclosing_state; 290 291 if (ompt_enabled.enabled) { 292 // get pointer to thread data structure 293 thr = __kmp_threads[*gtid]; 294 295 // save enclosing task state; set current state for task 296 enclosing_state = thr->th.ompt_thread_info.state; 297 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 298 299 // set task frame 300 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 301 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 302 } 303#endif 304 305 task(data); 306 307#if OMPT_SUPPORT 308 if (ompt_enabled.enabled) { 309 // clear task frame 310 ompt_frame->exit_frame = ompt_data_none; 311 312 // restore enclosing state 313 thr->th.ompt_thread_info.state = enclosing_state; 314 } 315#endif 316} 317 318#ifndef KMP_DEBUG 319static 320#endif /* KMP_DEBUG */ 321 void 322 __kmp_GOMP_parallel_microtask_wrapper(int *gtid, int *npr, 323 void (*task)(void *), void *data, 324 unsigned num_threads, ident_t *loc, 325 enum sched_type schedule, long start, 326 long end, long incr, 327 long chunk_size) { 328 // Initialize the loop worksharing construct. 329 330 KMP_DISPATCH_INIT(loc, *gtid, schedule, start, end, incr, chunk_size, 331 schedule != kmp_sch_static); 332 333#if OMPT_SUPPORT 334 kmp_info_t *thr; 335 ompt_frame_t *ompt_frame; 336 ompt_state_t enclosing_state; 337 338 if (ompt_enabled.enabled) { 339 thr = __kmp_threads[*gtid]; 340 // save enclosing task state; set current state for task 341 enclosing_state = thr->th.ompt_thread_info.state; 342 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 343 344 // set task frame 345 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 346 ompt_frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 347 } 348#endif 349 350 // Now invoke the microtask. 351 task(data); 352 353#if OMPT_SUPPORT 354 if (ompt_enabled.enabled) { 355 // clear task frame 356 ompt_frame->exit_frame = ompt_data_none; 357 358 // reset enclosing state 359 thr->th.ompt_thread_info.state = enclosing_state; 360 } 361#endif 362} 363 364#ifndef KMP_DEBUG 365static 366#endif /* KMP_DEBUG */ 367 void 368 __kmp_GOMP_fork_call(ident_t *loc, int gtid, void (*unwrapped_task)(void *), 369 microtask_t wrapper, int argc, ...) { 370 int rc; 371 kmp_info_t *thr = __kmp_threads[gtid]; 372 kmp_team_t *team = thr->th.th_team; 373 int tid = __kmp_tid_from_gtid(gtid); 374 375 va_list ap; 376 va_start(ap, argc); 377 378 rc = __kmp_fork_call(loc, gtid, fork_context_gnu, argc, wrapper, 379 __kmp_invoke_task_func, kmp_va_addr_of(ap)); 380 381 va_end(ap); 382 383 if (rc) { 384 __kmp_run_before_invoked_task(gtid, tid, thr, team); 385 } 386 387#if OMPT_SUPPORT 388 int ompt_team_size; 389 if (ompt_enabled.enabled) { 390 ompt_team_info_t *team_info = __ompt_get_teaminfo(0, NULL); 391 ompt_task_info_t *task_info = __ompt_get_task_info_object(0); 392 393 // implicit task callback 394 if (ompt_enabled.ompt_callback_implicit_task) { 395 ompt_team_size = __kmp_team_from_gtid(gtid)->t.t_nproc; 396 ompt_callbacks.ompt_callback(ompt_callback_implicit_task)( 397 ompt_scope_begin, &(team_info->parallel_data), 398 &(task_info->task_data), ompt_team_size, __kmp_tid_from_gtid(gtid), ompt_task_implicit); // TODO: Can this be ompt_task_initial? 399 task_info->thread_num = __kmp_tid_from_gtid(gtid); 400 } 401 thr->th.ompt_thread_info.state = ompt_state_work_parallel; 402 } 403#endif 404} 405 406static void __kmp_GOMP_serialized_parallel(ident_t *loc, kmp_int32 gtid, 407 void (*task)(void *)) { 408#if OMPT_SUPPORT 409 OMPT_STORE_RETURN_ADDRESS(gtid); 410#endif 411 __kmp_serialized_parallel(loc, gtid); 412} 413 414void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_START)(void (*task)(void *), 415 void *data, 416 unsigned num_threads) { 417 int gtid = __kmp_entry_gtid(); 418 419#if OMPT_SUPPORT 420 ompt_frame_t *parent_frame, *frame; 421 422 if (ompt_enabled.enabled) { 423 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 424 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 425 OMPT_STORE_RETURN_ADDRESS(gtid); 426 } 427#endif 428 429 MKLOC(loc, "GOMP_parallel_start"); 430 KA_TRACE(20, ("GOMP_parallel_start: T#%d\n", gtid)); 431 432 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 433 if (num_threads != 0) { 434 __kmp_push_num_threads(&loc, gtid, num_threads); 435 } 436 __kmp_GOMP_fork_call(&loc, gtid, task, 437 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 438 data); 439 } else { 440 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 441 } 442 443#if OMPT_SUPPORT 444 if (ompt_enabled.enabled) { 445 __ompt_get_task_info_internal(0, NULL, NULL, &frame, NULL, NULL); 446 frame->exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 447 } 448#endif 449} 450 451void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(void) { 452 int gtid = __kmp_get_gtid(); 453 kmp_info_t *thr; 454 455 thr = __kmp_threads[gtid]; 456 457 MKLOC(loc, "GOMP_parallel_end"); 458 KA_TRACE(20, ("GOMP_parallel_end: T#%d\n", gtid)); 459 460 if (!thr->th.th_team->t.t_serialized) { 461 __kmp_run_after_invoked_task(gtid, __kmp_tid_from_gtid(gtid), thr, 462 thr->th.th_team); 463 464#if OMPT_SUPPORT 465 if (ompt_enabled.enabled) { 466 // Implicit task is finished here, in the barrier we might schedule 467 // deferred tasks, 468 // these don't see the implicit task on the stack 469 OMPT_CUR_TASK_INFO(thr)->frame.exit_frame = ompt_data_none; 470 } 471#endif 472 473 __kmp_join_call(&loc, gtid 474#if OMPT_SUPPORT 475 , 476 fork_context_gnu 477#endif 478 ); 479 } else { 480 __kmpc_end_serialized_parallel(&loc, gtid); 481 } 482} 483 484// Loop worksharing constructs 485 486// The Gnu codegen passes in an exclusive upper bound for the overall range, 487// but the libguide dispatch code expects an inclusive upper bound, hence the 488// "end - incr" 5th argument to KMP_DISPATCH_INIT (and the " ub - str" 11th 489// argument to __kmp_GOMP_fork_call). 490// 491// Conversely, KMP_DISPATCH_NEXT returns and inclusive upper bound in *p_ub, 492// but the Gnu codegen expects an exclusive upper bound, so the adjustment 493// "*p_ub += stride" compensates for the discrepancy. 494// 495// Correction: the gnu codegen always adjusts the upper bound by +-1, not the 496// stride value. We adjust the dispatch parameters accordingly (by +-1), but 497// we still adjust p_ub by the actual stride value. 498// 499// The "runtime" versions do not take a chunk_sz parameter. 500// 501// The profile lib cannot support construct checking of unordered loops that 502// are predetermined by the compiler to be statically scheduled, as the gcc 503// codegen will not always emit calls to GOMP_loop_static_next() to get the 504// next iteration. Instead, it emits inline code to call omp_get_thread_num() 505// num and calculate the iteration space using the result. It doesn't do this 506// with ordered static loop, so they can be checked. 507 508#if OMPT_SUPPORT 509#define IF_OMPT_SUPPORT(code) code 510#else 511#define IF_OMPT_SUPPORT(code) 512#endif 513 514#define LOOP_START(func, schedule) \ 515 int func(long lb, long ub, long str, long chunk_sz, long *p_lb, \ 516 long *p_ub) { \ 517 int status; \ 518 long stride; \ 519 int gtid = __kmp_entry_gtid(); \ 520 MKLOC(loc, KMP_STR(func)); \ 521 KA_TRACE( \ 522 20, \ 523 (KMP_STR( \ 524 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 525 gtid, lb, ub, str, chunk_sz)); \ 526 \ 527 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 528 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 529 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 530 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 531 (schedule) != kmp_sch_static); \ 532 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 533 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 534 (kmp_int *)p_ub, (kmp_int *)&stride); \ 535 if (status) { \ 536 KMP_DEBUG_ASSERT(stride == str); \ 537 *p_ub += (str > 0) ? 1 : -1; \ 538 } \ 539 } else { \ 540 status = 0; \ 541 } \ 542 \ 543 KA_TRACE( \ 544 20, \ 545 (KMP_STR( \ 546 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 547 gtid, *p_lb, *p_ub, status)); \ 548 return status; \ 549 } 550 551#define LOOP_RUNTIME_START(func, schedule) \ 552 int func(long lb, long ub, long str, long *p_lb, long *p_ub) { \ 553 int status; \ 554 long stride; \ 555 long chunk_sz = 0; \ 556 int gtid = __kmp_entry_gtid(); \ 557 MKLOC(loc, KMP_STR(func)); \ 558 KA_TRACE( \ 559 20, \ 560 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 561 gtid, lb, ub, str, chunk_sz)); \ 562 \ 563 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 564 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 565 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 566 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \ 567 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 568 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 569 (kmp_int *)p_ub, (kmp_int *)&stride); \ 570 if (status) { \ 571 KMP_DEBUG_ASSERT(stride == str); \ 572 *p_ub += (str > 0) ? 1 : -1; \ 573 } \ 574 } else { \ 575 status = 0; \ 576 } \ 577 \ 578 KA_TRACE( \ 579 20, \ 580 (KMP_STR( \ 581 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 582 gtid, *p_lb, *p_ub, status)); \ 583 return status; \ 584 } 585 586#define KMP_DOACROSS_FINI(status, gtid) \ 587 if (!status && __kmp_threads[gtid]->th.th_dispatch->th_doacross_flags) { \ 588 __kmpc_doacross_fini(NULL, gtid); \ 589 } 590 591#define LOOP_NEXT(func, fini_code) \ 592 int func(long *p_lb, long *p_ub) { \ 593 int status; \ 594 long stride; \ 595 int gtid = __kmp_get_gtid(); \ 596 MKLOC(loc, KMP_STR(func)); \ 597 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 598 \ 599 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 600 fini_code status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 601 (kmp_int *)p_ub, (kmp_int *)&stride); \ 602 if (status) { \ 603 *p_ub += (stride > 0) ? 1 : -1; \ 604 } \ 605 KMP_DOACROSS_FINI(status, gtid) \ 606 \ 607 KA_TRACE( \ 608 20, \ 609 (KMP_STR(func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, stride 0x%lx, " \ 610 "returning %d\n", \ 611 gtid, *p_lb, *p_ub, stride, status)); \ 612 return status; \ 613 } 614 615LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_START), kmp_sch_static) 616LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT), {}) 617LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START), 618 kmp_sch_dynamic_chunked) 619LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START), 620 kmp_sch_dynamic_chunked) 621LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT), {}) 622LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT), {}) 623LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_START), 624 kmp_sch_guided_chunked) 625LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START), 626 kmp_sch_guided_chunked) 627LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT), {}) 628LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT), {}) 629LOOP_RUNTIME_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_START), 630 kmp_sch_runtime) 631LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT), {}) 632LOOP_RUNTIME_START( 633 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START), 634 kmp_sch_runtime) 635LOOP_RUNTIME_START( 636 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START), 637 kmp_sch_runtime) 638LOOP_NEXT( 639 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT), {}) 640LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT), {}) 641 642LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START), 643 kmp_ord_static) 644LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT), 645 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 646LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START), 647 kmp_ord_dynamic_chunked) 648LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT), 649 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 650LOOP_START(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START), 651 kmp_ord_guided_chunked) 652LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT), 653 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 654LOOP_RUNTIME_START( 655 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START), 656 kmp_ord_runtime) 657LOOP_NEXT(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT), 658 { KMP_DISPATCH_FINI_CHUNK(&loc, gtid); }) 659 660#define LOOP_DOACROSS_START(func, schedule) \ 661 bool func(unsigned ncounts, long *counts, long chunk_sz, long *p_lb, \ 662 long *p_ub) { \ 663 int status; \ 664 long stride, lb, ub, str; \ 665 int gtid = __kmp_entry_gtid(); \ 666 struct kmp_dim *dims = \ 667 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 668 MKLOC(loc, KMP_STR(func)); \ 669 for (unsigned i = 0; i < ncounts; ++i) { \ 670 dims[i].lo = 0; \ 671 dims[i].up = counts[i] - 1; \ 672 dims[i].st = 1; \ 673 } \ 674 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 675 lb = 0; \ 676 ub = counts[0]; \ 677 str = 1; \ 678 KA_TRACE(20, (KMP_STR(func) ": T#%d, ncounts %u, lb 0x%lx, ub 0x%lx, str " \ 679 "0x%lx, chunk_sz " \ 680 "0x%lx\n", \ 681 gtid, ncounts, lb, ub, str, chunk_sz)); \ 682 \ 683 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 684 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 685 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 686 (schedule) != kmp_sch_static); \ 687 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 688 (kmp_int *)p_ub, (kmp_int *)&stride); \ 689 if (status) { \ 690 KMP_DEBUG_ASSERT(stride == str); \ 691 *p_ub += (str > 0) ? 1 : -1; \ 692 } \ 693 } else { \ 694 status = 0; \ 695 } \ 696 KMP_DOACROSS_FINI(status, gtid); \ 697 \ 698 KA_TRACE( \ 699 20, \ 700 (KMP_STR( \ 701 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 702 gtid, *p_lb, *p_ub, status)); \ 703 __kmp_free(dims); \ 704 return status; \ 705 } 706 707#define LOOP_DOACROSS_RUNTIME_START(func, schedule) \ 708 int func(unsigned ncounts, long *counts, long *p_lb, long *p_ub) { \ 709 int status; \ 710 long stride, lb, ub, str; \ 711 long chunk_sz = 0; \ 712 int gtid = __kmp_entry_gtid(); \ 713 struct kmp_dim *dims = \ 714 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 715 MKLOC(loc, KMP_STR(func)); \ 716 for (unsigned i = 0; i < ncounts; ++i) { \ 717 dims[i].lo = 0; \ 718 dims[i].up = counts[i] - 1; \ 719 dims[i].st = 1; \ 720 } \ 721 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 722 lb = 0; \ 723 ub = counts[0]; \ 724 str = 1; \ 725 KA_TRACE( \ 726 20, \ 727 (KMP_STR(func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz %d\n", \ 728 gtid, lb, ub, str, chunk_sz)); \ 729 \ 730 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 731 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 732 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, TRUE); \ 733 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, (kmp_int *)p_lb, \ 734 (kmp_int *)p_ub, (kmp_int *)&stride); \ 735 if (status) { \ 736 KMP_DEBUG_ASSERT(stride == str); \ 737 *p_ub += (str > 0) ? 1 : -1; \ 738 } \ 739 } else { \ 740 status = 0; \ 741 } \ 742 KMP_DOACROSS_FINI(status, gtid); \ 743 \ 744 KA_TRACE( \ 745 20, \ 746 (KMP_STR( \ 747 func) " exit: T#%d, *p_lb 0x%lx, *p_ub 0x%lx, returning %d\n", \ 748 gtid, *p_lb, *p_ub, status)); \ 749 __kmp_free(dims); \ 750 return status; \ 751 } 752 753LOOP_DOACROSS_START( 754 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START), 755 kmp_sch_static) 756LOOP_DOACROSS_START( 757 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START), 758 kmp_sch_dynamic_chunked) 759LOOP_DOACROSS_START( 760 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START), 761 kmp_sch_guided_chunked) 762LOOP_DOACROSS_RUNTIME_START( 763 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START), 764 kmp_sch_runtime) 765 766void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END)(void) { 767 int gtid = __kmp_get_gtid(); 768 KA_TRACE(20, ("GOMP_loop_end: T#%d\n", gtid)) 769 770#if OMPT_SUPPORT && OMPT_OPTIONAL 771 ompt_frame_t *ompt_frame; 772 if (ompt_enabled.enabled) { 773 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 774 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 775 OMPT_STORE_RETURN_ADDRESS(gtid); 776 } 777#endif 778 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 779#if OMPT_SUPPORT && OMPT_OPTIONAL 780 if (ompt_enabled.enabled) { 781 ompt_frame->enter_frame = ompt_data_none; 782 } 783#endif 784 785 KA_TRACE(20, ("GOMP_loop_end exit: T#%d\n", gtid)) 786} 787 788void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_NOWAIT)(void) { 789 KA_TRACE(20, ("GOMP_loop_end_nowait: T#%d\n", __kmp_get_gtid())) 790} 791 792// Unsigned long long loop worksharing constructs 793// 794// These are new with gcc 4.4 795 796#define LOOP_START_ULL(func, schedule) \ 797 int func(int up, unsigned long long lb, unsigned long long ub, \ 798 unsigned long long str, unsigned long long chunk_sz, \ 799 unsigned long long *p_lb, unsigned long long *p_ub) { \ 800 int status; \ 801 long long str2 = up ? ((long long)str) : -((long long)str); \ 802 long long stride; \ 803 int gtid = __kmp_entry_gtid(); \ 804 MKLOC(loc, KMP_STR(func)); \ 805 \ 806 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 807 "0x%llx, chunk_sz 0x%llx\n", \ 808 gtid, up, lb, ub, str, chunk_sz)); \ 809 \ 810 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 811 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 812 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 813 (schedule) != kmp_sch_static); \ 814 status = \ 815 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 816 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 817 if (status) { \ 818 KMP_DEBUG_ASSERT(stride == str2); \ 819 *p_ub += (str > 0) ? 1 : -1; \ 820 } \ 821 } else { \ 822 status = 0; \ 823 } \ 824 \ 825 KA_TRACE( \ 826 20, \ 827 (KMP_STR( \ 828 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 829 gtid, *p_lb, *p_ub, status)); \ 830 return status; \ 831 } 832 833#define LOOP_RUNTIME_START_ULL(func, schedule) \ 834 int func(int up, unsigned long long lb, unsigned long long ub, \ 835 unsigned long long str, unsigned long long *p_lb, \ 836 unsigned long long *p_ub) { \ 837 int status; \ 838 long long str2 = up ? ((long long)str) : -((long long)str); \ 839 unsigned long long stride; \ 840 unsigned long long chunk_sz = 0; \ 841 int gtid = __kmp_entry_gtid(); \ 842 MKLOC(loc, KMP_STR(func)); \ 843 \ 844 KA_TRACE(20, (KMP_STR(func) ": T#%d, up %d, lb 0x%llx, ub 0x%llx, str " \ 845 "0x%llx, chunk_sz 0x%llx\n", \ 846 gtid, up, lb, ub, str, chunk_sz)); \ 847 \ 848 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 849 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 850 (str2 > 0) ? (ub - 1) : (ub + 1), str2, chunk_sz, \ 851 TRUE); \ 852 status = \ 853 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 854 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 855 if (status) { \ 856 KMP_DEBUG_ASSERT((long long)stride == str2); \ 857 *p_ub += (str > 0) ? 1 : -1; \ 858 } \ 859 } else { \ 860 status = 0; \ 861 } \ 862 \ 863 KA_TRACE( \ 864 20, \ 865 (KMP_STR( \ 866 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 867 gtid, *p_lb, *p_ub, status)); \ 868 return status; \ 869 } 870 871#define LOOP_NEXT_ULL(func, fini_code) \ 872 int func(unsigned long long *p_lb, unsigned long long *p_ub) { \ 873 int status; \ 874 long long stride; \ 875 int gtid = __kmp_get_gtid(); \ 876 MKLOC(loc, KMP_STR(func)); \ 877 KA_TRACE(20, (KMP_STR(func) ": T#%d\n", gtid)); \ 878 \ 879 fini_code status = \ 880 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 881 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 882 if (status) { \ 883 *p_ub += (stride > 0) ? 1 : -1; \ 884 } \ 885 \ 886 KA_TRACE( \ 887 20, \ 888 (KMP_STR( \ 889 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, stride 0x%llx, " \ 890 "returning %d\n", \ 891 gtid, *p_lb, *p_ub, stride, status)); \ 892 return status; \ 893 } 894 895LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START), 896 kmp_sch_static) 897LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT), {}) 898LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START), 899 kmp_sch_dynamic_chunked) 900LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT), {}) 901LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START), 902 kmp_sch_guided_chunked) 903LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT), {}) 904LOOP_START_ULL( 905 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START), 906 kmp_sch_dynamic_chunked) 907LOOP_NEXT_ULL( 908 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT), {}) 909LOOP_START_ULL( 910 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START), 911 kmp_sch_guided_chunked) 912LOOP_NEXT_ULL( 913 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT), {}) 914LOOP_RUNTIME_START_ULL( 915 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START), kmp_sch_runtime) 916LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT), {}) 917LOOP_RUNTIME_START_ULL( 918 KMP_EXPAND_NAME( 919 KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START), 920 kmp_sch_runtime) 921LOOP_RUNTIME_START_ULL( 922 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START), 923 kmp_sch_runtime) 924LOOP_NEXT_ULL( 925 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT), 926 {}) 927LOOP_NEXT_ULL( 928 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT), {}) 929 930LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START), 931 kmp_ord_static) 932LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT), 933 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 934LOOP_START_ULL( 935 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START), 936 kmp_ord_dynamic_chunked) 937LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT), 938 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 939LOOP_START_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START), 940 kmp_ord_guided_chunked) 941LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT), 942 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 943LOOP_RUNTIME_START_ULL( 944 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START), 945 kmp_ord_runtime) 946LOOP_NEXT_ULL(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT), 947 { KMP_DISPATCH_FINI_CHUNK_ULL(&loc, gtid); }) 948 949#define LOOP_DOACROSS_START_ULL(func, schedule) \ 950 int func(unsigned ncounts, unsigned long long *counts, \ 951 unsigned long long chunk_sz, unsigned long long *p_lb, \ 952 unsigned long long *p_ub) { \ 953 int status; \ 954 long long stride, str, lb, ub; \ 955 int gtid = __kmp_entry_gtid(); \ 956 struct kmp_dim *dims = \ 957 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 958 MKLOC(loc, KMP_STR(func)); \ 959 for (unsigned i = 0; i < ncounts; ++i) { \ 960 dims[i].lo = 0; \ 961 dims[i].up = counts[i] - 1; \ 962 dims[i].st = 1; \ 963 } \ 964 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 965 lb = 0; \ 966 ub = counts[0]; \ 967 str = 1; \ 968 \ 969 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 970 "0x%llx, chunk_sz 0x%llx\n", \ 971 gtid, lb, ub, str, chunk_sz)); \ 972 \ 973 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 974 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 975 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 976 (schedule) != kmp_sch_static); \ 977 status = \ 978 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 979 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 980 if (status) { \ 981 KMP_DEBUG_ASSERT(stride == str); \ 982 *p_ub += (str > 0) ? 1 : -1; \ 983 } \ 984 } else { \ 985 status = 0; \ 986 } \ 987 KMP_DOACROSS_FINI(status, gtid); \ 988 \ 989 KA_TRACE( \ 990 20, \ 991 (KMP_STR( \ 992 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 993 gtid, *p_lb, *p_ub, status)); \ 994 __kmp_free(dims); \ 995 return status; \ 996 } 997 998#define LOOP_DOACROSS_RUNTIME_START_ULL(func, schedule) \ 999 int func(unsigned ncounts, unsigned long long *counts, \ 1000 unsigned long long *p_lb, unsigned long long *p_ub) { \ 1001 int status; \ 1002 unsigned long long stride, str, lb, ub; \ 1003 unsigned long long chunk_sz = 0; \ 1004 int gtid = __kmp_entry_gtid(); \ 1005 struct kmp_dim *dims = \ 1006 (struct kmp_dim *)__kmp_allocate(sizeof(struct kmp_dim) * ncounts); \ 1007 MKLOC(loc, KMP_STR(func)); \ 1008 for (unsigned i = 0; i < ncounts; ++i) { \ 1009 dims[i].lo = 0; \ 1010 dims[i].up = counts[i] - 1; \ 1011 dims[i].st = 1; \ 1012 } \ 1013 __kmpc_doacross_init(&loc, gtid, (int)ncounts, dims); \ 1014 lb = 0; \ 1015 ub = counts[0]; \ 1016 str = 1; \ 1017 KA_TRACE(20, (KMP_STR(func) ": T#%d, lb 0x%llx, ub 0x%llx, str " \ 1018 "0x%llx, chunk_sz 0x%llx\n", \ 1019 gtid, lb, ub, str, chunk_sz)); \ 1020 \ 1021 if ((str > 0) ? (lb < ub) : (lb > ub)) { \ 1022 KMP_DISPATCH_INIT_ULL(&loc, gtid, (schedule), lb, \ 1023 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1024 TRUE); \ 1025 status = \ 1026 KMP_DISPATCH_NEXT_ULL(&loc, gtid, NULL, (kmp_uint64 *)p_lb, \ 1027 (kmp_uint64 *)p_ub, (kmp_int64 *)&stride); \ 1028 if (status) { \ 1029 KMP_DEBUG_ASSERT(stride == str); \ 1030 *p_ub += (str > 0) ? 1 : -1; \ 1031 } \ 1032 } else { \ 1033 status = 0; \ 1034 } \ 1035 KMP_DOACROSS_FINI(status, gtid); \ 1036 \ 1037 KA_TRACE( \ 1038 20, \ 1039 (KMP_STR( \ 1040 func) " exit: T#%d, *p_lb 0x%llx, *p_ub 0x%llx, returning %d\n", \ 1041 gtid, *p_lb, *p_ub, status)); \ 1042 __kmp_free(dims); \ 1043 return status; \ 1044 } 1045 1046LOOP_DOACROSS_START_ULL( 1047 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START), 1048 kmp_sch_static) 1049LOOP_DOACROSS_START_ULL( 1050 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START), 1051 kmp_sch_dynamic_chunked) 1052LOOP_DOACROSS_START_ULL( 1053 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START), 1054 kmp_sch_guided_chunked) 1055LOOP_DOACROSS_RUNTIME_START_ULL( 1056 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START), 1057 kmp_sch_runtime) 1058 1059// Combined parallel / loop worksharing constructs 1060// 1061// There are no ull versions (yet). 1062 1063#define PARALLEL_LOOP_START(func, schedule, ompt_pre, ompt_post) \ 1064 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1065 long ub, long str, long chunk_sz) { \ 1066 int gtid = __kmp_entry_gtid(); \ 1067 MKLOC(loc, KMP_STR(func)); \ 1068 KA_TRACE( \ 1069 20, \ 1070 (KMP_STR( \ 1071 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1072 gtid, lb, ub, str, chunk_sz)); \ 1073 \ 1074 ompt_pre(); \ 1075 \ 1076 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \ 1077 if (num_threads != 0) { \ 1078 __kmp_push_num_threads(&loc, gtid, num_threads); \ 1079 } \ 1080 __kmp_GOMP_fork_call(&loc, gtid, task, \ 1081 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1082 9, task, data, num_threads, &loc, (schedule), lb, \ 1083 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1084 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid)); \ 1085 } else { \ 1086 __kmp_GOMP_serialized_parallel(&loc, gtid, task); \ 1087 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid)); \ 1088 } \ 1089 \ 1090 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1091 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1092 (schedule) != kmp_sch_static); \ 1093 \ 1094 ompt_post(); \ 1095 \ 1096 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1097 } 1098 1099#if OMPT_SUPPORT && OMPT_OPTIONAL 1100 1101#define OMPT_LOOP_PRE() \ 1102 ompt_frame_t *parent_frame; \ 1103 if (ompt_enabled.enabled) { \ 1104 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); \ 1105 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); \ 1106 OMPT_STORE_RETURN_ADDRESS(gtid); \ 1107 } 1108 1109#define OMPT_LOOP_POST() \ 1110 if (ompt_enabled.enabled) { \ 1111 parent_frame->enter_frame = ompt_data_none; \ 1112 } 1113 1114#else 1115 1116#define OMPT_LOOP_PRE() 1117 1118#define OMPT_LOOP_POST() 1119 1120#endif 1121 1122PARALLEL_LOOP_START( 1123 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START), 1124 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1125PARALLEL_LOOP_START( 1126 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START), 1127 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1128PARALLEL_LOOP_START( 1129 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START), 1130 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1131PARALLEL_LOOP_START( 1132 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START), 1133 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1134 1135// Tasking constructs 1136 1137void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASK)(void (*func)(void *), void *data, 1138 void (*copy_func)(void *, void *), 1139 long arg_size, long arg_align, 1140 bool if_cond, unsigned gomp_flags, 1141 void **depend) { 1142 MKLOC(loc, "GOMP_task"); 1143 int gtid = __kmp_entry_gtid(); 1144 kmp_int32 flags = 0; 1145 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1146 1147 KA_TRACE(20, ("GOMP_task: T#%d\n", gtid)); 1148 1149 // The low-order bit is the "untied" flag 1150 if (!(gomp_flags & 1)) { 1151 input_flags->tiedness = 1; 1152 } 1153 // The second low-order bit is the "final" flag 1154 if (gomp_flags & 2) { 1155 input_flags->final = 1; 1156 } 1157 input_flags->native = 1; 1158 // __kmp_task_alloc() sets up all other flags 1159 1160 if (!if_cond) { 1161 arg_size = 0; 1162 } 1163 1164 kmp_task_t *task = __kmp_task_alloc( 1165 &loc, gtid, input_flags, sizeof(kmp_task_t), 1166 arg_size ? arg_size + arg_align - 1 : 0, (kmp_routine_entry_t)func); 1167 1168 if (arg_size > 0) { 1169 if (arg_align > 0) { 1170 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1171 arg_align * arg_align); 1172 } 1173 // else error?? 1174 1175 if (copy_func) { 1176 (*copy_func)(task->shareds, data); 1177 } else { 1178 KMP_MEMCPY(task->shareds, data, arg_size); 1179 } 1180 } 1181 1182#if OMPT_SUPPORT 1183 kmp_taskdata_t *current_task; 1184 if (ompt_enabled.enabled) { 1185 OMPT_STORE_RETURN_ADDRESS(gtid); 1186 current_task = __kmp_threads[gtid]->th.th_current_task; 1187 current_task->ompt_task_info.frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1188 } 1189#endif 1190 1191 if (if_cond) { 1192 if (gomp_flags & 8) { 1193 KMP_ASSERT(depend); 1194 const size_t ndeps = (kmp_intptr_t)depend[0]; 1195 const size_t nout = (kmp_intptr_t)depend[1]; 1196 kmp_depend_info_t dep_list[ndeps]; 1197 1198 for (size_t i = 0U; i < ndeps; i++) { 1199 dep_list[i].base_addr = (kmp_intptr_t)depend[2U + i]; 1200 dep_list[i].len = 0U; 1201 dep_list[i].flags.in = 1; 1202 dep_list[i].flags.out = (i < nout); 1203 } 1204 __kmpc_omp_task_with_deps(&loc, gtid, task, ndeps, dep_list, 0, NULL); 1205 } else { 1206 __kmpc_omp_task(&loc, gtid, task); 1207 } 1208 } else { 1209#if OMPT_SUPPORT 1210 ompt_thread_info_t oldInfo; 1211 kmp_info_t *thread; 1212 kmp_taskdata_t *taskdata; 1213 if (ompt_enabled.enabled) { 1214 // Store the threads states and restore them after the task 1215 thread = __kmp_threads[gtid]; 1216 taskdata = KMP_TASK_TO_TASKDATA(task); 1217 oldInfo = thread->th.ompt_thread_info; 1218 thread->th.ompt_thread_info.wait_id = 0; 1219 thread->th.ompt_thread_info.state = ompt_state_work_parallel; 1220 taskdata->ompt_task_info.frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1221 OMPT_STORE_RETURN_ADDRESS(gtid); 1222 } 1223#endif 1224 1225 __kmpc_omp_task_begin_if0(&loc, gtid, task); 1226 func(data); 1227 __kmpc_omp_task_complete_if0(&loc, gtid, task); 1228 1229#if OMPT_SUPPORT 1230 if (ompt_enabled.enabled) { 1231 thread->th.ompt_thread_info = oldInfo; 1232 taskdata->ompt_task_info.frame.exit_frame = ompt_data_none; 1233 } 1234#endif 1235 } 1236#if OMPT_SUPPORT 1237 if (ompt_enabled.enabled) { 1238 current_task->ompt_task_info.frame.enter_frame = ompt_data_none; 1239 } 1240#endif 1241 1242 KA_TRACE(20, ("GOMP_task exit: T#%d\n", gtid)); 1243} 1244 1245void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKWAIT)(void) { 1246 MKLOC(loc, "GOMP_taskwait"); 1247 int gtid = __kmp_entry_gtid(); 1248 1249#if OMPT_SUPPORT 1250 if (ompt_enabled.enabled) 1251 OMPT_STORE_RETURN_ADDRESS(gtid); 1252#endif 1253 1254 KA_TRACE(20, ("GOMP_taskwait: T#%d\n", gtid)); 1255 1256 __kmpc_omp_taskwait(&loc, gtid); 1257 1258 KA_TRACE(20, ("GOMP_taskwait exit: T#%d\n", gtid)); 1259} 1260 1261// Sections worksharing constructs 1262// 1263// For the sections construct, we initialize a dynamically scheduled loop 1264// worksharing construct with lb 1 and stride 1, and use the iteration #'s 1265// that its returns as sections ids. 1266// 1267// There are no special entry points for ordered sections, so we always use 1268// the dynamically scheduled workshare, even if the sections aren't ordered. 1269 1270unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_START)(unsigned count) { 1271 int status; 1272 kmp_int lb, ub, stride; 1273 int gtid = __kmp_entry_gtid(); 1274 MKLOC(loc, "GOMP_sections_start"); 1275 KA_TRACE(20, ("GOMP_sections_start: T#%d\n", gtid)); 1276 1277 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1278 1279 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1280 if (status) { 1281 KMP_DEBUG_ASSERT(stride == 1); 1282 KMP_DEBUG_ASSERT(lb > 0); 1283 KMP_ASSERT(lb == ub); 1284 } else { 1285 lb = 0; 1286 } 1287 1288 KA_TRACE(20, ("GOMP_sections_start exit: T#%d returning %u\n", gtid, 1289 (unsigned)lb)); 1290 return (unsigned)lb; 1291} 1292 1293unsigned KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_NEXT)(void) { 1294 int status; 1295 kmp_int lb, ub, stride; 1296 int gtid = __kmp_get_gtid(); 1297 MKLOC(loc, "GOMP_sections_next"); 1298 KA_TRACE(20, ("GOMP_sections_next: T#%d\n", gtid)); 1299 1300#if OMPT_SUPPORT 1301 OMPT_STORE_RETURN_ADDRESS(gtid); 1302#endif 1303 1304 status = KMP_DISPATCH_NEXT(&loc, gtid, NULL, &lb, &ub, &stride); 1305 if (status) { 1306 KMP_DEBUG_ASSERT(stride == 1); 1307 KMP_DEBUG_ASSERT(lb > 0); 1308 KMP_ASSERT(lb == ub); 1309 } else { 1310 lb = 0; 1311 } 1312 1313 KA_TRACE( 1314 20, ("GOMP_sections_next exit: T#%d returning %u\n", gtid, (unsigned)lb)); 1315 return (unsigned)lb; 1316} 1317 1318void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START)( 1319 void (*task)(void *), void *data, unsigned num_threads, unsigned count) { 1320 int gtid = __kmp_entry_gtid(); 1321 1322#if OMPT_SUPPORT 1323 ompt_frame_t *parent_frame; 1324 1325 if (ompt_enabled.enabled) { 1326 __ompt_get_task_info_internal(0, NULL, NULL, &parent_frame, NULL, NULL); 1327 parent_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1328 OMPT_STORE_RETURN_ADDRESS(gtid); 1329 } 1330#endif 1331 1332 MKLOC(loc, "GOMP_parallel_sections_start"); 1333 KA_TRACE(20, ("GOMP_parallel_sections_start: T#%d\n", gtid)); 1334 1335 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1336 if (num_threads != 0) { 1337 __kmp_push_num_threads(&loc, gtid, num_threads); 1338 } 1339 __kmp_GOMP_fork_call(&loc, gtid, task, 1340 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1341 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1342 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1343 } else { 1344 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1345 } 1346 1347#if OMPT_SUPPORT 1348 if (ompt_enabled.enabled) { 1349 parent_frame->enter_frame = ompt_data_none; 1350 } 1351#endif 1352 1353 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1354 1355 KA_TRACE(20, ("GOMP_parallel_sections_start exit: T#%d\n", gtid)); 1356} 1357 1358void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END)(void) { 1359 int gtid = __kmp_get_gtid(); 1360 KA_TRACE(20, ("GOMP_sections_end: T#%d\n", gtid)) 1361 1362#if OMPT_SUPPORT 1363 ompt_frame_t *ompt_frame; 1364 if (ompt_enabled.enabled) { 1365 __ompt_get_task_info_internal(0, NULL, NULL, &ompt_frame, NULL, NULL); 1366 ompt_frame->enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1367 OMPT_STORE_RETURN_ADDRESS(gtid); 1368 } 1369#endif 1370 __kmp_barrier(bs_plain_barrier, gtid, FALSE, 0, NULL, NULL); 1371#if OMPT_SUPPORT 1372 if (ompt_enabled.enabled) { 1373 ompt_frame->enter_frame = ompt_data_none; 1374 } 1375#endif 1376 1377 KA_TRACE(20, ("GOMP_sections_end exit: T#%d\n", gtid)) 1378} 1379 1380void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT)(void) { 1381 KA_TRACE(20, ("GOMP_sections_end_nowait: T#%d\n", __kmp_get_gtid())) 1382} 1383 1384// libgomp has an empty function for GOMP_taskyield as of 2013-10-10 1385void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKYIELD)(void) { 1386 KA_TRACE(20, ("GOMP_taskyield: T#%d\n", __kmp_get_gtid())) 1387 return; 1388} 1389 1390void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL)(void (*task)(void *), 1391 void *data, 1392 unsigned num_threads, 1393 unsigned int flags) { 1394 int gtid = __kmp_entry_gtid(); 1395 MKLOC(loc, "GOMP_parallel"); 1396 KA_TRACE(20, ("GOMP_parallel: T#%d\n", gtid)); 1397 1398#if OMPT_SUPPORT 1399 ompt_task_info_t *parent_task_info, *task_info; 1400 if (ompt_enabled.enabled) { 1401 parent_task_info = __ompt_get_task_info_object(0); 1402 parent_task_info->frame.enter_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1403 OMPT_STORE_RETURN_ADDRESS(gtid); 1404 } 1405#endif 1406 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1407 if (num_threads != 0) { 1408 __kmp_push_num_threads(&loc, gtid, num_threads); 1409 } 1410 if (flags != 0) { 1411 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); 1412 } 1413 __kmp_GOMP_fork_call(&loc, gtid, task, 1414 (microtask_t)__kmp_GOMP_microtask_wrapper, 2, task, 1415 data); 1416 } else { 1417 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1418 } 1419#if OMPT_SUPPORT 1420 if (ompt_enabled.enabled) { 1421 task_info = __ompt_get_task_info_object(0); 1422 task_info->frame.exit_frame.ptr = OMPT_GET_FRAME_ADDRESS(0); 1423 } 1424#endif 1425 task(data); 1426#if OMPT_SUPPORT 1427 if (ompt_enabled.enabled) { 1428 OMPT_STORE_RETURN_ADDRESS(gtid); 1429 } 1430#endif 1431 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1432#if OMPT_SUPPORT 1433 if (ompt_enabled.enabled) { 1434 task_info->frame.exit_frame = ompt_data_none; 1435 parent_task_info->frame.enter_frame = ompt_data_none; 1436 } 1437#endif 1438} 1439 1440void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_SECTIONS)(void (*task)(void *), 1441 void *data, 1442 unsigned num_threads, 1443 unsigned count, 1444 unsigned flags) { 1445 int gtid = __kmp_entry_gtid(); 1446 MKLOC(loc, "GOMP_parallel_sections"); 1447 KA_TRACE(20, ("GOMP_parallel_sections: T#%d\n", gtid)); 1448 1449#if OMPT_SUPPORT 1450 OMPT_STORE_RETURN_ADDRESS(gtid); 1451#endif 1452 1453 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { 1454 if (num_threads != 0) { 1455 __kmp_push_num_threads(&loc, gtid, num_threads); 1456 } 1457 if (flags != 0) { 1458 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); 1459 } 1460 __kmp_GOMP_fork_call(&loc, gtid, task, 1461 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, 9, 1462 task, data, num_threads, &loc, kmp_nm_dynamic_chunked, 1463 (kmp_int)1, (kmp_int)count, (kmp_int)1, (kmp_int)1); 1464 } else { 1465 __kmp_GOMP_serialized_parallel(&loc, gtid, task); 1466 } 1467 1468#if OMPT_SUPPORT 1469 OMPT_STORE_RETURN_ADDRESS(gtid); 1470#endif 1471 1472 KMP_DISPATCH_INIT(&loc, gtid, kmp_nm_dynamic_chunked, 1, count, 1, 1, TRUE); 1473 1474 task(data); 1475 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); 1476 KA_TRACE(20, ("GOMP_parallel_sections exit: T#%d\n", gtid)); 1477} 1478 1479#define PARALLEL_LOOP(func, schedule, ompt_pre, ompt_post) \ 1480 void func(void (*task)(void *), void *data, unsigned num_threads, long lb, \ 1481 long ub, long str, long chunk_sz, unsigned flags) { \ 1482 int gtid = __kmp_entry_gtid(); \ 1483 MKLOC(loc, KMP_STR(func)); \ 1484 KA_TRACE( \ 1485 20, \ 1486 (KMP_STR( \ 1487 func) ": T#%d, lb 0x%lx, ub 0x%lx, str 0x%lx, chunk_sz 0x%lx\n", \ 1488 gtid, lb, ub, str, chunk_sz)); \ 1489 \ 1490 ompt_pre(); \ 1491 if (__kmpc_ok_to_fork(&loc) && (num_threads != 1)) { \ 1492 if (num_threads != 0) { \ 1493 __kmp_push_num_threads(&loc, gtid, num_threads); \ 1494 } \ 1495 if (flags != 0) { \ 1496 __kmp_push_proc_bind(&loc, gtid, (kmp_proc_bind_t)flags); \ 1497 } \ 1498 __kmp_GOMP_fork_call(&loc, gtid, task, \ 1499 (microtask_t)__kmp_GOMP_parallel_microtask_wrapper, \ 1500 9, task, data, num_threads, &loc, (schedule), lb, \ 1501 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz); \ 1502 } else { \ 1503 __kmp_GOMP_serialized_parallel(&loc, gtid, task); \ 1504 } \ 1505 \ 1506 IF_OMPT_SUPPORT(OMPT_STORE_RETURN_ADDRESS(gtid);) \ 1507 KMP_DISPATCH_INIT(&loc, gtid, (schedule), lb, \ 1508 (str > 0) ? (ub - 1) : (ub + 1), str, chunk_sz, \ 1509 (schedule) != kmp_sch_static); \ 1510 task(data); \ 1511 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_END)(); \ 1512 ompt_post(); \ 1513 \ 1514 KA_TRACE(20, (KMP_STR(func) " exit: T#%d\n", gtid)); \ 1515 } 1516 1517PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC), 1518 kmp_sch_static, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1519PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC), 1520 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1521PARALLEL_LOOP( 1522 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED), 1523 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1524PARALLEL_LOOP( 1525 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC), 1526 kmp_sch_dynamic_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1527PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED), 1528 kmp_sch_guided_chunked, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1529PARALLEL_LOOP(KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME), 1530 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1531PARALLEL_LOOP( 1532 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME), 1533 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1534PARALLEL_LOOP( 1535 KMP_EXPAND_NAME(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME), 1536 kmp_sch_runtime, OMPT_LOOP_PRE, OMPT_LOOP_POST) 1537 1538void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_START)(void) { 1539 int gtid = __kmp_entry_gtid(); 1540 MKLOC(loc, "GOMP_taskgroup_start"); 1541 KA_TRACE(20, ("GOMP_taskgroup_start: T#%d\n", gtid)); 1542 1543#if OMPT_SUPPORT 1544 if (ompt_enabled.enabled) 1545 OMPT_STORE_RETURN_ADDRESS(gtid); 1546#endif 1547 1548 __kmpc_taskgroup(&loc, gtid); 1549 1550 return; 1551} 1552 1553void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKGROUP_END)(void) { 1554 int gtid = __kmp_get_gtid(); 1555 MKLOC(loc, "GOMP_taskgroup_end"); 1556 KA_TRACE(20, ("GOMP_taskgroup_end: T#%d\n", gtid)); 1557 1558#if OMPT_SUPPORT 1559 if (ompt_enabled.enabled) 1560 OMPT_STORE_RETURN_ADDRESS(gtid); 1561#endif 1562 1563 __kmpc_end_taskgroup(&loc, gtid); 1564 1565 return; 1566} 1567 1568static kmp_int32 __kmp_gomp_to_omp_cancellation_kind(int gomp_kind) { 1569 kmp_int32 cncl_kind = 0; 1570 switch (gomp_kind) { 1571 case 1: 1572 cncl_kind = cancel_parallel; 1573 break; 1574 case 2: 1575 cncl_kind = cancel_loop; 1576 break; 1577 case 4: 1578 cncl_kind = cancel_sections; 1579 break; 1580 case 8: 1581 cncl_kind = cancel_taskgroup; 1582 break; 1583 } 1584 return cncl_kind; 1585} 1586 1587// Return true if cancellation should take place, false otherwise 1588bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCELLATION_POINT)(int which) { 1589 int gtid = __kmp_get_gtid(); 1590 MKLOC(loc, "GOMP_cancellation_point"); 1591 KA_TRACE(20, ("GOMP_cancellation_point: T#%d which:%d\n", gtid, which)); 1592 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1593 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1594} 1595 1596// Return true if cancellation should take place, false otherwise 1597bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_CANCEL)(int which, bool do_cancel) { 1598 int gtid = __kmp_get_gtid(); 1599 MKLOC(loc, "GOMP_cancel"); 1600 KA_TRACE(20, ("GOMP_cancel: T#%d which:%d do_cancel:%d\n", gtid, which, 1601 (int)do_cancel)); 1602 kmp_int32 cncl_kind = __kmp_gomp_to_omp_cancellation_kind(which); 1603 1604 if (do_cancel == FALSE) { 1605 return __kmpc_cancellationpoint(&loc, gtid, cncl_kind); 1606 } else { 1607 return __kmpc_cancel(&loc, gtid, cncl_kind); 1608 } 1609} 1610 1611// Return true if cancellation should take place, false otherwise 1612bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_BARRIER_CANCEL)(void) { 1613 int gtid = __kmp_get_gtid(); 1614 KA_TRACE(20, ("GOMP_barrier_cancel: T#%d\n", gtid)); 1615 return __kmp_barrier_gomp_cancel(gtid); 1616} 1617 1618// Return true if cancellation should take place, false otherwise 1619bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL)(void) { 1620 int gtid = __kmp_get_gtid(); 1621 KA_TRACE(20, ("GOMP_sections_end_cancel: T#%d\n", gtid)); 1622 return __kmp_barrier_gomp_cancel(gtid); 1623} 1624 1625// Return true if cancellation should take place, false otherwise 1626bool KMP_EXPAND_NAME(KMP_API_NAME_GOMP_LOOP_END_CANCEL)(void) { 1627 int gtid = __kmp_get_gtid(); 1628 KA_TRACE(20, ("GOMP_loop_end_cancel: T#%d\n", gtid)); 1629 return __kmp_barrier_gomp_cancel(gtid); 1630} 1631 1632// All target functions are empty as of 2014-05-29 1633void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET)(int device, void (*fn)(void *), 1634 const void *openmp_target, 1635 size_t mapnum, void **hostaddrs, 1636 size_t *sizes, 1637 unsigned char *kinds) { 1638 return; 1639} 1640 1641void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_DATA)( 1642 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1643 size_t *sizes, unsigned char *kinds) { 1644 return; 1645} 1646 1647void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_END_DATA)(void) { return; } 1648 1649void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TARGET_UPDATE)( 1650 int device, const void *openmp_target, size_t mapnum, void **hostaddrs, 1651 size_t *sizes, unsigned char *kinds) { 1652 return; 1653} 1654 1655void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TEAMS)(unsigned int num_teams, 1656 unsigned int thread_limit) { 1657 return; 1658} 1659 1660// Task duplication function which copies src to dest (both are 1661// preallocated task structures) 1662static void __kmp_gomp_task_dup(kmp_task_t *dest, kmp_task_t *src, 1663 kmp_int32 last_private) { 1664 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(src); 1665 if (taskdata->td_copy_func) { 1666 (taskdata->td_copy_func)(dest->shareds, src->shareds); 1667 } 1668} 1669 1670#ifdef __cplusplus 1671} // extern "C" 1672#endif 1673 1674template <typename T> 1675void __GOMP_taskloop(void (*func)(void *), void *data, 1676 void (*copy_func)(void *, void *), long arg_size, 1677 long arg_align, unsigned gomp_flags, 1678 unsigned long num_tasks, int priority, T start, T end, 1679 T step) { 1680 typedef void (*p_task_dup_t)(kmp_task_t *, kmp_task_t *, kmp_int32); 1681 MKLOC(loc, "GOMP_taskloop"); 1682 int sched; 1683 T *loop_bounds; 1684 int gtid = __kmp_entry_gtid(); 1685 kmp_int32 flags = 0; 1686 int if_val = gomp_flags & (1u << 10); 1687 int nogroup = gomp_flags & (1u << 11); 1688 int up = gomp_flags & (1u << 8); 1689 p_task_dup_t task_dup = NULL; 1690 kmp_tasking_flags_t *input_flags = (kmp_tasking_flags_t *)&flags; 1691#ifdef KMP_DEBUG 1692 { 1693 char *buff; 1694 buff = __kmp_str_format( 1695 "GOMP_taskloop: T#%%d: func:%%p data:%%p copy_func:%%p " 1696 "arg_size:%%ld arg_align:%%ld gomp_flags:0x%%x num_tasks:%%lu " 1697 "priority:%%d start:%%%s end:%%%s step:%%%s\n", 1698 traits_t<T>::spec, traits_t<T>::spec, traits_t<T>::spec); 1699 KA_TRACE(20, (buff, gtid, func, data, copy_func, arg_size, arg_align, 1700 gomp_flags, num_tasks, priority, start, end, step)); 1701 __kmp_str_free(&buff); 1702 } 1703#endif 1704 KMP_ASSERT((size_t)arg_size >= 2 * sizeof(T)); 1705 KMP_ASSERT(arg_align > 0); 1706 // The low-order bit is the "untied" flag 1707 if (!(gomp_flags & 1)) { 1708 input_flags->tiedness = 1; 1709 } 1710 // The second low-order bit is the "final" flag 1711 if (gomp_flags & 2) { 1712 input_flags->final = 1; 1713 } 1714 // Negative step flag 1715 if (!up) { 1716 // If step is flagged as negative, but isn't properly sign extended 1717 // Then manually sign extend it. Could be a short, int, char embedded 1718 // in a long. So cannot assume any cast. 1719 if (step > 0) { 1720 for (int i = sizeof(T) * CHAR_BIT - 1; i >= 0L; --i) { 1721 // break at the first 1 bit 1722 if (step & ((T)1 << i)) 1723 break; 1724 step |= ((T)1 << i); 1725 } 1726 } 1727 } 1728 input_flags->native = 1; 1729 // Figure out if none/grainsize/num_tasks clause specified 1730 if (num_tasks > 0) { 1731 if (gomp_flags & (1u << 9)) 1732 sched = 1; // grainsize specified 1733 else 1734 sched = 2; // num_tasks specified 1735 // neither grainsize nor num_tasks specified 1736 } else { 1737 sched = 0; 1738 } 1739 1740 // __kmp_task_alloc() sets up all other flags 1741 kmp_task_t *task = 1742 __kmp_task_alloc(&loc, gtid, input_flags, sizeof(kmp_task_t), 1743 arg_size + arg_align - 1, (kmp_routine_entry_t)func); 1744 kmp_taskdata_t *taskdata = KMP_TASK_TO_TASKDATA(task); 1745 taskdata->td_copy_func = copy_func; 1746 taskdata->td_size_loop_bounds = sizeof(T); 1747 1748 // re-align shareds if needed and setup firstprivate copy constructors 1749 // through the task_dup mechanism 1750 task->shareds = (void *)((((size_t)task->shareds) + arg_align - 1) / 1751 arg_align * arg_align); 1752 if (copy_func) { 1753 task_dup = __kmp_gomp_task_dup; 1754 } 1755 KMP_MEMCPY(task->shareds, data, arg_size); 1756 1757 loop_bounds = (T *)task->shareds; 1758 loop_bounds[0] = start; 1759 loop_bounds[1] = end + (up ? -1 : 1); 1760 __kmpc_taskloop(&loc, gtid, task, if_val, (kmp_uint64 *)&(loop_bounds[0]), 1761 (kmp_uint64 *)&(loop_bounds[1]), (kmp_int64)step, nogroup, 1762 sched, (kmp_uint64)num_tasks, (void *)task_dup); 1763} 1764 1765// 4 byte version of GOMP_doacross_post 1766// This verison needs to create a temporary array which converts 4 byte 1767// integers into 8 byte integers 1768template <typename T, bool need_conversion = (sizeof(long) == 4)> 1769void __kmp_GOMP_doacross_post(T *count); 1770 1771template <> void __kmp_GOMP_doacross_post<long, true>(long *count) { 1772 int gtid = __kmp_entry_gtid(); 1773 kmp_info_t *th = __kmp_threads[gtid]; 1774 MKLOC(loc, "GOMP_doacross_post"); 1775 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1776 kmp_int64 *vec = 1777 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1778 for (kmp_int64 i = 0; i < num_dims; ++i) { 1779 vec[i] = (kmp_int64)count[i]; 1780 } 1781 __kmpc_doacross_post(&loc, gtid, vec); 1782 __kmp_thread_free(th, vec); 1783} 1784 1785// 8 byte versions of GOMP_doacross_post 1786// This version can just pass in the count array directly instead of creating 1787// a temporary array 1788template <> void __kmp_GOMP_doacross_post<long, false>(long *count) { 1789 int gtid = __kmp_entry_gtid(); 1790 MKLOC(loc, "GOMP_doacross_post"); 1791 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1792} 1793 1794template <typename T> void __kmp_GOMP_doacross_wait(T first, va_list args) { 1795 int gtid = __kmp_entry_gtid(); 1796 kmp_info_t *th = __kmp_threads[gtid]; 1797 MKLOC(loc, "GOMP_doacross_wait"); 1798 kmp_int64 num_dims = th->th.th_dispatch->th_doacross_info[0]; 1799 kmp_int64 *vec = 1800 (kmp_int64 *)__kmp_thread_malloc(th, sizeof(kmp_int64) * num_dims); 1801 vec[0] = (kmp_int64)first; 1802 for (kmp_int64 i = 1; i < num_dims; ++i) { 1803 T item = va_arg(args, T); 1804 vec[i] = (kmp_int64)item; 1805 } 1806 __kmpc_doacross_wait(&loc, gtid, vec); 1807 __kmp_thread_free(th, vec); 1808 return; 1809} 1810 1811#ifdef __cplusplus 1812extern "C" { 1813#endif // __cplusplus 1814 1815void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP)( 1816 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1817 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1818 int priority, long start, long end, long step) { 1819 __GOMP_taskloop<long>(func, data, copy_func, arg_size, arg_align, gomp_flags, 1820 num_tasks, priority, start, end, step); 1821} 1822 1823void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_TASKLOOP_ULL)( 1824 void (*func)(void *), void *data, void (*copy_func)(void *, void *), 1825 long arg_size, long arg_align, unsigned gomp_flags, unsigned long num_tasks, 1826 int priority, unsigned long long start, unsigned long long end, 1827 unsigned long long step) { 1828 __GOMP_taskloop<unsigned long long>(func, data, copy_func, arg_size, 1829 arg_align, gomp_flags, num_tasks, 1830 priority, start, end, step); 1831} 1832 1833void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_POST)(long *count) { 1834 __kmp_GOMP_doacross_post(count); 1835} 1836 1837void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_WAIT)(long first, ...) { 1838 va_list args; 1839 va_start(args, first); 1840 __kmp_GOMP_doacross_wait<long>(first, args); 1841 va_end(args); 1842} 1843 1844void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_POST)( 1845 unsigned long long *count) { 1846 int gtid = __kmp_entry_gtid(); 1847 MKLOC(loc, "GOMP_doacross_ull_post"); 1848 __kmpc_doacross_post(&loc, gtid, RCAST(kmp_int64 *, count)); 1849} 1850 1851void KMP_EXPAND_NAME(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT)( 1852 unsigned long long first, ...) { 1853 va_list args; 1854 va_start(args, first); 1855 __kmp_GOMP_doacross_wait<unsigned long long>(first, args); 1856 va_end(args); 1857} 1858 1859/* The following sections of code create aliases for the GOMP_* functions, then 1860 create versioned symbols using the assembler directive .symver. This is only 1861 pertinent for ELF .so library. The KMP_VERSION_SYMBOL macro is defined in 1862 kmp_os.h */ 1863 1864#ifdef KMP_USE_VERSION_SYMBOLS 1865// GOMP_1.0 versioned symbols 1866KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_END, 10, "GOMP_1.0"); 1867KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ATOMIC_START, 10, "GOMP_1.0"); 1868KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER, 10, "GOMP_1.0"); 1869KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_END, 10, "GOMP_1.0"); 1870KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_END, 10, "GOMP_1.0"); 1871KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_NAME_START, 10, "GOMP_1.0"); 1872KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CRITICAL_START, 10, "GOMP_1.0"); 1873KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1874KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DYNAMIC_START, 10, "GOMP_1.0"); 1875KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END, 10, "GOMP_1.0"); 1876KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_NOWAIT, 10, "GOMP_1.0"); 1877KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_NEXT, 10, "GOMP_1.0"); 1878KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_GUIDED_START, 10, "GOMP_1.0"); 1879KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_NEXT, 10, "GOMP_1.0"); 1880KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_DYNAMIC_START, 10, 1881 "GOMP_1.0"); 1882KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_NEXT, 10, "GOMP_1.0"); 1883KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_GUIDED_START, 10, "GOMP_1.0"); 1884KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_NEXT, 10, "GOMP_1.0"); 1885KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_RUNTIME_START, 10, 1886 "GOMP_1.0"); 1887KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_NEXT, 10, "GOMP_1.0"); 1888KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ORDERED_STATIC_START, 10, "GOMP_1.0"); 1889KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_NEXT, 10, "GOMP_1.0"); 1890KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_RUNTIME_START, 10, "GOMP_1.0"); 1891KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_NEXT, 10, "GOMP_1.0"); 1892KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_STATIC_START, 10, "GOMP_1.0"); 1893KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_END, 10, "GOMP_1.0"); 1894KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_ORDERED_START, 10, "GOMP_1.0"); 1895KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_END, 10, "GOMP_1.0"); 1896KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC_START, 10, 1897 "GOMP_1.0"); 1898KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED_START, 10, 1899 "GOMP_1.0"); 1900KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME_START, 10, 1901 "GOMP_1.0"); 1902KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC_START, 10, 1903 "GOMP_1.0"); 1904KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS_START, 10, "GOMP_1.0"); 1905KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_START, 10, "GOMP_1.0"); 1906KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END, 10, "GOMP_1.0"); 1907KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_NOWAIT, 10, "GOMP_1.0"); 1908KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_NEXT, 10, "GOMP_1.0"); 1909KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_START, 10, "GOMP_1.0"); 1910KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_END, 10, "GOMP_1.0"); 1911KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_COPY_START, 10, "GOMP_1.0"); 1912KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SINGLE_START, 10, "GOMP_1.0"); 1913 1914// GOMP_2.0 versioned symbols 1915KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASK, 20, "GOMP_2.0"); 1916KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKWAIT, 20, "GOMP_2.0"); 1917KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_NEXT, 20, "GOMP_2.0"); 1918KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DYNAMIC_START, 20, "GOMP_2.0"); 1919KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_NEXT, 20, "GOMP_2.0"); 1920KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_GUIDED_START, 20, "GOMP_2.0"); 1921KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_NEXT, 20, 1922 "GOMP_2.0"); 1923KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_DYNAMIC_START, 20, 1924 "GOMP_2.0"); 1925KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_NEXT, 20, 1926 "GOMP_2.0"); 1927KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_GUIDED_START, 20, 1928 "GOMP_2.0"); 1929KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_NEXT, 20, 1930 "GOMP_2.0"); 1931KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_RUNTIME_START, 20, 1932 "GOMP_2.0"); 1933KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_NEXT, 20, 1934 "GOMP_2.0"); 1935KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_ORDERED_STATIC_START, 20, 1936 "GOMP_2.0"); 1937KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_NEXT, 20, "GOMP_2.0"); 1938KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_RUNTIME_START, 20, "GOMP_2.0"); 1939KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_NEXT, 20, "GOMP_2.0"); 1940KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_STATIC_START, 20, "GOMP_2.0"); 1941 1942// GOMP_3.0 versioned symbols 1943KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKYIELD, 30, "GOMP_3.0"); 1944 1945// GOMP_4.0 versioned symbols 1946KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL, 40, "GOMP_4.0"); 1947KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_SECTIONS, 40, "GOMP_4.0"); 1948KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_DYNAMIC, 40, "GOMP_4.0"); 1949KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_GUIDED, 40, "GOMP_4.0"); 1950KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_RUNTIME, 40, "GOMP_4.0"); 1951KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_STATIC, 40, "GOMP_4.0"); 1952KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_START, 40, "GOMP_4.0"); 1953KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKGROUP_END, 40, "GOMP_4.0"); 1954KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_BARRIER_CANCEL, 40, "GOMP_4.0"); 1955KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCEL, 40, "GOMP_4.0"); 1956KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_CANCELLATION_POINT, 40, "GOMP_4.0"); 1957KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_END_CANCEL, 40, "GOMP_4.0"); 1958KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_SECTIONS_END_CANCEL, 40, "GOMP_4.0"); 1959KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET, 40, "GOMP_4.0"); 1960KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_DATA, 40, "GOMP_4.0"); 1961KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_END_DATA, 40, "GOMP_4.0"); 1962KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TARGET_UPDATE, 40, "GOMP_4.0"); 1963KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TEAMS, 40, "GOMP_4.0"); 1964 1965// GOMP_4.5 versioned symbols 1966KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP, 45, "GOMP_4.5"); 1967KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_TASKLOOP_ULL, 45, "GOMP_4.5"); 1968KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_POST, 45, "GOMP_4.5"); 1969KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_WAIT, 45, "GOMP_4.5"); 1970KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_STATIC_START, 45, 1971 "GOMP_4.5"); 1972KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_DYNAMIC_START, 45, 1973 "GOMP_4.5"); 1974KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_GUIDED_START, 45, 1975 "GOMP_4.5"); 1976KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_DOACROSS_RUNTIME_START, 45, 1977 "GOMP_4.5"); 1978KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_POST, 45, "GOMP_4.5"); 1979KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_DOACROSS_ULL_WAIT, 45, "GOMP_4.5"); 1980KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_STATIC_START, 45, 1981 "GOMP_4.5"); 1982KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_DYNAMIC_START, 45, 1983 "GOMP_4.5"); 1984KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_GUIDED_START, 45, 1985 "GOMP_4.5"); 1986KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_DOACROSS_RUNTIME_START, 45, 1987 "GOMP_4.5"); 1988KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_START, 45, 1989 "GOMP_4.5"); 1990KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_DYNAMIC_NEXT, 45, 1991 "GOMP_4.5"); 1992KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_START, 45, 1993 "GOMP_4.5"); 1994KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_GUIDED_NEXT, 45, 1995 "GOMP_4.5"); 1996KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_START, 45, 1997 "GOMP_4.5"); 1998KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_DYNAMIC_NEXT, 45, 1999 "GOMP_4.5"); 2000KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_START, 45, 2001 "GOMP_4.5"); 2002KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_GUIDED_NEXT, 45, 2003 "GOMP_4.5"); 2004KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_DYNAMIC, 45, 2005 "GOMP_4.5"); 2006KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_GUIDED, 45, 2007 "GOMP_4.5"); 2008 2009// GOMP_5.0 versioned symbols 2010KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_NEXT, 50, 2011 "GOMP_5.0"); 2012KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_MAYBE_NONMONOTONIC_RUNTIME_START, 50, 2013 "GOMP_5.0"); 2014KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_NEXT, 50, 2015 "GOMP_5.0"); 2016KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_NONMONOTONIC_RUNTIME_START, 50, 2017 "GOMP_5.0"); 2018KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_NEXT, 2019 50, "GOMP_5.0"); 2020KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_MAYBE_NONMONOTONIC_RUNTIME_START, 2021 50, "GOMP_5.0"); 2022KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_NEXT, 50, 2023 "GOMP_5.0"); 2024KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_LOOP_ULL_NONMONOTONIC_RUNTIME_START, 50, 2025 "GOMP_5.0"); 2026KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_NONMONOTONIC_RUNTIME, 50, 2027 "GOMP_5.0"); 2028KMP_VERSION_SYMBOL(KMP_API_NAME_GOMP_PARALLEL_LOOP_MAYBE_NONMONOTONIC_RUNTIME, 2029 50, "GOMP_5.0"); 2030 2031#endif // KMP_USE_VERSION_SYMBOLS 2032 2033#ifdef __cplusplus 2034} // extern "C" 2035#endif // __cplusplus 2036