1/* Cilk_abi.c -*-C++-*- 2 * 3 ************************************************************************* 4 * 5 * @copyright 6 * Copyright (C) 2010-2013, Intel Corporation 7 * All rights reserved. 8 * 9 * @copyright 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 14 * * Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * * Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in 18 * the documentation and/or other materials provided with the 19 * distribution. 20 * * Neither the name of Intel Corporation nor the names of its 21 * contributors may be used to endorse or promote products derived 22 * from this software without specific prior written permission. 23 * 24 * @copyright 25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 26 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 27 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 28 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 29 * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 30 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 31 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 32 * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 33 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 35 * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 36 * POSSIBILITY OF SUCH DAMAGE. 37 * 38 **************************************************************************/ 39 40/** 41 * @file cilk-abi.c 42 * 43 * @brief cilk-abi.c implements all of the entrypoints to the Intel Cilk 44 * Plus runtime. 45 */ 46 47/* 48 * Define this macro so that compiliation of this file generates the 49 * non-inlined versions of certain functions in cilk_api.h. 50 */ 51#include "internal/abi.h" 52#include "cilk/cilk_api.h" 53#include "cilk/cilk_undocumented.h" 54#include "cilktools/cilkscreen.h" 55 56#include "global_state.h" 57#include "os.h" 58#include "os_mutex.h" 59#include "bug.h" 60#include "local_state.h" 61#include "full_frame.h" 62#include "pedigrees.h" 63#include "scheduler.h" 64#include "sysdep.h" 65#include "except.h" 66#include "cilk_malloc.h" 67#include "record-replay.h" 68 69#include <errno.h> 70#include <string.h> 71#include <stdlib.h> 72 73#ifdef _MSC_VER 74/* Some versions of icc don't support limits.h on Linux if 75 gcc 4.3 or newer is installed. */ 76#include <limits.h> 77 78/* Declare _ReturnAddress compiler intrinsic */ 79void * _ReturnAddress(void); 80#pragma intrinsic(_ReturnAddress) 81 82#include "sysdep-win.h" // Needed for sysdep_init_module() 83#endif /* _WIN32 */ 84 85#include "metacall_impl.h" 86#include "reducer_impl.h" 87#include "cilk-ittnotify.h" 88#include "cilk-tbb-interop.h" 89 90#define TBB_INTEROP_DATA_DELAYED_UNTIL_BIND (void *)-1 91 92/** 93 * __cilkrts_bind_thread is a versioned entrypoint. The runtime should be 94 * exporting copies of __cilkrts_bind_version for the current and all previous 95 * versions of the ABI. 96 * 97 * This macro should always be set to generate a version to match the current 98 * version; __CILKRTS_ABI_VERSION. 99 */ 100#define BIND_THREAD_RTN __cilkrts_bind_thread_1 101 102static inline 103void enter_frame_internal(__cilkrts_stack_frame *sf, uint32_t version) 104{ 105 __cilkrts_worker *w = __cilkrts_get_tls_worker(); 106 if (w == 0) { /* slow path */ 107 w = BIND_THREAD_RTN(); 108 109 sf->flags = CILK_FRAME_LAST | (version << 24); 110 CILK_ASSERT((sf->flags & CILK_FRAME_FLAGS_MASK) == CILK_FRAME_LAST); 111 } else { 112 sf->flags = (version << 24); 113 CILK_ASSERT((sf->flags & CILK_FRAME_FLAGS_MASK) == 0); 114 } 115 sf->call_parent = w->current_stack_frame; 116 sf->worker = w; 117 w->current_stack_frame = sf; 118} 119 120CILK_ABI_VOID __cilkrts_enter_frame(__cilkrts_stack_frame *sf) 121{ 122 enter_frame_internal(sf, 0); 123} 124 125CILK_ABI_VOID __cilkrts_enter_frame_1(__cilkrts_stack_frame *sf) 126{ 127 enter_frame_internal(sf, 1); 128 sf->reserved = 0; 129} 130 131static inline 132void enter_frame_fast_internal(__cilkrts_stack_frame *sf, uint32_t version) 133{ 134 __cilkrts_worker *w = __cilkrts_get_tls_worker_fast(); 135 sf->flags = version << 24; 136 sf->call_parent = w->current_stack_frame; 137 sf->worker = w; 138 w->current_stack_frame = sf; 139} 140 141CILK_ABI_VOID __cilkrts_enter_frame_fast(__cilkrts_stack_frame *sf) 142{ 143 enter_frame_fast_internal(sf, 0); 144} 145 146CILK_ABI_VOID __cilkrts_enter_frame_fast_1(__cilkrts_stack_frame *sf) 147{ 148 enter_frame_fast_internal(sf, 1); 149 sf->reserved = 0; 150} 151 152/** 153 * A component of the THE protocol. __cilkrts_undo_detach checks whether 154 * this frame's parent has been stolen. If it hasn't, the frame can return 155 * normally. If the parent has been stolen, of if we suspect it might be, 156 * then __cilkrts_leave_frame() needs to call into the runtime. 157 * 158 * @note __cilkrts_undo_detach() is comparing the exception pointer against 159 * the tail pointer. The exception pointer is modified when another worker 160 * is considering whether it can steal a frame. The head pointer is updated 161 * to match when the worker lock is taken out and the thief is sure that 162 * it can complete the steal. If the steal cannot be completed, the thief 163 * will restore the exception pointer. 164 * 165 * @return true if undo-detach failed. 166 */ 167static int __cilkrts_undo_detach(__cilkrts_stack_frame *sf) 168{ 169 __cilkrts_worker *w = sf->worker; 170 __cilkrts_stack_frame *volatile *t = w->tail; 171 172/* DBGPRINTF("%d - __cilkrts_undo_detach - sf %p\n", w->self, sf); */ 173 174 --t; 175 w->tail = t; 176 /* On x86 the __sync_fetch_and_<op> family includes a 177 full memory barrier. In theory the sequence in the 178 second branch of the #if should be faster, but on 179 most x86 it is not. */ 180#if defined __i386__ || defined __x86_64__ 181 __sync_fetch_and_and(&sf->flags, ~CILK_FRAME_DETACHED); 182#else 183 __cilkrts_fence(); /* membar #StoreLoad */ 184 sf->flags &= ~CILK_FRAME_DETACHED; 185#endif 186 187 return __builtin_expect(t < w->exc, 0); 188} 189 190CILK_ABI_VOID __cilkrts_leave_frame(__cilkrts_stack_frame *sf) 191{ 192 __cilkrts_worker *w = sf->worker; 193 194/* DBGPRINTF("%d-%p __cilkrts_leave_frame - sf %p, flags: %x\n", w->self, GetWorkerFiber(w), sf, sf->flags); */ 195 196#ifdef _WIN32 197 /* if leave frame was called from our unwind handler, leave_frame should 198 proceed no further. */ 199 if (sf->flags & CILK_FRAME_UNWINDING) 200 { 201/* DBGPRINTF("%d - __cilkrts_leave_frame - aborting due to UNWINDING flag\n", w->self); */ 202 203 // If this is the frame of a spawn helper (indicated by the 204 // CILK_FRAME_DETACHED flag) we must update the pedigree. The pedigree 205 // points to nodes allocated on the stack. Failing to update it will 206 // result in a accvio/segfault if the pedigree is walked. This must happen 207 // for all spawn helper frames, even if we're processing an exception 208 if ((sf->flags & CILK_FRAME_DETACHED)) 209 { 210 update_pedigree_on_leave_frame(w, sf); 211 } 212 return; 213 } 214#endif 215 216#if CILK_LIB_DEBUG 217 /* ensure the caller popped itself */ 218 CILK_ASSERT(w->current_stack_frame != sf); 219#endif 220 221 /* The exiting function should have checked for zero flags, 222 so there is no check for flags == 0 here. */ 223 224#if CILK_LIB_DEBUG 225 if (__builtin_expect(sf->flags & (CILK_FRAME_EXITING|CILK_FRAME_UNSYNCHED), 0)) 226 __cilkrts_bug("W%u: function exiting with invalid flags %02x\n", 227 w->self, sf->flags); 228#endif 229 230 /* Must return normally if (1) the active function was called 231 and not spawned, or (2) the parent has never been stolen. */ 232 if ((sf->flags & CILK_FRAME_DETACHED)) { 233/* DBGPRINTF("%d - __cilkrts_leave_frame - CILK_FRAME_DETACHED\n", w->self); */ 234 235#ifndef _WIN32 236 if (__builtin_expect(sf->flags & CILK_FRAME_EXCEPTING, 0)) { 237// Pedigree will be updated in __cilkrts_leave_frame. We need the 238// pedigree before the update for record/replay 239// update_pedigree_on_leave_frame(w, sf); 240 __cilkrts_return_exception(sf); 241 /* If return_exception returns the caller is attached. 242 leave_frame is called from a cleanup (destructor) 243 for the frame object. The caller will reraise the 244 exception. */ 245 return; 246 } 247#endif 248 249 // During replay, check whether w was the last worker to continue 250 replay_wait_for_steal_if_parent_was_stolen(w); 251 252 // Attempt to undo the detach 253 if (__builtin_expect(__cilkrts_undo_detach(sf), 0)) { 254 // The update of pedigree for leaving the frame occurs 255 // inside this call if it does not return. 256 __cilkrts_c_THE_exception_check(w, sf); 257 } 258 259 update_pedigree_on_leave_frame(w, sf); 260 261 /* This path is taken when undo-detach wins the race with stealing. 262 Otherwise this strand terminates and the caller will be resumed 263 via setjmp at sync. */ 264 if (__builtin_expect(sf->flags & CILK_FRAME_FLAGS_MASK, 0)) 265 __cilkrts_bug("W%u: frame won undo-detach race with flags %02x\n", 266 w->self, sf->flags); 267 268 return; 269 } 270 271#if CILK_LIB_DEBUG 272 sf->flags |= CILK_FRAME_EXITING; 273#endif 274 275 if (__builtin_expect(sf->flags & CILK_FRAME_LAST, 0)) 276 __cilkrts_c_return_from_initial(w); /* does return */ 277 else if (sf->flags & CILK_FRAME_STOLEN) 278 __cilkrts_return(w); /* does return */ 279 280/* DBGPRINTF("%d-%p __cilkrts_leave_frame - returning, StackBase: %p\n", w->self, GetWorkerFiber(w)); */ 281} 282 283/* Caller must have called setjmp. */ 284CILK_ABI_VOID __cilkrts_sync(__cilkrts_stack_frame *sf) 285{ 286 __cilkrts_worker *w = sf->worker; 287/* DBGPRINTF("%d-%p __cilkrts_sync - sf %p\n", w->self, GetWorkerFiber(w), sf); */ 288 if (__builtin_expect(!(sf->flags & CILK_FRAME_UNSYNCHED), 0)) 289 __cilkrts_bug("W%u: double sync %p\n", w->self, sf); 290#ifndef _WIN32 291 if (__builtin_expect(sf->flags & CILK_FRAME_EXCEPTING, 0)) { 292 __cilkrts_c_sync_except(w, sf); 293 } 294#endif 295 296 __cilkrts_c_sync(w, sf); 297} 298 299/* 300 * __cilkrts_get_sf 301 * 302 * Debugging aid to provide access to the current __cilkrts_stack_frame. 303 * 304 * Not documented! 305 */ 306 307CILK_API_VOID_PTR 308__cilkrts_get_sf(void) 309{ 310 __cilkrts_worker *w = __cilkrts_get_tls_worker(); 311 if (0 == w) 312 return NULL; 313 314 return w->current_stack_frame; 315} 316 317/* Call with global lock held */ 318static __cilkrts_worker *find_free_worker(global_state_t *g) 319{ 320 __cilkrts_worker *w = 0; 321 int i; 322 323 // Scan the non-system workers looking for one which is free so we can 324 // use it. 325 for (i = g->P - 1; i < g->total_workers; ++i) { 326 w = g->workers[i]; 327 CILK_ASSERT(WORKER_SYSTEM != w->l->type); 328 if (w->l->type == WORKER_FREE) { 329 w->l->type = WORKER_USER; 330 w->l->team = w; 331 return w; 332 } 333 } 334 335 // If we ran out of workers, create a new one. It doesn't actually belong 336 // to the Cilk global state so nobody will ever try to steal from it. 337 w = (__cilkrts_worker *)__cilkrts_malloc(sizeof(*w)); 338 __cilkrts_cilkscreen_ignore_block(w, w+1); 339 make_worker(g, -1, w); 340 w->l->type = WORKER_USER; 341 w->l->team = w; 342 return w; 343} 344 345/* 346 * __cilkrts_bind_thread 347 * 348 * Exported function to bind a thread to the runtime. 349 * 350 * This function name should always have a trailing suffix for the latest ABI 351 * version. This means that code built with a new compiler will not load 352 * against an old copy of the runtime. 353 * 354 * Symbols for the function called by code compiled with old versions of the 355 * compiler are created in an OS-specific manner: 356 * - On Windows the old symbols are defined in the cilk-exports.def linker 357 * definitions file as aliases of BIND_THREAD_RTN 358 * - On Linux aliased symbols are created for BIND_THREAD_RTN in this file 359 * - On MacOS the alternate entrypoints are implemented and simply call 360 * BIND_THREAD_RTN. 361 */ 362CILK_ABI_WORKER_PTR BIND_THREAD_RTN(void) 363{ 364 __cilkrts_worker *w; 365 int start_cilkscreen = 0; 366#ifdef USE_ITTNOTIFY 367 static int unique_obj; 368#endif 369 370 // Cannot set this pointer until after __cilkrts_init_internal() call: 371 global_state_t* g; 372 373 ITT_SYNC_CREATE (&unique_obj, "Initialization"); 374 ITT_SYNC_PREPARE(&unique_obj); 375 ITT_SYNC_ACQUIRED(&unique_obj); 376 377 378 /* 1: Initialize and start the Cilk runtime */ 379 __cilkrts_init_internal(1); 380 381 /* 382 * 2: Choose a worker for this thread (fail if none left). The table of 383 * user workers is protected by the global OS mutex lock. 384 */ 385 g = cilkg_get_global_state(); 386 global_os_mutex_lock(); 387 if (__builtin_expect(g->work_done, 0)) 388 __cilkrts_bug("Attempt to enter Cilk while Cilk is shutting down"); 389 w = find_free_worker(g); 390 CILK_ASSERT(w); 391 392 __cilkrts_set_tls_worker(w); 393 __cilkrts_cilkscreen_establish_worker(w); 394 { 395 full_frame *ff = __cilkrts_make_full_frame(w, 0); 396 397 ff->fiber_self = cilk_fiber_allocate_from_thread(); 398 CILK_ASSERT(ff->fiber_self); 399 400 cilk_fiber_set_owner(ff->fiber_self, w); 401 cilk_fiber_tbb_interop_use_saved_stack_op_info(ff->fiber_self); 402 403 CILK_ASSERT(ff->join_counter == 0); 404 ff->join_counter = 1; 405 w->l->frame_ff = ff; 406 w->reducer_map = __cilkrts_make_reducer_map(w); 407 __cilkrts_set_leftmost_reducer_map(w->reducer_map, 1); 408 load_pedigree_leaf_into_user_worker(w); 409 } 410 411 // Make sure that the head and tail are reset, and saved_protected_tail 412 // allows all frames to be stolen. 413 // 414 // Note that we must NOT check w->exc, since workers that are trying to 415 // steal from it will be updating w->exc and we don't own the worker lock. 416 // It's not worth taking out the lock just for an assertion. 417 CILK_ASSERT(w->head == w->l->ltq); 418 CILK_ASSERT(w->tail == w->l->ltq); 419 CILK_ASSERT(w->protected_tail == w->ltq_limit); 420 421 // There may have been an old pending exception which was freed when the 422 // exception was caught outside of Cilk 423 w->l->pending_exception = NULL; 424 425 w->reserved = NULL; 426 427 // If we've already created a scheduling fiber for this worker, we'll just 428 // reuse it. If w->self < 0, it means that this is an ad-hoc user worker 429 // not known to the global state. Thus, we need to create a scheduling 430 // stack only if we don't already have one and w->self >= 0. 431 if (NULL == w->l->scheduling_fiber && w->self >= 0) 432 { 433 START_INTERVAL(w, INTERVAL_FIBER_ALLOCATE) { 434 // Create a scheduling fiber for this worker. 435 w->l->scheduling_fiber = 436 cilk_fiber_allocate_from_heap(CILK_SCHEDULING_STACK_SIZE); 437 cilk_fiber_reset_state(w->l->scheduling_fiber, 438 scheduler_fiber_proc_for_user_worker); 439 cilk_fiber_set_owner(w->l->scheduling_fiber, w); 440 } STOP_INTERVAL(w, INTERVAL_FIBER_ALLOCATE); 441 } 442 443 // If the scheduling fiber is NULL, we've either exceeded our quota for 444 // fibers or workers or we're out of memory, so we should lose parallelism 445 // by disallowing stealing. 446 if (NULL == w->l->scheduling_fiber) 447 __cilkrts_disallow_stealing(w, NULL); 448 449 start_cilkscreen = (0 == w->g->Q); 450 451 if (w->self != -1) { 452 // w->self != -1, means that w is a normal user worker and must be 453 // accounted for by the global state since other workers can steal from 454 // it. 455 456 // w->self == -1, means that w is an overflow worker and was created on 457 // demand. I.e., it does not need to be accounted for by the global 458 // state. 459 460 __cilkrts_enter_cilk(w->g); 461 } 462 463 global_os_mutex_unlock(); 464 465 /* If there's only 1 worker, the counts will be started in 466 * __cilkrts_scheduler */ 467 if (g->P > 1) 468 { 469 START_INTERVAL(w, INTERVAL_IN_SCHEDULER); 470 START_INTERVAL(w, INTERVAL_WORKING); 471 } 472 473 ITT_SYNC_RELEASING(&unique_obj); 474 475 /* Turn on Cilkscreen if this is the first worker. This needs to be done 476 * when we are NOT holding the os mutex. */ 477 if (start_cilkscreen) 478 __cilkrts_cilkscreen_enable_instrumentation(); 479 480 return w; 481} 482 483#ifndef _MSC_VER 484/* 485 * Define old version-specific symbols for binding threads (since they exist in 486 * all Cilk code). These aliases prohibit newly compiled code from loading an 487 * old version of the runtime. We can handle old code with a new runtime, but 488 * new code with an old runtime is verboten! 489 * 490 * For Windows, the aliased symbol is exported in cilk-exports.def. 491 */ 492#if defined(_DARWIN_C_SOURCE) || defined(__APPLE__) 493/** 494 * Mac OS X: Unfortunately, Darwin doesn't allow aliasing, so we just make a 495 * call and hope the optimizer does the right thing. 496 */ 497CILK_ABI_WORKER_PTR __cilkrts_bind_thread (void) { 498 return BIND_THREAD_RTN(); 499} 500#else 501 502/** 503 * Macro to convert a parameter to a string. Used on Linux or BSD. 504 */ 505#define STRINGIFY(x) #x 506 507/** 508 * Macro to generate an __attribute__ for an aliased name 509 */ 510#define ALIASED_NAME(x) __attribute__ ((alias (STRINGIFY(x)))) 511 512/** 513 * Linux or BSD: Use the alias attribute to make the labels for the versioned 514 * functions point to the same place in the code as the original. Using 515 * the two macros is annoying but required. 516 */ 517 518CILK_ABI_WORKER_PTR __cilkrts_bind_thread(void) 519 ALIASED_NAME(BIND_THREAD_RTN); 520 521#endif // defined _DARWIN_C_SOURCE || defined __APPLE__ 522#endif // !defined _MSC_VER 523 524CILK_API_SIZET 525__cilkrts_get_stack_size(void) { 526 return cilkg_get_stack_size(); 527} 528 529// Method for debugging. 530CILK_API_VOID __cilkrts_dump_stats(void) 531{ 532 // While the stats aren't protected by the global OS mutex, the table 533 // of workers is, so take out the global OS mutex while we're doing this 534 global_os_mutex_lock(); 535 if (cilkg_is_published()) { 536 global_state_t *g = cilkg_get_global_state(); 537 __cilkrts_dump_stats_to_stderr(g); 538 } 539 else { 540 __cilkrts_bug("Attempting to report Cilk stats before the runtime has started\n"); 541 } 542 global_os_mutex_unlock(); 543} 544 545#ifndef _WIN32 546CILK_ABI_THROWS_VOID __cilkrts_rethrow(__cilkrts_stack_frame *sf) 547{ 548 __cilkrts_gcc_rethrow(sf); 549} 550#endif 551 552/* 553 * __cilkrts_unwatch_stack 554 * 555 * Callback for TBB to tell us they don't want to watch the stack anymore 556 */ 557 558static __cilk_tbb_retcode __cilkrts_unwatch_stack(void *data) 559{ 560 __cilk_tbb_stack_op_thunk o; 561 562 // If the cilk_fiber wasn't available fetch it now 563 if (TBB_INTEROP_DATA_DELAYED_UNTIL_BIND == data) 564 { 565 full_frame *ff; 566 __cilkrts_worker *w = __cilkrts_get_tls_worker(); 567 if (NULL == w) 568 { 569 // Free any saved stack op information 570 cilk_fiber_tbb_interop_free_stack_op_info(); 571 572 return 0; /* Success! */ 573 } 574 575 __cilkrts_worker_lock(w); 576 ff = w->l->frame_ff; 577 __cilkrts_frame_lock(w,ff); 578 data = ff->fiber_self; 579 __cilkrts_frame_unlock(w,ff); 580 __cilkrts_worker_unlock(w); 581 } 582 583#if CILK_LIB_DEBUG /* Debug code */ 584 /* Get current stack */ 585 full_frame *ff; 586 __cilkrts_worker *w = __cilkrts_get_tls_worker(); 587 __cilkrts_worker_lock(w); 588 ff = w->l->frame_ff; 589 __cilkrts_frame_lock(w,ff); 590 CILK_ASSERT (data == ff->fiber_self); 591 __cilkrts_frame_unlock(w,ff); 592 __cilkrts_worker_unlock(w); 593#endif 594 595 /* Clear the callback information */ 596 o.data = NULL; 597 o.routine = NULL; 598 cilk_fiber_set_stack_op((cilk_fiber*)data, o); 599 600 // Note. Do *NOT* free any saved stack information here. If they want to 601 // free the saved stack op information, they'll do it when the thread is 602 // unbound 603 604 return 0; /* Success! */ 605} 606 607/* 608 * __cilkrts_watch_stack 609 * 610 * Called by TBB, defined by Cilk. 611 * 612 * Requests that Cilk invoke the stack op routine when it orphans a stack. 613 * Cilk sets *u to a thunk that TBB should call when it is no longer interested 614 * in watching the stack. 615 */ 616 617CILK_API_TBB_RETCODE 618__cilkrts_watch_stack(__cilk_tbb_unwatch_thunk *u, 619 __cilk_tbb_stack_op_thunk o) 620{ 621 cilk_fiber* current_fiber; 622 __cilkrts_worker *w; 623 624#ifdef _MSC_VER 625 // This may be called by TBB *before* the OS has given us our 626 // initialization call. Make sure the module is initialized. 627 sysdep_init_module(); 628#endif 629 630 // Fetch the __cilkrts_worker bound to this thread 631 w = __cilkrts_get_tls_worker(); 632 if (NULL == w) 633 { 634 // Save data for later. We'll deal with it when/if this thread binds 635 // to the runtime 636 cilk_fiber_tbb_interop_save_stack_op_info(o); 637 638 u->routine = __cilkrts_unwatch_stack; 639 u->data = TBB_INTEROP_DATA_DELAYED_UNTIL_BIND; 640 641 return 0; 642 } 643 644 /* Get current stack */ 645 __cilkrts_worker_lock(w); 646 current_fiber = w->l->frame_ff->fiber_self; 647 __cilkrts_worker_unlock(w); 648 649/* CILK_ASSERT( !sd->stack_op_data ); */ 650/* CILK_ASSERT( !sd->stack_op_routine ); */ 651 652 /* Give TBB our callback */ 653 u->routine = __cilkrts_unwatch_stack; 654 u->data = current_fiber; 655 /* Save the callback information */ 656 cilk_fiber_set_stack_op(current_fiber, o); 657 658 return 0; /* Success! */ 659} 660 661 662// This function must be called only within a continuation, within the stack 663// frame of the continuation itself. 664CILK_API_INT __cilkrts_synched(void) 665{ 666 __cilkrts_worker *w = __cilkrts_get_tls_worker(); 667 668 // If we don't have a worker, then we're synched by definition :o) 669 if (NULL == w) 670 return 1; 671 672 // Check to see if we are in a stolen continuation. If not, then 673 // we are synched. 674 uint32_t flags = w->current_stack_frame->flags; 675 if (0 == (flags & CILK_FRAME_UNSYNCHED)) 676 return 1; 677 678 // We are in a stolen continutation, but the join counter might have been 679 // decremented to one, making us synched again. Get the full frame so 680 // that we can check the join counter. ASSUME: frame_ff is stable (can be 681 // read without a lock) in a stolen continuation -- it can't be stolen 682 // while it's currently executing. 683 full_frame *ff = w->l->frame_ff; 684 685 // Make sure we have a full frame 686 // TBD: Don't think that we should ever not have a full frame here. 687 // CILK_ASSERT(NULL != ff); ? 688 if (NULL == ff) 689 return 1; 690 691 // We're synched if there are no outstanding children at this instant in 692 // time. Note that this is a known race, but it's ok since we're only 693 // reading. We can get false negatives, but not false positives. (I.e., 694 // we can read a non-one join_counter just before it goes to one, but the 695 // join_counter cannot go from one to greater than one while we're 696 // reading.) 697 return 1 == ff->join_counter; 698} 699 700 701 702 703CILK_API_INT 704__cilkrts_bump_loop_rank_internal(__cilkrts_worker* w) 705{ 706 // If we don't have a worker, then the runtime is not bound to this 707 // thread and there is no rank to increment 708 if (NULL == w) 709 return -1; 710 711 // We're at the start of the loop body. Advance the cilk_for loop 712 // body pedigree by following the parent link and updating its 713 // rank. 714 715 // Normally, we'd just write "w->pedigree.parent->rank++" 716 // But we need to cast away the "const". 717 ((__cilkrts_pedigree*) w->pedigree.parent)->rank++; 718 719 // Zero the worker's pedigree rank since this is the start of a new 720 // pedigree domain. 721 w->pedigree.rank = 0; 722 723 return 0; 724} 725 726CILK_ABI_VOID 727__cilkrts_save_fp_ctrl_state(__cilkrts_stack_frame *sf) 728{ 729 // Pass call onto OS/architecture dependent function 730 sysdep_save_fp_ctrl_state(sf); 731} 732 733/* end cilk-abi.c */ 734