1//===-- tsan_interceptors_libdispatch.cpp ---------------------------------===// 2// 3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4// See https://llvm.org/LICENSE.txt for license information. 5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6// 7//===----------------------------------------------------------------------===// 8// 9// This file is a part of ThreadSanitizer (TSan), a race detector. 10// 11// Support for intercepting libdispatch (GCD). 12//===----------------------------------------------------------------------===// 13 14#include "sanitizer_common/sanitizer_common.h" 15#include "interception/interception.h" 16#include "tsan_interceptors.h" 17#include "tsan_rtl.h" 18 19#include "BlocksRuntime/Block.h" 20#include "tsan_dispatch_defs.h" 21 22namespace __tsan { 23 typedef u16 uint16_t; 24 25typedef struct { 26 dispatch_queue_t queue; 27 void *orig_context; 28 dispatch_function_t orig_work; 29 bool free_context_in_callback; 30 bool submitted_synchronously; 31 bool is_barrier_block; 32 uptr non_queue_sync_object; 33} block_context_t; 34 35// The offsets of different fields of the dispatch_queue_t structure, exported 36// by libdispatch.dylib. 37extern "C" struct dispatch_queue_offsets_s { 38 const uint16_t dqo_version; 39 const uint16_t dqo_label; 40 const uint16_t dqo_label_size; 41 const uint16_t dqo_flags; 42 const uint16_t dqo_flags_size; 43 const uint16_t dqo_serialnum; 44 const uint16_t dqo_serialnum_size; 45 const uint16_t dqo_width; 46 const uint16_t dqo_width_size; 47 const uint16_t dqo_running; 48 const uint16_t dqo_running_size; 49 const uint16_t dqo_suspend_cnt; 50 const uint16_t dqo_suspend_cnt_size; 51 const uint16_t dqo_target_queue; 52 const uint16_t dqo_target_queue_size; 53 const uint16_t dqo_priority; 54 const uint16_t dqo_priority_size; 55} dispatch_queue_offsets; 56 57static bool IsQueueSerial(dispatch_queue_t q) { 58 CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2); 59 uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width); 60 CHECK_NE(width, 0); 61 return width == 1; 62} 63 64static dispatch_queue_t GetTargetQueueFromQueue(dispatch_queue_t q) { 65 CHECK_EQ(dispatch_queue_offsets.dqo_target_queue_size, 8); 66 dispatch_queue_t tq = *( 67 dispatch_queue_t *)(((uptr)q) + dispatch_queue_offsets.dqo_target_queue); 68 return tq; 69} 70 71static dispatch_queue_t GetTargetQueueFromSource(dispatch_source_t source) { 72 dispatch_queue_t tq = GetTargetQueueFromQueue((dispatch_queue_t)source); 73 CHECK_NE(tq, 0); 74 return tq; 75} 76 77static block_context_t *AllocContext(ThreadState *thr, uptr pc, 78 dispatch_queue_t queue, void *orig_context, 79 dispatch_function_t orig_work) { 80 block_context_t *new_context = 81 (block_context_t *)user_alloc_internal(thr, pc, sizeof(block_context_t)); 82 new_context->queue = queue; 83 new_context->orig_context = orig_context; 84 new_context->orig_work = orig_work; 85 new_context->free_context_in_callback = true; 86 new_context->submitted_synchronously = false; 87 new_context->is_barrier_block = false; 88 new_context->non_queue_sync_object = 0; 89 return new_context; 90} 91 92#define GET_QUEUE_SYNC_VARS(context, q) \ 93 bool is_queue_serial = q && IsQueueSerial(q); \ 94 uptr sync_ptr = (uptr)q ?: context->non_queue_sync_object; \ 95 uptr serial_sync = (uptr)sync_ptr; \ 96 uptr concurrent_sync = sync_ptr ? ((uptr)sync_ptr) + sizeof(uptr) : 0; \ 97 bool serial_task = context->is_barrier_block || is_queue_serial 98 99static void dispatch_sync_pre_execute(ThreadState *thr, uptr pc, 100 block_context_t *context) { 101 uptr submit_sync = (uptr)context; 102 Acquire(thr, pc, submit_sync); 103 104 dispatch_queue_t q = context->queue; 105 do { 106 GET_QUEUE_SYNC_VARS(context, q); 107 if (serial_sync) Acquire(thr, pc, serial_sync); 108 if (serial_task && concurrent_sync) Acquire(thr, pc, concurrent_sync); 109 110 if (q) q = GetTargetQueueFromQueue(q); 111 } while (q); 112} 113 114static void dispatch_sync_post_execute(ThreadState *thr, uptr pc, 115 block_context_t *context) { 116 uptr submit_sync = (uptr)context; 117 if (context->submitted_synchronously) Release(thr, pc, submit_sync); 118 119 dispatch_queue_t q = context->queue; 120 do { 121 GET_QUEUE_SYNC_VARS(context, q); 122 if (serial_task && serial_sync) Release(thr, pc, serial_sync); 123 if (!serial_task && concurrent_sync) Release(thr, pc, concurrent_sync); 124 125 if (q) q = GetTargetQueueFromQueue(q); 126 } while (q); 127} 128 129static void dispatch_callback_wrap(void *param) { 130 SCOPED_INTERCEPTOR_RAW(dispatch_callback_wrap); 131 block_context_t *context = (block_context_t *)param; 132 133 dispatch_sync_pre_execute(thr, pc, context); 134 135 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 136 context->orig_work(context->orig_context); 137 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 138 139 dispatch_sync_post_execute(thr, pc, context); 140 141 if (context->free_context_in_callback) user_free(thr, pc, context); 142} 143 144static void invoke_block(void *param) { 145 dispatch_block_t block = (dispatch_block_t)param; 146 block(); 147} 148 149static void invoke_and_release_block(void *param) { 150 dispatch_block_t block = (dispatch_block_t)param; 151 block(); 152 Block_release(block); 153} 154 155#define DISPATCH_INTERCEPT_ASYNC_B(name, barrier) \ 156 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \ 157 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 158 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 159 dispatch_block_t heap_block = Block_copy(block); \ 160 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 161 block_context_t *new_context = \ 162 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); \ 163 new_context->is_barrier_block = barrier; \ 164 Release(thr, pc, (uptr)new_context); \ 165 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 166 REAL(name##_f)(q, new_context, dispatch_callback_wrap); \ 167 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 168 } 169 170#define DISPATCH_INTERCEPT_SYNC_B(name, barrier) \ 171 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, \ 172 DISPATCH_NOESCAPE dispatch_block_t block) { \ 173 SCOPED_TSAN_INTERCEPTOR(name, q, block); \ 174 block_context_t new_context = { \ 175 q, block, &invoke_block, false, true, barrier, 0}; \ 176 Release(thr, pc, (uptr)&new_context); \ 177 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 178 REAL(name##_f)(q, &new_context, dispatch_callback_wrap); \ 179 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 180 Acquire(thr, pc, (uptr)&new_context); \ 181 } 182 183#define DISPATCH_INTERCEPT_ASYNC_F(name, barrier) \ 184 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 185 dispatch_function_t work) { \ 186 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 187 block_context_t *new_context = \ 188 AllocContext(thr, pc, q, context, work); \ 189 new_context->is_barrier_block = barrier; \ 190 Release(thr, pc, (uptr)new_context); \ 191 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 192 REAL(name)(q, new_context, dispatch_callback_wrap); \ 193 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 194 } 195 196#define DISPATCH_INTERCEPT_SYNC_F(name, barrier) \ 197 TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \ 198 dispatch_function_t work) { \ 199 SCOPED_TSAN_INTERCEPTOR(name, q, context, work); \ 200 block_context_t new_context = { \ 201 q, context, work, false, true, barrier, 0}; \ 202 Release(thr, pc, (uptr)&new_context); \ 203 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \ 204 REAL(name)(q, &new_context, dispatch_callback_wrap); \ 205 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \ 206 Acquire(thr, pc, (uptr)&new_context); \ 207 } 208 209#define DISPATCH_INTERCEPT(name, barrier) \ 210 DISPATCH_INTERCEPT_ASYNC_F(name##_async_f, barrier) \ 211 DISPATCH_INTERCEPT_ASYNC_B(name##_async, barrier) \ 212 DISPATCH_INTERCEPT_SYNC_F(name##_sync_f, barrier) \ 213 DISPATCH_INTERCEPT_SYNC_B(name##_sync, barrier) 214 215// We wrap dispatch_async, dispatch_sync and friends where we allocate a new 216// context, which is used to synchronize (we release the context before 217// submitting, and the callback acquires it before executing the original 218// callback). 219DISPATCH_INTERCEPT(dispatch, false) 220DISPATCH_INTERCEPT(dispatch_barrier, true) 221 222DECLARE_REAL(void, dispatch_after_f, dispatch_time_t when, 223 dispatch_queue_t queue, void *context, dispatch_function_t work) 224 225TSAN_INTERCEPTOR(void, dispatch_after, dispatch_time_t when, 226 dispatch_queue_t queue, dispatch_block_t block) { 227 SCOPED_TSAN_INTERCEPTOR(dispatch_after, when, queue, block); 228 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 229 dispatch_block_t heap_block = Block_copy(block); 230 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 231 block_context_t *new_context = 232 AllocContext(thr, pc, queue, heap_block, &invoke_and_release_block); 233 Release(thr, pc, (uptr)new_context); 234 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 235 REAL(dispatch_after_f)(when, queue, new_context, dispatch_callback_wrap); 236 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 237} 238 239TSAN_INTERCEPTOR(void, dispatch_after_f, dispatch_time_t when, 240 dispatch_queue_t queue, void *context, 241 dispatch_function_t work) { 242 SCOPED_TSAN_INTERCEPTOR(dispatch_after_f, when, queue, context, work); 243 WRAP(dispatch_after)(when, queue, ^(void) { 244 work(context); 245 }); 246} 247 248// GCD's dispatch_once implementation has a fast path that contains a racy read 249// and it's inlined into user's code. Furthermore, this fast path doesn't 250// establish a proper happens-before relations between the initialization and 251// code following the call to dispatch_once. We could deal with this in 252// instrumented code, but there's not much we can do about it in system 253// libraries. Let's disable the fast path (by never storing the value ~0 to 254// predicate), so the interceptor is always called, and let's add proper release 255// and acquire semantics. Since TSan does not see its own atomic stores, the 256// race on predicate won't be reported - the only accesses to it that TSan sees 257// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is 258// both a macro and a real function, we want to intercept the function, so we 259// need to undefine the macro. 260#undef dispatch_once 261TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate, 262 DISPATCH_NOESCAPE dispatch_block_t block) { 263 SCOPED_INTERCEPTOR_RAW(dispatch_once, predicate, block); 264 atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate); 265 u32 v = atomic_load(a, memory_order_acquire); 266 if (v == 0 && 267 atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) { 268 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 269 block(); 270 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 271 Release(thr, pc, (uptr)a); 272 atomic_store(a, 2, memory_order_release); 273 } else { 274 while (v != 2) { 275 internal_sched_yield(); 276 v = atomic_load(a, memory_order_acquire); 277 } 278 Acquire(thr, pc, (uptr)a); 279 } 280} 281 282#undef dispatch_once_f 283TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate, 284 void *context, dispatch_function_t function) { 285 SCOPED_INTERCEPTOR_RAW(dispatch_once_f, predicate, context, function); 286 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 287 WRAP(dispatch_once)(predicate, ^(void) { 288 function(context); 289 }); 290 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 291} 292 293TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal, 294 dispatch_semaphore_t dsema) { 295 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema); 296 Release(thr, pc, (uptr)dsema); 297 return REAL(dispatch_semaphore_signal)(dsema); 298} 299 300TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema, 301 dispatch_time_t timeout) { 302 SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout); 303 long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout); 304 if (result == 0) Acquire(thr, pc, (uptr)dsema); 305 return result; 306} 307 308TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group, 309 dispatch_time_t timeout) { 310 SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout); 311 long_t result = REAL(dispatch_group_wait)(group, timeout); 312 if (result == 0) Acquire(thr, pc, (uptr)group); 313 return result; 314} 315 316// Used, but not intercepted. 317extern "C" void dispatch_group_enter(dispatch_group_t group); 318 319TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) { 320 SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group); 321 // Acquired in the group notification callback in dispatch_group_notify[_f]. 322 Release(thr, pc, (uptr)group); 323 REAL(dispatch_group_leave)(group); 324} 325 326TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group, 327 dispatch_queue_t queue, dispatch_block_t block) { 328 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block); 329 dispatch_retain(group); 330 dispatch_group_enter(group); 331 __block dispatch_block_t block_copy = (dispatch_block_t)Block_copy(block); 332 WRAP(dispatch_async)(queue, ^(void) { 333 block_copy(); 334 Block_release(block_copy); 335 WRAP(dispatch_group_leave)(group); 336 dispatch_release(group); 337 }); 338} 339 340TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group, 341 dispatch_queue_t queue, void *context, 342 dispatch_function_t work) { 343 SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work); 344 dispatch_retain(group); 345 dispatch_group_enter(group); 346 WRAP(dispatch_async)(queue, ^(void) { 347 work(context); 348 WRAP(dispatch_group_leave)(group); 349 dispatch_release(group); 350 }); 351} 352 353DECLARE_REAL(void, dispatch_group_notify_f, dispatch_group_t group, 354 dispatch_queue_t q, void *context, dispatch_function_t work) 355 356TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group, 357 dispatch_queue_t q, dispatch_block_t block) { 358 SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block); 359 360 // To make sure the group is still available in the callback (otherwise 361 // it can be already destroyed). Will be released in the callback. 362 dispatch_retain(group); 363 364 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 365 dispatch_block_t heap_block = Block_copy(^(void) { 366 { 367 SCOPED_INTERCEPTOR_RAW(dispatch_read_callback); 368 // Released when leaving the group (dispatch_group_leave). 369 Acquire(thr, pc, (uptr)group); 370 } 371 dispatch_release(group); 372 block(); 373 }); 374 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 375 block_context_t *new_context = 376 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 377 new_context->is_barrier_block = true; 378 Release(thr, pc, (uptr)new_context); 379 REAL(dispatch_group_notify_f)(group, q, new_context, dispatch_callback_wrap); 380} 381 382TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group, 383 dispatch_queue_t q, void *context, dispatch_function_t work) { 384 WRAP(dispatch_group_notify)(group, q, ^(void) { work(context); }); 385} 386 387TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler, 388 dispatch_source_t source, dispatch_block_t handler) { 389 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler, source, handler); 390 if (handler == nullptr) 391 return REAL(dispatch_source_set_event_handler)(source, nullptr); 392 dispatch_queue_t q = GetTargetQueueFromSource(source); 393 __block block_context_t new_context = { 394 q, handler, &invoke_block, false, false, false, 0 }; 395 dispatch_block_t new_handler = Block_copy(^(void) { 396 new_context.orig_context = handler; // To explicitly capture "handler". 397 dispatch_callback_wrap(&new_context); 398 }); 399 uptr submit_sync = (uptr)&new_context; 400 Release(thr, pc, submit_sync); 401 REAL(dispatch_source_set_event_handler)(source, new_handler); 402 Block_release(new_handler); 403} 404 405TSAN_INTERCEPTOR(void, dispatch_source_set_event_handler_f, 406 dispatch_source_t source, dispatch_function_t handler) { 407 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_event_handler_f, source, handler); 408 if (handler == nullptr) 409 return REAL(dispatch_source_set_event_handler)(source, nullptr); 410 dispatch_block_t block = ^(void) { 411 handler(dispatch_get_context(source)); 412 }; 413 WRAP(dispatch_source_set_event_handler)(source, block); 414} 415 416TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler, 417 dispatch_source_t source, dispatch_block_t handler) { 418 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler, source, handler); 419 if (handler == nullptr) 420 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 421 dispatch_queue_t q = GetTargetQueueFromSource(source); 422 __block block_context_t new_context = { 423 q, handler, &invoke_block, false, false, false, 0}; 424 dispatch_block_t new_handler = Block_copy(^(void) { 425 new_context.orig_context = handler; // To explicitly capture "handler". 426 dispatch_callback_wrap(&new_context); 427 }); 428 uptr submit_sync = (uptr)&new_context; 429 Release(thr, pc, submit_sync); 430 REAL(dispatch_source_set_cancel_handler)(source, new_handler); 431 Block_release(new_handler); 432} 433 434TSAN_INTERCEPTOR(void, dispatch_source_set_cancel_handler_f, 435 dispatch_source_t source, dispatch_function_t handler) { 436 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_cancel_handler_f, source, 437 handler); 438 if (handler == nullptr) 439 return REAL(dispatch_source_set_cancel_handler)(source, nullptr); 440 dispatch_block_t block = ^(void) { 441 handler(dispatch_get_context(source)); 442 }; 443 WRAP(dispatch_source_set_cancel_handler)(source, block); 444} 445 446TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler, 447 dispatch_source_t source, dispatch_block_t handler) { 448 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler, source, 449 handler); 450 if (handler == nullptr) 451 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 452 dispatch_queue_t q = GetTargetQueueFromSource(source); 453 __block block_context_t new_context = { 454 q, handler, &invoke_block, false, false, false, 0}; 455 dispatch_block_t new_handler = Block_copy(^(void) { 456 new_context.orig_context = handler; // To explicitly capture "handler". 457 dispatch_callback_wrap(&new_context); 458 }); 459 uptr submit_sync = (uptr)&new_context; 460 Release(thr, pc, submit_sync); 461 REAL(dispatch_source_set_registration_handler)(source, new_handler); 462 Block_release(new_handler); 463} 464 465TSAN_INTERCEPTOR(void, dispatch_source_set_registration_handler_f, 466 dispatch_source_t source, dispatch_function_t handler) { 467 SCOPED_TSAN_INTERCEPTOR(dispatch_source_set_registration_handler_f, source, 468 handler); 469 if (handler == nullptr) 470 return REAL(dispatch_source_set_registration_handler)(source, nullptr); 471 dispatch_block_t block = ^(void) { 472 handler(dispatch_get_context(source)); 473 }; 474 WRAP(dispatch_source_set_registration_handler)(source, block); 475} 476 477TSAN_INTERCEPTOR(void, dispatch_apply, size_t iterations, 478 dispatch_queue_t queue, 479 DISPATCH_NOESCAPE void (^block)(size_t)) { 480 SCOPED_TSAN_INTERCEPTOR(dispatch_apply, iterations, queue, block); 481 482 u8 sync1, sync2; 483 uptr parent_to_child_sync = (uptr)&sync1; 484 uptr child_to_parent_sync = (uptr)&sync2; 485 486 Release(thr, pc, parent_to_child_sync); 487 void (^new_block)(size_t) = ^(size_t iteration) { 488 SCOPED_INTERCEPTOR_RAW(dispatch_apply); 489 Acquire(thr, pc, parent_to_child_sync); 490 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 491 block(iteration); 492 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 493 Release(thr, pc, child_to_parent_sync); 494 }; 495 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 496 REAL(dispatch_apply)(iterations, queue, new_block); 497 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 498 Acquire(thr, pc, child_to_parent_sync); 499} 500 501static void invoke_block_iteration(void *param, size_t iteration) { 502 auto block = (void (^)(size_t)) param; 503 block(iteration); 504} 505 506TSAN_INTERCEPTOR(void, dispatch_apply_f, size_t iterations, 507 dispatch_queue_t queue, void *context, 508 void (*work)(void *, size_t)) { 509 SCOPED_TSAN_INTERCEPTOR(dispatch_apply_f, iterations, queue, context, work); 510 511 // Unfortunately, we cannot delegate to dispatch_apply, since libdispatch 512 // implements dispatch_apply in terms of dispatch_apply_f. 513 u8 sync1, sync2; 514 uptr parent_to_child_sync = (uptr)&sync1; 515 uptr child_to_parent_sync = (uptr)&sync2; 516 517 Release(thr, pc, parent_to_child_sync); 518 void (^new_block)(size_t) = ^(size_t iteration) { 519 SCOPED_INTERCEPTOR_RAW(dispatch_apply_f); 520 Acquire(thr, pc, parent_to_child_sync); 521 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 522 work(context, iteration); 523 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 524 Release(thr, pc, child_to_parent_sync); 525 }; 526 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 527 REAL(dispatch_apply_f)(iterations, queue, new_block, invoke_block_iteration); 528 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 529 Acquire(thr, pc, child_to_parent_sync); 530} 531 532DECLARE_REAL_AND_INTERCEPTOR(void, free, void *ptr) 533DECLARE_REAL_AND_INTERCEPTOR(int, munmap, void *addr, long_t sz) 534 535TSAN_INTERCEPTOR(dispatch_data_t, dispatch_data_create, const void *buffer, 536 size_t size, dispatch_queue_t q, dispatch_block_t destructor) { 537 SCOPED_TSAN_INTERCEPTOR(dispatch_data_create, buffer, size, q, destructor); 538 if ((q == nullptr) || (destructor == DISPATCH_DATA_DESTRUCTOR_DEFAULT)) 539 return REAL(dispatch_data_create)(buffer, size, q, destructor); 540 541 if (destructor == DISPATCH_DATA_DESTRUCTOR_FREE) 542 destructor = ^(void) { WRAP(free)((void *)(uintptr_t)buffer); }; 543 else if (destructor == DISPATCH_DATA_DESTRUCTOR_MUNMAP) 544 destructor = ^(void) { WRAP(munmap)((void *)(uintptr_t)buffer, size); }; 545 546 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); 547 dispatch_block_t heap_block = Block_copy(destructor); 548 SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); 549 block_context_t *new_context = 550 AllocContext(thr, pc, q, heap_block, &invoke_and_release_block); 551 uptr submit_sync = (uptr)new_context; 552 Release(thr, pc, submit_sync); 553 return REAL(dispatch_data_create)(buffer, size, q, ^(void) { 554 dispatch_callback_wrap(new_context); 555 }); 556} 557 558typedef void (^fd_handler_t)(dispatch_data_t data, int error); 559typedef void (^cleanup_handler_t)(int error); 560 561TSAN_INTERCEPTOR(void, dispatch_read, dispatch_fd_t fd, size_t length, 562 dispatch_queue_t q, fd_handler_t h) { 563 SCOPED_TSAN_INTERCEPTOR(dispatch_read, fd, length, q, h); 564 __block block_context_t new_context = { 565 q, nullptr, &invoke_block, false, false, false, 0}; 566 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 567 new_context.orig_context = ^(void) { 568 h(data, error); 569 }; 570 dispatch_callback_wrap(&new_context); 571 }); 572 uptr submit_sync = (uptr)&new_context; 573 Release(thr, pc, submit_sync); 574 REAL(dispatch_read)(fd, length, q, new_h); 575 Block_release(new_h); 576} 577 578TSAN_INTERCEPTOR(void, dispatch_write, dispatch_fd_t fd, dispatch_data_t data, 579 dispatch_queue_t q, fd_handler_t h) { 580 SCOPED_TSAN_INTERCEPTOR(dispatch_write, fd, data, q, h); 581 __block block_context_t new_context = { 582 q, nullptr, &invoke_block, false, false, false, 0}; 583 fd_handler_t new_h = Block_copy(^(dispatch_data_t data, int error) { 584 new_context.orig_context = ^(void) { 585 h(data, error); 586 }; 587 dispatch_callback_wrap(&new_context); 588 }); 589 uptr submit_sync = (uptr)&new_context; 590 Release(thr, pc, submit_sync); 591 REAL(dispatch_write)(fd, data, q, new_h); 592 Block_release(new_h); 593} 594 595TSAN_INTERCEPTOR(void, dispatch_io_read, dispatch_io_t channel, off_t offset, 596 size_t length, dispatch_queue_t q, dispatch_io_handler_t h) { 597 SCOPED_TSAN_INTERCEPTOR(dispatch_io_read, channel, offset, length, q, h); 598 __block block_context_t new_context = { 599 q, nullptr, &invoke_block, false, false, false, 0}; 600 dispatch_io_handler_t new_h = 601 Block_copy(^(bool done, dispatch_data_t data, int error) { 602 new_context.orig_context = ^(void) { 603 h(done, data, error); 604 }; 605 dispatch_callback_wrap(&new_context); 606 }); 607 uptr submit_sync = (uptr)&new_context; 608 Release(thr, pc, submit_sync); 609 REAL(dispatch_io_read)(channel, offset, length, q, new_h); 610 Block_release(new_h); 611} 612 613TSAN_INTERCEPTOR(void, dispatch_io_write, dispatch_io_t channel, off_t offset, 614 dispatch_data_t data, dispatch_queue_t q, 615 dispatch_io_handler_t h) { 616 SCOPED_TSAN_INTERCEPTOR(dispatch_io_write, channel, offset, data, q, h); 617 __block block_context_t new_context = { 618 q, nullptr, &invoke_block, false, false, false, 0}; 619 dispatch_io_handler_t new_h = 620 Block_copy(^(bool done, dispatch_data_t data, int error) { 621 new_context.orig_context = ^(void) { 622 h(done, data, error); 623 }; 624 dispatch_callback_wrap(&new_context); 625 }); 626 uptr submit_sync = (uptr)&new_context; 627 Release(thr, pc, submit_sync); 628 REAL(dispatch_io_write)(channel, offset, data, q, new_h); 629 Block_release(new_h); 630} 631 632TSAN_INTERCEPTOR(void, dispatch_io_barrier, dispatch_io_t channel, 633 dispatch_block_t barrier) { 634 SCOPED_TSAN_INTERCEPTOR(dispatch_io_barrier, channel, barrier); 635 __block block_context_t new_context = { 636 nullptr, nullptr, &invoke_block, false, false, false, 0}; 637 new_context.non_queue_sync_object = (uptr)channel; 638 new_context.is_barrier_block = true; 639 dispatch_block_t new_block = Block_copy(^(void) { 640 new_context.orig_context = ^(void) { 641 barrier(); 642 }; 643 dispatch_callback_wrap(&new_context); 644 }); 645 uptr submit_sync = (uptr)&new_context; 646 Release(thr, pc, submit_sync); 647 REAL(dispatch_io_barrier)(channel, new_block); 648 Block_release(new_block); 649} 650 651TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create, dispatch_io_type_t type, 652 dispatch_fd_t fd, dispatch_queue_t q, cleanup_handler_t h) { 653 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create, type, fd, q, h); 654 __block dispatch_io_t new_channel = nullptr; 655 __block block_context_t new_context = { 656 q, nullptr, &invoke_block, false, false, false, 0}; 657 cleanup_handler_t new_h = Block_copy(^(int error) { 658 { 659 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 660 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 661 } 662 new_context.orig_context = ^(void) { 663 h(error); 664 }; 665 dispatch_callback_wrap(&new_context); 666 }); 667 uptr submit_sync = (uptr)&new_context; 668 Release(thr, pc, submit_sync); 669 new_channel = REAL(dispatch_io_create)(type, fd, q, new_h); 670 Block_release(new_h); 671 return new_channel; 672} 673 674TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_path, 675 dispatch_io_type_t type, const char *path, int oflag, 676 mode_t mode, dispatch_queue_t q, cleanup_handler_t h) { 677 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_path, type, path, oflag, mode, 678 q, h); 679 __block dispatch_io_t new_channel = nullptr; 680 __block block_context_t new_context = { 681 q, nullptr, &invoke_block, false, false, false, 0}; 682 cleanup_handler_t new_h = Block_copy(^(int error) { 683 { 684 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 685 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 686 } 687 new_context.orig_context = ^(void) { 688 h(error); 689 }; 690 dispatch_callback_wrap(&new_context); 691 }); 692 uptr submit_sync = (uptr)&new_context; 693 Release(thr, pc, submit_sync); 694 new_channel = 695 REAL(dispatch_io_create_with_path)(type, path, oflag, mode, q, new_h); 696 Block_release(new_h); 697 return new_channel; 698} 699 700TSAN_INTERCEPTOR(dispatch_io_t, dispatch_io_create_with_io, 701 dispatch_io_type_t type, dispatch_io_t io, dispatch_queue_t q, 702 cleanup_handler_t h) { 703 SCOPED_TSAN_INTERCEPTOR(dispatch_io_create_with_io, type, io, q, h); 704 __block dispatch_io_t new_channel = nullptr; 705 __block block_context_t new_context = { 706 q, nullptr, &invoke_block, false, false, false, 0}; 707 cleanup_handler_t new_h = Block_copy(^(int error) { 708 { 709 SCOPED_INTERCEPTOR_RAW(dispatch_io_create_callback); 710 Acquire(thr, pc, (uptr)new_channel); // Release() in dispatch_io_close. 711 } 712 new_context.orig_context = ^(void) { 713 h(error); 714 }; 715 dispatch_callback_wrap(&new_context); 716 }); 717 uptr submit_sync = (uptr)&new_context; 718 Release(thr, pc, submit_sync); 719 new_channel = REAL(dispatch_io_create_with_io)(type, io, q, new_h); 720 Block_release(new_h); 721 return new_channel; 722} 723 724TSAN_INTERCEPTOR(void, dispatch_io_close, dispatch_io_t channel, 725 dispatch_io_close_flags_t flags) { 726 SCOPED_TSAN_INTERCEPTOR(dispatch_io_close, channel, flags); 727 Release(thr, pc, (uptr)channel); // Acquire() in dispatch_io_create[_*]. 728 return REAL(dispatch_io_close)(channel, flags); 729} 730 731// Resuming a suspended queue needs to synchronize with all subsequent 732// executions of blocks in that queue. 733TSAN_INTERCEPTOR(void, dispatch_resume, dispatch_object_t o) { 734 SCOPED_TSAN_INTERCEPTOR(dispatch_resume, o); 735 Release(thr, pc, (uptr)o); // Synchronizes with the Acquire() on serial_sync 736 // in dispatch_sync_pre_execute 737 return REAL(dispatch_resume)(o); 738} 739 740void InitializeLibdispatchInterceptors() { 741 INTERCEPT_FUNCTION(dispatch_async); 742 INTERCEPT_FUNCTION(dispatch_async_f); 743 INTERCEPT_FUNCTION(dispatch_sync); 744 INTERCEPT_FUNCTION(dispatch_sync_f); 745 INTERCEPT_FUNCTION(dispatch_barrier_async); 746 INTERCEPT_FUNCTION(dispatch_barrier_async_f); 747 INTERCEPT_FUNCTION(dispatch_barrier_sync); 748 INTERCEPT_FUNCTION(dispatch_barrier_sync_f); 749 INTERCEPT_FUNCTION(dispatch_after); 750 INTERCEPT_FUNCTION(dispatch_after_f); 751 INTERCEPT_FUNCTION(dispatch_once); 752 INTERCEPT_FUNCTION(dispatch_once_f); 753 INTERCEPT_FUNCTION(dispatch_semaphore_signal); 754 INTERCEPT_FUNCTION(dispatch_semaphore_wait); 755 INTERCEPT_FUNCTION(dispatch_group_wait); 756 INTERCEPT_FUNCTION(dispatch_group_leave); 757 INTERCEPT_FUNCTION(dispatch_group_async); 758 INTERCEPT_FUNCTION(dispatch_group_async_f); 759 INTERCEPT_FUNCTION(dispatch_group_notify); 760 INTERCEPT_FUNCTION(dispatch_group_notify_f); 761 INTERCEPT_FUNCTION(dispatch_source_set_event_handler); 762 INTERCEPT_FUNCTION(dispatch_source_set_event_handler_f); 763 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler); 764 INTERCEPT_FUNCTION(dispatch_source_set_cancel_handler_f); 765 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler); 766 INTERCEPT_FUNCTION(dispatch_source_set_registration_handler_f); 767 INTERCEPT_FUNCTION(dispatch_apply); 768 INTERCEPT_FUNCTION(dispatch_apply_f); 769 INTERCEPT_FUNCTION(dispatch_data_create); 770 INTERCEPT_FUNCTION(dispatch_read); 771 INTERCEPT_FUNCTION(dispatch_write); 772 INTERCEPT_FUNCTION(dispatch_io_read); 773 INTERCEPT_FUNCTION(dispatch_io_write); 774 INTERCEPT_FUNCTION(dispatch_io_barrier); 775 INTERCEPT_FUNCTION(dispatch_io_create); 776 INTERCEPT_FUNCTION(dispatch_io_create_with_path); 777 INTERCEPT_FUNCTION(dispatch_io_create_with_io); 778 INTERCEPT_FUNCTION(dispatch_io_close); 779 INTERCEPT_FUNCTION(dispatch_resume); 780} 781 782} // namespace __tsan 783