1//===-- tsan_libdispatch_mac.cc -------------------------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file is a part of ThreadSanitizer (TSan), a race detector.
11//
12// Mac-specific libdispatch (GCD) support.
13//===----------------------------------------------------------------------===//
14
15#include "sanitizer_common/sanitizer_platform.h"
16#if SANITIZER_MAC
17
18#include "sanitizer_common/sanitizer_common.h"
19#include "interception/interception.h"
20#include "tsan_interceptors.h"
21#include "tsan_platform.h"
22#include "tsan_rtl.h"
23
24#include <Block.h>
25#include <dispatch/dispatch.h>
26#include <pthread.h>
27
28typedef long long_t;  // NOLINT
29
30namespace __tsan {
31
32typedef struct {
33  dispatch_queue_t queue;
34  void *orig_context;
35  dispatch_function_t orig_work;
36  uptr object_to_acquire;
37  dispatch_object_t object_to_release;
38} tsan_block_context_t;
39
40// The offsets of different fields of the dispatch_queue_t structure, exported
41// by libdispatch.dylib.
42extern "C" struct dispatch_queue_offsets_s {
43  const uint16_t dqo_version;
44  const uint16_t dqo_label;
45  const uint16_t dqo_label_size;
46  const uint16_t dqo_flags;
47  const uint16_t dqo_flags_size;
48  const uint16_t dqo_serialnum;
49  const uint16_t dqo_serialnum_size;
50  const uint16_t dqo_width;
51  const uint16_t dqo_width_size;
52  const uint16_t dqo_running;
53  const uint16_t dqo_running_size;
54  const uint16_t dqo_suspend_cnt;
55  const uint16_t dqo_suspend_cnt_size;
56  const uint16_t dqo_target_queue;
57  const uint16_t dqo_target_queue_size;
58  const uint16_t dqo_priority;
59  const uint16_t dqo_priority_size;
60} dispatch_queue_offsets;
61
62static bool IsQueueSerial(dispatch_queue_t q) {
63  CHECK_EQ(dispatch_queue_offsets.dqo_width_size, 2);
64  uptr width = *(uint16_t *)(((uptr)q) + dispatch_queue_offsets.dqo_width);
65  CHECK_NE(width, 0);
66  return width == 1;
67}
68
69static tsan_block_context_t *AllocContext(ThreadState *thr, uptr pc,
70                                          dispatch_queue_t queue,
71                                          void *orig_context,
72                                          dispatch_function_t orig_work) {
73  tsan_block_context_t *new_context =
74      (tsan_block_context_t *)user_alloc(thr, pc, sizeof(tsan_block_context_t));
75  new_context->queue = queue;
76  new_context->orig_context = orig_context;
77  new_context->orig_work = orig_work;
78  new_context->object_to_acquire = (uptr)new_context;
79  new_context->object_to_release = nullptr;
80  return new_context;
81}
82
83static void dispatch_callback_wrap_acquire(void *param) {
84  SCOPED_INTERCEPTOR_RAW(dispatch_async_f_callback_wrap);
85  tsan_block_context_t *context = (tsan_block_context_t *)param;
86  Acquire(thr, pc, context->object_to_acquire);
87
88  // Extra retain/release is required for dispatch groups. We use the group
89  // itself to synchronize, but in a notification (dispatch_group_notify
90  // callback), it may be disposed already. To solve this, we retain the group
91  // and release it here.
92  if (context->object_to_release) dispatch_release(context->object_to_release);
93
94  // In serial queues, work items can be executed on different threads, we need
95  // to explicitly synchronize on the queue itself.
96  if (IsQueueSerial(context->queue)) Acquire(thr, pc, (uptr)context->queue);
97  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
98  context->orig_work(context->orig_context);
99  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
100  if (IsQueueSerial(context->queue)) Release(thr, pc, (uptr)context->queue);
101  user_free(thr, pc, context);
102}
103
104static void invoke_and_release_block(void *param) {
105  dispatch_block_t block = (dispatch_block_t)param;
106  block();
107  Block_release(block);
108}
109
110#define DISPATCH_INTERCEPT_B(name)                                           \
111  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, dispatch_block_t block) { \
112    SCOPED_TSAN_INTERCEPTOR(name, q, block);                                 \
113    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
114    dispatch_block_t heap_block = Block_copy(block);                         \
115    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
116    tsan_block_context_t *new_context =                                      \
117        AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);     \
118    Release(thr, pc, (uptr)new_context);                                     \
119    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
120    REAL(name##_f)(q, new_context, dispatch_callback_wrap_acquire);          \
121    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
122  }
123
124#define DISPATCH_INTERCEPT_F(name)                                \
125  TSAN_INTERCEPTOR(void, name, dispatch_queue_t q, void *context, \
126                   dispatch_function_t work) {                    \
127    SCOPED_TSAN_INTERCEPTOR(name, q, context, work);              \
128    tsan_block_context_t *new_context =                           \
129        AllocContext(thr, pc, q, context, work);                  \
130    Release(thr, pc, (uptr)new_context);                          \
131    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START(); \
132    REAL(name)(q, new_context, dispatch_callback_wrap_acquire);   \
133    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END(); \
134  }
135
136// We wrap dispatch_async, dispatch_sync and friends where we allocate a new
137// context, which is used to synchronize (we release the context before
138// submitting, and the callback acquires it before executing the original
139// callback).
140DISPATCH_INTERCEPT_B(dispatch_async)
141DISPATCH_INTERCEPT_B(dispatch_barrier_async)
142DISPATCH_INTERCEPT_F(dispatch_async_f)
143DISPATCH_INTERCEPT_F(dispatch_barrier_async_f)
144DISPATCH_INTERCEPT_B(dispatch_sync)
145DISPATCH_INTERCEPT_B(dispatch_barrier_sync)
146DISPATCH_INTERCEPT_F(dispatch_sync_f)
147DISPATCH_INTERCEPT_F(dispatch_barrier_sync_f)
148
149// GCD's dispatch_once implementation has a fast path that contains a racy read
150// and it's inlined into user's code. Furthermore, this fast path doesn't
151// establish a proper happens-before relations between the initialization and
152// code following the call to dispatch_once. We could deal with this in
153// instrumented code, but there's not much we can do about it in system
154// libraries. Let's disable the fast path (by never storing the value ~0 to
155// predicate), so the interceptor is always called, and let's add proper release
156// and acquire semantics. Since TSan does not see its own atomic stores, the
157// race on predicate won't be reported - the only accesses to it that TSan sees
158// are the loads on the fast path. Loads don't race. Secondly, dispatch_once is
159// both a macro and a real function, we want to intercept the function, so we
160// need to undefine the macro.
161#undef dispatch_once
162TSAN_INTERCEPTOR(void, dispatch_once, dispatch_once_t *predicate,
163                 dispatch_block_t block) {
164  SCOPED_TSAN_INTERCEPTOR(dispatch_once, predicate, block);
165  atomic_uint32_t *a = reinterpret_cast<atomic_uint32_t *>(predicate);
166  u32 v = atomic_load(a, memory_order_acquire);
167  if (v == 0 &&
168      atomic_compare_exchange_strong(a, &v, 1, memory_order_relaxed)) {
169    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
170    block();
171    SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
172    Release(thr, pc, (uptr)a);
173    atomic_store(a, 2, memory_order_release);
174  } else {
175    while (v != 2) {
176      internal_sched_yield();
177      v = atomic_load(a, memory_order_acquire);
178    }
179    Acquire(thr, pc, (uptr)a);
180  }
181}
182
183#undef dispatch_once_f
184TSAN_INTERCEPTOR(void, dispatch_once_f, dispatch_once_t *predicate,
185                 void *context, dispatch_function_t function) {
186  SCOPED_TSAN_INTERCEPTOR(dispatch_once_f, predicate, context, function);
187  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
188  WRAP(dispatch_once)(predicate, ^(void) {
189    function(context);
190  });
191  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
192}
193
194TSAN_INTERCEPTOR(long_t, dispatch_semaphore_signal,
195                 dispatch_semaphore_t dsema) {
196  SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_signal, dsema);
197  Release(thr, pc, (uptr)dsema);
198  return REAL(dispatch_semaphore_signal)(dsema);
199}
200
201TSAN_INTERCEPTOR(long_t, dispatch_semaphore_wait, dispatch_semaphore_t dsema,
202                 dispatch_time_t timeout) {
203  SCOPED_TSAN_INTERCEPTOR(dispatch_semaphore_wait, dsema, timeout);
204  long_t result = REAL(dispatch_semaphore_wait)(dsema, timeout);
205  if (result == 0) Acquire(thr, pc, (uptr)dsema);
206  return result;
207}
208
209TSAN_INTERCEPTOR(long_t, dispatch_group_wait, dispatch_group_t group,
210                 dispatch_time_t timeout) {
211  SCOPED_TSAN_INTERCEPTOR(dispatch_group_wait, group, timeout);
212  long_t result = REAL(dispatch_group_wait)(group, timeout);
213  if (result == 0) Acquire(thr, pc, (uptr)group);
214  return result;
215}
216
217TSAN_INTERCEPTOR(void, dispatch_group_leave, dispatch_group_t group) {
218  SCOPED_TSAN_INTERCEPTOR(dispatch_group_leave, group);
219  Release(thr, pc, (uptr)group);
220  REAL(dispatch_group_leave)(group);
221}
222
223TSAN_INTERCEPTOR(void, dispatch_group_async, dispatch_group_t group,
224                 dispatch_queue_t queue, dispatch_block_t block) {
225  SCOPED_TSAN_INTERCEPTOR(dispatch_group_async, group, queue, block);
226  dispatch_retain(group);
227  dispatch_group_enter(group);
228  WRAP(dispatch_async)(queue, ^(void) {
229    block();
230    WRAP(dispatch_group_leave)(group);
231    dispatch_release(group);
232  });
233}
234
235TSAN_INTERCEPTOR(void, dispatch_group_async_f, dispatch_group_t group,
236                 dispatch_queue_t queue, void *context,
237                 dispatch_function_t work) {
238  SCOPED_TSAN_INTERCEPTOR(dispatch_group_async_f, group, queue, context, work);
239  dispatch_retain(group);
240  dispatch_group_enter(group);
241  WRAP(dispatch_async)(queue, ^(void) {
242    work(context);
243    WRAP(dispatch_group_leave)(group);
244    dispatch_release(group);
245  });
246}
247
248TSAN_INTERCEPTOR(void, dispatch_group_notify, dispatch_group_t group,
249                 dispatch_queue_t q, dispatch_block_t block) {
250  SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify, group, q, block);
251  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_START();
252  dispatch_block_t heap_block = Block_copy(block);
253  SCOPED_TSAN_INTERCEPTOR_USER_CALLBACK_END();
254  tsan_block_context_t *new_context =
255      AllocContext(thr, pc, q, heap_block, &invoke_and_release_block);
256  new_context->object_to_acquire = (uptr)group;
257
258  // Will be released in dispatch_callback_wrap_acquire.
259  new_context->object_to_release = group;
260  dispatch_retain(group);
261
262  Release(thr, pc, (uptr)group);
263  REAL(dispatch_group_notify_f)(group, q, new_context,
264                                dispatch_callback_wrap_acquire);
265}
266
267TSAN_INTERCEPTOR(void, dispatch_group_notify_f, dispatch_group_t group,
268                 dispatch_queue_t q, void *context, dispatch_function_t work) {
269  SCOPED_TSAN_INTERCEPTOR(dispatch_group_notify_f, group, q, context, work);
270  tsan_block_context_t *new_context = AllocContext(thr, pc, q, context, work);
271  new_context->object_to_acquire = (uptr)group;
272
273  // Will be released in dispatch_callback_wrap_acquire.
274  new_context->object_to_release = group;
275  dispatch_retain(group);
276
277  Release(thr, pc, (uptr)group);
278  REAL(dispatch_group_notify_f)(group, q, new_context,
279                                dispatch_callback_wrap_acquire);
280}
281
282}  // namespace __tsan
283
284#endif  // SANITIZER_MAC
285