1//===----------------------------------------------------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8#ifndef LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
9#define LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
10
11/* cxa_guard_impl.h - Implements the C++ runtime support for function local
12 * static guards.
13 * The layout of the guard object is the same across ARM and Itanium.
14 *
15 * The first "guard byte" (which is checked by the compiler) is set only upon
16 * the completion of cxa release.
17 *
18 * The second "init byte" does the rest of the bookkeeping. It tracks if
19 * initialization is complete or pending, and if there are waiting threads.
20 *
21 * If the guard variable is 64-bits and the platforms supplies a 32-bit thread
22 * identifier, it is used to detect recursive initialization. The thread ID of
23 * the thread currently performing initialization is stored in the second word.
24 *
25 *  Guard Object Layout:
26 * -------------------------------------------------------------------------
27 * |a: guard byte | a+1: init byte | a+2 : unused ... | a+4: thread-id ... |
28 * ------------------------------------------------------------------------
29 *
30 *  Access Protocol:
31 *    For each implementation the guard byte is checked and set before accessing
32 *    the init byte.
33 *
34 *  Overall Design:
35 *    The implementation was designed to allow each implementation to be tested
36 *    independent of the C++ runtime or platform support.
37 *
38 */
39
40#include "__cxxabi_config.h"
41#include "include/atomic_support.h"
42#include <unistd.h>
43#include <sys/types.h>
44#if defined(__has_include)
45# if __has_include(<sys/syscall.h>)
46#   include <sys/syscall.h>
47# endif
48#endif
49
50#include <stdlib.h>
51#include <__threading_support>
52#ifndef _LIBCXXABI_HAS_NO_THREADS
53#if defined(__unix__) && !defined(__ANDROID__) && defined(__ELF__) && defined(_LIBCXXABI_HAS_COMMENT_LIB_PRAGMA)
54#pragma comment(lib, "pthread")
55#endif
56#endif
57
58// To make testing possible, this header is included from both cxa_guard.cpp
59// and a number of tests.
60//
61// For this reason we place everything in an anonymous namespace -- even though
62// we're in a header. We want the actual implementation and the tests to have
63// unique definitions of the types in this header (since the tests may depend
64// on function local statics).
65//
66// To enforce this either `BUILDING_CXA_GUARD` or `TESTING_CXA_GUARD` must be
67// defined when including this file. Only `src/cxa_guard.cpp` should define
68// the former.
69#ifdef BUILDING_CXA_GUARD
70# include "abort_message.h"
71# define ABORT_WITH_MESSAGE(...) ::abort_message(__VA_ARGS__)
72#elif defined(TESTING_CXA_GUARD)
73# define ABORT_WITH_MESSAGE(...) ::abort()
74#else
75# error "Either BUILDING_CXA_GUARD or TESTING_CXA_GUARD must be defined"
76#endif
77
78#if __has_feature(thread_sanitizer)
79extern "C" void __tsan_acquire(void*);
80extern "C" void __tsan_release(void*);
81#else
82#define __tsan_acquire(addr) ((void)0)
83#define __tsan_release(addr) ((void)0)
84#endif
85
86namespace __cxxabiv1 {
87// Use an anonymous namespace to ensure that the tests and actual implementation
88// have unique definitions of these symbols.
89namespace {
90
91//===----------------------------------------------------------------------===//
92//                          Misc Utilities
93//===----------------------------------------------------------------------===//
94
95template <class T, T(*Init)()>
96struct LazyValue {
97  LazyValue() : is_init(false) {}
98
99  T& get() {
100    if (!is_init) {
101      value = Init();
102      is_init = true;
103    }
104    return value;
105  }
106 private:
107  T value;
108  bool is_init = false;
109};
110
111//===----------------------------------------------------------------------===//
112//                       PlatformGetThreadID
113//===----------------------------------------------------------------------===//
114
115#if defined(__APPLE__) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
116uint32_t PlatformThreadID() {
117  static_assert(sizeof(mach_port_t) == sizeof(uint32_t), "");
118  return static_cast<uint32_t>(
119      pthread_mach_thread_np(std::__libcpp_thread_get_current_id()));
120}
121#elif defined(SYS_gettid) && defined(_LIBCPP_HAS_THREAD_API_PTHREAD)
122uint32_t PlatformThreadID() {
123  static_assert(sizeof(pid_t) == sizeof(uint32_t), "");
124  return static_cast<uint32_t>(syscall(SYS_gettid));
125}
126#else
127constexpr uint32_t (*PlatformThreadID)() = nullptr;
128#endif
129
130
131constexpr bool PlatformSupportsThreadID() {
132#ifdef __clang__
133#pragma clang diagnostic push
134#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
135#endif
136  return +PlatformThreadID != nullptr;
137#ifdef __clang__
138#pragma clang diagnostic pop
139#endif
140}
141
142//===----------------------------------------------------------------------===//
143//                          GuardBase
144//===----------------------------------------------------------------------===//
145
146enum class AcquireResult {
147  INIT_IS_DONE,
148  INIT_IS_PENDING,
149};
150constexpr AcquireResult INIT_IS_DONE = AcquireResult::INIT_IS_DONE;
151constexpr AcquireResult INIT_IS_PENDING = AcquireResult::INIT_IS_PENDING;
152
153static constexpr uint8_t UNSET = 0;
154static constexpr uint8_t COMPLETE_BIT = (1 << 0);
155static constexpr uint8_t PENDING_BIT = (1 << 1);
156static constexpr uint8_t WAITING_BIT = (1 << 2);
157
158template <class Derived>
159struct GuardObject {
160  GuardObject() = delete;
161  GuardObject(GuardObject const&) = delete;
162  GuardObject& operator=(GuardObject const&) = delete;
163
164  explicit GuardObject(uint32_t* g)
165      : base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
166        init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
167        thread_id_address(nullptr) {}
168
169  explicit GuardObject(uint64_t* g)
170      : base_address(g), guard_byte_address(reinterpret_cast<uint8_t*>(g)),
171        init_byte_address(reinterpret_cast<uint8_t*>(g) + 1),
172        thread_id_address(reinterpret_cast<uint32_t*>(g) + 1) {}
173
174public:
175  /// Implements __cxa_guard_acquire
176  AcquireResult cxa_guard_acquire() {
177    AtomicInt<uint8_t> guard_byte(guard_byte_address);
178    if (guard_byte.load(std::_AO_Acquire) != UNSET)
179      return INIT_IS_DONE;
180    return derived()->acquire_init_byte();
181  }
182
183  /// Implements __cxa_guard_release
184  void cxa_guard_release() {
185    AtomicInt<uint8_t> guard_byte(guard_byte_address);
186    // Store complete first, so that when release wakes other folks, they see
187    // it as having been completed.
188    guard_byte.store(COMPLETE_BIT, std::_AO_Release);
189    derived()->release_init_byte();
190  }
191
192  /// Implements __cxa_guard_abort
193  void cxa_guard_abort() { derived()->abort_init_byte(); }
194
195public:
196  /// base_address - the address of the original guard object.
197  void* const base_address;
198  /// The address of the guord byte at offset 0.
199  uint8_t* const guard_byte_address;
200  /// The address of the byte used by the implementation during initialization.
201  uint8_t* const init_byte_address;
202  /// An optional address storing an identifier for the thread performing initialization.
203  /// It's used to detect recursive initialization.
204  uint32_t* const thread_id_address;
205
206private:
207  Derived* derived() { return static_cast<Derived*>(this); }
208};
209
210//===----------------------------------------------------------------------===//
211//                    Single Threaded Implementation
212//===----------------------------------------------------------------------===//
213
214struct InitByteNoThreads : GuardObject<InitByteNoThreads> {
215  using GuardObject::GuardObject;
216
217  AcquireResult acquire_init_byte() {
218    if (*init_byte_address == COMPLETE_BIT)
219      return INIT_IS_DONE;
220    if (*init_byte_address & PENDING_BIT)
221      ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
222    *init_byte_address = PENDING_BIT;
223    return INIT_IS_PENDING;
224  }
225
226  void release_init_byte() { *init_byte_address = COMPLETE_BIT; }
227  void abort_init_byte() { *init_byte_address = UNSET; }
228};
229
230
231//===----------------------------------------------------------------------===//
232//                     Global Mutex Implementation
233//===----------------------------------------------------------------------===//
234
235struct LibcppMutex;
236struct LibcppCondVar;
237
238#ifndef _LIBCXXABI_HAS_NO_THREADS
239struct LibcppMutex {
240  LibcppMutex() = default;
241  LibcppMutex(LibcppMutex const&) = delete;
242  LibcppMutex& operator=(LibcppMutex const&) = delete;
243
244  bool lock() { return std::__libcpp_mutex_lock(&mutex); }
245  bool unlock() { return std::__libcpp_mutex_unlock(&mutex); }
246
247private:
248  friend struct LibcppCondVar;
249  std::__libcpp_mutex_t mutex = _LIBCPP_MUTEX_INITIALIZER;
250};
251
252struct LibcppCondVar {
253  LibcppCondVar() = default;
254  LibcppCondVar(LibcppCondVar const&) = delete;
255  LibcppCondVar& operator=(LibcppCondVar const&) = delete;
256
257  bool wait(LibcppMutex& mut) {
258    return std::__libcpp_condvar_wait(&cond, &mut.mutex);
259  }
260  bool broadcast() { return std::__libcpp_condvar_broadcast(&cond); }
261
262private:
263  std::__libcpp_condvar_t cond = _LIBCPP_CONDVAR_INITIALIZER;
264};
265#else
266struct LibcppMutex {};
267struct LibcppCondVar {};
268#endif // !defined(_LIBCXXABI_HAS_NO_THREADS)
269
270
271template <class Mutex, class CondVar, Mutex& global_mutex, CondVar& global_cond,
272          uint32_t (*GetThreadID)() = PlatformThreadID>
273struct InitByteGlobalMutex
274    : GuardObject<InitByteGlobalMutex<Mutex, CondVar, global_mutex, global_cond,
275                                    GetThreadID>> {
276
277  using BaseT = typename InitByteGlobalMutex::GuardObject;
278  using BaseT::BaseT;
279
280  explicit InitByteGlobalMutex(uint32_t *g)
281    : BaseT(g), has_thread_id_support(false) {}
282  explicit InitByteGlobalMutex(uint64_t *g)
283    : BaseT(g), has_thread_id_support(PlatformSupportsThreadID()) {}
284
285public:
286  AcquireResult acquire_init_byte() {
287    LockGuard g("__cxa_guard_acquire");
288    // Check for possible recursive initialization.
289    if (has_thread_id_support && (*init_byte_address & PENDING_BIT)) {
290      if (*thread_id_address == current_thread_id.get())
291       ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
292    }
293
294    // Wait until the pending bit is not set.
295    while (*init_byte_address & PENDING_BIT) {
296      *init_byte_address |= WAITING_BIT;
297      global_cond.wait(global_mutex);
298    }
299
300    if (*init_byte_address == COMPLETE_BIT)
301      return INIT_IS_DONE;
302
303    if (has_thread_id_support)
304      *thread_id_address = current_thread_id.get();
305
306    *init_byte_address = PENDING_BIT;
307    return INIT_IS_PENDING;
308  }
309
310  void release_init_byte() {
311    bool has_waiting;
312    {
313      LockGuard g("__cxa_guard_release");
314      has_waiting = *init_byte_address & WAITING_BIT;
315      *init_byte_address = COMPLETE_BIT;
316    }
317    if (has_waiting) {
318      if (global_cond.broadcast()) {
319        ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_release");
320      }
321    }
322  }
323
324  void abort_init_byte() {
325    bool has_waiting;
326    {
327      LockGuard g("__cxa_guard_abort");
328      if (has_thread_id_support)
329        *thread_id_address = 0;
330      has_waiting = *init_byte_address & WAITING_BIT;
331      *init_byte_address = UNSET;
332    }
333    if (has_waiting) {
334      if (global_cond.broadcast()) {
335        ABORT_WITH_MESSAGE("%s failed to broadcast", "__cxa_guard_abort");
336      }
337    }
338  }
339
340private:
341  using BaseT::init_byte_address;
342  using BaseT::thread_id_address;
343  const bool has_thread_id_support;
344  LazyValue<uint32_t, GetThreadID> current_thread_id;
345
346private:
347  struct LockGuard {
348    LockGuard() = delete;
349    LockGuard(LockGuard const&) = delete;
350    LockGuard& operator=(LockGuard const&) = delete;
351
352    explicit LockGuard(const char* calling_func)
353        : calling_func(calling_func)  {
354      if (global_mutex.lock())
355        ABORT_WITH_MESSAGE("%s failed to acquire mutex", calling_func);
356    }
357
358    ~LockGuard() {
359      if (global_mutex.unlock())
360        ABORT_WITH_MESSAGE("%s failed to release mutex", calling_func);
361    }
362
363  private:
364    const char* const calling_func;
365  };
366};
367
368//===----------------------------------------------------------------------===//
369//                         Futex Implementation
370//===----------------------------------------------------------------------===//
371
372#if defined(SYS_futex)
373void PlatformFutexWait(int* addr, int expect) {
374  constexpr int WAIT = 0;
375  syscall(SYS_futex, addr, WAIT, expect, 0);
376  __tsan_acquire(addr);
377}
378void PlatformFutexWake(int* addr) {
379  constexpr int WAKE = 1;
380  __tsan_release(addr);
381  syscall(SYS_futex, addr, WAKE, INT_MAX);
382}
383#else
384constexpr void (*PlatformFutexWait)(int*, int) = nullptr;
385constexpr void (*PlatformFutexWake)(int*) = nullptr;
386#endif
387
388constexpr bool PlatformSupportsFutex() {
389#ifdef __clang__
390#pragma clang diagnostic push
391#pragma clang diagnostic ignored "-Wtautological-pointer-compare"
392#endif
393  return +PlatformFutexWait != nullptr;
394#ifdef __clang__
395#pragma clang diagnostic pop
396#endif
397}
398
399/// InitByteFutex - Manages initialization using atomics and the futex syscall
400/// for waiting and waking.
401template <void (*Wait)(int*, int) = PlatformFutexWait,
402          void (*Wake)(int*) = PlatformFutexWake,
403          uint32_t (*GetThreadIDArg)() = PlatformThreadID>
404struct InitByteFutex : GuardObject<InitByteFutex<Wait, Wake, GetThreadIDArg>> {
405  using BaseT = typename InitByteFutex::GuardObject;
406
407  /// ARM Constructor
408  explicit InitByteFutex(uint32_t *g) : BaseT(g),
409    init_byte(this->init_byte_address),
410    has_thread_id_support(this->thread_id_address && GetThreadIDArg),
411    thread_id(this->thread_id_address) {}
412
413  /// Itanium Constructor
414  explicit InitByteFutex(uint64_t *g) : BaseT(g),
415    init_byte(this->init_byte_address),
416    has_thread_id_support(this->thread_id_address && GetThreadIDArg),
417    thread_id(this->thread_id_address) {}
418
419public:
420  AcquireResult acquire_init_byte() {
421    while (true) {
422      uint8_t last_val = UNSET;
423      if (init_byte.compare_exchange(&last_val, PENDING_BIT, std::_AO_Acq_Rel,
424                                     std::_AO_Acquire)) {
425        if (has_thread_id_support) {
426          thread_id.store(current_thread_id.get(), std::_AO_Relaxed);
427        }
428        return INIT_IS_PENDING;
429      }
430
431      if (last_val == COMPLETE_BIT)
432        return INIT_IS_DONE;
433
434      if (last_val & PENDING_BIT) {
435
436        // Check for recursive initialization
437        if (has_thread_id_support && thread_id.load(std::_AO_Relaxed) == current_thread_id.get()) {
438            ABORT_WITH_MESSAGE("__cxa_guard_acquire detected recursive initialization");
439        }
440
441        if ((last_val & WAITING_BIT) == 0) {
442          // This compare exchange can fail for several reasons
443          // (1) another thread finished the whole thing before we got here
444          // (2) another thread set the waiting bit we were trying to thread
445          // (3) another thread had an exception and failed to finish
446          if (!init_byte.compare_exchange(&last_val, PENDING_BIT | WAITING_BIT,
447                                          std::_AO_Acq_Rel, std::_AO_Release)) {
448            // (1) success, via someone else's work!
449            if (last_val == COMPLETE_BIT)
450              return INIT_IS_DONE;
451
452            // (3) someone else, bailed on doing the work, retry from the start!
453            if (last_val == UNSET)
454              continue;
455
456            // (2) the waiting bit got set, so we are happy to keep waiting
457          }
458        }
459        wait_on_initialization();
460      }
461    }
462  }
463
464  void release_init_byte() {
465    uint8_t old = init_byte.exchange(COMPLETE_BIT, std::_AO_Acq_Rel);
466    if (old & WAITING_BIT)
467      wake_all();
468  }
469
470  void abort_init_byte() {
471    if (has_thread_id_support)
472      thread_id.store(0, std::_AO_Relaxed);
473
474    uint8_t old = init_byte.exchange(0, std::_AO_Acq_Rel);
475    if (old & WAITING_BIT)
476      wake_all();
477  }
478
479private:
480  /// Use the futex to wait on the current guard variable. Futex expects a
481  /// 32-bit 4-byte aligned address as the first argument, so we have to use use
482  /// the base address of the guard variable (not the init byte).
483  void wait_on_initialization() {
484    Wait(static_cast<int*>(this->base_address),
485         expected_value_for_futex(PENDING_BIT | WAITING_BIT));
486  }
487  void wake_all() { Wake(static_cast<int*>(this->base_address)); }
488
489private:
490  AtomicInt<uint8_t> init_byte;
491
492  const bool has_thread_id_support;
493  // Unsafe to use unless has_thread_id_support
494  AtomicInt<uint32_t> thread_id;
495  LazyValue<uint32_t, GetThreadIDArg> current_thread_id;
496
497  /// Create the expected integer value for futex `wait(int* addr, int expected)`.
498  /// We pass the base address as the first argument, So this function creates
499  /// an zero-initialized integer  with `b` copied at the correct offset.
500  static int expected_value_for_futex(uint8_t b) {
501    int dest_val = 0;
502    std::memcpy(reinterpret_cast<char*>(&dest_val) + 1, &b, 1);
503    return dest_val;
504  }
505
506  static_assert(Wait != nullptr && Wake != nullptr, "");
507};
508
509//===----------------------------------------------------------------------===//
510//
511//===----------------------------------------------------------------------===//
512
513template <class T>
514struct GlobalStatic {
515  static T instance;
516};
517template <class T>
518_LIBCPP_SAFE_STATIC T GlobalStatic<T>::instance = {};
519
520enum class Implementation {
521  NoThreads,
522  GlobalLock,
523  Futex
524};
525
526template <Implementation Impl>
527struct SelectImplementation;
528
529template <>
530struct SelectImplementation<Implementation::NoThreads> {
531  using type = InitByteNoThreads;
532};
533
534template <>
535struct SelectImplementation<Implementation::GlobalLock> {
536  using type = InitByteGlobalMutex<
537      LibcppMutex, LibcppCondVar, GlobalStatic<LibcppMutex>::instance,
538      GlobalStatic<LibcppCondVar>::instance, PlatformThreadID>;
539};
540
541template <>
542struct SelectImplementation<Implementation::Futex> {
543  using type =
544      InitByteFutex<PlatformFutexWait, PlatformFutexWake, PlatformThreadID>;
545};
546
547// TODO(EricWF): We should prefer the futex implementation when available. But
548// it should be done in a separate step from adding the implementation.
549constexpr Implementation CurrentImplementation =
550#if defined(_LIBCXXABI_HAS_NO_THREADS)
551    Implementation::NoThreads;
552#elif defined(_LIBCXXABI_USE_FUTEX)
553    Implementation::Futex;
554#else
555   Implementation::GlobalLock;
556#endif
557
558static_assert(CurrentImplementation != Implementation::Futex
559           || PlatformSupportsFutex(), "Futex selected but not supported");
560
561using SelectedImplementation =
562    SelectImplementation<CurrentImplementation>::type;
563
564} // end namespace
565} // end namespace __cxxabiv1
566
567#endif // LIBCXXABI_SRC_INCLUDE_CXA_GUARD_IMPL_H
568