1#ifndef JEMALLOC_INTERNAL_MUTEX_H
2#define JEMALLOC_INTERNAL_MUTEX_H
3
4#include "jemalloc/internal/atomic.h"
5#include "jemalloc/internal/mutex_prof.h"
6#include "jemalloc/internal/tsd.h"
7#include "jemalloc/internal/witness.h"
8
9typedef enum {
10	/* Can only acquire one mutex of a given witness rank at a time. */
11	malloc_mutex_rank_exclusive,
12	/*
13	 * Can acquire multiple mutexes of the same witness rank, but in
14	 * address-ascending order only.
15	 */
16	malloc_mutex_address_ordered
17} malloc_mutex_lock_order_t;
18
19typedef struct malloc_mutex_s malloc_mutex_t;
20struct malloc_mutex_s {
21	union {
22		struct {
23			/*
24			 * prof_data is defined first to reduce cacheline
25			 * bouncing: the data is not touched by the mutex holder
26			 * during unlocking, while might be modified by
27			 * contenders.  Having it before the mutex itself could
28			 * avoid prefetching a modified cacheline (for the
29			 * unlocking thread).
30			 */
31			mutex_prof_data_t	prof_data;
32#ifdef _WIN32
33#  if _WIN32_WINNT >= 0x0600
34			SRWLOCK         	lock;
35#  else
36			CRITICAL_SECTION	lock;
37#  endif
38#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
39			os_unfair_lock		lock;
40#elif (defined(JEMALLOC_MUTEX_INIT_CB))
41			pthread_mutex_t		lock;
42			malloc_mutex_t		*postponed_next;
43#else
44			pthread_mutex_t		lock;
45#endif
46			/*
47			 * Hint flag to avoid exclusive cache line contention
48			 * during spin waiting
49			 */
50			atomic_b_t		locked;
51		};
52		/*
53		 * We only touch witness when configured w/ debug.  However we
54		 * keep the field in a union when !debug so that we don't have
55		 * to pollute the code base with #ifdefs, while avoid paying the
56		 * memory cost.
57		 */
58#if !defined(JEMALLOC_DEBUG)
59		witness_t			witness;
60		malloc_mutex_lock_order_t	lock_order;
61#endif
62	};
63
64#if defined(JEMALLOC_DEBUG)
65	witness_t			witness;
66	malloc_mutex_lock_order_t	lock_order;
67#endif
68};
69
70/*
71 * Based on benchmark results, a fixed spin with this amount of retries works
72 * well for our critical sections.
73 */
74#define MALLOC_MUTEX_MAX_SPIN 250
75
76#ifdef _WIN32
77#  if _WIN32_WINNT >= 0x0600
78#    define MALLOC_MUTEX_LOCK(m)    AcquireSRWLockExclusive(&(m)->lock)
79#    define MALLOC_MUTEX_UNLOCK(m)  ReleaseSRWLockExclusive(&(m)->lock)
80#    define MALLOC_MUTEX_TRYLOCK(m) (!TryAcquireSRWLockExclusive(&(m)->lock))
81#  else
82#    define MALLOC_MUTEX_LOCK(m)    EnterCriticalSection(&(m)->lock)
83#    define MALLOC_MUTEX_UNLOCK(m)  LeaveCriticalSection(&(m)->lock)
84#    define MALLOC_MUTEX_TRYLOCK(m) (!TryEnterCriticalSection(&(m)->lock))
85#  endif
86#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
87#    define MALLOC_MUTEX_LOCK(m)    os_unfair_lock_lock(&(m)->lock)
88#    define MALLOC_MUTEX_UNLOCK(m)  os_unfair_lock_unlock(&(m)->lock)
89#    define MALLOC_MUTEX_TRYLOCK(m) (!os_unfair_lock_trylock(&(m)->lock))
90#else
91#    define MALLOC_MUTEX_LOCK(m)    pthread_mutex_lock(&(m)->lock)
92#    define MALLOC_MUTEX_UNLOCK(m)  pthread_mutex_unlock(&(m)->lock)
93#    define MALLOC_MUTEX_TRYLOCK(m) (pthread_mutex_trylock(&(m)->lock) != 0)
94#endif
95
96#define LOCK_PROF_DATA_INITIALIZER					\
97    {NSTIME_ZERO_INITIALIZER, NSTIME_ZERO_INITIALIZER, 0, 0, 0,		\
98	    ATOMIC_INIT(0), 0, NULL, 0}
99
100#ifdef _WIN32
101#  define MALLOC_MUTEX_INITIALIZER
102#elif (defined(JEMALLOC_OS_UNFAIR_LOCK))
103#  if defined(JEMALLOC_DEBUG)
104#    define MALLOC_MUTEX_INITIALIZER					\
105  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}}, \
106         WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
107#  else
108#    define MALLOC_MUTEX_INITIALIZER                      \
109  {{{LOCK_PROF_DATA_INITIALIZER, OS_UNFAIR_LOCK_INIT, ATOMIC_INIT(false)}},  \
110      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
111#  endif
112#elif (defined(JEMALLOC_MUTEX_INIT_CB))
113#  if (defined(JEMALLOC_DEBUG))
114#     define MALLOC_MUTEX_INITIALIZER					\
115      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
116           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
117#  else
118#     define MALLOC_MUTEX_INITIALIZER					\
119      {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, NULL, ATOMIC_INIT(false)}},	\
120           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
121#  endif
122
123#else
124#    define MALLOC_MUTEX_TYPE PTHREAD_MUTEX_DEFAULT
125#  if defined(JEMALLOC_DEBUG)
126#    define MALLOC_MUTEX_INITIALIZER					\
127     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}}, \
128           WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT), 0}
129#  else
130#    define MALLOC_MUTEX_INITIALIZER                          \
131     {{{LOCK_PROF_DATA_INITIALIZER, PTHREAD_MUTEX_INITIALIZER, ATOMIC_INIT(false)}},	\
132      WITNESS_INITIALIZER("mutex", WITNESS_RANK_OMIT)}
133#  endif
134#endif
135
136#ifdef JEMALLOC_LAZY_LOCK
137extern bool isthreaded;
138#endif
139
140bool malloc_mutex_init(malloc_mutex_t *mutex, const char *name,
141    witness_rank_t rank, malloc_mutex_lock_order_t lock_order);
142void malloc_mutex_prefork(tsdn_t *tsdn, malloc_mutex_t *mutex);
143void malloc_mutex_postfork_parent(tsdn_t *tsdn, malloc_mutex_t *mutex);
144void malloc_mutex_postfork_child(tsdn_t *tsdn, malloc_mutex_t *mutex);
145bool malloc_mutex_first_thread(void);
146bool malloc_mutex_boot(void);
147void malloc_mutex_prof_data_reset(tsdn_t *tsdn, malloc_mutex_t *mutex);
148
149void malloc_mutex_lock_slow(malloc_mutex_t *mutex);
150
151static inline void
152malloc_mutex_lock_final(malloc_mutex_t *mutex) {
153	MALLOC_MUTEX_LOCK(mutex);
154	atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
155}
156
157static inline bool
158malloc_mutex_trylock_final(malloc_mutex_t *mutex) {
159	return MALLOC_MUTEX_TRYLOCK(mutex);
160}
161
162static inline void
163mutex_owner_stats_update(tsdn_t *tsdn, malloc_mutex_t *mutex) {
164	if (config_stats) {
165		mutex_prof_data_t *data = &mutex->prof_data;
166		data->n_lock_ops++;
167		if (data->prev_owner != tsdn) {
168			data->prev_owner = tsdn;
169			data->n_owner_switches++;
170		}
171	}
172}
173
174/* Trylock: return false if the lock is successfully acquired. */
175static inline bool
176malloc_mutex_trylock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
177	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
178	if (isthreaded) {
179		if (malloc_mutex_trylock_final(mutex)) {
180			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
181			return true;
182		}
183		mutex_owner_stats_update(tsdn, mutex);
184	}
185	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
186
187	return false;
188}
189
190/* Aggregate lock prof data. */
191static inline void
192malloc_mutex_prof_merge(mutex_prof_data_t *sum, mutex_prof_data_t *data) {
193	nstime_add(&sum->tot_wait_time, &data->tot_wait_time);
194	if (nstime_compare(&sum->max_wait_time, &data->max_wait_time) < 0) {
195		nstime_copy(&sum->max_wait_time, &data->max_wait_time);
196	}
197
198	sum->n_wait_times += data->n_wait_times;
199	sum->n_spin_acquired += data->n_spin_acquired;
200
201	if (sum->max_n_thds < data->max_n_thds) {
202		sum->max_n_thds = data->max_n_thds;
203	}
204	uint32_t cur_n_waiting_thds = atomic_load_u32(&sum->n_waiting_thds,
205	    ATOMIC_RELAXED);
206	uint32_t new_n_waiting_thds = cur_n_waiting_thds + atomic_load_u32(
207	    &data->n_waiting_thds, ATOMIC_RELAXED);
208	atomic_store_u32(&sum->n_waiting_thds, new_n_waiting_thds,
209	    ATOMIC_RELAXED);
210	sum->n_owner_switches += data->n_owner_switches;
211	sum->n_lock_ops += data->n_lock_ops;
212}
213
214static inline void
215malloc_mutex_lock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
216	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
217	if (isthreaded) {
218		if (malloc_mutex_trylock_final(mutex)) {
219			malloc_mutex_lock_slow(mutex);
220			atomic_store_b(&mutex->locked, true, ATOMIC_RELAXED);
221		}
222		mutex_owner_stats_update(tsdn, mutex);
223	}
224	witness_lock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
225}
226
227static inline void
228malloc_mutex_unlock(tsdn_t *tsdn, malloc_mutex_t *mutex) {
229	atomic_store_b(&mutex->locked, false, ATOMIC_RELAXED);
230	witness_unlock(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
231	if (isthreaded) {
232		MALLOC_MUTEX_UNLOCK(mutex);
233	}
234}
235
236static inline void
237malloc_mutex_assert_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
238	witness_assert_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
239}
240
241static inline void
242malloc_mutex_assert_not_owner(tsdn_t *tsdn, malloc_mutex_t *mutex) {
243	witness_assert_not_owner(tsdn_witness_tsdp_get(tsdn), &mutex->witness);
244}
245
246/* Copy the prof data from mutex for processing. */
247static inline void
248malloc_mutex_prof_read(tsdn_t *tsdn, mutex_prof_data_t *data,
249    malloc_mutex_t *mutex) {
250	mutex_prof_data_t *source = &mutex->prof_data;
251	/* Can only read holding the mutex. */
252	malloc_mutex_assert_owner(tsdn, mutex);
253
254	/*
255	 * Not *really* allowed (we shouldn't be doing non-atomic loads of
256	 * atomic data), but the mutex protection makes this safe, and writing
257	 * a member-for-member copy is tedious for this situation.
258	 */
259	*data = *source;
260	/* n_wait_thds is not reported (modified w/o locking). */
261	atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
262}
263
264static inline void
265malloc_mutex_prof_accum(tsdn_t *tsdn, mutex_prof_data_t *data,
266    malloc_mutex_t *mutex) {
267	mutex_prof_data_t *source = &mutex->prof_data;
268	/* Can only read holding the mutex. */
269	malloc_mutex_assert_owner(tsdn, mutex);
270
271	nstime_add(&data->tot_wait_time, &source->tot_wait_time);
272	if (nstime_compare(&source->max_wait_time, &data->max_wait_time) > 0) {
273		nstime_copy(&data->max_wait_time, &source->max_wait_time);
274	}
275	data->n_wait_times += source->n_wait_times;
276	data->n_spin_acquired += source->n_spin_acquired;
277	if (data->max_n_thds < source->max_n_thds) {
278		data->max_n_thds = source->max_n_thds;
279	}
280	/* n_wait_thds is not reported. */
281	atomic_store_u32(&data->n_waiting_thds, 0, ATOMIC_RELAXED);
282	data->n_owner_switches += source->n_owner_switches;
283	data->n_lock_ops += source->n_lock_ops;
284}
285
286#endif /* JEMALLOC_INTERNAL_MUTEX_H */
287