1299587Sjasone/******************************************************************************/ 2299587Sjasone#ifdef JEMALLOC_H_TYPES 3299587Sjasone 4299587Sjasonetypedef struct witness_s witness_t; 5299587Sjasonetypedef unsigned witness_rank_t; 6299587Sjasonetypedef ql_head(witness_t) witness_list_t; 7299587Sjasonetypedef int witness_comp_t (const witness_t *, const witness_t *); 8299587Sjasone 9299587Sjasone/* 10299587Sjasone * Lock ranks. Witnesses with rank WITNESS_RANK_OMIT are completely ignored by 11299587Sjasone * the witness machinery. 12299587Sjasone */ 13299587Sjasone#define WITNESS_RANK_OMIT 0U 14299587Sjasone 15299587Sjasone#define WITNESS_RANK_INIT 1U 16299587Sjasone#define WITNESS_RANK_CTL 1U 17299587Sjasone#define WITNESS_RANK_ARENAS 2U 18299587Sjasone 19299587Sjasone#define WITNESS_RANK_PROF_DUMP 3U 20299587Sjasone#define WITNESS_RANK_PROF_BT2GCTX 4U 21299587Sjasone#define WITNESS_RANK_PROF_TDATAS 5U 22299587Sjasone#define WITNESS_RANK_PROF_TDATA 6U 23299587Sjasone#define WITNESS_RANK_PROF_GCTX 7U 24299587Sjasone 25299587Sjasone#define WITNESS_RANK_ARENA 8U 26299587Sjasone#define WITNESS_RANK_ARENA_CHUNKS 9U 27299587Sjasone#define WITNESS_RANK_ARENA_NODE_CACHE 10 28299587Sjasone 29299587Sjasone#define WITNESS_RANK_BASE 11U 30299587Sjasone 31299587Sjasone#define WITNESS_RANK_LEAF 0xffffffffU 32299587Sjasone#define WITNESS_RANK_ARENA_BIN WITNESS_RANK_LEAF 33299587Sjasone#define WITNESS_RANK_ARENA_HUGE WITNESS_RANK_LEAF 34299587Sjasone#define WITNESS_RANK_DSS WITNESS_RANK_LEAF 35299587Sjasone#define WITNESS_RANK_PROF_ACTIVE WITNESS_RANK_LEAF 36299587Sjasone#define WITNESS_RANK_PROF_DUMP_SEQ WITNESS_RANK_LEAF 37299587Sjasone#define WITNESS_RANK_PROF_GDUMP WITNESS_RANK_LEAF 38299587Sjasone#define WITNESS_RANK_PROF_NEXT_THR_UID WITNESS_RANK_LEAF 39299587Sjasone#define WITNESS_RANK_PROF_THREAD_ACTIVE_INIT WITNESS_RANK_LEAF 40299587Sjasone 41299587Sjasone#define WITNESS_INITIALIZER(rank) {"initializer", rank, NULL, {NULL, NULL}} 42299587Sjasone 43299587Sjasone#endif /* JEMALLOC_H_TYPES */ 44299587Sjasone/******************************************************************************/ 45299587Sjasone#ifdef JEMALLOC_H_STRUCTS 46299587Sjasone 47299587Sjasonestruct witness_s { 48299587Sjasone /* Name, used for printing lock order reversal messages. */ 49299587Sjasone const char *name; 50299587Sjasone 51299587Sjasone /* 52299587Sjasone * Witness rank, where 0 is lowest and UINT_MAX is highest. Witnesses 53299587Sjasone * must be acquired in order of increasing rank. 54299587Sjasone */ 55299587Sjasone witness_rank_t rank; 56299587Sjasone 57299587Sjasone /* 58299587Sjasone * If two witnesses are of equal rank and they have the samp comp 59299587Sjasone * function pointer, it is called as a last attempt to differentiate 60299587Sjasone * between witnesses of equal rank. 61299587Sjasone */ 62299587Sjasone witness_comp_t *comp; 63299587Sjasone 64299587Sjasone /* Linkage for thread's currently owned locks. */ 65299587Sjasone ql_elm(witness_t) link; 66299587Sjasone}; 67299587Sjasone 68299587Sjasone#endif /* JEMALLOC_H_STRUCTS */ 69299587Sjasone/******************************************************************************/ 70299587Sjasone#ifdef JEMALLOC_H_EXTERNS 71299587Sjasone 72299587Sjasonevoid witness_init(witness_t *witness, const char *name, witness_rank_t rank, 73299587Sjasone witness_comp_t *comp); 74299587Sjasone#ifdef JEMALLOC_JET 75299587Sjasonetypedef void (witness_lock_error_t)(const witness_list_t *, const witness_t *); 76299587Sjasoneextern witness_lock_error_t *witness_lock_error; 77299587Sjasone#else 78299587Sjasonevoid witness_lock_error(const witness_list_t *witnesses, 79299587Sjasone const witness_t *witness); 80299587Sjasone#endif 81299587Sjasone#ifdef JEMALLOC_JET 82299587Sjasonetypedef void (witness_owner_error_t)(const witness_t *); 83299587Sjasoneextern witness_owner_error_t *witness_owner_error; 84299587Sjasone#else 85299587Sjasonevoid witness_owner_error(const witness_t *witness); 86299587Sjasone#endif 87299587Sjasone#ifdef JEMALLOC_JET 88299587Sjasonetypedef void (witness_not_owner_error_t)(const witness_t *); 89299587Sjasoneextern witness_not_owner_error_t *witness_not_owner_error; 90299587Sjasone#else 91299587Sjasonevoid witness_not_owner_error(const witness_t *witness); 92299587Sjasone#endif 93299587Sjasone#ifdef JEMALLOC_JET 94299587Sjasonetypedef void (witness_lockless_error_t)(const witness_list_t *); 95299587Sjasoneextern witness_lockless_error_t *witness_lockless_error; 96299587Sjasone#else 97299587Sjasonevoid witness_lockless_error(const witness_list_t *witnesses); 98299587Sjasone#endif 99299587Sjasone 100299587Sjasonevoid witnesses_cleanup(tsd_t *tsd); 101299587Sjasonevoid witness_fork_cleanup(tsd_t *tsd); 102299587Sjasonevoid witness_prefork(tsd_t *tsd); 103299587Sjasonevoid witness_postfork_parent(tsd_t *tsd); 104299587Sjasonevoid witness_postfork_child(tsd_t *tsd); 105299587Sjasone 106299587Sjasone#endif /* JEMALLOC_H_EXTERNS */ 107299587Sjasone/******************************************************************************/ 108299587Sjasone#ifdef JEMALLOC_H_INLINES 109299587Sjasone 110299587Sjasone#ifndef JEMALLOC_ENABLE_INLINE 111299587Sjasonevoid witness_assert_owner(tsdn_t *tsdn, const witness_t *witness); 112299587Sjasonevoid witness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness); 113299587Sjasonevoid witness_assert_lockless(tsdn_t *tsdn); 114299587Sjasonevoid witness_lock(tsdn_t *tsdn, witness_t *witness); 115299587Sjasonevoid witness_unlock(tsdn_t *tsdn, witness_t *witness); 116299587Sjasone#endif 117299587Sjasone 118299587Sjasone#if (defined(JEMALLOC_ENABLE_INLINE) || defined(JEMALLOC_MUTEX_C_)) 119299587SjasoneJEMALLOC_INLINE void 120299587Sjasonewitness_assert_owner(tsdn_t *tsdn, const witness_t *witness) 121299587Sjasone{ 122299587Sjasone tsd_t *tsd; 123299587Sjasone witness_list_t *witnesses; 124299587Sjasone witness_t *w; 125299587Sjasone 126299587Sjasone if (!config_debug) 127299587Sjasone return; 128299587Sjasone 129299587Sjasone if (tsdn_null(tsdn)) 130299587Sjasone return; 131299587Sjasone tsd = tsdn_tsd(tsdn); 132299587Sjasone if (witness->rank == WITNESS_RANK_OMIT) 133299587Sjasone return; 134299587Sjasone 135299587Sjasone witnesses = tsd_witnessesp_get(tsd); 136299587Sjasone ql_foreach(w, witnesses, link) { 137299587Sjasone if (w == witness) 138299587Sjasone return; 139299587Sjasone } 140299587Sjasone witness_owner_error(witness); 141299587Sjasone} 142299587Sjasone 143299587SjasoneJEMALLOC_INLINE void 144299587Sjasonewitness_assert_not_owner(tsdn_t *tsdn, const witness_t *witness) 145299587Sjasone{ 146299587Sjasone tsd_t *tsd; 147299587Sjasone witness_list_t *witnesses; 148299587Sjasone witness_t *w; 149299587Sjasone 150299587Sjasone if (!config_debug) 151299587Sjasone return; 152299587Sjasone 153299587Sjasone if (tsdn_null(tsdn)) 154299587Sjasone return; 155299587Sjasone tsd = tsdn_tsd(tsdn); 156299587Sjasone if (witness->rank == WITNESS_RANK_OMIT) 157299587Sjasone return; 158299587Sjasone 159299587Sjasone witnesses = tsd_witnessesp_get(tsd); 160299587Sjasone ql_foreach(w, witnesses, link) { 161299587Sjasone if (w == witness) 162299587Sjasone witness_not_owner_error(witness); 163299587Sjasone } 164299587Sjasone} 165299587Sjasone 166299587SjasoneJEMALLOC_INLINE void 167299587Sjasonewitness_assert_lockless(tsdn_t *tsdn) 168299587Sjasone{ 169299587Sjasone tsd_t *tsd; 170299587Sjasone witness_list_t *witnesses; 171299587Sjasone witness_t *w; 172299587Sjasone 173299587Sjasone if (!config_debug) 174299587Sjasone return; 175299587Sjasone 176299587Sjasone if (tsdn_null(tsdn)) 177299587Sjasone return; 178299587Sjasone tsd = tsdn_tsd(tsdn); 179299587Sjasone 180299587Sjasone witnesses = tsd_witnessesp_get(tsd); 181299587Sjasone w = ql_last(witnesses, link); 182299587Sjasone if (w != NULL) 183299587Sjasone witness_lockless_error(witnesses); 184299587Sjasone} 185299587Sjasone 186299587SjasoneJEMALLOC_INLINE void 187299587Sjasonewitness_lock(tsdn_t *tsdn, witness_t *witness) 188299587Sjasone{ 189299587Sjasone tsd_t *tsd; 190299587Sjasone witness_list_t *witnesses; 191299587Sjasone witness_t *w; 192299587Sjasone 193299587Sjasone if (!config_debug) 194299587Sjasone return; 195299587Sjasone 196299587Sjasone if (tsdn_null(tsdn)) 197299587Sjasone return; 198299587Sjasone tsd = tsdn_tsd(tsdn); 199299587Sjasone if (witness->rank == WITNESS_RANK_OMIT) 200299587Sjasone return; 201299587Sjasone 202299587Sjasone witness_assert_not_owner(tsdn, witness); 203299587Sjasone 204299587Sjasone witnesses = tsd_witnessesp_get(tsd); 205299587Sjasone w = ql_last(witnesses, link); 206299587Sjasone if (w == NULL) { 207299587Sjasone /* No other locks; do nothing. */ 208299587Sjasone } else if (tsd_witness_fork_get(tsd) && w->rank <= witness->rank) { 209299587Sjasone /* Forking, and relaxed ranking satisfied. */ 210299587Sjasone } else if (w->rank > witness->rank) { 211299587Sjasone /* Not forking, rank order reversal. */ 212299587Sjasone witness_lock_error(witnesses, witness); 213299587Sjasone } else if (w->rank == witness->rank && (w->comp == NULL || w->comp != 214299587Sjasone witness->comp || w->comp(w, witness) > 0)) { 215299587Sjasone /* 216299587Sjasone * Missing/incompatible comparison function, or comparison 217299587Sjasone * function indicates rank order reversal. 218299587Sjasone */ 219299587Sjasone witness_lock_error(witnesses, witness); 220299587Sjasone } 221299587Sjasone 222299587Sjasone ql_elm_new(witness, link); 223299587Sjasone ql_tail_insert(witnesses, witness, link); 224299587Sjasone} 225299587Sjasone 226299587SjasoneJEMALLOC_INLINE void 227299587Sjasonewitness_unlock(tsdn_t *tsdn, witness_t *witness) 228299587Sjasone{ 229299587Sjasone tsd_t *tsd; 230299587Sjasone witness_list_t *witnesses; 231299587Sjasone 232299587Sjasone if (!config_debug) 233299587Sjasone return; 234299587Sjasone 235299587Sjasone if (tsdn_null(tsdn)) 236299587Sjasone return; 237299587Sjasone tsd = tsdn_tsd(tsdn); 238299587Sjasone if (witness->rank == WITNESS_RANK_OMIT) 239299587Sjasone return; 240299587Sjasone 241299587Sjasone witness_assert_owner(tsdn, witness); 242299587Sjasone 243299587Sjasone witnesses = tsd_witnessesp_get(tsd); 244299587Sjasone ql_remove(witnesses, witness, link); 245299587Sjasone} 246299587Sjasone#endif 247299587Sjasone 248299587Sjasone#endif /* JEMALLOC_H_INLINES */ 249299587Sjasone/******************************************************************************/ 250