quarantine.c revision 235238
1234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 2234370Sjasone 3235238Sjasone/* 4235238Sjasone * quarantine pointers close to NULL are used to encode state information that 5235238Sjasone * is used for cleaning up during thread shutdown. 6235238Sjasone */ 7235238Sjasone#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 8235238Sjasone#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 9235238Sjasone#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 10235238Sjasone 11234370Sjasone/******************************************************************************/ 12234370Sjasone/* Data. */ 13234370Sjasone 14235238Sjasonetypedef struct quarantine_obj_s quarantine_obj_t; 15234370Sjasonetypedef struct quarantine_s quarantine_t; 16234370Sjasone 17235238Sjasonestruct quarantine_obj_s { 18235238Sjasone void *ptr; 19235238Sjasone size_t usize; 20235238Sjasone}; 21235238Sjasone 22234370Sjasonestruct quarantine_s { 23235238Sjasone size_t curbytes; 24235238Sjasone size_t curobjs; 25235238Sjasone size_t first; 26234370Sjasone#define LG_MAXOBJS_INIT 10 27235238Sjasone size_t lg_maxobjs; 28235238Sjasone quarantine_obj_t objs[1]; /* Dynamically sized ring buffer. */ 29234370Sjasone}; 30234370Sjasone 31234370Sjasonestatic void quarantine_cleanup(void *arg); 32234370Sjasone 33234370Sjasonemalloc_tsd_data(static, quarantine, quarantine_t *, NULL) 34234370Sjasonemalloc_tsd_funcs(JEMALLOC_INLINE, quarantine, quarantine_t *, NULL, 35234370Sjasone quarantine_cleanup) 36234370Sjasone 37234370Sjasone/******************************************************************************/ 38234370Sjasone/* Function prototypes for non-inline static functions. */ 39234370Sjasone 40234370Sjasonestatic quarantine_t *quarantine_init(size_t lg_maxobjs); 41234370Sjasonestatic quarantine_t *quarantine_grow(quarantine_t *quarantine); 42234370Sjasonestatic void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); 43234370Sjasone 44234370Sjasone/******************************************************************************/ 45234370Sjasone 46234370Sjasonestatic quarantine_t * 47234370Sjasonequarantine_init(size_t lg_maxobjs) 48234370Sjasone{ 49234370Sjasone quarantine_t *quarantine; 50234370Sjasone 51234370Sjasone quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + 52235238Sjasone ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); 53234370Sjasone if (quarantine == NULL) 54234370Sjasone return (NULL); 55234370Sjasone quarantine->curbytes = 0; 56234370Sjasone quarantine->curobjs = 0; 57234370Sjasone quarantine->first = 0; 58234370Sjasone quarantine->lg_maxobjs = lg_maxobjs; 59234370Sjasone 60234370Sjasone quarantine_tsd_set(&quarantine); 61234370Sjasone 62234370Sjasone return (quarantine); 63234370Sjasone} 64234370Sjasone 65234370Sjasonestatic quarantine_t * 66234370Sjasonequarantine_grow(quarantine_t *quarantine) 67234370Sjasone{ 68234370Sjasone quarantine_t *ret; 69234370Sjasone 70234370Sjasone ret = quarantine_init(quarantine->lg_maxobjs + 1); 71234370Sjasone if (ret == NULL) 72234370Sjasone return (quarantine); 73234370Sjasone 74234370Sjasone ret->curbytes = quarantine->curbytes; 75235238Sjasone ret->curobjs = quarantine->curobjs; 76235238Sjasone if (quarantine->first + quarantine->curobjs <= (ZU(1) << 77234370Sjasone quarantine->lg_maxobjs)) { 78234370Sjasone /* objs ring buffer data are contiguous. */ 79234370Sjasone memcpy(ret->objs, &quarantine->objs[quarantine->first], 80235238Sjasone quarantine->curobjs * sizeof(quarantine_obj_t)); 81234370Sjasone } else { 82234370Sjasone /* objs ring buffer data wrap around. */ 83235238Sjasone size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 84234370Sjasone quarantine->first; 85235238Sjasone size_t ncopy_b = quarantine->curobjs - ncopy_a; 86235238Sjasone 87235238Sjasone memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 88235238Sjasone * sizeof(quarantine_obj_t)); 89235238Sjasone memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 90235238Sjasone sizeof(quarantine_obj_t)); 91234370Sjasone } 92234370Sjasone 93234370Sjasone return (ret); 94234370Sjasone} 95234370Sjasone 96234370Sjasonestatic void 97234370Sjasonequarantine_drain(quarantine_t *quarantine, size_t upper_bound) 98234370Sjasone{ 99234370Sjasone 100234370Sjasone while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) { 101235238Sjasone quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 102235238Sjasone assert(obj->usize == isalloc(obj->ptr, config_prof)); 103235238Sjasone idalloc(obj->ptr); 104235238Sjasone quarantine->curbytes -= obj->usize; 105234370Sjasone quarantine->curobjs--; 106234370Sjasone quarantine->first = (quarantine->first + 1) & ((ZU(1) << 107234370Sjasone quarantine->lg_maxobjs) - 1); 108234370Sjasone } 109234370Sjasone} 110234370Sjasone 111234370Sjasonevoid 112234370Sjasonequarantine(void *ptr) 113234370Sjasone{ 114234370Sjasone quarantine_t *quarantine; 115234370Sjasone size_t usize = isalloc(ptr, config_prof); 116234370Sjasone 117234543Sjasone cassert(config_fill); 118234370Sjasone assert(opt_quarantine); 119234370Sjasone 120234370Sjasone quarantine = *quarantine_tsd_get(); 121235238Sjasone if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { 122235238Sjasone if (quarantine == NULL) { 123235238Sjasone if ((quarantine = quarantine_init(LG_MAXOBJS_INIT)) == 124235238Sjasone NULL) { 125235238Sjasone idalloc(ptr); 126235238Sjasone return; 127235238Sjasone } 128235238Sjasone } else { 129235238Sjasone if (quarantine == QUARANTINE_STATE_PURGATORY) { 130235238Sjasone /* 131235238Sjasone * Make a note that quarantine() was called 132235238Sjasone * after quarantine_cleanup() was called. 133235238Sjasone */ 134235238Sjasone quarantine = QUARANTINE_STATE_REINCARNATED; 135235238Sjasone quarantine_tsd_set(&quarantine); 136235238Sjasone } 137235238Sjasone idalloc(ptr); 138235238Sjasone return; 139235238Sjasone } 140234370Sjasone } 141234370Sjasone /* 142234370Sjasone * Drain one or more objects if the quarantine size limit would be 143234370Sjasone * exceeded by appending ptr. 144234370Sjasone */ 145234370Sjasone if (quarantine->curbytes + usize > opt_quarantine) { 146234370Sjasone size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 147234370Sjasone - usize : 0; 148234370Sjasone quarantine_drain(quarantine, upper_bound); 149234370Sjasone } 150234370Sjasone /* Grow the quarantine ring buffer if it's full. */ 151234370Sjasone if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 152234370Sjasone quarantine = quarantine_grow(quarantine); 153234370Sjasone /* quarantine_grow() must free a slot if it fails to grow. */ 154234370Sjasone assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 155234370Sjasone /* Append ptr if its size doesn't exceed the quarantine size. */ 156234370Sjasone if (quarantine->curbytes + usize <= opt_quarantine) { 157234370Sjasone size_t offset = (quarantine->first + quarantine->curobjs) & 158234370Sjasone ((ZU(1) << quarantine->lg_maxobjs) - 1); 159235238Sjasone quarantine_obj_t *obj = &quarantine->objs[offset]; 160235238Sjasone obj->ptr = ptr; 161235238Sjasone obj->usize = usize; 162234370Sjasone quarantine->curbytes += usize; 163234370Sjasone quarantine->curobjs++; 164234370Sjasone if (opt_junk) 165234370Sjasone memset(ptr, 0x5a, usize); 166234370Sjasone } else { 167234370Sjasone assert(quarantine->curbytes == 0); 168234370Sjasone idalloc(ptr); 169234370Sjasone } 170234370Sjasone} 171234370Sjasone 172234370Sjasonestatic void 173234370Sjasonequarantine_cleanup(void *arg) 174234370Sjasone{ 175234370Sjasone quarantine_t *quarantine = *(quarantine_t **)arg; 176234370Sjasone 177235238Sjasone if (quarantine == QUARANTINE_STATE_REINCARNATED) { 178235238Sjasone /* 179235238Sjasone * Another destructor deallocated memory after this destructor 180235238Sjasone * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY 181235238Sjasone * in order to receive another callback. 182235238Sjasone */ 183235238Sjasone quarantine = QUARANTINE_STATE_PURGATORY; 184235238Sjasone quarantine_tsd_set(&quarantine); 185235238Sjasone } else if (quarantine == QUARANTINE_STATE_PURGATORY) { 186235238Sjasone /* 187235238Sjasone * The previous time this destructor was called, we set the key 188235238Sjasone * to QUARANTINE_STATE_PURGATORY so that other destructors 189235238Sjasone * wouldn't cause re-creation of the quarantine. This time, do 190235238Sjasone * nothing, so that the destructor will not be called again. 191235238Sjasone */ 192235238Sjasone } else if (quarantine != NULL) { 193234370Sjasone quarantine_drain(quarantine, 0); 194234370Sjasone idalloc(quarantine); 195235238Sjasone quarantine = QUARANTINE_STATE_PURGATORY; 196235238Sjasone quarantine_tsd_set(&quarantine); 197234370Sjasone } 198234370Sjasone} 199234370Sjasone 200234370Sjasonebool 201234370Sjasonequarantine_boot(void) 202234370Sjasone{ 203234370Sjasone 204234543Sjasone cassert(config_fill); 205234370Sjasone 206234370Sjasone if (quarantine_tsd_boot()) 207234370Sjasone return (true); 208234370Sjasone 209234370Sjasone return (false); 210234370Sjasone} 211