quarantine.c revision 251300
1251300Sjasone#define JEMALLOC_QUARANTINE_C_ 2234370Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3234370Sjasone 4235238Sjasone/* 5235238Sjasone * quarantine pointers close to NULL are used to encode state information that 6235238Sjasone * is used for cleaning up during thread shutdown. 7235238Sjasone */ 8235238Sjasone#define QUARANTINE_STATE_REINCARNATED ((quarantine_t *)(uintptr_t)1) 9235238Sjasone#define QUARANTINE_STATE_PURGATORY ((quarantine_t *)(uintptr_t)2) 10235238Sjasone#define QUARANTINE_STATE_MAX QUARANTINE_STATE_PURGATORY 11235238Sjasone 12234370Sjasone/******************************************************************************/ 13234370Sjasone/* Data. */ 14234370Sjasone 15251300Sjasonemalloc_tsd_data(, quarantine, quarantine_t *, NULL) 16234370Sjasone 17234370Sjasone/******************************************************************************/ 18234370Sjasone/* Function prototypes for non-inline static functions. */ 19234370Sjasone 20234370Sjasonestatic quarantine_t *quarantine_grow(quarantine_t *quarantine); 21251300Sjasonestatic void quarantine_drain_one(quarantine_t *quarantine); 22234370Sjasonestatic void quarantine_drain(quarantine_t *quarantine, size_t upper_bound); 23234370Sjasone 24234370Sjasone/******************************************************************************/ 25234370Sjasone 26251300Sjasonequarantine_t * 27234370Sjasonequarantine_init(size_t lg_maxobjs) 28234370Sjasone{ 29234370Sjasone quarantine_t *quarantine; 30234370Sjasone 31234370Sjasone quarantine = (quarantine_t *)imalloc(offsetof(quarantine_t, objs) + 32235238Sjasone ((ZU(1) << lg_maxobjs) * sizeof(quarantine_obj_t))); 33234370Sjasone if (quarantine == NULL) 34234370Sjasone return (NULL); 35234370Sjasone quarantine->curbytes = 0; 36234370Sjasone quarantine->curobjs = 0; 37234370Sjasone quarantine->first = 0; 38234370Sjasone quarantine->lg_maxobjs = lg_maxobjs; 39234370Sjasone 40234370Sjasone quarantine_tsd_set(&quarantine); 41234370Sjasone 42234370Sjasone return (quarantine); 43234370Sjasone} 44234370Sjasone 45234370Sjasonestatic quarantine_t * 46234370Sjasonequarantine_grow(quarantine_t *quarantine) 47234370Sjasone{ 48234370Sjasone quarantine_t *ret; 49234370Sjasone 50234370Sjasone ret = quarantine_init(quarantine->lg_maxobjs + 1); 51251300Sjasone if (ret == NULL) { 52251300Sjasone quarantine_drain_one(quarantine); 53234370Sjasone return (quarantine); 54251300Sjasone } 55234370Sjasone 56234370Sjasone ret->curbytes = quarantine->curbytes; 57235238Sjasone ret->curobjs = quarantine->curobjs; 58235238Sjasone if (quarantine->first + quarantine->curobjs <= (ZU(1) << 59234370Sjasone quarantine->lg_maxobjs)) { 60234370Sjasone /* objs ring buffer data are contiguous. */ 61234370Sjasone memcpy(ret->objs, &quarantine->objs[quarantine->first], 62235238Sjasone quarantine->curobjs * sizeof(quarantine_obj_t)); 63234370Sjasone } else { 64234370Sjasone /* objs ring buffer data wrap around. */ 65235238Sjasone size_t ncopy_a = (ZU(1) << quarantine->lg_maxobjs) - 66234370Sjasone quarantine->first; 67235238Sjasone size_t ncopy_b = quarantine->curobjs - ncopy_a; 68235238Sjasone 69235238Sjasone memcpy(ret->objs, &quarantine->objs[quarantine->first], ncopy_a 70235238Sjasone * sizeof(quarantine_obj_t)); 71235238Sjasone memcpy(&ret->objs[ncopy_a], quarantine->objs, ncopy_b * 72235238Sjasone sizeof(quarantine_obj_t)); 73234370Sjasone } 74251300Sjasone idalloc(quarantine); 75234370Sjasone 76234370Sjasone return (ret); 77234370Sjasone} 78234370Sjasone 79234370Sjasonestatic void 80251300Sjasonequarantine_drain_one(quarantine_t *quarantine) 81251300Sjasone{ 82251300Sjasone quarantine_obj_t *obj = &quarantine->objs[quarantine->first]; 83251300Sjasone assert(obj->usize == isalloc(obj->ptr, config_prof)); 84251300Sjasone idalloc(obj->ptr); 85251300Sjasone quarantine->curbytes -= obj->usize; 86251300Sjasone quarantine->curobjs--; 87251300Sjasone quarantine->first = (quarantine->first + 1) & ((ZU(1) << 88251300Sjasone quarantine->lg_maxobjs) - 1); 89251300Sjasone} 90251300Sjasone 91251300Sjasonestatic void 92234370Sjasonequarantine_drain(quarantine_t *quarantine, size_t upper_bound) 93234370Sjasone{ 94234370Sjasone 95251300Sjasone while (quarantine->curbytes > upper_bound && quarantine->curobjs > 0) 96251300Sjasone quarantine_drain_one(quarantine); 97234370Sjasone} 98234370Sjasone 99234370Sjasonevoid 100234370Sjasonequarantine(void *ptr) 101234370Sjasone{ 102234370Sjasone quarantine_t *quarantine; 103234370Sjasone size_t usize = isalloc(ptr, config_prof); 104234370Sjasone 105234543Sjasone cassert(config_fill); 106234370Sjasone assert(opt_quarantine); 107234370Sjasone 108234370Sjasone quarantine = *quarantine_tsd_get(); 109235238Sjasone if ((uintptr_t)quarantine <= (uintptr_t)QUARANTINE_STATE_MAX) { 110251300Sjasone if (quarantine == QUARANTINE_STATE_PURGATORY) { 111251300Sjasone /* 112251300Sjasone * Make a note that quarantine() was called after 113251300Sjasone * quarantine_cleanup() was called. 114251300Sjasone */ 115251300Sjasone quarantine = QUARANTINE_STATE_REINCARNATED; 116251300Sjasone quarantine_tsd_set(&quarantine); 117235238Sjasone } 118251300Sjasone idalloc(ptr); 119251300Sjasone return; 120234370Sjasone } 121234370Sjasone /* 122234370Sjasone * Drain one or more objects if the quarantine size limit would be 123234370Sjasone * exceeded by appending ptr. 124234370Sjasone */ 125234370Sjasone if (quarantine->curbytes + usize > opt_quarantine) { 126234370Sjasone size_t upper_bound = (opt_quarantine >= usize) ? opt_quarantine 127234370Sjasone - usize : 0; 128234370Sjasone quarantine_drain(quarantine, upper_bound); 129234370Sjasone } 130234370Sjasone /* Grow the quarantine ring buffer if it's full. */ 131234370Sjasone if (quarantine->curobjs == (ZU(1) << quarantine->lg_maxobjs)) 132234370Sjasone quarantine = quarantine_grow(quarantine); 133234370Sjasone /* quarantine_grow() must free a slot if it fails to grow. */ 134234370Sjasone assert(quarantine->curobjs < (ZU(1) << quarantine->lg_maxobjs)); 135234370Sjasone /* Append ptr if its size doesn't exceed the quarantine size. */ 136234370Sjasone if (quarantine->curbytes + usize <= opt_quarantine) { 137234370Sjasone size_t offset = (quarantine->first + quarantine->curobjs) & 138234370Sjasone ((ZU(1) << quarantine->lg_maxobjs) - 1); 139235238Sjasone quarantine_obj_t *obj = &quarantine->objs[offset]; 140235238Sjasone obj->ptr = ptr; 141235238Sjasone obj->usize = usize; 142234370Sjasone quarantine->curbytes += usize; 143234370Sjasone quarantine->curobjs++; 144234370Sjasone if (opt_junk) 145234370Sjasone memset(ptr, 0x5a, usize); 146234370Sjasone } else { 147234370Sjasone assert(quarantine->curbytes == 0); 148234370Sjasone idalloc(ptr); 149234370Sjasone } 150234370Sjasone} 151234370Sjasone 152251300Sjasonevoid 153234370Sjasonequarantine_cleanup(void *arg) 154234370Sjasone{ 155234370Sjasone quarantine_t *quarantine = *(quarantine_t **)arg; 156234370Sjasone 157235238Sjasone if (quarantine == QUARANTINE_STATE_REINCARNATED) { 158235238Sjasone /* 159235238Sjasone * Another destructor deallocated memory after this destructor 160235238Sjasone * was called. Reset quarantine to QUARANTINE_STATE_PURGATORY 161235238Sjasone * in order to receive another callback. 162235238Sjasone */ 163235238Sjasone quarantine = QUARANTINE_STATE_PURGATORY; 164235238Sjasone quarantine_tsd_set(&quarantine); 165235238Sjasone } else if (quarantine == QUARANTINE_STATE_PURGATORY) { 166235238Sjasone /* 167235238Sjasone * The previous time this destructor was called, we set the key 168235238Sjasone * to QUARANTINE_STATE_PURGATORY so that other destructors 169235238Sjasone * wouldn't cause re-creation of the quarantine. This time, do 170235238Sjasone * nothing, so that the destructor will not be called again. 171235238Sjasone */ 172235238Sjasone } else if (quarantine != NULL) { 173234370Sjasone quarantine_drain(quarantine, 0); 174234370Sjasone idalloc(quarantine); 175235238Sjasone quarantine = QUARANTINE_STATE_PURGATORY; 176235238Sjasone quarantine_tsd_set(&quarantine); 177234370Sjasone } 178234370Sjasone} 179234370Sjasone 180234370Sjasonebool 181234370Sjasonequarantine_boot(void) 182234370Sjasone{ 183234370Sjasone 184234543Sjasone cassert(config_fill); 185234370Sjasone 186234370Sjasone if (quarantine_tsd_boot()) 187234370Sjasone return (true); 188234370Sjasone 189234370Sjasone return (false); 190234370Sjasone} 191