malloc.c revision 114625
118334Speter/*- 290075Sobrien * Copyright (c) 1983 Regents of the University of California. 390075Sobrien * All rights reserved. 418334Speter * 590075Sobrien * Redistribution and use in source and binary forms, with or without 618334Speter * modification, are permitted provided that the following conditions 790075Sobrien * are met: 890075Sobrien * 1. Redistributions of source code must retain the above copyright 990075Sobrien * notice, this list of conditions and the following disclaimer. 1090075Sobrien * 2. Redistributions in binary form must reproduce the above copyright 1118334Speter * notice, this list of conditions and the following disclaimer in the 1290075Sobrien * documentation and/or other materials provided with the distribution. 1390075Sobrien * 3. All advertising materials mentioning features or use of this software 1490075Sobrien * must display the following acknowledgement: 1590075Sobrien * This product includes software developed by the University of 1618334Speter * California, Berkeley and its contributors. 1718334Speter * 4. Neither the name of the University nor the names of its contributors 1890075Sobrien * may be used to endorse or promote products derived from this software 1990075Sobrien * without specific prior written permission. 2090075Sobrien * 2118334Speter * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 2218334Speter * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 2318334Speter * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 2450397Sobrien * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 2518334Speter * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 2650397Sobrien * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 2790075Sobrien * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 2818334Speter * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 2918334Speter * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 3050397Sobrien * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 3150397Sobrien * SUCH DAMAGE. 3290075Sobrien */ 3318334Speter 3490075Sobrien#if defined(LIBC_SCCS) && !defined(lint) 3590075Sobrien/*static char *sccsid = "from: @(#)malloc.c 5.11 (Berkeley) 2/23/91";*/ 3618334Speterstatic char *rcsid = "$FreeBSD: head/libexec/rtld-elf/malloc.c 114625 2003-05-04 00:56:00Z obrien $"; 3790075Sobrien#endif /* LIBC_SCCS and not lint */ 3890075Sobrien 3990075Sobrien/* 4090075Sobrien * malloc.c (Caltech) 2/21/82 4118334Speter * Chris Kingsley, kingsley@cit-20. 4250397Sobrien * 4390075Sobrien * This is a very fast storage allocator. It allocates blocks of a small 4418334Speter * number of different sizes, and keeps free lists of each size. Blocks that 4518334Speter * don't exactly fit are passed up to the next larger size. In this 4618334Speter * implementation, the available sizes are 2^n-4 (or 2^n-10) bytes long. 4790075Sobrien * This is designed for use in a virtual memory environment. 4818334Speter */ 4918334Speter 5018334Speter#include <sys/types.h> 5190075Sobrien#include <err.h> 5218334Speter#include <paths.h> 5390075Sobrien#include <stdarg.h> 5490075Sobrien#include <stdio.h> 5590075Sobrien#include <stdlib.h> 5690075Sobrien#include <string.h> 5790075Sobrien#include <stddef.h> 5890075Sobrien#include <unistd.h> 5990075Sobrien#include <sys/param.h> 6090075Sobrien#include <sys/mman.h> 6190075Sobrien#ifndef BSD 6218334Speter#define MAP_COPY MAP_PRIVATE 6318334Speter#define MAP_FILE 0 6418334Speter#define MAP_ANON 0 6518334Speter#endif 6618334Speter 6718334Speter#ifndef BSD /* Need do better than this */ 6818334Speter#define NEED_DEV_ZERO 1 6918334Speter#endif 7018334Speter 7118334Speterstatic void morecore(); 7290075Sobrienstatic int findbucket(); 7390075Sobrien 7490075Sobrien/* 7590075Sobrien * Pre-allocate mmap'ed pages 7690075Sobrien */ 7790075Sobrien#define NPOOLPAGES (32*1024/pagesz) 7890075Sobrienstatic caddr_t pagepool_start, pagepool_end; 7990075Sobrienstatic int morepages(); 8090075Sobrien 8190075Sobrien/* 8290075Sobrien * The overhead on a block is at least 4 bytes. When free, this space 8318334Speter * contains a pointer to the next free block, and the bottom two bits must 8418334Speter * be zero. When in use, the first byte is set to MAGIC, and the second 8518334Speter * byte is the size index. The remaining bytes are for alignment. 8618334Speter * If range checking is enabled then a second word holds the size of the 8718334Speter * requested block, less 1, rounded up to a multiple of sizeof(RMAGIC). 8818334Speter * The order of elements is critical: ov_magic must overlay the low order 8918334Speter * bits of ov_next, and ov_magic can not be a valid ov_next bit pattern. 9018334Speter */ 9118334Speterunion overhead { 9290075Sobrien union overhead *ov_next; /* when free */ 9318334Speter struct { 9418334Speter u_char ovu_magic; /* magic number */ 9518334Speter u_char ovu_index; /* bucket # */ 9618334Speter#ifdef RCHECK 9790075Sobrien u_short ovu_rmagic; /* range magic number */ 9890075Sobrien u_int ovu_size; /* actual block size */ 9990075Sobrien#endif 10090075Sobrien } ovu; 10190075Sobrien#define ov_magic ovu.ovu_magic 10290075Sobrien#define ov_index ovu.ovu_index 10390075Sobrien#define ov_rmagic ovu.ovu_rmagic 10490075Sobrien#define ov_size ovu.ovu_size 10590075Sobrien}; 10690075Sobrien 10790075Sobrien#define MAGIC 0xef /* magic # on accounting info */ 10890075Sobrien#define RMAGIC 0x5555 /* magic # on range info */ 10990075Sobrien 11090075Sobrien#ifdef RCHECK 11190075Sobrien#define RSLOP sizeof (u_short) 11290075Sobrien#else 11318334Speter#define RSLOP 0 11490075Sobrien#endif 11590075Sobrien 11690075Sobrien/* 11790075Sobrien * nextf[i] is the pointer to the next free block of size 2^(i+3). The 11890075Sobrien * smallest allocatable block is 8 bytes. The overhead information 11990075Sobrien * precedes the data area returned to the user. 12090075Sobrien */ 12190075Sobrien#define NBUCKETS 30 12290075Sobrienstatic union overhead *nextf[NBUCKETS]; 12390075Sobrien 12490075Sobrienstatic int pagesz; /* page size */ 12590075Sobrienstatic int pagebucket; /* page size bucket */ 12690075Sobrien 12790075Sobrien#ifdef MSTATS 12890075Sobrien/* 12990075Sobrien * nmalloc[i] is the difference between the number of mallocs and frees 13090075Sobrien * for a given block size. 13190075Sobrien */ 13218334Speterstatic u_int nmalloc[NBUCKETS]; 13318334Speter#include <stdio.h> 13418334Speter#endif 13518334Speter 13618334Speter#if defined(MALLOC_DEBUG) || defined(RCHECK) 13718334Speter#define ASSERT(p) if (!(p)) botch("p") 13818334Speter#include <stdio.h> 13918334Speterstatic void 14018334Speterbotch(s) 14118334Speter char *s; 14218334Speter{ 14318334Speter fprintf(stderr, "\r\nassertion botched: %s\r\n", s); 14418334Speter (void) fflush(stderr); /* just in case user buffered it */ 14518334Speter abort(); 14618334Speter} 14718334Speter#else 14818334Speter#define ASSERT(p) 14918334Speter#endif 15090075Sobrien 15190075Sobrien/* Debugging stuff */ 15218334Speterstatic void xprintf(const char *, ...); 15318334Speter#define TRACE() xprintf("TRACE %s:%d\n", __FILE__, __LINE__) 15418334Speter 15518334Spetervoid * 15618334Spetermalloc(nbytes) 15718334Speter size_t nbytes; 15890075Sobrien{ 15990075Sobrien register union overhead *op; 16090075Sobrien register int bucket; 16190075Sobrien register long n; 16290075Sobrien register unsigned amt; 16390075Sobrien 16490075Sobrien /* 16590075Sobrien * First time malloc is called, setup page size and 16690075Sobrien * align break pointer so all data will be page aligned. 16790075Sobrien */ 16890075Sobrien if (pagesz == 0) { 16918334Speter pagesz = n = getpagesize(); 17018334Speter if (morepages(NPOOLPAGES) == 0) 17118334Speter return NULL; 17218334Speter op = (union overhead *)(pagepool_start); 17318334Speter n = n - sizeof (*op) - ((long)op & (n - 1)); 17418334Speter if (n < 0) 17518334Speter n += pagesz; 17690075Sobrien if (n) { 17718334Speter pagepool_start += n; 17818334Speter } 17918334Speter bucket = 0; 18018334Speter amt = 8; 18118334Speter while ((unsigned)pagesz > amt) { 18290075Sobrien amt <<= 1; 18318334Speter bucket++; 18490075Sobrien } 18590075Sobrien pagebucket = bucket; 18690075Sobrien } 18790075Sobrien /* 18818334Speter * Convert amount of memory requested into closest block size 18990075Sobrien * stored in hash buckets which satisfies request. 19018334Speter * Account for space used per block for accounting. 19118334Speter */ 19218334Speter if (nbytes <= (unsigned long)(n = pagesz - sizeof (*op) - RSLOP)) { 19318334Speter#ifndef RCHECK 19418334Speter amt = 8; /* size of first bucket */ 19518334Speter bucket = 0; 19618334Speter#else 19718334Speter amt = 16; /* size of first bucket */ 19818334Speter bucket = 1; 19918334Speter#endif 20018334Speter n = -(sizeof (*op) + RSLOP); 20118334Speter } else { 20218334Speter amt = pagesz; 20318334Speter bucket = pagebucket; 20418334Speter } 20518334Speter while (nbytes > amt + n) { 20618334Speter amt <<= 1; 20718334Speter if (amt == 0) 20818334Speter return (NULL); 20990075Sobrien bucket++; 21018334Speter } 21190075Sobrien /* 21218334Speter * If nothing in hash bucket right now, 21318334Speter * request more memory from the system. 21418334Speter */ 21518334Speter if ((op = nextf[bucket]) == NULL) { 21618334Speter morecore(bucket); 21790075Sobrien if ((op = nextf[bucket]) == NULL) 21818334Speter return (NULL); 21918334Speter } 22018334Speter /* remove from linked list */ 22118334Speter nextf[bucket] = op->ov_next; 22218334Speter op->ov_magic = MAGIC; 22390075Sobrien op->ov_index = bucket; 22490075Sobrien#ifdef MSTATS 22590075Sobrien nmalloc[bucket]++; 22690075Sobrien#endif 22790075Sobrien#ifdef RCHECK 22890075Sobrien /* 22990075Sobrien * Record allocated size of block and 23090075Sobrien * bound space with magic numbers. 23190075Sobrien */ 23290075Sobrien op->ov_size = (nbytes + RSLOP - 1) & ~(RSLOP - 1); 23390075Sobrien op->ov_rmagic = RMAGIC; 23490075Sobrien *(u_short *)((caddr_t)(op + 1) + op->ov_size) = RMAGIC; 23590075Sobrien#endif 23690075Sobrien return ((char *)(op + 1)); 23790075Sobrien} 23890075Sobrien 23990075Sobrien/* 24090075Sobrien * Allocate more memory to the indicated bucket. 24118334Speter */ 24218334Speterstatic void 24318334Spetermorecore(bucket) 24452284Sobrien int bucket; 24518334Speter{ 24618334Speter register union overhead *op; 24718334Speter register int sz; /* size of desired block */ 24818334Speter int amt; /* amount to allocate */ 24990075Sobrien int nblks; /* how many blocks we get */ 25018334Speter 25118334Speter /* 25218334Speter * sbrk_size <= 0 only for big, FLUFFY, requests (about 25318334Speter * 2^30 bytes on a VAX, I think) or for a negative arg. 25418334Speter */ 25590075Sobrien sz = 1 << (bucket + 3); 25618334Speter#ifdef MALLOC_DEBUG 25718334Speter ASSERT(sz > 0); 25818334Speter#else 25918334Speter if (sz <= 0) 26018334Speter return; 26152284Sobrien#endif 26252284Sobrien if (sz < pagesz) { 26352284Sobrien amt = pagesz; 26452284Sobrien nblks = amt / sz; 26552284Sobrien } else { 26652284Sobrien amt = sz + pagesz; 26752284Sobrien nblks = 1; 26852284Sobrien } 26952284Sobrien if (amt > pagepool_end - pagepool_start) 27052284Sobrien if (morepages(amt/pagesz + NPOOLPAGES) == 0) 27152284Sobrien return; 27252284Sobrien op = (union overhead *)pagepool_start; 27352284Sobrien pagepool_start += amt; 27452284Sobrien 27552284Sobrien /* 27690075Sobrien * Add new memory allocated to that on 27790075Sobrien * free list for this hash bucket. 27852284Sobrien */ 27952284Sobrien nextf[bucket] = op; 28052284Sobrien while (--nblks > 0) { 28152284Sobrien op->ov_next = (union overhead *)((caddr_t)op + sz); 28252284Sobrien op = (union overhead *)((caddr_t)op + sz); 28352284Sobrien } 28452284Sobrien} 28590075Sobrien 28690075Sobrienvoid 28752284Sobrienfree(cp) 28852284Sobrien void *cp; 28990075Sobrien{ 29052284Sobrien register int size; 29152284Sobrien register union overhead *op; 29252284Sobrien 29352284Sobrien if (cp == NULL) 29452284Sobrien return; 29590075Sobrien op = (union overhead *)((caddr_t)cp - sizeof (union overhead)); 29690075Sobrien#ifdef MALLOC_DEBUG 29718334Speter ASSERT(op->ov_magic == MAGIC); /* make sure it was in use */ 29818334Speter#else 29918334Speter if (op->ov_magic != MAGIC) 30018334Speter return; /* sanity */ 30118334Speter#endif 30218334Speter#ifdef RCHECK 30390075Sobrien ASSERT(op->ov_rmagic == RMAGIC); 30490075Sobrien ASSERT(*(u_short *)((caddr_t)(op + 1) + op->ov_size) == RMAGIC); 30590075Sobrien#endif 30618334Speter size = op->ov_index; 30790075Sobrien ASSERT(size < NBUCKETS); 30890075Sobrien op->ov_next = nextf[size]; /* also clobbers ov_magic */ 30990075Sobrien nextf[size] = op; 31090075Sobrien#ifdef MSTATS 31190075Sobrien nmalloc[size]--; 31290075Sobrien#endif 31390075Sobrien} 31490075Sobrien 31590075Sobrien/* 31690075Sobrien * When a program attempts "storage compaction" as mentioned in the 31790075Sobrien * old malloc man page, it realloc's an already freed block. Usually 31890075Sobrien * this is the last block it freed; occasionally it might be farther 31918334Speter * back. We have to search all the free lists for the block in order 32018334Speter * to determine its bucket: 1st we make one pass thru the lists 32118334Speter * checking only the first block in each; if that fails we search 32218334Speter * ``realloc_srchlen'' blocks in each list for a match (the variable 32318334Speter * is extern so the caller can modify it). If that fails we just copy 32418334Speter * however many bytes was given to realloc() and hope it's not huge. 32518334Speter */ 32618334Speterint realloc_srchlen = 4; /* 4 should be plenty, -1 =>'s whole list */ 32718334Speter 32818334Spetervoid * 32918334Speterrealloc(cp, nbytes) 33018334Speter void *cp; 33118334Speter size_t nbytes; 33218334Speter{ 33318334Speter register u_int onb; 33418334Speter register int i; 33518334Speter union overhead *op; 33690075Sobrien char *res; 33718334Speter int was_alloced = 0; 33890075Sobrien 33990075Sobrien if (cp == NULL) 34018334Speter return (malloc(nbytes)); 34118334Speter op = (union overhead *)((caddr_t)cp - sizeof (union overhead)); 34218334Speter if (op->ov_magic == MAGIC) { 34390075Sobrien was_alloced++; 34490075Sobrien i = op->ov_index; 34518334Speter } else { 34618334Speter /* 34718334Speter * Already free, doing "compaction". 34890075Sobrien * 34918334Speter * Search for the old block of memory on the 35090075Sobrien * free list. First, check the most common 35190075Sobrien * case (last element free'd), then (this failing) 35290075Sobrien * the last ``realloc_srchlen'' items free'd. 35390075Sobrien * If all lookups fail, then assume the size of 35490075Sobrien * the memory block being realloc'd is the 35590075Sobrien * largest possible (so that all "nbytes" of new 35690075Sobrien * memory are copied into). Note that this could cause 35790075Sobrien * a memory fault if the old area was tiny, and the moon 35890075Sobrien * is gibbous. However, that is very unlikely. 35918334Speter */ 36018334Speter if ((i = findbucket(op, 1)) < 0 && 36190075Sobrien (i = findbucket(op, realloc_srchlen)) < 0) 36290075Sobrien i = NBUCKETS; 36390075Sobrien } 36418334Speter onb = 1 << (i + 3); 36518334Speter if (onb < (u_int)pagesz) 36690075Sobrien onb -= sizeof (*op) + RSLOP; 36790075Sobrien else 36890075Sobrien onb += pagesz - sizeof (*op) - RSLOP; 36990075Sobrien /* avoid the copy if same size block */ 37090075Sobrien if (was_alloced) { 37190075Sobrien if (i) { 37290075Sobrien i = 1 << (i + 2); 37318334Speter if (i < pagesz) 37418334Speter i -= sizeof (*op) + RSLOP; 37518334Speter else 37618334Speter i += pagesz - sizeof (*op) - RSLOP; 37790075Sobrien } 37890075Sobrien if (nbytes <= onb && nbytes > (size_t)i) { 37990075Sobrien#ifdef RCHECK 38090075Sobrien op->ov_size = (nbytes + RSLOP - 1) & ~(RSLOP - 1); 38190075Sobrien *(u_short *)((caddr_t)(op + 1) + op->ov_size) = RMAGIC; 38290075Sobrien#endif 38390075Sobrien return(cp); 38490075Sobrien } else 38518334Speter free(cp); 38690075Sobrien } 38718334Speter if ((res = malloc(nbytes)) == NULL) 38818334Speter return (NULL); 38918334Speter if (cp != res) /* common optimization if "compacting" */ 39018334Speter bcopy(cp, res, (nbytes < onb) ? nbytes : onb); 39190075Sobrien return (res); 39290075Sobrien} 39390075Sobrien 39490075Sobrien/* 39590075Sobrien * Search ``srchlen'' elements of each free list for a block whose 39690075Sobrien * header starts at ``freep''. If srchlen is -1 search the whole list. 39790075Sobrien * Return bucket number, or -1 if not found. 39890075Sobrien */ 39990075Sobrienstatic int 40090075Sobrienfindbucket(freep, srchlen) 40190075Sobrien union overhead *freep; 40290075Sobrien int srchlen; 40390075Sobrien{ 40490075Sobrien register union overhead *p; 40518334Speter register int i, j; 40618334Speter 40790075Sobrien for (i = 0; i < NBUCKETS; i++) { 40890075Sobrien j = 0; 40990075Sobrien for (p = nextf[i]; p && j != srchlen; p = p->ov_next) { 41090075Sobrien if (p == freep) 41118334Speter return (i); 41250397Sobrien j++; 41350397Sobrien } 41418334Speter } 41590075Sobrien return (-1); 41690075Sobrien} 41718334Speter 41890075Sobrien#ifdef MSTATS 41918334Speter/* 42090075Sobrien * mstats - print out statistics about malloc 42118334Speter * 42218334Speter * Prints two lines of numbers, one showing the length of the free list 42318334Speter * for each size category, the second showing the number of mallocs - 42418334Speter * frees for each size category. 42518334Speter */ 42618334Spetermstats(s) 42750397Sobrien char *s; 42890075Sobrien{ 42990075Sobrien register int i, j; 43090075Sobrien register union overhead *p; 43190075Sobrien int totfree = 0, 43290075Sobrien totused = 0; 43350397Sobrien 43450397Sobrien fprintf(stderr, "Memory allocation statistics %s\nfree:\t", s); 43518334Speter for (i = 0; i < NBUCKETS; i++) { 43618334Speter for (j = 0, p = nextf[i]; p; p = p->ov_next, j++) 43718334Speter ; 43890075Sobrien fprintf(stderr, " %d", j); 43990075Sobrien totfree += j * (1 << (i + 3)); 44090075Sobrien } 44190075Sobrien fprintf(stderr, "\nused:\t"); 44290075Sobrien for (i = 0; i < NBUCKETS; i++) { 44390075Sobrien fprintf(stderr, " %d", nmalloc[i]); 44490075Sobrien totused += nmalloc[i] * (1 << (i + 3)); 44590075Sobrien } 44690075Sobrien fprintf(stderr, "\n\tTotal in use: %d, total free: %d\n", 44790075Sobrien totused, totfree); 44890075Sobrien} 44990075Sobrien#endif 45090075Sobrien 45190075Sobrien 45290075Sobrienstatic int 45390075Sobrienmorepages(n) 45490075Sobrienint n; 45590075Sobrien{ 45690075Sobrien int fd = -1; 45790075Sobrien int offset; 45890075Sobrien 45990075Sobrien#ifdef NEED_DEV_ZERO 46090075Sobrien fd = open(_PATH_DEVZERO, O_RDWR, 0); 46118334Speter if (fd == -1) 46218334Speter perror(_PATH_DEVZERO); 46390075Sobrien#endif 46490075Sobrien 46518334Speter if (pagepool_end - pagepool_start > pagesz) { 46690075Sobrien caddr_t addr = (caddr_t) 46718334Speter (((long)pagepool_start + pagesz - 1) & ~(pagesz - 1)); 46890075Sobrien if (munmap(addr, pagepool_end - addr) != 0) 46990075Sobrien warn("morepages: munmap %p", addr); 47090075Sobrien } 47190075Sobrien 47290075Sobrien offset = (long)pagepool_start - ((long)pagepool_start & ~(pagesz - 1)); 47390075Sobrien 47418334Speter if ((pagepool_start = mmap(0, n * pagesz, 47590075Sobrien PROT_READ|PROT_WRITE, 47690075Sobrien MAP_ANON|MAP_COPY, fd, 0)) == (caddr_t)-1) { 47790075Sobrien xprintf("Cannot map anonymous memory"); 47890075Sobrien return 0; 47990075Sobrien } 48090075Sobrien pagepool_end = pagepool_start + n * pagesz; 48190075Sobrien pagepool_start += offset; 48290075Sobrien 48390075Sobrien#ifdef NEED_DEV_ZERO 48490075Sobrien close(fd); 48518334Speter#endif 48690075Sobrien return n; 48790075Sobrien} 48818334Speter 48990075Sobrien/* 49090075Sobrien * Non-mallocing printf, for use by malloc itself. 49190075Sobrien */ 49290075Sobrienstatic void 49390075Sobrienxprintf(const char *fmt, ...) 49490075Sobrien{ 49590075Sobrien char buf[256]; 49690075Sobrien va_list ap; 49790075Sobrien 49850397Sobrien va_start(ap, fmt); 49950397Sobrien vsprintf(buf, fmt, ap); 50090075Sobrien (void)write(STDOUT_FILENO, buf, strlen(buf)); 50190075Sobrien va_end(ap); 50250397Sobrien} 50318334Speter