1286866Sjasone#define JEMALLOC_PAGES_C_ 2286866Sjasone#include "jemalloc/internal/jemalloc_internal.h" 3286866Sjasone 4299587Sjasone#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 5299587Sjasone#include <sys/sysctl.h> 6299587Sjasone#endif 7299587Sjasone 8286866Sjasone/******************************************************************************/ 9299587Sjasone/* Data. */ 10286866Sjasone 11299587Sjasone#ifndef _WIN32 12299587Sjasone# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) 13299587Sjasone# define PAGES_PROT_DECOMMIT (PROT_NONE) 14299587Sjasonestatic int mmap_flags; 15299587Sjasone#endif 16299587Sjasonestatic bool os_overcommits; 17299587Sjasone 18299587Sjasone/******************************************************************************/ 19299587Sjasone 20286866Sjasonevoid * 21299587Sjasonepages_map(void *addr, size_t size, bool *commit) 22286866Sjasone{ 23286866Sjasone void *ret; 24286866Sjasone 25286866Sjasone assert(size != 0); 26286866Sjasone 27299587Sjasone if (os_overcommits) 28299587Sjasone *commit = true; 29299587Sjasone 30286866Sjasone#ifdef _WIN32 31286866Sjasone /* 32286866Sjasone * If VirtualAlloc can't allocate at the given address when one is 33286866Sjasone * given, it fails and returns NULL. 34286866Sjasone */ 35299587Sjasone ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0), 36286866Sjasone PAGE_READWRITE); 37286866Sjasone#else 38286866Sjasone /* 39286866Sjasone * We don't use MAP_FIXED here, because it can cause the *replacement* 40286866Sjasone * of existing mappings, and we only want to create new mappings. 41286866Sjasone */ 42299587Sjasone { 43299587Sjasone int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 44299587Sjasone 45299587Sjasone ret = mmap(addr, size, prot, mmap_flags, -1, 0); 46299587Sjasone } 47286866Sjasone assert(ret != NULL); 48286866Sjasone 49286866Sjasone if (ret == MAP_FAILED) 50286866Sjasone ret = NULL; 51286866Sjasone else if (addr != NULL && ret != addr) { 52286866Sjasone /* 53286866Sjasone * We succeeded in mapping memory, but not in the right place. 54286866Sjasone */ 55286866Sjasone pages_unmap(ret, size); 56286866Sjasone ret = NULL; 57286866Sjasone } 58286866Sjasone#endif 59286866Sjasone assert(ret == NULL || (addr == NULL && ret != addr) 60286866Sjasone || (addr != NULL && ret == addr)); 61286866Sjasone return (ret); 62286866Sjasone} 63286866Sjasone 64286866Sjasonevoid 65286866Sjasonepages_unmap(void *addr, size_t size) 66286866Sjasone{ 67286866Sjasone 68286866Sjasone#ifdef _WIN32 69286866Sjasone if (VirtualFree(addr, 0, MEM_RELEASE) == 0) 70286866Sjasone#else 71286866Sjasone if (munmap(addr, size) == -1) 72286866Sjasone#endif 73286866Sjasone { 74286866Sjasone char buf[BUFERROR_BUF]; 75286866Sjasone 76286866Sjasone buferror(get_errno(), buf, sizeof(buf)); 77286866Sjasone malloc_printf("<jemalloc>: Error in " 78286866Sjasone#ifdef _WIN32 79286866Sjasone "VirtualFree" 80286866Sjasone#else 81286866Sjasone "munmap" 82286866Sjasone#endif 83286866Sjasone "(): %s\n", buf); 84286866Sjasone if (opt_abort) 85286866Sjasone abort(); 86286866Sjasone } 87286866Sjasone} 88286866Sjasone 89286866Sjasonevoid * 90299587Sjasonepages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, 91299587Sjasone bool *commit) 92286866Sjasone{ 93286866Sjasone void *ret = (void *)((uintptr_t)addr + leadsize); 94286866Sjasone 95286866Sjasone assert(alloc_size >= leadsize + size); 96286866Sjasone#ifdef _WIN32 97286866Sjasone { 98286866Sjasone void *new_addr; 99286866Sjasone 100286866Sjasone pages_unmap(addr, alloc_size); 101299587Sjasone new_addr = pages_map(ret, size, commit); 102286866Sjasone if (new_addr == ret) 103286866Sjasone return (ret); 104286866Sjasone if (new_addr) 105286866Sjasone pages_unmap(new_addr, size); 106286866Sjasone return (NULL); 107286866Sjasone } 108286866Sjasone#else 109286866Sjasone { 110286866Sjasone size_t trailsize = alloc_size - leadsize - size; 111286866Sjasone 112286866Sjasone if (leadsize != 0) 113286866Sjasone pages_unmap(addr, leadsize); 114286866Sjasone if (trailsize != 0) 115286866Sjasone pages_unmap((void *)((uintptr_t)ret + size), trailsize); 116286866Sjasone return (ret); 117286866Sjasone } 118286866Sjasone#endif 119286866Sjasone} 120286866Sjasone 121286866Sjasonestatic bool 122286866Sjasonepages_commit_impl(void *addr, size_t size, bool commit) 123286866Sjasone{ 124286866Sjasone 125299587Sjasone if (os_overcommits) 126299587Sjasone return (true); 127299587Sjasone 128299587Sjasone#ifdef _WIN32 129299587Sjasone return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, 130299587Sjasone PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); 131299587Sjasone#else 132299587Sjasone { 133299587Sjasone int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 134299587Sjasone void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, 135299587Sjasone -1, 0); 136286866Sjasone if (result == MAP_FAILED) 137286866Sjasone return (true); 138286866Sjasone if (result != addr) { 139286866Sjasone /* 140286866Sjasone * We succeeded in mapping memory, but not in the right 141286866Sjasone * place. 142286866Sjasone */ 143286866Sjasone pages_unmap(result, size); 144286866Sjasone return (true); 145286866Sjasone } 146286866Sjasone return (false); 147286866Sjasone } 148286866Sjasone#endif 149286866Sjasone} 150286866Sjasone 151286866Sjasonebool 152286866Sjasonepages_commit(void *addr, size_t size) 153286866Sjasone{ 154286866Sjasone 155286866Sjasone return (pages_commit_impl(addr, size, true)); 156286866Sjasone} 157286866Sjasone 158286866Sjasonebool 159286866Sjasonepages_decommit(void *addr, size_t size) 160286866Sjasone{ 161286866Sjasone 162286866Sjasone return (pages_commit_impl(addr, size, false)); 163286866Sjasone} 164286866Sjasone 165286866Sjasonebool 166286866Sjasonepages_purge(void *addr, size_t size) 167286866Sjasone{ 168286866Sjasone bool unzeroed; 169286866Sjasone 170286866Sjasone#ifdef _WIN32 171286866Sjasone VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); 172286866Sjasone unzeroed = true; 173286866Sjasone#elif defined(JEMALLOC_HAVE_MADVISE) 174286866Sjasone# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 175286866Sjasone# define JEMALLOC_MADV_PURGE MADV_DONTNEED 176286866Sjasone# define JEMALLOC_MADV_ZEROS true 177286866Sjasone# elif defined(JEMALLOC_PURGE_MADVISE_FREE) 178286866Sjasone# define JEMALLOC_MADV_PURGE MADV_FREE 179286866Sjasone# define JEMALLOC_MADV_ZEROS false 180286866Sjasone# else 181286866Sjasone# error "No madvise(2) flag defined for purging unused dirty pages." 182286866Sjasone# endif 183286866Sjasone int err = madvise(addr, size, JEMALLOC_MADV_PURGE); 184286866Sjasone unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); 185286866Sjasone# undef JEMALLOC_MADV_PURGE 186286866Sjasone# undef JEMALLOC_MADV_ZEROS 187286866Sjasone#else 188286866Sjasone /* Last resort no-op. */ 189286866Sjasone unzeroed = true; 190286866Sjasone#endif 191286866Sjasone return (unzeroed); 192286866Sjasone} 193286866Sjasone 194299587Sjasone#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 195299587Sjasonestatic bool 196299587Sjasoneos_overcommits_sysctl(void) 197299587Sjasone{ 198299587Sjasone int vm_overcommit; 199299587Sjasone size_t sz; 200299587Sjasone 201299587Sjasone sz = sizeof(vm_overcommit); 202299587Sjasone if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) 203299587Sjasone return (false); /* Error. */ 204299587Sjasone 205299587Sjasone return ((vm_overcommit & 0x3) == 0); 206299587Sjasone} 207299587Sjasone#endif 208299587Sjasone 209299587Sjasone#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY 210299587Sjasonestatic bool 211299587Sjasoneos_overcommits_proc(void) 212299587Sjasone{ 213299587Sjasone int fd; 214299587Sjasone char buf[1]; 215299587Sjasone ssize_t nread; 216299587Sjasone 217299587Sjasone fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); 218299587Sjasone if (fd == -1) 219299587Sjasone return (false); /* Error. */ 220299587Sjasone 221299587Sjasone nread = read(fd, &buf, sizeof(buf)); 222299587Sjasone if (nread < 1) 223299587Sjasone return (false); /* Error. */ 224299587Sjasone /* 225299587Sjasone * /proc/sys/vm/overcommit_memory meanings: 226299587Sjasone * 0: Heuristic overcommit. 227299587Sjasone * 1: Always overcommit. 228299587Sjasone * 2: Never overcommit. 229299587Sjasone */ 230299587Sjasone return (buf[0] == '0' || buf[0] == '1'); 231299587Sjasone} 232299587Sjasone#endif 233299587Sjasone 234299587Sjasonevoid 235299587Sjasonepages_boot(void) 236299587Sjasone{ 237299587Sjasone 238299587Sjasone#ifndef _WIN32 239299587Sjasone mmap_flags = MAP_PRIVATE | MAP_ANON; 240299587Sjasone#endif 241299587Sjasone 242299587Sjasone#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 243299587Sjasone os_overcommits = os_overcommits_sysctl(); 244299587Sjasone#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY) 245299587Sjasone os_overcommits = os_overcommits_proc(); 246299587Sjasone# ifdef MAP_NORESERVE 247299587Sjasone if (os_overcommits) 248299587Sjasone mmap_flags |= MAP_NORESERVE; 249299587Sjasone# endif 250299587Sjasone#else 251299587Sjasone os_overcommits = false; 252299587Sjasone#endif 253299587Sjasone} 254