1#define JEMALLOC_PAGES_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3
| 1#define JEMALLOC_PAGES_C_ 2#include "jemalloc/internal/jemalloc_internal.h" 3
|
| 4#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 5#include <sys/sysctl.h> 6#endif 7
|
4/******************************************************************************/
| 8/******************************************************************************/
|
| 9/* Data. */
|
5
| 10
|
| 11#ifndef _WIN32 12# define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE) 13# define PAGES_PROT_DECOMMIT (PROT_NONE) 14static int mmap_flags; 15#endif 16static bool os_overcommits; 17 18/******************************************************************************/ 19
|
6void *
| 20void *
|
7pages_map(void *addr, size_t size)
| 21pages_map(void *addr, size_t size, bool *commit)
|
8{ 9 void *ret; 10 11 assert(size != 0); 12
| 22{ 23 void *ret; 24 25 assert(size != 0); 26
|
| 27 if (os_overcommits) 28 *commit = true; 29
|
13#ifdef _WIN32 14 /* 15 * If VirtualAlloc can't allocate at the given address when one is 16 * given, it fails and returns NULL. 17 */
| 30#ifdef _WIN32 31 /* 32 * If VirtualAlloc can't allocate at the given address when one is 33 * given, it fails and returns NULL. 34 */
|
18 ret = VirtualAlloc(addr, size, MEM_COMMIT | MEM_RESERVE,
| 35 ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
|
19 PAGE_READWRITE); 20#else 21 /* 22 * We don't use MAP_FIXED here, because it can cause the *replacement* 23 * of existing mappings, and we only want to create new mappings. 24 */
| 36 PAGE_READWRITE); 37#else 38 /* 39 * We don't use MAP_FIXED here, because it can cause the *replacement* 40 * of existing mappings, and we only want to create new mappings. 41 */
|
25 ret = mmap(addr, size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, 26 -1, 0);
| 42 { 43 int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 44 45 ret = mmap(addr, size, prot, mmap_flags, -1, 0); 46 }
|
27 assert(ret != NULL); 28 29 if (ret == MAP_FAILED) 30 ret = NULL; 31 else if (addr != NULL && ret != addr) { 32 /* 33 * We succeeded in mapping memory, but not in the right place. 34 */ 35 pages_unmap(ret, size); 36 ret = NULL; 37 } 38#endif 39 assert(ret == NULL || (addr == NULL && ret != addr) 40 || (addr != NULL && ret == addr)); 41 return (ret); 42} 43 44void 45pages_unmap(void *addr, size_t size) 46{ 47 48#ifdef _WIN32 49 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) 50#else 51 if (munmap(addr, size) == -1) 52#endif 53 { 54 char buf[BUFERROR_BUF]; 55 56 buferror(get_errno(), buf, sizeof(buf)); 57 malloc_printf("<jemalloc>: Error in " 58#ifdef _WIN32 59 "VirtualFree" 60#else 61 "munmap" 62#endif 63 "(): %s\n", buf); 64 if (opt_abort) 65 abort(); 66 } 67} 68 69void *
| 47 assert(ret != NULL); 48 49 if (ret == MAP_FAILED) 50 ret = NULL; 51 else if (addr != NULL && ret != addr) { 52 /* 53 * We succeeded in mapping memory, but not in the right place. 54 */ 55 pages_unmap(ret, size); 56 ret = NULL; 57 } 58#endif 59 assert(ret == NULL || (addr == NULL && ret != addr) 60 || (addr != NULL && ret == addr)); 61 return (ret); 62} 63 64void 65pages_unmap(void *addr, size_t size) 66{ 67 68#ifdef _WIN32 69 if (VirtualFree(addr, 0, MEM_RELEASE) == 0) 70#else 71 if (munmap(addr, size) == -1) 72#endif 73 { 74 char buf[BUFERROR_BUF]; 75 76 buferror(get_errno(), buf, sizeof(buf)); 77 malloc_printf("<jemalloc>: Error in " 78#ifdef _WIN32 79 "VirtualFree" 80#else 81 "munmap" 82#endif 83 "(): %s\n", buf); 84 if (opt_abort) 85 abort(); 86 } 87} 88 89void *
|
70pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size)
| 90pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size, 91 bool *commit)
|
71{ 72 void *ret = (void *)((uintptr_t)addr + leadsize); 73 74 assert(alloc_size >= leadsize + size); 75#ifdef _WIN32 76 { 77 void *new_addr; 78 79 pages_unmap(addr, alloc_size);
| 92{ 93 void *ret = (void *)((uintptr_t)addr + leadsize); 94 95 assert(alloc_size >= leadsize + size); 96#ifdef _WIN32 97 { 98 void *new_addr; 99 100 pages_unmap(addr, alloc_size);
|
80 new_addr = pages_map(ret, size);
| 101 new_addr = pages_map(ret, size, commit);
|
81 if (new_addr == ret) 82 return (ret); 83 if (new_addr) 84 pages_unmap(new_addr, size); 85 return (NULL); 86 } 87#else 88 { 89 size_t trailsize = alloc_size - leadsize - size; 90 91 if (leadsize != 0) 92 pages_unmap(addr, leadsize); 93 if (trailsize != 0) 94 pages_unmap((void *)((uintptr_t)ret + size), trailsize); 95 return (ret); 96 } 97#endif 98} 99 100static bool 101pages_commit_impl(void *addr, size_t size, bool commit) 102{ 103
| 102 if (new_addr == ret) 103 return (ret); 104 if (new_addr) 105 pages_unmap(new_addr, size); 106 return (NULL); 107 } 108#else 109 { 110 size_t trailsize = alloc_size - leadsize - size; 111 112 if (leadsize != 0) 113 pages_unmap(addr, leadsize); 114 if (trailsize != 0) 115 pages_unmap((void *)((uintptr_t)ret + size), trailsize); 116 return (ret); 117 } 118#endif 119} 120 121static bool 122pages_commit_impl(void *addr, size_t size, bool commit) 123{ 124
|
104#ifndef _WIN32 105 /* 106 * The following decommit/commit implementation is functional, but 107 * always disabled because it doesn't add value beyong improved 108 * debugging (at the cost of extra system calls) on systems that 109 * overcommit. 110 */ 111 if (false) { 112 int prot = commit ? (PROT_READ | PROT_WRITE) : PROT_NONE; 113 void *result = mmap(addr, size, prot, MAP_PRIVATE | MAP_ANON | 114 MAP_FIXED, -1, 0);
| 125 if (os_overcommits) 126 return (true); 127 128#ifdef _WIN32 129 return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT, 130 PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT))); 131#else 132 { 133 int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT; 134 void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED, 135 -1, 0);
|
115 if (result == MAP_FAILED) 116 return (true); 117 if (result != addr) { 118 /* 119 * We succeeded in mapping memory, but not in the right 120 * place. 121 */ 122 pages_unmap(result, size); 123 return (true); 124 } 125 return (false); 126 } 127#endif
| 136 if (result == MAP_FAILED) 137 return (true); 138 if (result != addr) { 139 /* 140 * We succeeded in mapping memory, but not in the right 141 * place. 142 */ 143 pages_unmap(result, size); 144 return (true); 145 } 146 return (false); 147 } 148#endif
|
128 return (true);
| |
129} 130 131bool 132pages_commit(void *addr, size_t size) 133{ 134 135 return (pages_commit_impl(addr, size, true)); 136} 137 138bool 139pages_decommit(void *addr, size_t size) 140{ 141 142 return (pages_commit_impl(addr, size, false)); 143} 144 145bool 146pages_purge(void *addr, size_t size) 147{ 148 bool unzeroed; 149 150#ifdef _WIN32 151 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); 152 unzeroed = true; 153#elif defined(JEMALLOC_HAVE_MADVISE) 154# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 155# define JEMALLOC_MADV_PURGE MADV_DONTNEED 156# define JEMALLOC_MADV_ZEROS true 157# elif defined(JEMALLOC_PURGE_MADVISE_FREE) 158# define JEMALLOC_MADV_PURGE MADV_FREE 159# define JEMALLOC_MADV_ZEROS false 160# else 161# error "No madvise(2) flag defined for purging unused dirty pages." 162# endif 163 int err = madvise(addr, size, JEMALLOC_MADV_PURGE); 164 unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); 165# undef JEMALLOC_MADV_PURGE 166# undef JEMALLOC_MADV_ZEROS 167#else 168 /* Last resort no-op. */ 169 unzeroed = true; 170#endif 171 return (unzeroed); 172} 173
| 149} 150 151bool 152pages_commit(void *addr, size_t size) 153{ 154 155 return (pages_commit_impl(addr, size, true)); 156} 157 158bool 159pages_decommit(void *addr, size_t size) 160{ 161 162 return (pages_commit_impl(addr, size, false)); 163} 164 165bool 166pages_purge(void *addr, size_t size) 167{ 168 bool unzeroed; 169 170#ifdef _WIN32 171 VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE); 172 unzeroed = true; 173#elif defined(JEMALLOC_HAVE_MADVISE) 174# ifdef JEMALLOC_PURGE_MADVISE_DONTNEED 175# define JEMALLOC_MADV_PURGE MADV_DONTNEED 176# define JEMALLOC_MADV_ZEROS true 177# elif defined(JEMALLOC_PURGE_MADVISE_FREE) 178# define JEMALLOC_MADV_PURGE MADV_FREE 179# define JEMALLOC_MADV_ZEROS false 180# else 181# error "No madvise(2) flag defined for purging unused dirty pages." 182# endif 183 int err = madvise(addr, size, JEMALLOC_MADV_PURGE); 184 unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0); 185# undef JEMALLOC_MADV_PURGE 186# undef JEMALLOC_MADV_ZEROS 187#else 188 /* Last resort no-op. */ 189 unzeroed = true; 190#endif 191 return (unzeroed); 192} 193
|
| 194#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 195static bool 196os_overcommits_sysctl(void) 197{ 198 int vm_overcommit; 199 size_t sz; 200 201 sz = sizeof(vm_overcommit); 202 if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0) 203 return (false); /* Error. */ 204 205 return ((vm_overcommit & 0x3) == 0); 206} 207#endif 208 209#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY 210static bool 211os_overcommits_proc(void) 212{ 213 int fd; 214 char buf[1]; 215 ssize_t nread; 216 217 fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY); 218 if (fd == -1) 219 return (false); /* Error. */ 220 221 nread = read(fd, &buf, sizeof(buf)); 222 if (nread < 1) 223 return (false); /* Error. */ 224 /* 225 * /proc/sys/vm/overcommit_memory meanings: 226 * 0: Heuristic overcommit. 227 * 1: Always overcommit. 228 * 2: Never overcommit. 229 */ 230 return (buf[0] == '0' || buf[0] == '1'); 231} 232#endif 233 234void 235pages_boot(void) 236{ 237 238#ifndef _WIN32 239 mmap_flags = MAP_PRIVATE | MAP_ANON; 240#endif 241 242#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT 243 os_overcommits = os_overcommits_sysctl(); 244#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY) 245 os_overcommits = os_overcommits_proc(); 246# ifdef MAP_NORESERVE 247 if (os_overcommits) 248 mmap_flags |= MAP_NORESERVE; 249# endif 250#else 251 os_overcommits = false; 252#endif 253}
|
| |