1#define	JEMALLOC_PAGES_C_
2#include "jemalloc/internal/jemalloc_internal.h"
3
4#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
5#include <sys/sysctl.h>
6#endif
7
8/******************************************************************************/
9/* Data. */
10
11#ifndef _WIN32
12#  define PAGES_PROT_COMMIT (PROT_READ | PROT_WRITE)
13#  define PAGES_PROT_DECOMMIT (PROT_NONE)
14static int	mmap_flags;
15#endif
16static bool	os_overcommits;
17
18/******************************************************************************/
19
20void *
21pages_map(void *addr, size_t size, bool *commit)
22{
23	void *ret;
24
25	assert(size != 0);
26
27	if (os_overcommits)
28		*commit = true;
29
30#ifdef _WIN32
31	/*
32	 * If VirtualAlloc can't allocate at the given address when one is
33	 * given, it fails and returns NULL.
34	 */
35	ret = VirtualAlloc(addr, size, MEM_RESERVE | (*commit ? MEM_COMMIT : 0),
36	    PAGE_READWRITE);
37#else
38	/*
39	 * We don't use MAP_FIXED here, because it can cause the *replacement*
40	 * of existing mappings, and we only want to create new mappings.
41	 */
42	{
43		int prot = *commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
44
45		ret = mmap(addr, size, prot, mmap_flags, -1, 0);
46	}
47	assert(ret != NULL);
48
49	if (ret == MAP_FAILED)
50		ret = NULL;
51	else if (addr != NULL && ret != addr) {
52		/*
53		 * We succeeded in mapping memory, but not in the right place.
54		 */
55		pages_unmap(ret, size);
56		ret = NULL;
57	}
58#endif
59	assert(ret == NULL || (addr == NULL && ret != addr)
60	    || (addr != NULL && ret == addr));
61	return (ret);
62}
63
64void
65pages_unmap(void *addr, size_t size)
66{
67
68#ifdef _WIN32
69	if (VirtualFree(addr, 0, MEM_RELEASE) == 0)
70#else
71	if (munmap(addr, size) == -1)
72#endif
73	{
74		char buf[BUFERROR_BUF];
75
76		buferror(get_errno(), buf, sizeof(buf));
77		malloc_printf("<jemalloc>: Error in "
78#ifdef _WIN32
79		              "VirtualFree"
80#else
81		              "munmap"
82#endif
83		              "(): %s\n", buf);
84		if (opt_abort)
85			abort();
86	}
87}
88
89void *
90pages_trim(void *addr, size_t alloc_size, size_t leadsize, size_t size,
91    bool *commit)
92{
93	void *ret = (void *)((uintptr_t)addr + leadsize);
94
95	assert(alloc_size >= leadsize + size);
96#ifdef _WIN32
97	{
98		void *new_addr;
99
100		pages_unmap(addr, alloc_size);
101		new_addr = pages_map(ret, size, commit);
102		if (new_addr == ret)
103			return (ret);
104		if (new_addr)
105			pages_unmap(new_addr, size);
106		return (NULL);
107	}
108#else
109	{
110		size_t trailsize = alloc_size - leadsize - size;
111
112		if (leadsize != 0)
113			pages_unmap(addr, leadsize);
114		if (trailsize != 0)
115			pages_unmap((void *)((uintptr_t)ret + size), trailsize);
116		return (ret);
117	}
118#endif
119}
120
121static bool
122pages_commit_impl(void *addr, size_t size, bool commit)
123{
124
125	if (os_overcommits)
126		return (true);
127
128#ifdef _WIN32
129	return (commit ? (addr != VirtualAlloc(addr, size, MEM_COMMIT,
130	    PAGE_READWRITE)) : (!VirtualFree(addr, size, MEM_DECOMMIT)));
131#else
132	{
133		int prot = commit ? PAGES_PROT_COMMIT : PAGES_PROT_DECOMMIT;
134		void *result = mmap(addr, size, prot, mmap_flags | MAP_FIXED,
135		    -1, 0);
136		if (result == MAP_FAILED)
137			return (true);
138		if (result != addr) {
139			/*
140			 * We succeeded in mapping memory, but not in the right
141			 * place.
142			 */
143			pages_unmap(result, size);
144			return (true);
145		}
146		return (false);
147	}
148#endif
149}
150
151bool
152pages_commit(void *addr, size_t size)
153{
154
155	return (pages_commit_impl(addr, size, true));
156}
157
158bool
159pages_decommit(void *addr, size_t size)
160{
161
162	return (pages_commit_impl(addr, size, false));
163}
164
165bool
166pages_purge(void *addr, size_t size)
167{
168	bool unzeroed;
169
170#ifdef _WIN32
171	VirtualAlloc(addr, size, MEM_RESET, PAGE_READWRITE);
172	unzeroed = true;
173#elif defined(JEMALLOC_HAVE_MADVISE)
174#  ifdef JEMALLOC_PURGE_MADVISE_DONTNEED
175#    define JEMALLOC_MADV_PURGE MADV_DONTNEED
176#    define JEMALLOC_MADV_ZEROS true
177#  elif defined(JEMALLOC_PURGE_MADVISE_FREE)
178#    define JEMALLOC_MADV_PURGE MADV_FREE
179#    define JEMALLOC_MADV_ZEROS false
180#  else
181#    error "No madvise(2) flag defined for purging unused dirty pages."
182#  endif
183	int err = madvise(addr, size, JEMALLOC_MADV_PURGE);
184	unzeroed = (!JEMALLOC_MADV_ZEROS || err != 0);
185#  undef JEMALLOC_MADV_PURGE
186#  undef JEMALLOC_MADV_ZEROS
187#else
188	/* Last resort no-op. */
189	unzeroed = true;
190#endif
191	return (unzeroed);
192}
193
194#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
195static bool
196os_overcommits_sysctl(void)
197{
198	int vm_overcommit;
199	size_t sz;
200
201	sz = sizeof(vm_overcommit);
202	if (sysctlbyname("vm.overcommit", &vm_overcommit, &sz, NULL, 0) != 0)
203		return (false); /* Error. */
204
205	return ((vm_overcommit & 0x3) == 0);
206}
207#endif
208
209#ifdef JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY
210static bool
211os_overcommits_proc(void)
212{
213	int fd;
214	char buf[1];
215	ssize_t nread;
216
217	fd = open("/proc/sys/vm/overcommit_memory", O_RDONLY);
218	if (fd == -1)
219		return (false); /* Error. */
220
221	nread = read(fd, &buf, sizeof(buf));
222	if (nread < 1)
223		return (false); /* Error. */
224	/*
225	 * /proc/sys/vm/overcommit_memory meanings:
226	 * 0: Heuristic overcommit.
227	 * 1: Always overcommit.
228	 * 2: Never overcommit.
229	 */
230	return (buf[0] == '0' || buf[0] == '1');
231}
232#endif
233
234void
235pages_boot(void)
236{
237
238#ifndef _WIN32
239	mmap_flags = MAP_PRIVATE | MAP_ANON;
240#endif
241
242#ifdef JEMALLOC_SYSCTL_VM_OVERCOMMIT
243	os_overcommits = os_overcommits_sysctl();
244#elif defined(JEMALLOC_PROC_SYS_VM_OVERCOMMIT_MEMORY)
245	os_overcommits = os_overcommits_proc();
246#  ifdef MAP_NORESERVE
247	if (os_overcommits)
248		mmap_flags |= MAP_NORESERVE;
249#  endif
250#else
251	os_overcommits = false;
252#endif
253}
254