1// SPDX-License-Identifier: GPL-2.0
2/*
3 * hugepage-madvise:
4 *
5 * Basic functional testing of madvise MADV_DONTNEED and MADV_REMOVE
6 * on hugetlb mappings.
7 *
8 * Before running this test, make sure the administrator has pre-allocated
9 * at least MIN_FREE_PAGES hugetlb pages and they are free.  In addition,
10 * the test takes an argument that is the path to a file in a hugetlbfs
11 * filesystem.  Therefore, a hugetlbfs filesystem must be mounted on some
12 * directory.
13 */
14
15#define _GNU_SOURCE
16#include <stdlib.h>
17#include <stdio.h>
18#include <unistd.h>
19#include <sys/mman.h>
20#include <fcntl.h>
21#include "vm_util.h"
22#include "../kselftest.h"
23
24#define MIN_FREE_PAGES	20
25#define NR_HUGE_PAGES	10	/* common number of pages to map/allocate */
26
27#define validate_free_pages(exp_free)					\
28	do {								\
29		int fhp = get_free_hugepages();				\
30		if (fhp != (exp_free)) {				\
31			printf("Unexpected number of free huge "	\
32				"pages line %d\n", __LINE__);		\
33			exit(1);					\
34		}							\
35	} while (0)
36
37unsigned long huge_page_size;
38unsigned long base_page_size;
39
40void write_fault_pages(void *addr, unsigned long nr_pages)
41{
42	unsigned long i;
43
44	for (i = 0; i < nr_pages; i++)
45		*((unsigned long *)(addr + (i * huge_page_size))) = i;
46}
47
48void read_fault_pages(void *addr, unsigned long nr_pages)
49{
50	volatile unsigned long dummy = 0;
51	unsigned long i;
52
53	for (i = 0; i < nr_pages; i++) {
54		dummy += *((unsigned long *)(addr + (i * huge_page_size)));
55
56		/* Prevent the compiler from optimizing out the entire loop: */
57		asm volatile("" : "+r" (dummy));
58	}
59}
60
61int main(int argc, char **argv)
62{
63	unsigned long free_hugepages;
64	void *addr, *addr2;
65	int fd;
66	int ret;
67
68	huge_page_size = default_huge_page_size();
69	if (!huge_page_size) {
70		printf("Unable to determine huge page size, exiting!\n");
71		exit(1);
72	}
73	base_page_size = sysconf(_SC_PAGE_SIZE);
74	if (!huge_page_size) {
75		printf("Unable to determine base page size, exiting!\n");
76		exit(1);
77	}
78
79	free_hugepages = get_free_hugepages();
80	if (free_hugepages < MIN_FREE_PAGES) {
81		printf("Not enough free huge pages to test, exiting!\n");
82		exit(KSFT_SKIP);
83	}
84
85	fd = memfd_create(argv[0], MFD_HUGETLB);
86	if (fd < 0) {
87		perror("memfd_create() failed");
88		exit(1);
89	}
90
91	/*
92	 * Test validity of MADV_DONTNEED addr and length arguments.  mmap
93	 * size is NR_HUGE_PAGES + 2.  One page at the beginning and end of
94	 * the mapping will be unmapped so we KNOW there is nothing mapped
95	 * there.
96	 */
97	addr = mmap(NULL, (NR_HUGE_PAGES + 2) * huge_page_size,
98			PROT_READ | PROT_WRITE,
99			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
100			-1, 0);
101	if (addr == MAP_FAILED) {
102		perror("mmap");
103		exit(1);
104	}
105	if (munmap(addr, huge_page_size) ||
106			munmap(addr + (NR_HUGE_PAGES + 1) * huge_page_size,
107				huge_page_size)) {
108		perror("munmap");
109		exit(1);
110	}
111	addr = addr + huge_page_size;
112
113	write_fault_pages(addr, NR_HUGE_PAGES);
114	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
115
116	/* addr before mapping should fail */
117	ret = madvise(addr - base_page_size, NR_HUGE_PAGES * huge_page_size,
118		MADV_DONTNEED);
119	if (!ret) {
120		printf("Unexpected success of madvise call with invalid addr line %d\n",
121				__LINE__);
122			exit(1);
123	}
124
125	/* addr + length after mapping should fail */
126	ret = madvise(addr, (NR_HUGE_PAGES * huge_page_size) + base_page_size,
127		MADV_DONTNEED);
128	if (!ret) {
129		printf("Unexpected success of madvise call with invalid length line %d\n",
130				__LINE__);
131			exit(1);
132	}
133
134	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
135
136	/*
137	 * Test alignment of MADV_DONTNEED addr and length arguments
138	 */
139	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
140			PROT_READ | PROT_WRITE,
141			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
142			-1, 0);
143	if (addr == MAP_FAILED) {
144		perror("mmap");
145		exit(1);
146	}
147	write_fault_pages(addr, NR_HUGE_PAGES);
148	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
149
150	/* addr is not huge page size aligned and should fail */
151	ret = madvise(addr + base_page_size,
152			NR_HUGE_PAGES * huge_page_size - base_page_size,
153			MADV_DONTNEED);
154	if (!ret) {
155		printf("Unexpected success of madvise call with unaligned start address %d\n",
156				__LINE__);
157			exit(1);
158	}
159
160	/* addr + length should be aligned down to huge page size */
161	if (madvise(addr,
162			((NR_HUGE_PAGES - 1) * huge_page_size) + base_page_size,
163			MADV_DONTNEED)) {
164		perror("madvise");
165		exit(1);
166	}
167
168	/* should free all but last page in mapping */
169	validate_free_pages(free_hugepages - 1);
170
171	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
172	validate_free_pages(free_hugepages);
173
174	/*
175	 * Test MADV_DONTNEED on anonymous private mapping
176	 */
177	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
178			PROT_READ | PROT_WRITE,
179			MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB,
180			-1, 0);
181	if (addr == MAP_FAILED) {
182		perror("mmap");
183		exit(1);
184	}
185	write_fault_pages(addr, NR_HUGE_PAGES);
186	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
187
188	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
189		perror("madvise");
190		exit(1);
191	}
192
193	/* should free all pages in mapping */
194	validate_free_pages(free_hugepages);
195
196	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
197
198	/*
199	 * Test MADV_DONTNEED on private mapping of hugetlb file
200	 */
201	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
202		perror("fallocate");
203		exit(1);
204	}
205	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
206
207	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
208			PROT_READ | PROT_WRITE,
209			MAP_PRIVATE, fd, 0);
210	if (addr == MAP_FAILED) {
211		perror("mmap");
212		exit(1);
213	}
214
215	/* read should not consume any pages */
216	read_fault_pages(addr, NR_HUGE_PAGES);
217	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
218
219	/* madvise should not free any pages */
220	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
221		perror("madvise");
222		exit(1);
223	}
224	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
225
226	/* writes should allocate private pages */
227	write_fault_pages(addr, NR_HUGE_PAGES);
228	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
229
230	/* madvise should free private pages */
231	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
232		perror("madvise");
233		exit(1);
234	}
235	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
236
237	/* writes should allocate private pages */
238	write_fault_pages(addr, NR_HUGE_PAGES);
239	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
240
241	/*
242	 * The fallocate below certainly should free the pages associated
243	 * with the file.  However, pages in the private mapping are also
244	 * freed.  This is not the 'correct' behavior, but is expected
245	 * because this is how it has worked since the initial hugetlb
246	 * implementation.
247	 */
248	if (fallocate(fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
249					0, NR_HUGE_PAGES * huge_page_size)) {
250		perror("fallocate");
251		exit(1);
252	}
253	validate_free_pages(free_hugepages);
254
255	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
256
257	/*
258	 * Test MADV_DONTNEED on shared mapping of hugetlb file
259	 */
260	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
261		perror("fallocate");
262		exit(1);
263	}
264	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
265
266	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
267			PROT_READ | PROT_WRITE,
268			MAP_SHARED, fd, 0);
269	if (addr == MAP_FAILED) {
270		perror("mmap");
271		exit(1);
272	}
273
274	/* write should not consume any pages */
275	write_fault_pages(addr, NR_HUGE_PAGES);
276	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
277
278	/* madvise should not free any pages */
279	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
280		perror("madvise");
281		exit(1);
282	}
283	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
284
285	/*
286	 * Test MADV_REMOVE on shared mapping of hugetlb file
287	 *
288	 * madvise is same as hole punch and should free all pages.
289	 */
290	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) {
291		perror("madvise");
292		exit(1);
293	}
294	validate_free_pages(free_hugepages);
295	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
296
297	/*
298	 * Test MADV_REMOVE on shared and private mapping of hugetlb file
299	 */
300	if (fallocate(fd, 0, 0, NR_HUGE_PAGES * huge_page_size)) {
301		perror("fallocate");
302		exit(1);
303	}
304	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
305
306	addr = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
307			PROT_READ | PROT_WRITE,
308			MAP_SHARED, fd, 0);
309	if (addr == MAP_FAILED) {
310		perror("mmap");
311		exit(1);
312	}
313
314	/* shared write should not consume any additional pages */
315	write_fault_pages(addr, NR_HUGE_PAGES);
316	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
317
318	addr2 = mmap(NULL, NR_HUGE_PAGES * huge_page_size,
319			PROT_READ | PROT_WRITE,
320			MAP_PRIVATE, fd, 0);
321	if (addr2 == MAP_FAILED) {
322		perror("mmap");
323		exit(1);
324	}
325
326	/* private read should not consume any pages */
327	read_fault_pages(addr2, NR_HUGE_PAGES);
328	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
329
330	/* private write should consume additional pages */
331	write_fault_pages(addr2, NR_HUGE_PAGES);
332	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
333
334	/* madvise of shared mapping should not free any pages */
335	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
336		perror("madvise");
337		exit(1);
338	}
339	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
340
341	/* madvise of private mapping should free private pages */
342	if (madvise(addr2, NR_HUGE_PAGES * huge_page_size, MADV_DONTNEED)) {
343		perror("madvise");
344		exit(1);
345	}
346	validate_free_pages(free_hugepages - NR_HUGE_PAGES);
347
348	/* private write should consume additional pages again */
349	write_fault_pages(addr2, NR_HUGE_PAGES);
350	validate_free_pages(free_hugepages - (2 * NR_HUGE_PAGES));
351
352	/*
353	 * madvise should free both file and private pages although this is
354	 * not correct.  private pages should not be freed, but this is
355	 * expected.  See comment associated with FALLOC_FL_PUNCH_HOLE call.
356	 */
357	if (madvise(addr, NR_HUGE_PAGES * huge_page_size, MADV_REMOVE)) {
358		perror("madvise");
359		exit(1);
360	}
361	validate_free_pages(free_hugepages);
362
363	(void)munmap(addr, NR_HUGE_PAGES * huge_page_size);
364	(void)munmap(addr2, NR_HUGE_PAGES * huge_page_size);
365
366	close(fd);
367	return 0;
368}
369