1#include <fcntl.h>
2#include <errno.h>
3#include <stdio.h>
4#include <stdlib.h>
5#include <unistd.h>
6#include <dirent.h>
7#include <sys/ioctl.h>
8#include <sys/mman.h>
9#include <sys/stat.h>
10#include <sys/types.h>
11#include <pthread.h>
12#include <assert.h>
13#include <mm/gup_test.h>
14#include "../kselftest.h"
15#include "vm_util.h"
16
17#define MB (1UL << 20)
18
19/* Just the flags we need, copied from mm.h: */
20#define FOLL_WRITE	0x01	/* check pte is writable */
21#define FOLL_TOUCH	0x02	/* mark page accessed */
22
23#define GUP_TEST_FILE "/sys/kernel/debug/gup_test"
24
25static unsigned long cmd = GUP_FAST_BENCHMARK;
26static int gup_fd, repeats = 1;
27static unsigned long size = 128 * MB;
28/* Serialize prints */
29static pthread_mutex_t print_mutex = PTHREAD_MUTEX_INITIALIZER;
30
31static char *cmd_to_str(unsigned long cmd)
32{
33	switch (cmd) {
34	case GUP_FAST_BENCHMARK:
35		return "GUP_FAST_BENCHMARK";
36	case PIN_FAST_BENCHMARK:
37		return "PIN_FAST_BENCHMARK";
38	case PIN_LONGTERM_BENCHMARK:
39		return "PIN_LONGTERM_BENCHMARK";
40	case GUP_BASIC_TEST:
41		return "GUP_BASIC_TEST";
42	case PIN_BASIC_TEST:
43		return "PIN_BASIC_TEST";
44	case DUMP_USER_PAGES_TEST:
45		return "DUMP_USER_PAGES_TEST";
46	}
47	return "Unknown command";
48}
49
50void *gup_thread(void *data)
51{
52	struct gup_test gup = *(struct gup_test *)data;
53	int i, status;
54
55	/* Only report timing information on the *_BENCHMARK commands: */
56	if ((cmd == PIN_FAST_BENCHMARK) || (cmd == GUP_FAST_BENCHMARK) ||
57	     (cmd == PIN_LONGTERM_BENCHMARK)) {
58		for (i = 0; i < repeats; i++) {
59			gup.size = size;
60			status = ioctl(gup_fd, cmd, &gup);
61			if (status)
62				break;
63
64			pthread_mutex_lock(&print_mutex);
65			ksft_print_msg("%s: Time: get:%lld put:%lld us",
66				       cmd_to_str(cmd), gup.get_delta_usec,
67				       gup.put_delta_usec);
68			if (gup.size != size)
69				ksft_print_msg(", truncated (size: %lld)", gup.size);
70			ksft_print_msg("\n");
71			pthread_mutex_unlock(&print_mutex);
72		}
73	} else {
74		gup.size = size;
75		status = ioctl(gup_fd, cmd, &gup);
76		if (status)
77			goto return_;
78
79		pthread_mutex_lock(&print_mutex);
80		ksft_print_msg("%s: done\n", cmd_to_str(cmd));
81		if (gup.size != size)
82			ksft_print_msg("Truncated (size: %lld)\n", gup.size);
83		pthread_mutex_unlock(&print_mutex);
84	}
85
86return_:
87	ksft_test_result(!status, "ioctl status %d\n", status);
88	return NULL;
89}
90
91int main(int argc, char **argv)
92{
93	struct gup_test gup = { 0 };
94	int filed, i, opt, nr_pages = 1, thp = -1, write = 1, nthreads = 1, ret;
95	int flags = MAP_PRIVATE, touch = 0;
96	char *file = "/dev/zero";
97	pthread_t *tid;
98	char *p;
99
100	while ((opt = getopt(argc, argv, "m:r:n:F:f:abcj:tTLUuwWSHpz")) != -1) {
101		switch (opt) {
102		case 'a':
103			cmd = PIN_FAST_BENCHMARK;
104			break;
105		case 'b':
106			cmd = PIN_BASIC_TEST;
107			break;
108		case 'L':
109			cmd = PIN_LONGTERM_BENCHMARK;
110			break;
111		case 'c':
112			cmd = DUMP_USER_PAGES_TEST;
113			/*
114			 * Dump page 0 (index 1). May be overridden later, by
115			 * user's non-option arguments.
116			 *
117			 * .which_pages is zero-based, so that zero can mean "do
118			 * nothing".
119			 */
120			gup.which_pages[0] = 1;
121			break;
122		case 'p':
123			/* works only with DUMP_USER_PAGES_TEST */
124			gup.test_flags |= GUP_TEST_FLAG_DUMP_PAGES_USE_PIN;
125			break;
126		case 'F':
127			/* strtol, so you can pass flags in hex form */
128			gup.gup_flags = strtol(optarg, 0, 0);
129			break;
130		case 'j':
131			nthreads = atoi(optarg);
132			break;
133		case 'm':
134			size = atoi(optarg) * MB;
135			break;
136		case 'r':
137			repeats = atoi(optarg);
138			break;
139		case 'n':
140			nr_pages = atoi(optarg);
141			break;
142		case 't':
143			thp = 1;
144			break;
145		case 'T':
146			thp = 0;
147			break;
148		case 'U':
149			cmd = GUP_BASIC_TEST;
150			break;
151		case 'u':
152			cmd = GUP_FAST_BENCHMARK;
153			break;
154		case 'w':
155			write = 1;
156			break;
157		case 'W':
158			write = 0;
159			break;
160		case 'f':
161			file = optarg;
162			break;
163		case 'S':
164			flags &= ~MAP_PRIVATE;
165			flags |= MAP_SHARED;
166			break;
167		case 'H':
168			flags |= (MAP_HUGETLB | MAP_ANONYMOUS);
169			break;
170		case 'z':
171			/* fault pages in gup, do not fault in userland */
172			touch = 1;
173			break;
174		default:
175			ksft_exit_fail_msg("Wrong argument\n");
176		}
177	}
178
179	if (optind < argc) {
180		int extra_arg_count = 0;
181		/*
182		 * For example:
183		 *
184		 *   ./gup_test -c 0 1 0x1001
185		 *
186		 * ...to dump pages 0, 1, and 4097
187		 */
188
189		while ((optind < argc) &&
190		       (extra_arg_count < GUP_TEST_MAX_PAGES_TO_DUMP)) {
191			/*
192			 * Do the 1-based indexing here, so that the user can
193			 * use normal 0-based indexing on the command line.
194			 */
195			long page_index = strtol(argv[optind], 0, 0) + 1;
196
197			gup.which_pages[extra_arg_count] = page_index;
198			extra_arg_count++;
199			optind++;
200		}
201	}
202
203	ksft_print_header();
204	ksft_set_plan(nthreads);
205
206	filed = open(file, O_RDWR|O_CREAT, 0664);
207	if (filed < 0)
208		ksft_exit_fail_msg("Unable to open %s: %s\n", file, strerror(errno));
209
210	gup.nr_pages_per_call = nr_pages;
211	if (write)
212		gup.gup_flags |= FOLL_WRITE;
213
214	gup_fd = open(GUP_TEST_FILE, O_RDWR);
215	if (gup_fd == -1) {
216		switch (errno) {
217		case EACCES:
218			if (getuid())
219				ksft_print_msg("Please run this test as root\n");
220			break;
221		case ENOENT:
222			if (opendir("/sys/kernel/debug") == NULL)
223				ksft_print_msg("mount debugfs at /sys/kernel/debug\n");
224			ksft_print_msg("check if CONFIG_GUP_TEST is enabled in kernel config\n");
225			break;
226		default:
227			ksft_print_msg("failed to open %s: %s\n", GUP_TEST_FILE, strerror(errno));
228			break;
229		}
230		ksft_test_result_skip("Please run this test as root\n");
231		return ksft_exit_pass();
232	}
233
234	p = mmap(NULL, size, PROT_READ | PROT_WRITE, flags, filed, 0);
235	if (p == MAP_FAILED)
236		ksft_exit_fail_msg("mmap: %s\n", strerror(errno));
237	gup.addr = (unsigned long)p;
238
239	if (thp == 1)
240		madvise(p, size, MADV_HUGEPAGE);
241	else if (thp == 0)
242		madvise(p, size, MADV_NOHUGEPAGE);
243
244	/*
245	 * FOLL_TOUCH, in gup_test, is used as an either/or case: either
246	 * fault pages in from the kernel via FOLL_TOUCH, or fault them
247	 * in here, from user space. This allows comparison of performance
248	 * between those two cases.
249	 */
250	if (touch) {
251		gup.gup_flags |= FOLL_TOUCH;
252	} else {
253		for (; (unsigned long)p < gup.addr + size; p += psize())
254			p[0] = 0;
255	}
256
257	tid = malloc(sizeof(pthread_t) * nthreads);
258	assert(tid);
259	for (i = 0; i < nthreads; i++) {
260		ret = pthread_create(&tid[i], NULL, gup_thread, &gup);
261		assert(ret == 0);
262	}
263	for (i = 0; i < nthreads; i++) {
264		ret = pthread_join(tid[i], NULL);
265		assert(ret == 0);
266	}
267
268	free(tid);
269
270	return ksft_exit_pass();
271}
272