1#include <linux/kernel.h>
2#include <linux/mm.h>
3#include <linux/slab.h>
4#include <linux/uaccess.h>
5#include <linux/ktime.h>
6#include <linux/debugfs.h>
7#include <linux/highmem.h>
8#include "gup_test.h"
9
10static void put_back_pages(unsigned int cmd, struct page **pages,
11			   unsigned long nr_pages, unsigned int gup_test_flags)
12{
13	unsigned long i;
14
15	switch (cmd) {
16	case GUP_FAST_BENCHMARK:
17	case GUP_BASIC_TEST:
18		for (i = 0; i < nr_pages; i++)
19			put_page(pages[i]);
20		break;
21
22	case PIN_FAST_BENCHMARK:
23	case PIN_BASIC_TEST:
24	case PIN_LONGTERM_BENCHMARK:
25		unpin_user_pages(pages, nr_pages);
26		break;
27	case DUMP_USER_PAGES_TEST:
28		if (gup_test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN) {
29			unpin_user_pages(pages, nr_pages);
30		} else {
31			for (i = 0; i < nr_pages; i++)
32				put_page(pages[i]);
33
34		}
35		break;
36	}
37}
38
39static void verify_dma_pinned(unsigned int cmd, struct page **pages,
40			      unsigned long nr_pages)
41{
42	unsigned long i;
43	struct folio *folio;
44
45	switch (cmd) {
46	case PIN_FAST_BENCHMARK:
47	case PIN_BASIC_TEST:
48	case PIN_LONGTERM_BENCHMARK:
49		for (i = 0; i < nr_pages; i++) {
50			folio = page_folio(pages[i]);
51
52			if (WARN(!folio_maybe_dma_pinned(folio),
53				 "pages[%lu] is NOT dma-pinned\n", i)) {
54
55				dump_page(&folio->page, "gup_test failure");
56				break;
57			} else if (cmd == PIN_LONGTERM_BENCHMARK &&
58				WARN(!folio_is_longterm_pinnable(folio),
59				     "pages[%lu] is NOT pinnable but pinned\n",
60				     i)) {
61				dump_page(&folio->page, "gup_test failure");
62				break;
63			}
64		}
65		break;
66	}
67}
68
69static void dump_pages_test(struct gup_test *gup, struct page **pages,
70			    unsigned long nr_pages)
71{
72	unsigned int index_to_dump;
73	unsigned int i;
74
75	/*
76	 * Zero out any user-supplied page index that is out of range. Remember:
77	 * .which_pages[] contains a 1-based set of page indices.
78	 */
79	for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
80		if (gup->which_pages[i] > nr_pages) {
81			pr_warn("ZEROING due to out of range: .which_pages[%u]: %u\n",
82				i, gup->which_pages[i]);
83			gup->which_pages[i] = 0;
84		}
85	}
86
87	for (i = 0; i < GUP_TEST_MAX_PAGES_TO_DUMP; i++) {
88		index_to_dump = gup->which_pages[i];
89
90		if (index_to_dump) {
91			index_to_dump--; // Decode from 1-based, to 0-based
92			pr_info("---- page #%u, starting from user virt addr: 0x%llx\n",
93				index_to_dump, gup->addr);
94			dump_page(pages[index_to_dump],
95				  "gup_test: dump_pages() test");
96		}
97	}
98}
99
100static int __gup_test_ioctl(unsigned int cmd,
101		struct gup_test *gup)
102{
103	ktime_t start_time, end_time;
104	unsigned long i, nr_pages, addr, next;
105	long nr;
106	struct page **pages;
107	int ret = 0;
108	bool needs_mmap_lock =
109		cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
110
111	if (gup->size > ULONG_MAX)
112		return -EINVAL;
113
114	nr_pages = gup->size / PAGE_SIZE;
115	pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
116	if (!pages)
117		return -ENOMEM;
118
119	if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
120		ret = -EINTR;
121		goto free_pages;
122	}
123
124	i = 0;
125	nr = gup->nr_pages_per_call;
126	start_time = ktime_get();
127	for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
128		if (nr != gup->nr_pages_per_call)
129			break;
130
131		next = addr + nr * PAGE_SIZE;
132		if (next > gup->addr + gup->size) {
133			next = gup->addr + gup->size;
134			nr = (next - addr) / PAGE_SIZE;
135		}
136
137		switch (cmd) {
138		case GUP_FAST_BENCHMARK:
139			nr = get_user_pages_fast(addr, nr, gup->gup_flags,
140						 pages + i);
141			break;
142		case GUP_BASIC_TEST:
143			nr = get_user_pages(addr, nr, gup->gup_flags, pages + i);
144			break;
145		case PIN_FAST_BENCHMARK:
146			nr = pin_user_pages_fast(addr, nr, gup->gup_flags,
147						 pages + i);
148			break;
149		case PIN_BASIC_TEST:
150			nr = pin_user_pages(addr, nr, gup->gup_flags, pages + i);
151			break;
152		case PIN_LONGTERM_BENCHMARK:
153			nr = pin_user_pages(addr, nr,
154					    gup->gup_flags | FOLL_LONGTERM,
155					    pages + i);
156			break;
157		case DUMP_USER_PAGES_TEST:
158			if (gup->test_flags & GUP_TEST_FLAG_DUMP_PAGES_USE_PIN)
159				nr = pin_user_pages(addr, nr, gup->gup_flags,
160						    pages + i);
161			else
162				nr = get_user_pages(addr, nr, gup->gup_flags,
163						    pages + i);
164			break;
165		default:
166			ret = -EINVAL;
167			goto unlock;
168		}
169
170		if (nr <= 0)
171			break;
172		i += nr;
173	}
174	end_time = ktime_get();
175
176	/* Shifting the meaning of nr_pages: now it is actual number pinned: */
177	nr_pages = i;
178
179	gup->get_delta_usec = ktime_us_delta(end_time, start_time);
180	gup->size = addr - gup->addr;
181
182	/*
183	 * Take an un-benchmark-timed moment to verify DMA pinned
184	 * state: print a warning if any non-dma-pinned pages are found:
185	 */
186	verify_dma_pinned(cmd, pages, nr_pages);
187
188	if (cmd == DUMP_USER_PAGES_TEST)
189		dump_pages_test(gup, pages, nr_pages);
190
191	start_time = ktime_get();
192
193	put_back_pages(cmd, pages, nr_pages, gup->test_flags);
194
195	end_time = ktime_get();
196	gup->put_delta_usec = ktime_us_delta(end_time, start_time);
197
198unlock:
199	if (needs_mmap_lock)
200		mmap_read_unlock(current->mm);
201free_pages:
202	kvfree(pages);
203	return ret;
204}
205
206static DEFINE_MUTEX(pin_longterm_test_mutex);
207static struct page **pin_longterm_test_pages;
208static unsigned long pin_longterm_test_nr_pages;
209
210static inline void pin_longterm_test_stop(void)
211{
212	if (pin_longterm_test_pages) {
213		if (pin_longterm_test_nr_pages)
214			unpin_user_pages(pin_longterm_test_pages,
215					 pin_longterm_test_nr_pages);
216		kvfree(pin_longterm_test_pages);
217		pin_longterm_test_pages = NULL;
218		pin_longterm_test_nr_pages = 0;
219	}
220}
221
222static inline int pin_longterm_test_start(unsigned long arg)
223{
224	long nr_pages, cur_pages, addr, remaining_pages;
225	int gup_flags = FOLL_LONGTERM;
226	struct pin_longterm_test args;
227	struct page **pages;
228	int ret = 0;
229	bool fast;
230
231	if (pin_longterm_test_pages)
232		return -EINVAL;
233
234	if (copy_from_user(&args, (void __user *)arg, sizeof(args)))
235		return -EFAULT;
236
237	if (args.flags &
238	    ~(PIN_LONGTERM_TEST_FLAG_USE_WRITE|PIN_LONGTERM_TEST_FLAG_USE_FAST))
239		return -EINVAL;
240	if (!IS_ALIGNED(args.addr | args.size, PAGE_SIZE))
241		return -EINVAL;
242	if (args.size > LONG_MAX)
243		return -EINVAL;
244	nr_pages = args.size / PAGE_SIZE;
245	if (!nr_pages)
246		return -EINVAL;
247
248	pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
249	if (!pages)
250		return -ENOMEM;
251
252	if (args.flags & PIN_LONGTERM_TEST_FLAG_USE_WRITE)
253		gup_flags |= FOLL_WRITE;
254	fast = !!(args.flags & PIN_LONGTERM_TEST_FLAG_USE_FAST);
255
256	if (!fast && mmap_read_lock_killable(current->mm)) {
257		kvfree(pages);
258		return -EINTR;
259	}
260
261	pin_longterm_test_pages = pages;
262	pin_longterm_test_nr_pages = 0;
263
264	while (nr_pages - pin_longterm_test_nr_pages) {
265		remaining_pages = nr_pages - pin_longterm_test_nr_pages;
266		addr = args.addr + pin_longterm_test_nr_pages * PAGE_SIZE;
267
268		if (fast)
269			cur_pages = pin_user_pages_fast(addr, remaining_pages,
270							gup_flags, pages);
271		else
272			cur_pages = pin_user_pages(addr, remaining_pages,
273						   gup_flags, pages);
274		if (cur_pages < 0) {
275			pin_longterm_test_stop();
276			ret = cur_pages;
277			break;
278		}
279		pin_longterm_test_nr_pages += cur_pages;
280		pages += cur_pages;
281	}
282
283	if (!fast)
284		mmap_read_unlock(current->mm);
285	return ret;
286}
287
288static inline int pin_longterm_test_read(unsigned long arg)
289{
290	__u64 user_addr;
291	unsigned long i;
292
293	if (!pin_longterm_test_pages)
294		return -EINVAL;
295
296	if (copy_from_user(&user_addr, (void __user *)arg, sizeof(user_addr)))
297		return -EFAULT;
298
299	for (i = 0; i < pin_longterm_test_nr_pages; i++) {
300		void *addr = kmap_local_page(pin_longterm_test_pages[i]);
301		unsigned long ret;
302
303		ret = copy_to_user((void __user *)(unsigned long)user_addr, addr,
304				   PAGE_SIZE);
305		kunmap_local(addr);
306		if (ret)
307			return -EFAULT;
308		user_addr += PAGE_SIZE;
309	}
310	return 0;
311}
312
313static long pin_longterm_test_ioctl(struct file *filep, unsigned int cmd,
314				    unsigned long arg)
315{
316	int ret = -EINVAL;
317
318	if (mutex_lock_killable(&pin_longterm_test_mutex))
319		return -EINTR;
320
321	switch (cmd) {
322	case PIN_LONGTERM_TEST_START:
323		ret = pin_longterm_test_start(arg);
324		break;
325	case PIN_LONGTERM_TEST_STOP:
326		pin_longterm_test_stop();
327		ret = 0;
328		break;
329	case PIN_LONGTERM_TEST_READ:
330		ret = pin_longterm_test_read(arg);
331		break;
332	}
333
334	mutex_unlock(&pin_longterm_test_mutex);
335	return ret;
336}
337
338static long gup_test_ioctl(struct file *filep, unsigned int cmd,
339		unsigned long arg)
340{
341	struct gup_test gup;
342	int ret;
343
344	switch (cmd) {
345	case GUP_FAST_BENCHMARK:
346	case PIN_FAST_BENCHMARK:
347	case PIN_LONGTERM_BENCHMARK:
348	case GUP_BASIC_TEST:
349	case PIN_BASIC_TEST:
350	case DUMP_USER_PAGES_TEST:
351		break;
352	case PIN_LONGTERM_TEST_START:
353	case PIN_LONGTERM_TEST_STOP:
354	case PIN_LONGTERM_TEST_READ:
355		return pin_longterm_test_ioctl(filep, cmd, arg);
356	default:
357		return -EINVAL;
358	}
359
360	if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
361		return -EFAULT;
362
363	ret = __gup_test_ioctl(cmd, &gup);
364	if (ret)
365		return ret;
366
367	if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
368		return -EFAULT;
369
370	return 0;
371}
372
373static int gup_test_release(struct inode *inode, struct file *file)
374{
375	pin_longterm_test_stop();
376
377	return 0;
378}
379
380static const struct file_operations gup_test_fops = {
381	.open = nonseekable_open,
382	.unlocked_ioctl = gup_test_ioctl,
383	.compat_ioctl = compat_ptr_ioctl,
384	.release = gup_test_release,
385};
386
387static int __init gup_test_init(void)
388{
389	debugfs_create_file_unsafe("gup_test", 0600, NULL, NULL,
390				   &gup_test_fops);
391
392	return 0;
393}
394
395late_initcall(gup_test_init);
396