1/**************************************************************************
2 *
3 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27/*
28 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com>
29 */
30/*
31 * Copyright (c) 2013 The FreeBSD Foundation
32 * All rights reserved.
33 *
34 * Portions of this software were developed by Konstantin Belousov
35 * <kib@FreeBSD.org> under sponsorship from the FreeBSD Foundation.
36 */
37
38#include <sys/cdefs.h>
39__FBSDID("$FreeBSD$");
40
41#include "opt_vm.h"
42
43#include <dev/drm2/drmP.h>
44#include <dev/drm2/ttm/ttm_module.h>
45#include <dev/drm2/ttm/ttm_bo_driver.h>
46#include <dev/drm2/ttm/ttm_placement.h>
47
48#include <vm/vm.h>
49#include <vm/vm_page.h>
50
51#define TTM_BO_VM_NUM_PREFAULT 16
52
53RB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb,
54    ttm_bo_cmp_rb_tree_items);
55
56int
57ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a,
58    struct ttm_buffer_object *b)
59{
60
61	if (a->vm_node->start < b->vm_node->start) {
62		return (-1);
63	} else if (a->vm_node->start > b->vm_node->start) {
64		return (1);
65	} else {
66		return (0);
67	}
68}
69
70static struct ttm_buffer_object *ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev,
71						     unsigned long page_start,
72						     unsigned long num_pages)
73{
74	unsigned long cur_offset;
75	struct ttm_buffer_object *bo;
76	struct ttm_buffer_object *best_bo = NULL;
77
78	bo = RB_ROOT(&bdev->addr_space_rb);
79	while (bo != NULL) {
80		cur_offset = bo->vm_node->start;
81		if (page_start >= cur_offset) {
82			best_bo = bo;
83			if (page_start == cur_offset)
84				break;
85			bo = RB_RIGHT(bo, vm_rb);
86		} else
87			bo = RB_LEFT(bo, vm_rb);
88	}
89
90	if (unlikely(best_bo == NULL))
91		return NULL;
92
93	if (unlikely((best_bo->vm_node->start + best_bo->num_pages) <
94		     (page_start + num_pages)))
95		return NULL;
96
97	return best_bo;
98}
99
100static int
101ttm_bo_vm_fault(vm_object_t vm_obj, vm_ooffset_t offset,
102    int prot, vm_page_t *mres)
103{
104
105	struct ttm_buffer_object *bo = vm_obj->handle;
106	struct ttm_bo_device *bdev = bo->bdev;
107	struct ttm_tt *ttm = NULL;
108	vm_page_t m, m1, oldm;
109	int ret;
110	int retval = VM_PAGER_OK;
111	struct ttm_mem_type_manager *man =
112		&bdev->man[bo->mem.mem_type];
113
114	vm_object_pip_add(vm_obj, 1);
115	oldm = *mres;
116	if (oldm != NULL) {
117		vm_page_lock(oldm);
118		vm_page_remove(oldm);
119		vm_page_unlock(oldm);
120		*mres = NULL;
121	} else
122		oldm = NULL;
123retry:
124	VM_OBJECT_UNLOCK(vm_obj);
125	m = NULL;
126
127reserve:
128	ret = ttm_bo_reserve(bo, false, false, false, 0);
129	if (unlikely(ret != 0)) {
130		if (ret == -EBUSY) {
131			kern_yield(0);
132			goto reserve;
133		}
134	}
135
136	if (bdev->driver->fault_reserve_notify) {
137		ret = bdev->driver->fault_reserve_notify(bo);
138		switch (ret) {
139		case 0:
140			break;
141		case -EBUSY:
142		case -ERESTART:
143		case -EINTR:
144			kern_yield(0);
145			goto reserve;
146		default:
147			retval = VM_PAGER_ERROR;
148			goto out_unlock;
149		}
150	}
151
152	/*
153	 * Wait for buffer data in transit, due to a pipelined
154	 * move.
155	 */
156
157	mtx_lock(&bdev->fence_lock);
158	if (test_bit(TTM_BO_PRIV_FLAG_MOVING, &bo->priv_flags)) {
159		/*
160		 * Here, the behavior differs between Linux and FreeBSD.
161		 *
162		 * On Linux, the wait is interruptible (3rd argument to
163		 * ttm_bo_wait). There must be some mechanism to resume
164		 * page fault handling, once the signal is processed.
165		 *
166		 * On FreeBSD, the wait is uninteruptible. This is not a
167		 * problem as we can't end up with an unkillable process
168		 * here, because the wait will eventually time out.
169		 *
170		 * An example of this situation is the Xorg process
171		 * which uses SIGALRM internally. The signal could
172		 * interrupt the wait, causing the page fault to fail
173		 * and the process to receive SIGSEGV.
174		 */
175		ret = ttm_bo_wait(bo, false, false, false);
176		mtx_unlock(&bdev->fence_lock);
177		if (unlikely(ret != 0)) {
178			retval = VM_PAGER_ERROR;
179			goto out_unlock;
180		}
181	} else
182		mtx_unlock(&bdev->fence_lock);
183
184	ret = ttm_mem_io_lock(man, true);
185	if (unlikely(ret != 0)) {
186		retval = VM_PAGER_ERROR;
187		goto out_unlock;
188	}
189	ret = ttm_mem_io_reserve_vm(bo);
190	if (unlikely(ret != 0)) {
191		retval = VM_PAGER_ERROR;
192		goto out_io_unlock;
193	}
194
195	/*
196	 * Strictly, we're not allowed to modify vma->vm_page_prot here,
197	 * since the mmap_sem is only held in read mode. However, we
198	 * modify only the caching bits of vma->vm_page_prot and
199	 * consider those bits protected by
200	 * the bo->mutex, as we should be the only writers.
201	 * There shouldn't really be any readers of these bits except
202	 * within vm_insert_mixed()? fork?
203	 *
204	 * TODO: Add a list of vmas to the bo, and change the
205	 * vma->vm_page_prot when the object changes caching policy, with
206	 * the correct locks held.
207	 */
208	if (!bo->mem.bus.is_iomem) {
209		/* Allocate all page at once, most common usage */
210		ttm = bo->ttm;
211		if (ttm->bdev->driver->ttm_tt_populate(ttm)) {
212			retval = VM_PAGER_ERROR;
213			goto out_io_unlock;
214		}
215	}
216
217	if (bo->mem.bus.is_iomem) {
218		m = vm_phys_fictitious_to_vm_page(bo->mem.bus.base +
219		    bo->mem.bus.offset + offset);
220		pmap_page_set_memattr(m, ttm_io_prot(bo->mem.placement));
221	} else {
222		ttm = bo->ttm;
223		m = ttm->pages[OFF_TO_IDX(offset)];
224		if (unlikely(!m)) {
225			retval = VM_PAGER_ERROR;
226			goto out_io_unlock;
227		}
228		pmap_page_set_memattr(m,
229		    (bo->mem.placement & TTM_PL_FLAG_CACHED) ?
230		    VM_MEMATTR_WRITE_BACK : ttm_io_prot(bo->mem.placement));
231	}
232
233	VM_OBJECT_LOCK(vm_obj);
234	if ((m->flags & VPO_BUSY) != 0) {
235		vm_page_sleep(m, "ttmpbs");
236		ttm_mem_io_unlock(man);
237		ttm_bo_unreserve(bo);
238		goto retry;
239	}
240	m->valid = VM_PAGE_BITS_ALL;
241	*mres = m;
242	m1 = vm_page_lookup(vm_obj, OFF_TO_IDX(offset));
243	if (m1 == NULL) {
244		vm_page_insert(m, vm_obj, OFF_TO_IDX(offset));
245	} else {
246		KASSERT(m == m1,
247		    ("inconsistent insert bo %p m %p m1 %p offset %jx",
248		    bo, m, m1, (uintmax_t)offset));
249	}
250	vm_page_busy(m);
251
252	if (oldm != NULL) {
253		vm_page_lock(oldm);
254		vm_page_free(oldm);
255		vm_page_unlock(oldm);
256	}
257
258out_io_unlock1:
259	ttm_mem_io_unlock(man);
260out_unlock1:
261	ttm_bo_unreserve(bo);
262	vm_object_pip_wakeup(vm_obj);
263	return (retval);
264
265out_io_unlock:
266	VM_OBJECT_LOCK(vm_obj);
267	goto out_io_unlock1;
268
269out_unlock:
270	VM_OBJECT_LOCK(vm_obj);
271	goto out_unlock1;
272}
273
274static int
275ttm_bo_vm_ctor(void *handle, vm_ooffset_t size, vm_prot_t prot,
276    vm_ooffset_t foff, struct ucred *cred, u_short *color)
277{
278
279	/*
280	 * On Linux, a reference to the buffer object is acquired here.
281	 * The reason is that this function is not called when the
282	 * mmap() is initialized, but only when a process forks for
283	 * instance. Therefore on Linux, the reference on the bo is
284	 * acquired either in ttm_bo_mmap() or ttm_bo_vm_open(). It's
285	 * then released in ttm_bo_vm_close().
286	 *
287	 * Here, this function is called during mmap() intialization.
288	 * Thus, the reference acquired in ttm_bo_mmap_single() is
289	 * sufficient.
290	 */
291
292	*color = 0;
293	return (0);
294}
295
296static void
297ttm_bo_vm_dtor(void *handle)
298{
299	struct ttm_buffer_object *bo = handle;
300
301	ttm_bo_unref(&bo);
302}
303
304static struct cdev_pager_ops ttm_pager_ops = {
305	.cdev_pg_fault = ttm_bo_vm_fault,
306	.cdev_pg_ctor = ttm_bo_vm_ctor,
307	.cdev_pg_dtor = ttm_bo_vm_dtor
308};
309
310int
311ttm_bo_mmap_single(struct ttm_bo_device *bdev, vm_ooffset_t *offset, vm_size_t size,
312    struct vm_object **obj_res, int nprot)
313{
314	struct ttm_bo_driver *driver;
315	struct ttm_buffer_object *bo;
316	struct vm_object *vm_obj;
317	int ret;
318
319	rw_wlock(&bdev->vm_lock);
320	bo = ttm_bo_vm_lookup_rb(bdev, OFF_TO_IDX(*offset), OFF_TO_IDX(size));
321	if (likely(bo != NULL))
322		refcount_acquire(&bo->kref);
323	rw_wunlock(&bdev->vm_lock);
324
325	if (unlikely(bo == NULL)) {
326		printf("[TTM] Could not find buffer object to map\n");
327		return (EINVAL);
328	}
329
330	driver = bo->bdev->driver;
331	if (unlikely(!driver->verify_access)) {
332		ret = EPERM;
333		goto out_unref;
334	}
335	ret = -driver->verify_access(bo);
336	if (unlikely(ret != 0))
337		goto out_unref;
338
339	vm_obj = cdev_pager_allocate(bo, OBJT_MGTDEVICE, &ttm_pager_ops,
340	    size, nprot, 0, curthread->td_ucred);
341	if (vm_obj == NULL) {
342		ret = EINVAL;
343		goto out_unref;
344	}
345	/*
346	 * Note: We're transferring the bo reference to vm_obj->handle here.
347	 */
348	*offset = 0;
349	*obj_res = vm_obj;
350	return 0;
351out_unref:
352	ttm_bo_unref(&bo);
353	return ret;
354}
355
356void
357ttm_bo_release_mmap(struct ttm_buffer_object *bo)
358{
359	vm_object_t vm_obj;
360	vm_page_t m;
361	int i;
362
363	vm_obj = cdev_pager_lookup(bo);
364	if (vm_obj == NULL)
365		return;
366
367	VM_OBJECT_LOCK(vm_obj);
368retry:
369	for (i = 0; i < bo->num_pages; i++) {
370		m = vm_page_lookup(vm_obj, i);
371		if (m == NULL)
372			continue;
373		if (vm_page_sleep_if_busy(m, true, "ttm_unm"))
374			goto retry;
375		cdev_pager_free_page(vm_obj, m);
376	}
377	VM_OBJECT_UNLOCK(vm_obj);
378
379	vm_object_deallocate(vm_obj);
380}
381
382#if 0
383int ttm_fbdev_mmap(struct vm_area_struct *vma, struct ttm_buffer_object *bo)
384{
385	if (vma->vm_pgoff != 0)
386		return -EACCES;
387
388	vma->vm_ops = &ttm_bo_vm_ops;
389	vma->vm_private_data = ttm_bo_reference(bo);
390	vma->vm_flags |= VM_IO | VM_MIXEDMAP | VM_DONTEXPAND;
391	return 0;
392}
393
394ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp,
395		  const char __user *wbuf, char __user *rbuf, size_t count,
396		  loff_t *f_pos, bool write)
397{
398	struct ttm_buffer_object *bo;
399	struct ttm_bo_driver *driver;
400	struct ttm_bo_kmap_obj map;
401	unsigned long dev_offset = (*f_pos >> PAGE_SHIFT);
402	unsigned long kmap_offset;
403	unsigned long kmap_end;
404	unsigned long kmap_num;
405	size_t io_size;
406	unsigned int page_offset;
407	char *virtual;
408	int ret;
409	bool no_wait = false;
410	bool dummy;
411
412	read_lock(&bdev->vm_lock);
413	bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1);
414	if (likely(bo != NULL))
415		ttm_bo_reference(bo);
416	read_unlock(&bdev->vm_lock);
417
418	if (unlikely(bo == NULL))
419		return -EFAULT;
420
421	driver = bo->bdev->driver;
422	if (unlikely(!driver->verify_access)) {
423		ret = -EPERM;
424		goto out_unref;
425	}
426
427	ret = driver->verify_access(bo, filp);
428	if (unlikely(ret != 0))
429		goto out_unref;
430
431	kmap_offset = dev_offset - bo->vm_node->start;
432	if (unlikely(kmap_offset >= bo->num_pages)) {
433		ret = -EFBIG;
434		goto out_unref;
435	}
436
437	page_offset = *f_pos & ~PAGE_MASK;
438	io_size = bo->num_pages - kmap_offset;
439	io_size = (io_size << PAGE_SHIFT) - page_offset;
440	if (count < io_size)
441		io_size = count;
442
443	kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
444	kmap_num = kmap_end - kmap_offset + 1;
445
446	ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
447
448	switch (ret) {
449	case 0:
450		break;
451	case -EBUSY:
452		ret = -EAGAIN;
453		goto out_unref;
454	default:
455		goto out_unref;
456	}
457
458	ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
459	if (unlikely(ret != 0)) {
460		ttm_bo_unreserve(bo);
461		goto out_unref;
462	}
463
464	virtual = ttm_kmap_obj_virtual(&map, &dummy);
465	virtual += page_offset;
466
467	if (write)
468		ret = copy_from_user(virtual, wbuf, io_size);
469	else
470		ret = copy_to_user(rbuf, virtual, io_size);
471
472	ttm_bo_kunmap(&map);
473	ttm_bo_unreserve(bo);
474	ttm_bo_unref(&bo);
475
476	if (unlikely(ret != 0))
477		return -EFBIG;
478
479	*f_pos += io_size;
480
481	return io_size;
482out_unref:
483	ttm_bo_unref(&bo);
484	return ret;
485}
486
487ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *bo, const char __user *wbuf,
488			char __user *rbuf, size_t count, loff_t *f_pos,
489			bool write)
490{
491	struct ttm_bo_kmap_obj map;
492	unsigned long kmap_offset;
493	unsigned long kmap_end;
494	unsigned long kmap_num;
495	size_t io_size;
496	unsigned int page_offset;
497	char *virtual;
498	int ret;
499	bool no_wait = false;
500	bool dummy;
501
502	kmap_offset = (*f_pos >> PAGE_SHIFT);
503	if (unlikely(kmap_offset >= bo->num_pages))
504		return -EFBIG;
505
506	page_offset = *f_pos & ~PAGE_MASK;
507	io_size = bo->num_pages - kmap_offset;
508	io_size = (io_size << PAGE_SHIFT) - page_offset;
509	if (count < io_size)
510		io_size = count;
511
512	kmap_end = (*f_pos + count - 1) >> PAGE_SHIFT;
513	kmap_num = kmap_end - kmap_offset + 1;
514
515	ret = ttm_bo_reserve(bo, true, no_wait, false, 0);
516
517	switch (ret) {
518	case 0:
519		break;
520	case -EBUSY:
521		return -EAGAIN;
522	default:
523		return ret;
524	}
525
526	ret = ttm_bo_kmap(bo, kmap_offset, kmap_num, &map);
527	if (unlikely(ret != 0)) {
528		ttm_bo_unreserve(bo);
529		return ret;
530	}
531
532	virtual = ttm_kmap_obj_virtual(&map, &dummy);
533	virtual += page_offset;
534
535	if (write)
536		ret = copy_from_user(virtual, wbuf, io_size);
537	else
538		ret = copy_to_user(rbuf, virtual, io_size);
539
540	ttm_bo_kunmap(&map);
541	ttm_bo_unreserve(bo);
542	ttm_bo_unref(&bo);
543
544	if (unlikely(ret != 0))
545		return ret;
546
547	*f_pos += io_size;
548
549	return io_size;
550}
551#endif
552