1/*
2 *  arch/s390/lib/uaccess_pt.c
3 *
4 *  User access functions based on page table walks for enhanced
5 *  system layout without hardware support.
6 *
7 *    Copyright IBM Corp. 2006
8 *    Author(s): Gerald Schaefer (gerald.schaefer@de.ibm.com)
9 */
10
11#include <linux/errno.h>
12#include <linux/hardirq.h>
13#include <linux/mm.h>
14#include <asm/uaccess.h>
15#include <asm/futex.h>
16#include "uaccess.h"
17
18static int __handle_fault(struct mm_struct *mm, unsigned long address,
19			  int write_access)
20{
21	struct vm_area_struct *vma;
22	int ret = -EFAULT;
23
24	if (in_atomic())
25		return ret;
26	down_read(&mm->mmap_sem);
27	vma = find_vma(mm, address);
28	if (unlikely(!vma))
29		goto out;
30	if (unlikely(vma->vm_start > address)) {
31		if (!(vma->vm_flags & VM_GROWSDOWN))
32			goto out;
33		if (expand_stack(vma, address))
34			goto out;
35	}
36
37	if (!write_access) {
38		/* page not present, check vm flags */
39		if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
40			goto out;
41	} else {
42		if (!(vma->vm_flags & VM_WRITE))
43			goto out;
44	}
45
46survive:
47	switch (handle_mm_fault(mm, vma, address, write_access)) {
48	case VM_FAULT_MINOR:
49		current->min_flt++;
50		break;
51	case VM_FAULT_MAJOR:
52		current->maj_flt++;
53		break;
54	case VM_FAULT_SIGBUS:
55		goto out_sigbus;
56	case VM_FAULT_OOM:
57		goto out_of_memory;
58	default:
59		BUG();
60	}
61	ret = 0;
62out:
63	up_read(&mm->mmap_sem);
64	return ret;
65
66out_of_memory:
67	up_read(&mm->mmap_sem);
68	if (is_init(current)) {
69		yield();
70		down_read(&mm->mmap_sem);
71		goto survive;
72	}
73	printk("VM: killing process %s\n", current->comm);
74	return ret;
75
76out_sigbus:
77	up_read(&mm->mmap_sem);
78	current->thread.prot_addr = address;
79	current->thread.trap_no = 0x11;
80	force_sig(SIGBUS, current);
81	return ret;
82}
83
84static size_t __user_copy_pt(unsigned long uaddr, void *kptr,
85			     size_t n, int write_user)
86{
87	struct mm_struct *mm = current->mm;
88	unsigned long offset, pfn, done, size;
89	pgd_t *pgd;
90	pmd_t *pmd;
91	pte_t *pte;
92	void *from, *to;
93
94	done = 0;
95retry:
96	spin_lock(&mm->page_table_lock);
97	do {
98		pgd = pgd_offset(mm, uaddr);
99		if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
100			goto fault;
101
102		pmd = pmd_offset(pgd, uaddr);
103		if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
104			goto fault;
105
106		pte = pte_offset_map(pmd, uaddr);
107		if (!pte || !pte_present(*pte) ||
108		    (write_user && !pte_write(*pte)))
109			goto fault;
110
111		pfn = pte_pfn(*pte);
112		if (!pfn_valid(pfn))
113			goto out;
114
115		offset = uaddr & (PAGE_SIZE - 1);
116		size = min(n - done, PAGE_SIZE - offset);
117		if (write_user) {
118			to = (void *)((pfn << PAGE_SHIFT) + offset);
119			from = kptr + done;
120		} else {
121			from = (void *)((pfn << PAGE_SHIFT) + offset);
122			to = kptr + done;
123		}
124		memcpy(to, from, size);
125		done += size;
126		uaddr += size;
127	} while (done < n);
128out:
129	spin_unlock(&mm->page_table_lock);
130	return n - done;
131fault:
132	spin_unlock(&mm->page_table_lock);
133	if (__handle_fault(mm, uaddr, write_user))
134		return n - done;
135	goto retry;
136}
137
138/*
139 * Do DAT for user address by page table walk, return kernel address.
140 * This function needs to be called with current->mm->page_table_lock held.
141 */
142static unsigned long __dat_user_addr(unsigned long uaddr)
143{
144	struct mm_struct *mm = current->mm;
145	unsigned long pfn, ret;
146	pgd_t *pgd;
147	pmd_t *pmd;
148	pte_t *pte;
149	int rc;
150
151	ret = 0;
152retry:
153	pgd = pgd_offset(mm, uaddr);
154	if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
155		goto fault;
156
157	pmd = pmd_offset(pgd, uaddr);
158	if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
159		goto fault;
160
161	pte = pte_offset_map(pmd, uaddr);
162	if (!pte || !pte_present(*pte))
163		goto fault;
164
165	pfn = pte_pfn(*pte);
166	if (!pfn_valid(pfn))
167		goto out;
168
169	ret = (pfn << PAGE_SHIFT) + (uaddr & (PAGE_SIZE - 1));
170out:
171	return ret;
172fault:
173	spin_unlock(&mm->page_table_lock);
174	rc = __handle_fault(mm, uaddr, 0);
175	spin_lock(&mm->page_table_lock);
176	if (rc)
177		goto out;
178	goto retry;
179}
180
181size_t copy_from_user_pt(size_t n, const void __user *from, void *to)
182{
183	size_t rc;
184
185	if (segment_eq(get_fs(), KERNEL_DS)) {
186		memcpy(to, (void __kernel __force *) from, n);
187		return 0;
188	}
189	rc = __user_copy_pt((unsigned long) from, to, n, 0);
190	if (unlikely(rc))
191		memset(to + n - rc, 0, rc);
192	return rc;
193}
194
195size_t copy_to_user_pt(size_t n, void __user *to, const void *from)
196{
197	if (segment_eq(get_fs(), KERNEL_DS)) {
198		memcpy((void __kernel __force *) to, from, n);
199		return 0;
200	}
201	return __user_copy_pt((unsigned long) to, (void *) from, n, 1);
202}
203
204static size_t clear_user_pt(size_t n, void __user *to)
205{
206	long done, size, ret;
207
208	if (segment_eq(get_fs(), KERNEL_DS)) {
209		memset((void __kernel __force *) to, 0, n);
210		return 0;
211	}
212	done = 0;
213	do {
214		if (n - done > PAGE_SIZE)
215			size = PAGE_SIZE;
216		else
217			size = n - done;
218		ret = __user_copy_pt((unsigned long) to + done,
219				      &empty_zero_page, size, 1);
220		done += size;
221		if (ret)
222			return ret + n - done;
223	} while (done < n);
224	return 0;
225}
226
227static size_t strnlen_user_pt(size_t count, const char __user *src)
228{
229	char *addr;
230	unsigned long uaddr = (unsigned long) src;
231	struct mm_struct *mm = current->mm;
232	unsigned long offset, pfn, done, len;
233	pgd_t *pgd;
234	pmd_t *pmd;
235	pte_t *pte;
236	size_t len_str;
237
238	if (segment_eq(get_fs(), KERNEL_DS))
239		return strnlen((const char __kernel __force *) src, count) + 1;
240	done = 0;
241retry:
242	spin_lock(&mm->page_table_lock);
243	do {
244		pgd = pgd_offset(mm, uaddr);
245		if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
246			goto fault;
247
248		pmd = pmd_offset(pgd, uaddr);
249		if (pmd_none(*pmd) || unlikely(pmd_bad(*pmd)))
250			goto fault;
251
252		pte = pte_offset_map(pmd, uaddr);
253		if (!pte || !pte_present(*pte))
254			goto fault;
255
256		pfn = pte_pfn(*pte);
257		if (!pfn_valid(pfn)) {
258			done = -1;
259			goto out;
260		}
261
262		offset = uaddr & (PAGE_SIZE-1);
263		addr = (char *)(pfn << PAGE_SHIFT) + offset;
264		len = min(count - done, PAGE_SIZE - offset);
265		len_str = strnlen(addr, len);
266		done += len_str;
267		uaddr += len_str;
268	} while ((len_str == len) && (done < count));
269out:
270	spin_unlock(&mm->page_table_lock);
271	return done + 1;
272fault:
273	spin_unlock(&mm->page_table_lock);
274	if (__handle_fault(mm, uaddr, 0)) {
275		return 0;
276	}
277	goto retry;
278}
279
280static size_t strncpy_from_user_pt(size_t count, const char __user *src,
281				   char *dst)
282{
283	size_t n = strnlen_user_pt(count, src);
284
285	if (!n)
286		return -EFAULT;
287	if (n > count)
288		n = count;
289	if (segment_eq(get_fs(), KERNEL_DS)) {
290		memcpy(dst, (const char __kernel __force *) src, n);
291		if (dst[n-1] == '\0')
292			return n-1;
293		else
294			return n;
295	}
296	if (__user_copy_pt((unsigned long) src, dst, n, 0))
297		return -EFAULT;
298	if (dst[n-1] == '\0')
299		return n-1;
300	else
301		return n;
302}
303
304static size_t copy_in_user_pt(size_t n, void __user *to,
305			      const void __user *from)
306{
307	struct mm_struct *mm = current->mm;
308	unsigned long offset_from, offset_to, offset_max, pfn_from, pfn_to,
309		      uaddr, done, size;
310	unsigned long uaddr_from = (unsigned long) from;
311	unsigned long uaddr_to = (unsigned long) to;
312	pgd_t *pgd_from, *pgd_to;
313	pmd_t *pmd_from, *pmd_to;
314	pte_t *pte_from, *pte_to;
315	int write_user;
316
317	done = 0;
318retry:
319	spin_lock(&mm->page_table_lock);
320	do {
321		pgd_from = pgd_offset(mm, uaddr_from);
322		if (pgd_none(*pgd_from) || unlikely(pgd_bad(*pgd_from))) {
323			uaddr = uaddr_from;
324			write_user = 0;
325			goto fault;
326		}
327		pgd_to = pgd_offset(mm, uaddr_to);
328		if (pgd_none(*pgd_to) || unlikely(pgd_bad(*pgd_to))) {
329			uaddr = uaddr_to;
330			write_user = 1;
331			goto fault;
332		}
333
334		pmd_from = pmd_offset(pgd_from, uaddr_from);
335		if (pmd_none(*pmd_from) || unlikely(pmd_bad(*pmd_from))) {
336			uaddr = uaddr_from;
337			write_user = 0;
338			goto fault;
339		}
340		pmd_to = pmd_offset(pgd_to, uaddr_to);
341		if (pmd_none(*pmd_to) || unlikely(pmd_bad(*pmd_to))) {
342			uaddr = uaddr_to;
343			write_user = 1;
344			goto fault;
345		}
346
347		pte_from = pte_offset_map(pmd_from, uaddr_from);
348		if (!pte_from || !pte_present(*pte_from)) {
349			uaddr = uaddr_from;
350			write_user = 0;
351			goto fault;
352		}
353		pte_to = pte_offset_map(pmd_to, uaddr_to);
354		if (!pte_to || !pte_present(*pte_to) || !pte_write(*pte_to)) {
355			uaddr = uaddr_to;
356			write_user = 1;
357			goto fault;
358		}
359
360		pfn_from = pte_pfn(*pte_from);
361		if (!pfn_valid(pfn_from))
362			goto out;
363		pfn_to = pte_pfn(*pte_to);
364		if (!pfn_valid(pfn_to))
365			goto out;
366
367		offset_from = uaddr_from & (PAGE_SIZE-1);
368		offset_to = uaddr_from & (PAGE_SIZE-1);
369		offset_max = max(offset_from, offset_to);
370		size = min(n - done, PAGE_SIZE - offset_max);
371
372		memcpy((void *)(pfn_to << PAGE_SHIFT) + offset_to,
373		       (void *)(pfn_from << PAGE_SHIFT) + offset_from, size);
374		done += size;
375		uaddr_from += size;
376		uaddr_to += size;
377	} while (done < n);
378out:
379	spin_unlock(&mm->page_table_lock);
380	return n - done;
381fault:
382	spin_unlock(&mm->page_table_lock);
383	if (__handle_fault(mm, uaddr, write_user))
384		return n - done;
385	goto retry;
386}
387
388#define __futex_atomic_op(insn, ret, oldval, newval, uaddr, oparg)	\
389	asm volatile("0: l   %1,0(%6)\n"				\
390		     "1: " insn						\
391		     "2: cs  %1,%2,0(%6)\n"				\
392		     "3: jl  1b\n"					\
393		     "   lhi %0,0\n"					\
394		     "4:\n"						\
395		     EX_TABLE(0b,4b) EX_TABLE(2b,4b) EX_TABLE(3b,4b)	\
396		     : "=d" (ret), "=&d" (oldval), "=&d" (newval),	\
397		       "=m" (*uaddr)					\
398		     : "0" (-EFAULT), "d" (oparg), "a" (uaddr),		\
399		       "m" (*uaddr) : "cc" );
400
401int futex_atomic_op_pt(int op, int __user *uaddr, int oparg, int *old)
402{
403	int oldval = 0, newval, ret;
404
405	spin_lock(&current->mm->page_table_lock);
406	uaddr = (int __user *) __dat_user_addr((unsigned long) uaddr);
407	if (!uaddr) {
408		spin_unlock(&current->mm->page_table_lock);
409		return -EFAULT;
410	}
411	get_page(virt_to_page(uaddr));
412	spin_unlock(&current->mm->page_table_lock);
413	switch (op) {
414	case FUTEX_OP_SET:
415		__futex_atomic_op("lr %2,%5\n",
416				  ret, oldval, newval, uaddr, oparg);
417		break;
418	case FUTEX_OP_ADD:
419		__futex_atomic_op("lr %2,%1\nar %2,%5\n",
420				  ret, oldval, newval, uaddr, oparg);
421		break;
422	case FUTEX_OP_OR:
423		__futex_atomic_op("lr %2,%1\nor %2,%5\n",
424				  ret, oldval, newval, uaddr, oparg);
425		break;
426	case FUTEX_OP_ANDN:
427		__futex_atomic_op("lr %2,%1\nnr %2,%5\n",
428				  ret, oldval, newval, uaddr, oparg);
429		break;
430	case FUTEX_OP_XOR:
431		__futex_atomic_op("lr %2,%1\nxr %2,%5\n",
432				  ret, oldval, newval, uaddr, oparg);
433		break;
434	default:
435		ret = -ENOSYS;
436	}
437	put_page(virt_to_page(uaddr));
438	*old = oldval;
439	return ret;
440}
441
442int futex_atomic_cmpxchg_pt(int __user *uaddr, int oldval, int newval)
443{
444	int ret;
445
446	spin_lock(&current->mm->page_table_lock);
447	uaddr = (int __user *) __dat_user_addr((unsigned long) uaddr);
448	if (!uaddr) {
449		spin_unlock(&current->mm->page_table_lock);
450		return -EFAULT;
451	}
452	get_page(virt_to_page(uaddr));
453	spin_unlock(&current->mm->page_table_lock);
454	asm volatile("   cs   %1,%4,0(%5)\n"
455		     "0: lr   %0,%1\n"
456		     "1:\n"
457		     EX_TABLE(0b,1b)
458		     : "=d" (ret), "+d" (oldval), "=m" (*uaddr)
459		     : "0" (-EFAULT), "d" (newval), "a" (uaddr), "m" (*uaddr)
460		     : "cc", "memory" );
461	put_page(virt_to_page(uaddr));
462	return ret;
463}
464
465struct uaccess_ops uaccess_pt = {
466	.copy_from_user		= copy_from_user_pt,
467	.copy_from_user_small	= copy_from_user_pt,
468	.copy_to_user		= copy_to_user_pt,
469	.copy_to_user_small	= copy_to_user_pt,
470	.copy_in_user		= copy_in_user_pt,
471	.clear_user		= clear_user_pt,
472	.strnlen_user		= strnlen_user_pt,
473	.strncpy_from_user	= strncpy_from_user_pt,
474	.futex_atomic_op	= futex_atomic_op_pt,
475	.futex_atomic_cmpxchg	= futex_atomic_cmpxchg_pt,
476};
477