1#ifndef __i386_UACCESS_H
2#define __i386_UACCESS_H
3
4/*
5 * User space memory access functions
6 */
7#include <linux/config.h>
8#include <linux/sched.h>
9#include <linux/prefetch.h>
10#include <asm/page.h>
11
12#define VERIFY_READ 0
13#define VERIFY_WRITE 1
14
15/*
16 * The fs value determines whether argument validity checking should be
17 * performed or not.  If get_fs() == USER_DS, checking is performed, with
18 * get_fs() == KERNEL_DS, checking is bypassed.
19 *
20 * For historical reasons, these macros are grossly misnamed.
21 */
22
23#define MAKE_MM_SEG(s)	((mm_segment_t) { (s) })
24
25
26#define KERNEL_DS	MAKE_MM_SEG(0xFFFFFFFF)
27#define USER_DS		MAKE_MM_SEG(PAGE_OFFSET)
28
29#define get_ds()	(KERNEL_DS)
30#define get_fs()	(current->addr_limit)
31#define set_fs(x)	(current->addr_limit = (x))
32
33#define segment_eq(a,b)	((a).seg == (b).seg)
34
35extern int __verify_write(const void *, unsigned long);
36
37#define __addr_ok(addr) ((unsigned long)(addr) < (current->addr_limit.seg))
38
39/*
40 * Uhhuh, this needs 33-bit arithmetic. We have a carry..
41 */
42#define __range_ok(addr,size) ({ \
43	unsigned long flag,sum; \
44	asm("addl %3,%1 ; sbbl %0,%0; cmpl %1,%4; sbbl $0,%0" \
45		:"=&r" (flag), "=r" (sum) \
46		:"1" (addr),"g" ((int)(size)),"g" (current->addr_limit.seg)); \
47	flag; })
48
49#ifdef CONFIG_X86_WP_WORKS_OK
50
51#define access_ok(type,addr,size) (__range_ok(addr,size) == 0)
52
53#else
54
55#define access_ok(type,addr,size) ( (__range_ok(addr,size) == 0) && \
56			 ((type) == VERIFY_READ || boot_cpu_data.wp_works_ok || \
57			 segment_eq(get_fs(),KERNEL_DS) || \
58			  __verify_write((void *)(addr),(size))))
59
60#endif
61
62static inline int verify_area(int type, const void * addr, unsigned long size)
63{
64	return access_ok(type,addr,size) ? 0 : -EFAULT;
65}
66
67
68/*
69 * The exception table consists of pairs of addresses: the first is the
70 * address of an instruction that is allowed to fault, and the second is
71 * the address at which the program should continue.  No registers are
72 * modified, so it is entirely up to the continuation code to figure out
73 * what to do.
74 *
75 * All the routines below use bits of fixup code that are out of line
76 * with the main instruction path.  This means when everything is well,
77 * we don't even have to jump over them.  Further, they do not intrude
78 * on our cache or tlb entries.
79 */
80
81struct exception_table_entry
82{
83	unsigned long insn, fixup;
84};
85
86/* Returns 0 if exception not found and fixup otherwise.  */
87extern unsigned long search_exception_table(unsigned long);
88
89
90/*
91 * These are the main single-value transfer routines.  They automatically
92 * use the right size if we just have the right pointer type.
93 *
94 * This gets kind of ugly. We want to return _two_ values in "get_user()"
95 * and yet we don't want to do any pointers, because that is too much
96 * of a performance impact. Thus we have a few rather ugly macros here,
97 * and hide all the uglyness from the user.
98 *
99 * The "__xxx" versions of the user access functions are versions that
100 * do not verify the address space, that must have been done previously
101 * with a separate "access_ok()" call (this is used when we do multiple
102 * accesses to the same area of user memory).
103 */
104
105extern void __get_user_1(void);
106extern void __get_user_2(void);
107extern void __get_user_4(void);
108
109#define __get_user_x(size,ret,x,ptr) \
110	__asm__ __volatile__("call __get_user_" #size \
111		:"=a" (ret),"=d" (x) \
112		:"0" (ptr))
113
114/* Careful: we have to cast the result to the type of the pointer for sign reasons */
115#define get_user(x,ptr)							\
116({	int __ret_gu,__val_gu;						\
117	switch(sizeof (*(ptr))) {					\
118	case 1:  __get_user_x(1,__ret_gu,__val_gu,ptr); break;		\
119	case 2:  __get_user_x(2,__ret_gu,__val_gu,ptr); break;		\
120	case 4:  __get_user_x(4,__ret_gu,__val_gu,ptr); break;		\
121	default: __get_user_x(X,__ret_gu,__val_gu,ptr); break;		\
122	}								\
123	(x) = (__typeof__(*(ptr)))__val_gu;				\
124	__ret_gu;							\
125})
126
127extern void __put_user_1(void);
128extern void __put_user_2(void);
129extern void __put_user_4(void);
130extern void __put_user_8(void);
131
132extern void __put_user_bad(void);
133
134#define put_user(x,ptr)							\
135  __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
136
137#define __get_user(x,ptr) \
138  __get_user_nocheck((x),(ptr),sizeof(*(ptr)))
139#define __put_user(x,ptr) \
140  __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr)))
141
142#define __put_user_nocheck(x,ptr,size)			\
143({							\
144	long __pu_err;					\
145	__put_user_size((x),(ptr),(size),__pu_err);	\
146	__pu_err;					\
147})
148
149
150#define __put_user_check(x,ptr,size)			\
151({							\
152	long __pu_err = -EFAULT;					\
153	__typeof__(*(ptr)) *__pu_addr = (ptr);		\
154	if (access_ok(VERIFY_WRITE,__pu_addr,size))	\
155		__put_user_size((x),__pu_addr,(size),__pu_err);	\
156	__pu_err;					\
157})
158
159#define __put_user_u64(x, addr, err)				\
160	__asm__ __volatile__(					\
161		"1:	movl %%eax,0(%2)\n"			\
162		"2:	movl %%edx,4(%2)\n"			\
163		"3:\n"						\
164		".section .fixup,\"ax\"\n"			\
165		"4:	movl %3,%0\n"				\
166		"	jmp 3b\n"				\
167		".previous\n"					\
168		".section __ex_table,\"a\"\n"			\
169		"	.align 4\n"				\
170		"	.long 1b,4b\n"				\
171		"	.long 2b,4b\n"				\
172		".previous"					\
173		: "=r"(err)					\
174		: "A" (x), "r" (addr), "i"(-EFAULT), "0"(err))
175
176#define __put_user_size(x,ptr,size,retval)				\
177do {									\
178	retval = 0;							\
179	switch (size) {							\
180	  case 1: __put_user_asm(x,ptr,retval,"b","b","iq"); break;	\
181	  case 2: __put_user_asm(x,ptr,retval,"w","w","ir"); break;	\
182	  case 4: __put_user_asm(x,ptr,retval,"l","","ir"); break;	\
183	  case 8: __put_user_u64(x,ptr,retval); break;			\
184	  default: __put_user_bad();					\
185	}								\
186} while (0)
187
188struct __large_struct { unsigned long buf[100]; };
189#define __m(x) (*(struct __large_struct *)(x))
190
191/*
192 * Tell gcc we read from memory instead of writing: this is because
193 * we do not write to any memory gcc knows about, so there are no
194 * aliasing issues.
195 */
196#define __put_user_asm(x, addr, err, itype, rtype, ltype)	\
197	__asm__ __volatile__(					\
198		"1:	mov"itype" %"rtype"1,%2\n"		\
199		"2:\n"						\
200		".section .fixup,\"ax\"\n"			\
201		"3:	movl %3,%0\n"				\
202		"	jmp 2b\n"				\
203		".previous\n"					\
204		".section __ex_table,\"a\"\n"			\
205		"	.align 4\n"				\
206		"	.long 1b,3b\n"				\
207		".previous"					\
208		: "=r"(err)					\
209		: ltype (x), "m"(__m(addr)), "i"(-EFAULT), "0"(err))
210
211
212#define __get_user_nocheck(x,ptr,size)				\
213({								\
214	long __gu_err, __gu_val;				\
215	__get_user_size(__gu_val,(ptr),(size),__gu_err);	\
216	(x) = (__typeof__(*(ptr)))__gu_val;			\
217	__gu_err;						\
218})
219
220extern long __get_user_bad(void);
221
222#define __get_user_size(x,ptr,size,retval)				\
223do {									\
224	retval = 0;							\
225	switch (size) {							\
226	  case 1: __get_user_asm(x,ptr,retval,"b","b","=q"); break;	\
227	  case 2: __get_user_asm(x,ptr,retval,"w","w","=r"); break;	\
228	  case 4: __get_user_asm(x,ptr,retval,"l","","=r"); break;	\
229	  default: (x) = __get_user_bad();				\
230	}								\
231} while (0)
232
233#define __get_user_asm(x, addr, err, itype, rtype, ltype)	\
234	__asm__ __volatile__(					\
235		"1:	mov"itype" %2,%"rtype"1\n"		\
236		"2:\n"						\
237		".section .fixup,\"ax\"\n"			\
238		"3:	movl %3,%0\n"				\
239		"	xor"itype" %"rtype"1,%"rtype"1\n"	\
240		"	jmp 2b\n"				\
241		".previous\n"					\
242		".section __ex_table,\"a\"\n"			\
243		"	.align 4\n"				\
244		"	.long 1b,3b\n"				\
245		".previous"					\
246		: "=r"(err), ltype (x)				\
247		: "m"(__m(addr)), "i"(-EFAULT), "0"(err))
248
249
250/*
251 * Copy To/From Userspace
252 */
253
254/* Generic arbitrary sized copy.  */
255#define __copy_user(to,from,size)					\
256do {									\
257	int __d0, __d1;							\
258	__asm__ __volatile__(						\
259		"0:	rep; movsl\n"					\
260		"	movl %3,%0\n"					\
261		"1:	rep; movsb\n"					\
262		"2:\n"							\
263		".section .fixup,\"ax\"\n"				\
264		"3:	lea 0(%3,%0,4),%0\n"				\
265		"	jmp 2b\n"					\
266		".previous\n"						\
267		".section __ex_table,\"a\"\n"				\
268		"	.align 4\n"					\
269		"	.long 0b,3b\n"					\
270		"	.long 1b,2b\n"					\
271		".previous"						\
272		: "=&c"(size), "=&D" (__d0), "=&S" (__d1)		\
273		: "r"(size & 3), "0"(size / 4), "1"(to), "2"(from)	\
274		: "memory");						\
275} while (0)
276
277#define __copy_user_zeroing(to,from,size)				\
278do {									\
279	int __d0, __d1;							\
280	__asm__ __volatile__(						\
281		"0:	rep; movsl\n"					\
282		"	movl %3,%0\n"					\
283		"1:	rep; movsb\n"					\
284		"2:\n"							\
285		".section .fixup,\"ax\"\n"				\
286		"3:	lea 0(%3,%0,4),%0\n"				\
287		"4:	pushl %0\n"					\
288		"	pushl %%eax\n"					\
289		"	xorl %%eax,%%eax\n"				\
290		"	rep; stosb\n"					\
291		"	popl %%eax\n"					\
292		"	popl %0\n"					\
293		"	jmp 2b\n"					\
294		".previous\n"						\
295		".section __ex_table,\"a\"\n"				\
296		"	.align 4\n"					\
297		"	.long 0b,3b\n"					\
298		"	.long 1b,4b\n"					\
299		".previous"						\
300		: "=&c"(size), "=&D" (__d0), "=&S" (__d1)		\
301		: "r"(size & 3), "0"(size / 4), "1"(to), "2"(from)	\
302		: "memory");						\
303} while (0)
304
305/* We let the __ versions of copy_from/to_user inline, because they're often
306 * used in fast paths and have only a small space overhead.
307 */
308static inline unsigned long
309__generic_copy_from_user_nocheck(void *to, const void *from, unsigned long n)
310{
311	__copy_user_zeroing(to,from,n);
312	return n;
313}
314
315static inline unsigned long
316__generic_copy_to_user_nocheck(void *to, const void *from, unsigned long n)
317{
318	__copy_user(to,from,n);
319	return n;
320}
321
322
323/* Optimize just a little bit when we know the size of the move. */
324#define __constant_copy_user(to, from, size)			\
325do {								\
326	int __d0, __d1;						\
327	switch (size & 3) {					\
328	default:						\
329		__asm__ __volatile__(				\
330			"0:	rep; movsl\n"			\
331			"1:\n"					\
332			".section .fixup,\"ax\"\n"		\
333			"2:	shl $2,%0\n"			\
334			"	jmp 1b\n"			\
335			".previous\n"				\
336			".section __ex_table,\"a\"\n"		\
337			"	.align 4\n"			\
338			"	.long 0b,2b\n"			\
339			".previous"				\
340			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
341			: "1"(from), "2"(to), "0"(size/4)	\
342			: "memory");				\
343		break;						\
344	case 1:							\
345		__asm__ __volatile__(				\
346			"0:	rep; movsl\n"			\
347			"1:	movsb\n"			\
348			"2:\n"					\
349			".section .fixup,\"ax\"\n"		\
350			"3:	shl $2,%0\n"			\
351			"4:	incl %0\n"			\
352			"	jmp 2b\n"			\
353			".previous\n"				\
354			".section __ex_table,\"a\"\n"		\
355			"	.align 4\n"			\
356			"	.long 0b,3b\n"			\
357			"	.long 1b,4b\n"			\
358			".previous"				\
359			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
360			: "1"(from), "2"(to), "0"(size/4)	\
361			: "memory");				\
362		break;						\
363	case 2:							\
364		__asm__ __volatile__(				\
365			"0:	rep; movsl\n"			\
366			"1:	movsw\n"			\
367			"2:\n"					\
368			".section .fixup,\"ax\"\n"		\
369			"3:	shl $2,%0\n"			\
370			"4:	addl $2,%0\n"			\
371			"	jmp 2b\n"			\
372			".previous\n"				\
373			".section __ex_table,\"a\"\n"		\
374			"	.align 4\n"			\
375			"	.long 0b,3b\n"			\
376			"	.long 1b,4b\n"			\
377			".previous"				\
378			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
379			: "1"(from), "2"(to), "0"(size/4)	\
380			: "memory");				\
381		break;						\
382	case 3:							\
383		__asm__ __volatile__(				\
384			"0:	rep; movsl\n"			\
385			"1:	movsw\n"			\
386			"2:	movsb\n"			\
387			"3:\n"					\
388			".section .fixup,\"ax\"\n"		\
389			"4:	shl $2,%0\n"			\
390			"5:	addl $2,%0\n"			\
391			"6:	incl %0\n"			\
392			"	jmp 3b\n"			\
393			".previous\n"				\
394			".section __ex_table,\"a\"\n"		\
395			"	.align 4\n"			\
396			"	.long 0b,4b\n"			\
397			"	.long 1b,5b\n"			\
398			"	.long 2b,6b\n"			\
399			".previous"				\
400			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
401			: "1"(from), "2"(to), "0"(size/4)	\
402			: "memory");				\
403		break;						\
404	}							\
405} while (0)
406
407/* Optimize just a little bit when we know the size of the move. */
408#define __constant_copy_user_zeroing(to, from, size)		\
409do {								\
410	int __d0, __d1;						\
411	switch (size & 3) {					\
412	default:						\
413		__asm__ __volatile__(				\
414			"0:	rep; movsl\n"			\
415			"1:\n"					\
416			".section .fixup,\"ax\"\n"		\
417			"2:	pushl %0\n"			\
418			"	pushl %%eax\n"			\
419			"	xorl %%eax,%%eax\n"		\
420			"	rep; stosl\n"			\
421			"	popl %%eax\n"			\
422			"	popl %0\n"			\
423			"	shl $2,%0\n"			\
424			"	jmp 1b\n"			\
425			".previous\n"				\
426			".section __ex_table,\"a\"\n"		\
427			"	.align 4\n"			\
428			"	.long 0b,2b\n"			\
429			".previous"				\
430			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
431			: "1"(from), "2"(to), "0"(size/4)	\
432			: "memory");				\
433		break;						\
434	case 1:							\
435		__asm__ __volatile__(				\
436			"0:	rep; movsl\n"			\
437			"1:	movsb\n"			\
438			"2:\n"					\
439			".section .fixup,\"ax\"\n"		\
440			"3:	pushl %0\n"			\
441			"	pushl %%eax\n"			\
442			"	xorl %%eax,%%eax\n"		\
443			"	rep; stosl\n"			\
444			"	stosb\n"			\
445			"	popl %%eax\n"			\
446			"	popl %0\n"			\
447			"	shl $2,%0\n"			\
448			"	incl %0\n"			\
449			"	jmp 2b\n"			\
450			"4:	pushl %%eax\n"			\
451			"	xorl %%eax,%%eax\n"		\
452			"	stosb\n"			\
453			"	popl %%eax\n"			\
454			"	incl %0\n"			\
455			"	jmp 2b\n"			\
456			".previous\n"				\
457			".section __ex_table,\"a\"\n"		\
458			"	.align 4\n"			\
459			"	.long 0b,3b\n"			\
460			"	.long 1b,4b\n"			\
461			".previous"				\
462			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
463			: "1"(from), "2"(to), "0"(size/4)	\
464			: "memory");				\
465		break;						\
466	case 2:							\
467		__asm__ __volatile__(				\
468			"0:	rep; movsl\n"			\
469			"1:	movsw\n"			\
470			"2:\n"					\
471			".section .fixup,\"ax\"\n"		\
472			"3:	pushl %0\n"			\
473			"	pushl %%eax\n"			\
474			"	xorl %%eax,%%eax\n"		\
475			"	rep; stosl\n"			\
476			"	stosw\n"			\
477			"	popl %%eax\n"			\
478			"	popl %0\n"			\
479			"	shl $2,%0\n"			\
480			"	addl $2,%0\n"			\
481			"	jmp 2b\n"			\
482			"4:	pushl %%eax\n"			\
483			"	xorl %%eax,%%eax\n"		\
484			"	stosw\n"			\
485			"	popl %%eax\n"			\
486			"	addl $2,%0\n"			\
487			"	jmp 2b\n"			\
488			".previous\n"				\
489			".section __ex_table,\"a\"\n"		\
490			"	.align 4\n"			\
491			"	.long 0b,3b\n"			\
492			"	.long 1b,4b\n"			\
493			".previous"				\
494			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
495			: "1"(from), "2"(to), "0"(size/4)	\
496			: "memory");				\
497		break;						\
498	case 3:							\
499		__asm__ __volatile__(				\
500			"0:	rep; movsl\n"			\
501			"1:	movsw\n"			\
502			"2:	movsb\n"			\
503			"3:\n"					\
504			".section .fixup,\"ax\"\n"		\
505			"4:	pushl %0\n"			\
506			"	pushl %%eax\n"			\
507			"	xorl %%eax,%%eax\n"		\
508			"	rep; stosl\n"			\
509			"	stosw\n"			\
510			"	stosb\n"			\
511			"	popl %%eax\n"			\
512			"	popl %0\n"			\
513			"	shl $2,%0\n"			\
514			"	addl $3,%0\n"			\
515			"	jmp 2b\n"			\
516			"5:	pushl %%eax\n"			\
517			"	xorl %%eax,%%eax\n"		\
518			"	stosw\n"			\
519			"	stosb\n"			\
520			"	popl %%eax\n"			\
521			"	addl $3,%0\n"			\
522			"	jmp 2b\n"			\
523			"6:	pushl %%eax\n"			\
524			"	xorl %%eax,%%eax\n"		\
525			"	stosb\n"			\
526			"	popl %%eax\n"			\
527			"	incl %0\n"			\
528			"	jmp 3b\n"			\
529			".previous\n"				\
530			".section __ex_table,\"a\"\n"		\
531			"	.align 4\n"			\
532			"	.long 0b,4b\n"			\
533			"	.long 1b,5b\n"			\
534			"	.long 2b,6b\n"			\
535			".previous"				\
536			: "=c"(size), "=&S" (__d0), "=&D" (__d1)\
537			: "1"(from), "2"(to), "0"(size/4)	\
538			: "memory");				\
539		break;						\
540	}							\
541} while (0)
542
543unsigned long __generic_copy_to_user(void *, const void *, unsigned long);
544unsigned long __generic_copy_from_user(void *, const void *, unsigned long);
545
546static inline unsigned long
547__constant_copy_to_user(void *to, const void *from, unsigned long n)
548{
549	prefetch(from);
550	if (access_ok(VERIFY_WRITE, to, n))
551		__constant_copy_user(to,from,n);
552	return n;
553}
554
555static inline unsigned long
556__constant_copy_from_user(void *to, const void *from, unsigned long n)
557{
558	if (access_ok(VERIFY_READ, from, n))
559		__constant_copy_user_zeroing(to,from,n);
560	else
561		memset(to, 0, n);
562	return n;
563}
564
565static inline unsigned long
566__constant_copy_to_user_nocheck(void *to, const void *from, unsigned long n)
567{
568	__constant_copy_user(to,from,n);
569	return n;
570}
571
572static inline unsigned long
573__constant_copy_from_user_nocheck(void *to, const void *from, unsigned long n)
574{
575	__constant_copy_user_zeroing(to,from,n);
576	return n;
577}
578
579#define copy_to_user(to,from,n)				\
580	(__builtin_constant_p(n) ?			\
581	 __constant_copy_to_user((to),(from),(n)) :	\
582	 __generic_copy_to_user((to),(from),(n)))
583
584#define copy_from_user(to,from,n)			\
585	(__builtin_constant_p(n) ?			\
586	 __constant_copy_from_user((to),(from),(n)) :	\
587	 __generic_copy_from_user((to),(from),(n)))
588
589#define __copy_to_user(to,from,n)			\
590	(__builtin_constant_p(n) ?			\
591	 __constant_copy_to_user_nocheck((to),(from),(n)) :	\
592	 __generic_copy_to_user_nocheck((to),(from),(n)))
593
594#define __copy_from_user(to,from,n)			\
595	(__builtin_constant_p(n) ?			\
596	 __constant_copy_from_user_nocheck((to),(from),(n)) :	\
597	 __generic_copy_from_user_nocheck((to),(from),(n)))
598
599long strncpy_from_user(char *dst, const char *src, long count);
600long __strncpy_from_user(char *dst, const char *src, long count);
601#define strlen_user(str) strnlen_user(str, ~0UL >> 1)
602long strnlen_user(const char *str, long n);
603unsigned long clear_user(void *mem, unsigned long len);
604unsigned long __clear_user(void *mem, unsigned long len);
605
606#endif /* __i386_UACCESS_H */
607