1/* 2 * User address space access functions. 3 * The non inlined parts of asm-i386/uaccess.h are here. 4 * 5 * Copyright 1997 Andi Kleen <ak@muc.de> 6 * Copyright 1997 Linus Torvalds 7 */ 8#include <linux/mm.h> 9#include <linux/highmem.h> 10#include <linux/blkdev.h> 11#include <linux/module.h> 12#include <linux/backing-dev.h> 13#include <linux/interrupt.h> 14#include <asm/uaccess.h> 15#include <asm/mmx.h> 16 17#ifdef CONFIG_X86_INTEL_USERCOPY 18/* 19 * Alignment at which movsl is preferred for bulk memory copies. 20 */ 21struct movsl_mask movsl_mask __read_mostly; 22#endif 23 24static inline int __movsl_is_ok(unsigned long a1, unsigned long a2, unsigned long n) 25{ 26#ifdef CONFIG_X86_INTEL_USERCOPY 27 if (n >= 64 && ((a1 ^ a2) & movsl_mask.mask)) 28 return 0; 29#endif 30 return 1; 31} 32#define movsl_is_ok(a1, a2, n) \ 33 __movsl_is_ok((unsigned long)(a1), (unsigned long)(a2), (n)) 34 35/* 36 * Copy a null terminated string from userspace. 37 */ 38 39#define __do_strncpy_from_user(dst, src, count, res) \ 40do { \ 41 int __d0, __d1, __d2; \ 42 might_fault(); \ 43 __asm__ __volatile__( \ 44 " testl %1,%1\n" \ 45 " jz 2f\n" \ 46 "0: lodsb\n" \ 47 " stosb\n" \ 48 " testb %%al,%%al\n" \ 49 " jz 1f\n" \ 50 " decl %1\n" \ 51 " jnz 0b\n" \ 52 "1: subl %1,%0\n" \ 53 "2:\n" \ 54 ".section .fixup,\"ax\"\n" \ 55 "3: movl %5,%0\n" \ 56 " jmp 2b\n" \ 57 ".previous\n" \ 58 _ASM_EXTABLE(0b,3b) \ 59 : "=&d"(res), "=&c"(count), "=&a" (__d0), "=&S" (__d1), \ 60 "=&D" (__d2) \ 61 : "i"(-EFAULT), "0"(count), "1"(count), "3"(src), "4"(dst) \ 62 : "memory"); \ 63} while (0) 64 65/** 66 * __strncpy_from_user: - Copy a NUL terminated string from userspace, with less checking. 67 * @dst: Destination address, in kernel space. This buffer must be at 68 * least @count bytes long. 69 * @src: Source address, in user space. 70 * @count: Maximum number of bytes to copy, including the trailing NUL. 71 * 72 * Copies a NUL-terminated string from userspace to kernel space. 73 * Caller must check the specified block with access_ok() before calling 74 * this function. 75 * 76 * On success, returns the length of the string (not including the trailing 77 * NUL). 78 * 79 * If access to userspace fails, returns -EFAULT (some data may have been 80 * copied). 81 * 82 * If @count is smaller than the length of the string, copies @count bytes 83 * and returns @count. 84 */ 85long 86__strncpy_from_user(char *dst, const char __user *src, long count) 87{ 88 long res; 89 __do_strncpy_from_user(dst, src, count, res); 90 return res; 91} 92EXPORT_SYMBOL(__strncpy_from_user); 93 94/** 95 * strncpy_from_user: - Copy a NUL terminated string from userspace. 96 * @dst: Destination address, in kernel space. This buffer must be at 97 * least @count bytes long. 98 * @src: Source address, in user space. 99 * @count: Maximum number of bytes to copy, including the trailing NUL. 100 * 101 * Copies a NUL-terminated string from userspace to kernel space. 102 * 103 * On success, returns the length of the string (not including the trailing 104 * NUL). 105 * 106 * If access to userspace fails, returns -EFAULT (some data may have been 107 * copied). 108 * 109 * If @count is smaller than the length of the string, copies @count bytes 110 * and returns @count. 111 */ 112long 113strncpy_from_user(char *dst, const char __user *src, long count) 114{ 115 long res = -EFAULT; 116 if (access_ok(VERIFY_READ, src, 1)) 117 __do_strncpy_from_user(dst, src, count, res); 118 return res; 119} 120EXPORT_SYMBOL(strncpy_from_user); 121 122/* 123 * Zero Userspace 124 */ 125 126#define __do_clear_user(addr,size) \ 127do { \ 128 int __d0; \ 129 might_fault(); \ 130 __asm__ __volatile__( \ 131 "0: rep; stosl\n" \ 132 " movl %2,%0\n" \ 133 "1: rep; stosb\n" \ 134 "2:\n" \ 135 ".section .fixup,\"ax\"\n" \ 136 "3: lea 0(%2,%0,4),%0\n" \ 137 " jmp 2b\n" \ 138 ".previous\n" \ 139 _ASM_EXTABLE(0b,3b) \ 140 _ASM_EXTABLE(1b,2b) \ 141 : "=&c"(size), "=&D" (__d0) \ 142 : "r"(size & 3), "0"(size / 4), "1"(addr), "a"(0)); \ 143} while (0) 144 145/** 146 * clear_user: - Zero a block of memory in user space. 147 * @to: Destination address, in user space. 148 * @n: Number of bytes to zero. 149 * 150 * Zero a block of memory in user space. 151 * 152 * Returns number of bytes that could not be cleared. 153 * On success, this will be zero. 154 */ 155unsigned long 156clear_user(void __user *to, unsigned long n) 157{ 158 might_fault(); 159 if (access_ok(VERIFY_WRITE, to, n)) 160 __do_clear_user(to, n); 161 return n; 162} 163EXPORT_SYMBOL(clear_user); 164 165/** 166 * __clear_user: - Zero a block of memory in user space, with less checking. 167 * @to: Destination address, in user space. 168 * @n: Number of bytes to zero. 169 * 170 * Zero a block of memory in user space. Caller must check 171 * the specified block with access_ok() before calling this function. 172 * 173 * Returns number of bytes that could not be cleared. 174 * On success, this will be zero. 175 */ 176unsigned long 177__clear_user(void __user *to, unsigned long n) 178{ 179 __do_clear_user(to, n); 180 return n; 181} 182EXPORT_SYMBOL(__clear_user); 183 184/** 185 * strnlen_user: - Get the size of a string in user space. 186 * @s: The string to measure. 187 * @n: The maximum valid length 188 * 189 * Get the size of a NUL-terminated string in user space. 190 * 191 * Returns the size of the string INCLUDING the terminating NUL. 192 * On exception, returns 0. 193 * If the string is too long, returns a value greater than @n. 194 */ 195long strnlen_user(const char __user *s, long n) 196{ 197 unsigned long mask = -__addr_ok(s); 198 unsigned long res, tmp; 199 200 might_fault(); 201 202 __asm__ __volatile__( 203 " testl %0, %0\n" 204 " jz 3f\n" 205 " andl %0,%%ecx\n" 206 "0: repne; scasb\n" 207 " setne %%al\n" 208 " subl %%ecx,%0\n" 209 " addl %0,%%eax\n" 210 "1:\n" 211 ".section .fixup,\"ax\"\n" 212 "2: xorl %%eax,%%eax\n" 213 " jmp 1b\n" 214 "3: movb $1,%%al\n" 215 " jmp 1b\n" 216 ".previous\n" 217 ".section __ex_table,\"a\"\n" 218 " .align 4\n" 219 " .long 0b,2b\n" 220 ".previous" 221 :"=&r" (n), "=&D" (s), "=&a" (res), "=&c" (tmp) 222 :"0" (n), "1" (s), "2" (0), "3" (mask) 223 :"cc"); 224 return res & mask; 225} 226EXPORT_SYMBOL(strnlen_user); 227 228#ifdef CONFIG_X86_INTEL_USERCOPY 229static unsigned long 230__copy_user_intel(void __user *to, const void *from, unsigned long size) 231{ 232 int d0, d1; 233 __asm__ __volatile__( 234 " .align 2,0x90\n" 235 "1: movl 32(%4), %%eax\n" 236 " cmpl $67, %0\n" 237 " jbe 3f\n" 238 "2: movl 64(%4), %%eax\n" 239 " .align 2,0x90\n" 240 "3: movl 0(%4), %%eax\n" 241 "4: movl 4(%4), %%edx\n" 242 "5: movl %%eax, 0(%3)\n" 243 "6: movl %%edx, 4(%3)\n" 244 "7: movl 8(%4), %%eax\n" 245 "8: movl 12(%4),%%edx\n" 246 "9: movl %%eax, 8(%3)\n" 247 "10: movl %%edx, 12(%3)\n" 248 "11: movl 16(%4), %%eax\n" 249 "12: movl 20(%4), %%edx\n" 250 "13: movl %%eax, 16(%3)\n" 251 "14: movl %%edx, 20(%3)\n" 252 "15: movl 24(%4), %%eax\n" 253 "16: movl 28(%4), %%edx\n" 254 "17: movl %%eax, 24(%3)\n" 255 "18: movl %%edx, 28(%3)\n" 256 "19: movl 32(%4), %%eax\n" 257 "20: movl 36(%4), %%edx\n" 258 "21: movl %%eax, 32(%3)\n" 259 "22: movl %%edx, 36(%3)\n" 260 "23: movl 40(%4), %%eax\n" 261 "24: movl 44(%4), %%edx\n" 262 "25: movl %%eax, 40(%3)\n" 263 "26: movl %%edx, 44(%3)\n" 264 "27: movl 48(%4), %%eax\n" 265 "28: movl 52(%4), %%edx\n" 266 "29: movl %%eax, 48(%3)\n" 267 "30: movl %%edx, 52(%3)\n" 268 "31: movl 56(%4), %%eax\n" 269 "32: movl 60(%4), %%edx\n" 270 "33: movl %%eax, 56(%3)\n" 271 "34: movl %%edx, 60(%3)\n" 272 " addl $-64, %0\n" 273 " addl $64, %4\n" 274 " addl $64, %3\n" 275 " cmpl $63, %0\n" 276 " ja 1b\n" 277 "35: movl %0, %%eax\n" 278 " shrl $2, %0\n" 279 " andl $3, %%eax\n" 280 " cld\n" 281 "99: rep; movsl\n" 282 "36: movl %%eax, %0\n" 283 "37: rep; movsb\n" 284 "100:\n" 285 ".section .fixup,\"ax\"\n" 286 "101: lea 0(%%eax,%0,4),%0\n" 287 " jmp 100b\n" 288 ".previous\n" 289 ".section __ex_table,\"a\"\n" 290 " .align 4\n" 291 " .long 1b,100b\n" 292 " .long 2b,100b\n" 293 " .long 3b,100b\n" 294 " .long 4b,100b\n" 295 " .long 5b,100b\n" 296 " .long 6b,100b\n" 297 " .long 7b,100b\n" 298 " .long 8b,100b\n" 299 " .long 9b,100b\n" 300 " .long 10b,100b\n" 301 " .long 11b,100b\n" 302 " .long 12b,100b\n" 303 " .long 13b,100b\n" 304 " .long 14b,100b\n" 305 " .long 15b,100b\n" 306 " .long 16b,100b\n" 307 " .long 17b,100b\n" 308 " .long 18b,100b\n" 309 " .long 19b,100b\n" 310 " .long 20b,100b\n" 311 " .long 21b,100b\n" 312 " .long 22b,100b\n" 313 " .long 23b,100b\n" 314 " .long 24b,100b\n" 315 " .long 25b,100b\n" 316 " .long 26b,100b\n" 317 " .long 27b,100b\n" 318 " .long 28b,100b\n" 319 " .long 29b,100b\n" 320 " .long 30b,100b\n" 321 " .long 31b,100b\n" 322 " .long 32b,100b\n" 323 " .long 33b,100b\n" 324 " .long 34b,100b\n" 325 " .long 35b,100b\n" 326 " .long 36b,100b\n" 327 " .long 37b,100b\n" 328 " .long 99b,101b\n" 329 ".previous" 330 : "=&c"(size), "=&D" (d0), "=&S" (d1) 331 : "1"(to), "2"(from), "0"(size) 332 : "eax", "edx", "memory"); 333 return size; 334} 335 336static unsigned long 337__copy_user_zeroing_intel(void *to, const void __user *from, unsigned long size) 338{ 339 int d0, d1; 340 __asm__ __volatile__( 341 " .align 2,0x90\n" 342 "0: movl 32(%4), %%eax\n" 343 " cmpl $67, %0\n" 344 " jbe 2f\n" 345 "1: movl 64(%4), %%eax\n" 346 " .align 2,0x90\n" 347 "2: movl 0(%4), %%eax\n" 348 "21: movl 4(%4), %%edx\n" 349 " movl %%eax, 0(%3)\n" 350 " movl %%edx, 4(%3)\n" 351 "3: movl 8(%4), %%eax\n" 352 "31: movl 12(%4),%%edx\n" 353 " movl %%eax, 8(%3)\n" 354 " movl %%edx, 12(%3)\n" 355 "4: movl 16(%4), %%eax\n" 356 "41: movl 20(%4), %%edx\n" 357 " movl %%eax, 16(%3)\n" 358 " movl %%edx, 20(%3)\n" 359 "10: movl 24(%4), %%eax\n" 360 "51: movl 28(%4), %%edx\n" 361 " movl %%eax, 24(%3)\n" 362 " movl %%edx, 28(%3)\n" 363 "11: movl 32(%4), %%eax\n" 364 "61: movl 36(%4), %%edx\n" 365 " movl %%eax, 32(%3)\n" 366 " movl %%edx, 36(%3)\n" 367 "12: movl 40(%4), %%eax\n" 368 "71: movl 44(%4), %%edx\n" 369 " movl %%eax, 40(%3)\n" 370 " movl %%edx, 44(%3)\n" 371 "13: movl 48(%4), %%eax\n" 372 "81: movl 52(%4), %%edx\n" 373 " movl %%eax, 48(%3)\n" 374 " movl %%edx, 52(%3)\n" 375 "14: movl 56(%4), %%eax\n" 376 "91: movl 60(%4), %%edx\n" 377 " movl %%eax, 56(%3)\n" 378 " movl %%edx, 60(%3)\n" 379 " addl $-64, %0\n" 380 " addl $64, %4\n" 381 " addl $64, %3\n" 382 " cmpl $63, %0\n" 383 " ja 0b\n" 384 "5: movl %0, %%eax\n" 385 " shrl $2, %0\n" 386 " andl $3, %%eax\n" 387 " cld\n" 388 "6: rep; movsl\n" 389 " movl %%eax,%0\n" 390 "7: rep; movsb\n" 391 "8:\n" 392 ".section .fixup,\"ax\"\n" 393 "9: lea 0(%%eax,%0,4),%0\n" 394 "16: pushl %0\n" 395 " pushl %%eax\n" 396 " xorl %%eax,%%eax\n" 397 " rep; stosb\n" 398 " popl %%eax\n" 399 " popl %0\n" 400 " jmp 8b\n" 401 ".previous\n" 402 ".section __ex_table,\"a\"\n" 403 " .align 4\n" 404 " .long 0b,16b\n" 405 " .long 1b,16b\n" 406 " .long 2b,16b\n" 407 " .long 21b,16b\n" 408 " .long 3b,16b\n" 409 " .long 31b,16b\n" 410 " .long 4b,16b\n" 411 " .long 41b,16b\n" 412 " .long 10b,16b\n" 413 " .long 51b,16b\n" 414 " .long 11b,16b\n" 415 " .long 61b,16b\n" 416 " .long 12b,16b\n" 417 " .long 71b,16b\n" 418 " .long 13b,16b\n" 419 " .long 81b,16b\n" 420 " .long 14b,16b\n" 421 " .long 91b,16b\n" 422 " .long 6b,9b\n" 423 " .long 7b,16b\n" 424 ".previous" 425 : "=&c"(size), "=&D" (d0), "=&S" (d1) 426 : "1"(to), "2"(from), "0"(size) 427 : "eax", "edx", "memory"); 428 return size; 429} 430 431/* 432 * Non Temporal Hint version of __copy_user_zeroing_intel. It is cache aware. 433 * hyoshiok@miraclelinux.com 434 */ 435 436static unsigned long __copy_user_zeroing_intel_nocache(void *to, 437 const void __user *from, unsigned long size) 438{ 439 int d0, d1; 440 441 __asm__ __volatile__( 442 " .align 2,0x90\n" 443 "0: movl 32(%4), %%eax\n" 444 " cmpl $67, %0\n" 445 " jbe 2f\n" 446 "1: movl 64(%4), %%eax\n" 447 " .align 2,0x90\n" 448 "2: movl 0(%4), %%eax\n" 449 "21: movl 4(%4), %%edx\n" 450 " movnti %%eax, 0(%3)\n" 451 " movnti %%edx, 4(%3)\n" 452 "3: movl 8(%4), %%eax\n" 453 "31: movl 12(%4),%%edx\n" 454 " movnti %%eax, 8(%3)\n" 455 " movnti %%edx, 12(%3)\n" 456 "4: movl 16(%4), %%eax\n" 457 "41: movl 20(%4), %%edx\n" 458 " movnti %%eax, 16(%3)\n" 459 " movnti %%edx, 20(%3)\n" 460 "10: movl 24(%4), %%eax\n" 461 "51: movl 28(%4), %%edx\n" 462 " movnti %%eax, 24(%3)\n" 463 " movnti %%edx, 28(%3)\n" 464 "11: movl 32(%4), %%eax\n" 465 "61: movl 36(%4), %%edx\n" 466 " movnti %%eax, 32(%3)\n" 467 " movnti %%edx, 36(%3)\n" 468 "12: movl 40(%4), %%eax\n" 469 "71: movl 44(%4), %%edx\n" 470 " movnti %%eax, 40(%3)\n" 471 " movnti %%edx, 44(%3)\n" 472 "13: movl 48(%4), %%eax\n" 473 "81: movl 52(%4), %%edx\n" 474 " movnti %%eax, 48(%3)\n" 475 " movnti %%edx, 52(%3)\n" 476 "14: movl 56(%4), %%eax\n" 477 "91: movl 60(%4), %%edx\n" 478 " movnti %%eax, 56(%3)\n" 479 " movnti %%edx, 60(%3)\n" 480 " addl $-64, %0\n" 481 " addl $64, %4\n" 482 " addl $64, %3\n" 483 " cmpl $63, %0\n" 484 " ja 0b\n" 485 " sfence \n" 486 "5: movl %0, %%eax\n" 487 " shrl $2, %0\n" 488 " andl $3, %%eax\n" 489 " cld\n" 490 "6: rep; movsl\n" 491 " movl %%eax,%0\n" 492 "7: rep; movsb\n" 493 "8:\n" 494 ".section .fixup,\"ax\"\n" 495 "9: lea 0(%%eax,%0,4),%0\n" 496 "16: pushl %0\n" 497 " pushl %%eax\n" 498 " xorl %%eax,%%eax\n" 499 " rep; stosb\n" 500 " popl %%eax\n" 501 " popl %0\n" 502 " jmp 8b\n" 503 ".previous\n" 504 ".section __ex_table,\"a\"\n" 505 " .align 4\n" 506 " .long 0b,16b\n" 507 " .long 1b,16b\n" 508 " .long 2b,16b\n" 509 " .long 21b,16b\n" 510 " .long 3b,16b\n" 511 " .long 31b,16b\n" 512 " .long 4b,16b\n" 513 " .long 41b,16b\n" 514 " .long 10b,16b\n" 515 " .long 51b,16b\n" 516 " .long 11b,16b\n" 517 " .long 61b,16b\n" 518 " .long 12b,16b\n" 519 " .long 71b,16b\n" 520 " .long 13b,16b\n" 521 " .long 81b,16b\n" 522 " .long 14b,16b\n" 523 " .long 91b,16b\n" 524 " .long 6b,9b\n" 525 " .long 7b,16b\n" 526 ".previous" 527 : "=&c"(size), "=&D" (d0), "=&S" (d1) 528 : "1"(to), "2"(from), "0"(size) 529 : "eax", "edx", "memory"); 530 return size; 531} 532 533static unsigned long __copy_user_intel_nocache(void *to, 534 const void __user *from, unsigned long size) 535{ 536 int d0, d1; 537 538 __asm__ __volatile__( 539 " .align 2,0x90\n" 540 "0: movl 32(%4), %%eax\n" 541 " cmpl $67, %0\n" 542 " jbe 2f\n" 543 "1: movl 64(%4), %%eax\n" 544 " .align 2,0x90\n" 545 "2: movl 0(%4), %%eax\n" 546 "21: movl 4(%4), %%edx\n" 547 " movnti %%eax, 0(%3)\n" 548 " movnti %%edx, 4(%3)\n" 549 "3: movl 8(%4), %%eax\n" 550 "31: movl 12(%4),%%edx\n" 551 " movnti %%eax, 8(%3)\n" 552 " movnti %%edx, 12(%3)\n" 553 "4: movl 16(%4), %%eax\n" 554 "41: movl 20(%4), %%edx\n" 555 " movnti %%eax, 16(%3)\n" 556 " movnti %%edx, 20(%3)\n" 557 "10: movl 24(%4), %%eax\n" 558 "51: movl 28(%4), %%edx\n" 559 " movnti %%eax, 24(%3)\n" 560 " movnti %%edx, 28(%3)\n" 561 "11: movl 32(%4), %%eax\n" 562 "61: movl 36(%4), %%edx\n" 563 " movnti %%eax, 32(%3)\n" 564 " movnti %%edx, 36(%3)\n" 565 "12: movl 40(%4), %%eax\n" 566 "71: movl 44(%4), %%edx\n" 567 " movnti %%eax, 40(%3)\n" 568 " movnti %%edx, 44(%3)\n" 569 "13: movl 48(%4), %%eax\n" 570 "81: movl 52(%4), %%edx\n" 571 " movnti %%eax, 48(%3)\n" 572 " movnti %%edx, 52(%3)\n" 573 "14: movl 56(%4), %%eax\n" 574 "91: movl 60(%4), %%edx\n" 575 " movnti %%eax, 56(%3)\n" 576 " movnti %%edx, 60(%3)\n" 577 " addl $-64, %0\n" 578 " addl $64, %4\n" 579 " addl $64, %3\n" 580 " cmpl $63, %0\n" 581 " ja 0b\n" 582 " sfence \n" 583 "5: movl %0, %%eax\n" 584 " shrl $2, %0\n" 585 " andl $3, %%eax\n" 586 " cld\n" 587 "6: rep; movsl\n" 588 " movl %%eax,%0\n" 589 "7: rep; movsb\n" 590 "8:\n" 591 ".section .fixup,\"ax\"\n" 592 "9: lea 0(%%eax,%0,4),%0\n" 593 "16: jmp 8b\n" 594 ".previous\n" 595 ".section __ex_table,\"a\"\n" 596 " .align 4\n" 597 " .long 0b,16b\n" 598 " .long 1b,16b\n" 599 " .long 2b,16b\n" 600 " .long 21b,16b\n" 601 " .long 3b,16b\n" 602 " .long 31b,16b\n" 603 " .long 4b,16b\n" 604 " .long 41b,16b\n" 605 " .long 10b,16b\n" 606 " .long 51b,16b\n" 607 " .long 11b,16b\n" 608 " .long 61b,16b\n" 609 " .long 12b,16b\n" 610 " .long 71b,16b\n" 611 " .long 13b,16b\n" 612 " .long 81b,16b\n" 613 " .long 14b,16b\n" 614 " .long 91b,16b\n" 615 " .long 6b,9b\n" 616 " .long 7b,16b\n" 617 ".previous" 618 : "=&c"(size), "=&D" (d0), "=&S" (d1) 619 : "1"(to), "2"(from), "0"(size) 620 : "eax", "edx", "memory"); 621 return size; 622} 623 624#else 625 626/* 627 * Leave these declared but undefined. They should not be any references to 628 * them 629 */ 630unsigned long __copy_user_zeroing_intel(void *to, const void __user *from, 631 unsigned long size); 632unsigned long __copy_user_intel(void __user *to, const void *from, 633 unsigned long size); 634unsigned long __copy_user_zeroing_intel_nocache(void *to, 635 const void __user *from, unsigned long size); 636#endif /* CONFIG_X86_INTEL_USERCOPY */ 637 638/* Generic arbitrary sized copy. */ 639#define __copy_user(to, from, size) \ 640do { \ 641 int __d0, __d1, __d2; \ 642 __asm__ __volatile__( \ 643 " cmp $7,%0\n" \ 644 " jbe 1f\n" \ 645 " movl %1,%0\n" \ 646 " negl %0\n" \ 647 " andl $7,%0\n" \ 648 " subl %0,%3\n" \ 649 "4: rep; movsb\n" \ 650 " movl %3,%0\n" \ 651 " shrl $2,%0\n" \ 652 " andl $3,%3\n" \ 653 " .align 2,0x90\n" \ 654 "0: rep; movsl\n" \ 655 " movl %3,%0\n" \ 656 "1: rep; movsb\n" \ 657 "2:\n" \ 658 ".section .fixup,\"ax\"\n" \ 659 "5: addl %3,%0\n" \ 660 " jmp 2b\n" \ 661 "3: lea 0(%3,%0,4),%0\n" \ 662 " jmp 2b\n" \ 663 ".previous\n" \ 664 ".section __ex_table,\"a\"\n" \ 665 " .align 4\n" \ 666 " .long 4b,5b\n" \ 667 " .long 0b,3b\n" \ 668 " .long 1b,2b\n" \ 669 ".previous" \ 670 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ 671 : "3"(size), "0"(size), "1"(to), "2"(from) \ 672 : "memory"); \ 673} while (0) 674 675#define __copy_user_zeroing(to, from, size) \ 676do { \ 677 int __d0, __d1, __d2; \ 678 __asm__ __volatile__( \ 679 " cmp $7,%0\n" \ 680 " jbe 1f\n" \ 681 " movl %1,%0\n" \ 682 " negl %0\n" \ 683 " andl $7,%0\n" \ 684 " subl %0,%3\n" \ 685 "4: rep; movsb\n" \ 686 " movl %3,%0\n" \ 687 " shrl $2,%0\n" \ 688 " andl $3,%3\n" \ 689 " .align 2,0x90\n" \ 690 "0: rep; movsl\n" \ 691 " movl %3,%0\n" \ 692 "1: rep; movsb\n" \ 693 "2:\n" \ 694 ".section .fixup,\"ax\"\n" \ 695 "5: addl %3,%0\n" \ 696 " jmp 6f\n" \ 697 "3: lea 0(%3,%0,4),%0\n" \ 698 "6: pushl %0\n" \ 699 " pushl %%eax\n" \ 700 " xorl %%eax,%%eax\n" \ 701 " rep; stosb\n" \ 702 " popl %%eax\n" \ 703 " popl %0\n" \ 704 " jmp 2b\n" \ 705 ".previous\n" \ 706 ".section __ex_table,\"a\"\n" \ 707 " .align 4\n" \ 708 " .long 4b,5b\n" \ 709 " .long 0b,3b\n" \ 710 " .long 1b,6b\n" \ 711 ".previous" \ 712 : "=&c"(size), "=&D" (__d0), "=&S" (__d1), "=r"(__d2) \ 713 : "3"(size), "0"(size), "1"(to), "2"(from) \ 714 : "memory"); \ 715} while (0) 716 717unsigned long __copy_to_user_ll(void __user *to, const void *from, 718 unsigned long n) 719{ 720#ifndef CONFIG_X86_WP_WORKS_OK 721 if (unlikely(boot_cpu_data.wp_works_ok == 0) && 722 ((unsigned long)to) < TASK_SIZE) { 723 /* 724 * When we are in an atomic section (see 725 * mm/filemap.c:file_read_actor), return the full 726 * length to take the slow path. 727 */ 728 if (in_atomic()) 729 return n; 730 731 /* 732 * CPU does not honor the WP bit when writing 733 * from supervisory mode, and due to preemption or SMP, 734 * the page tables can change at any time. 735 * Do it manually. Manfred <manfred@colorfullife.com> 736 */ 737 while (n) { 738 unsigned long offset = ((unsigned long)to)%PAGE_SIZE; 739 unsigned long len = PAGE_SIZE - offset; 740 int retval; 741 struct page *pg; 742 void *maddr; 743 744 if (len > n) 745 len = n; 746 747survive: 748 down_read(¤t->mm->mmap_sem); 749 retval = get_user_pages(current, current->mm, 750 (unsigned long)to, 1, 1, 0, &pg, NULL); 751 752 if (retval == -ENOMEM && is_global_init(current)) { 753 up_read(¤t->mm->mmap_sem); 754 congestion_wait(BLK_RW_ASYNC, HZ/50); 755 goto survive; 756 } 757 758 if (retval != 1) { 759 up_read(¤t->mm->mmap_sem); 760 break; 761 } 762 763 maddr = kmap_atomic(pg, KM_USER0); 764 memcpy(maddr + offset, from, len); 765 kunmap_atomic(maddr, KM_USER0); 766 set_page_dirty_lock(pg); 767 put_page(pg); 768 up_read(¤t->mm->mmap_sem); 769 770 from += len; 771 to += len; 772 n -= len; 773 } 774 return n; 775 } 776#endif 777 if (movsl_is_ok(to, from, n)) 778 __copy_user(to, from, n); 779 else 780 n = __copy_user_intel(to, from, n); 781 return n; 782} 783EXPORT_SYMBOL(__copy_to_user_ll); 784 785unsigned long __copy_from_user_ll(void *to, const void __user *from, 786 unsigned long n) 787{ 788 if (movsl_is_ok(to, from, n)) 789 __copy_user_zeroing(to, from, n); 790 else 791 n = __copy_user_zeroing_intel(to, from, n); 792 return n; 793} 794EXPORT_SYMBOL(__copy_from_user_ll); 795 796unsigned long __copy_from_user_ll_nozero(void *to, const void __user *from, 797 unsigned long n) 798{ 799 if (movsl_is_ok(to, from, n)) 800 __copy_user(to, from, n); 801 else 802 n = __copy_user_intel((void __user *)to, 803 (const void *)from, n); 804 return n; 805} 806EXPORT_SYMBOL(__copy_from_user_ll_nozero); 807 808unsigned long __copy_from_user_ll_nocache(void *to, const void __user *from, 809 unsigned long n) 810{ 811#ifdef CONFIG_X86_INTEL_USERCOPY 812 if (n > 64 && cpu_has_xmm2) 813 n = __copy_user_zeroing_intel_nocache(to, from, n); 814 else 815 __copy_user_zeroing(to, from, n); 816#else 817 __copy_user_zeroing(to, from, n); 818#endif 819 return n; 820} 821EXPORT_SYMBOL(__copy_from_user_ll_nocache); 822 823unsigned long __copy_from_user_ll_nocache_nozero(void *to, const void __user *from, 824 unsigned long n) 825{ 826#ifdef CONFIG_X86_INTEL_USERCOPY 827 if (n > 64 && cpu_has_xmm2) 828 n = __copy_user_intel_nocache(to, from, n); 829 else 830 __copy_user(to, from, n); 831#else 832 __copy_user(to, from, n); 833#endif 834 return n; 835} 836EXPORT_SYMBOL(__copy_from_user_ll_nocache_nozero); 837 838/** 839 * copy_to_user: - Copy a block of data into user space. 840 * @to: Destination address, in user space. 841 * @from: Source address, in kernel space. 842 * @n: Number of bytes to copy. 843 * 844 * Context: User context only. This function may sleep. 845 * 846 * Copy data from kernel space to user space. 847 * 848 * Returns number of bytes that could not be copied. 849 * On success, this will be zero. 850 */ 851unsigned long 852copy_to_user(void __user *to, const void *from, unsigned long n) 853{ 854 if (access_ok(VERIFY_WRITE, to, n)) 855 n = __copy_to_user(to, from, n); 856 return n; 857} 858EXPORT_SYMBOL(copy_to_user); 859 860/** 861 * copy_from_user: - Copy a block of data from user space. 862 * @to: Destination address, in kernel space. 863 * @from: Source address, in user space. 864 * @n: Number of bytes to copy. 865 * 866 * Context: User context only. This function may sleep. 867 * 868 * Copy data from user space to kernel space. 869 * 870 * Returns number of bytes that could not be copied. 871 * On success, this will be zero. 872 * 873 * If some data could not be copied, this function will pad the copied 874 * data to the requested size using zero bytes. 875 */ 876unsigned long 877_copy_from_user(void *to, const void __user *from, unsigned long n) 878{ 879 if (access_ok(VERIFY_READ, from, n)) 880 n = __copy_from_user(to, from, n); 881 else 882 memset(to, 0, n); 883 return n; 884} 885EXPORT_SYMBOL(_copy_from_user); 886 887void copy_from_user_overflow(void) 888{ 889 WARN(1, "Buffer overflow detected!\n"); 890} 891EXPORT_SYMBOL(copy_from_user_overflow); 892