1/* 2 * linux/arch/arm/lib/uaccess.S 3 * 4 * Copyright (C) 1995, 1996,1997,1998 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * Routines to block copy data to/from user memory 11 * These are highly optimised both for the 4k page size 12 * and for various alignments. 13 */ 14#include <linux/linkage.h> 15#include <asm/assembler.h> 16#include <asm/errno.h> 17 18 .text 19 20#define PAGE_SHIFT 12 21 22/* Prototype: int __copy_to_user(void *to, const char *from, size_t n) 23 * Purpose : copy a block to user memory from kernel memory 24 * Params : to - user memory 25 * : from - kernel memory 26 * : n - number of bytes to copy 27 * Returns : Number of bytes NOT copied. 28 */ 29 30.Lc2u_dest_not_aligned: 31 rsb ip, ip, #4 32 cmp ip, #2 33 ldrb r3, [r1], #1 34USER( strbt r3, [r0], #1) @ May fault 35 ldrgeb r3, [r1], #1 36USER( strgebt r3, [r0], #1) @ May fault 37 ldrgtb r3, [r1], #1 38USER( strgtbt r3, [r0], #1) @ May fault 39 sub r2, r2, ip 40 b .Lc2u_dest_aligned 41 42ENTRY(__copy_to_user) 43 stmfd sp!, {r2, r4 - r7, lr} 44 cmp r2, #4 45 blt .Lc2u_not_enough 46 ands ip, r0, #3 47 bne .Lc2u_dest_not_aligned 48.Lc2u_dest_aligned: 49 50 ands ip, r1, #3 51 bne .Lc2u_src_not_aligned 52/* 53 * Seeing as there has to be at least 8 bytes to copy, we can 54 * copy one word, and force a user-mode page fault... 55 */ 56 57.Lc2u_0fupi: subs r2, r2, #4 58 addmi ip, r2, #4 59 bmi .Lc2u_0nowords 60 ldr r3, [r1], #4 61USER( strt r3, [r0], #4) @ May fault 62 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 63 rsb ip, ip, #0 64 movs ip, ip, lsr #32 - PAGE_SHIFT 65 beq .Lc2u_0fupi 66/* 67 * ip = max no. of bytes to copy before needing another "strt" insn 68 */ 69 cmp r2, ip 70 movlt ip, r2 71 sub r2, r2, ip 72 subs ip, ip, #32 73 blt .Lc2u_0rem8lp 74 75.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6} 76 stmia r0!, {r3 - r6} @ Shouldnt fault 77 ldmia r1!, {r3 - r6} 78 subs ip, ip, #32 79 stmia r0!, {r3 - r6} @ Shouldnt fault 80 bpl .Lc2u_0cpy8lp 81 82.Lc2u_0rem8lp: cmn ip, #16 83 ldmgeia r1!, {r3 - r6} 84 stmgeia r0!, {r3 - r6} @ Shouldnt fault 85 tst ip, #8 86 ldmneia r1!, {r3 - r4} 87 stmneia r0!, {r3 - r4} @ Shouldnt fault 88 tst ip, #4 89 ldrne r3, [r1], #4 90 strnet r3, [r0], #4 @ Shouldnt fault 91 ands ip, ip, #3 92 beq .Lc2u_0fupi 93.Lc2u_0nowords: teq ip, #0 94 beq .Lc2u_finished 95.Lc2u_nowords: cmp ip, #2 96 ldrb r3, [r1], #1 97USER( strbt r3, [r0], #1) @ May fault 98 ldrgeb r3, [r1], #1 99USER( strgebt r3, [r0], #1) @ May fault 100 ldrgtb r3, [r1], #1 101USER( strgtbt r3, [r0], #1) @ May fault 102 b .Lc2u_finished 103 104.Lc2u_not_enough: 105 movs ip, r2 106 bne .Lc2u_nowords 107.Lc2u_finished: mov r0, #0 108 ldmfd sp!, {r2, r4 - r7, pc} 109 110.Lc2u_src_not_aligned: 111 bic r1, r1, #3 112 ldr r7, [r1], #4 113 cmp ip, #2 114 bgt .Lc2u_3fupi 115 beq .Lc2u_2fupi 116.Lc2u_1fupi: subs r2, r2, #4 117 addmi ip, r2, #4 118 bmi .Lc2u_1nowords 119 mov r3, r7, pull #8 120 ldr r7, [r1], #4 121 orr r3, r3, r7, push #24 122USER( strt r3, [r0], #4) @ May fault 123 mov ip, r0, lsl #32 - PAGE_SHIFT 124 rsb ip, ip, #0 125 movs ip, ip, lsr #32 - PAGE_SHIFT 126 beq .Lc2u_1fupi 127 cmp r2, ip 128 movlt ip, r2 129 sub r2, r2, ip 130 subs ip, ip, #16 131 blt .Lc2u_1rem8lp 132 133.Lc2u_1cpy8lp: mov r3, r7, pull #8 134 ldmia r1!, {r4 - r7} 135 subs ip, ip, #16 136 orr r3, r3, r4, push #24 137 mov r4, r4, pull #8 138 orr r4, r4, r5, push #24 139 mov r5, r5, pull #8 140 orr r5, r5, r6, push #24 141 mov r6, r6, pull #8 142 orr r6, r6, r7, push #24 143 stmia r0!, {r3 - r6} @ Shouldnt fault 144 bpl .Lc2u_1cpy8lp 145 146.Lc2u_1rem8lp: tst ip, #8 147 movne r3, r7, pull #8 148 ldmneia r1!, {r4, r7} 149 orrne r3, r3, r4, push #24 150 movne r4, r4, pull #8 151 orrne r4, r4, r7, push #24 152 stmneia r0!, {r3 - r4} @ Shouldnt fault 153 tst ip, #4 154 movne r3, r7, pull #8 155 ldrne r7, [r1], #4 156 orrne r3, r3, r7, push #24 157 strnet r3, [r0], #4 @ Shouldnt fault 158 ands ip, ip, #3 159 beq .Lc2u_1fupi 160.Lc2u_1nowords: mov r3, r7, get_byte_1 161 teq ip, #0 162 beq .Lc2u_finished 163 cmp ip, #2 164USER( strbt r3, [r0], #1) @ May fault 165 movge r3, r7, get_byte_2 166USER( strgebt r3, [r0], #1) @ May fault 167 movgt r3, r7, get_byte_3 168USER( strgtbt r3, [r0], #1) @ May fault 169 b .Lc2u_finished 170 171.Lc2u_2fupi: subs r2, r2, #4 172 addmi ip, r2, #4 173 bmi .Lc2u_2nowords 174 mov r3, r7, pull #16 175 ldr r7, [r1], #4 176 orr r3, r3, r7, push #16 177USER( strt r3, [r0], #4) @ May fault 178 mov ip, r0, lsl #32 - PAGE_SHIFT 179 rsb ip, ip, #0 180 movs ip, ip, lsr #32 - PAGE_SHIFT 181 beq .Lc2u_2fupi 182 cmp r2, ip 183 movlt ip, r2 184 sub r2, r2, ip 185 subs ip, ip, #16 186 blt .Lc2u_2rem8lp 187 188.Lc2u_2cpy8lp: mov r3, r7, pull #16 189 ldmia r1!, {r4 - r7} 190 subs ip, ip, #16 191 orr r3, r3, r4, push #16 192 mov r4, r4, pull #16 193 orr r4, r4, r5, push #16 194 mov r5, r5, pull #16 195 orr r5, r5, r6, push #16 196 mov r6, r6, pull #16 197 orr r6, r6, r7, push #16 198 stmia r0!, {r3 - r6} @ Shouldnt fault 199 bpl .Lc2u_2cpy8lp 200 201.Lc2u_2rem8lp: tst ip, #8 202 movne r3, r7, pull #16 203 ldmneia r1!, {r4, r7} 204 orrne r3, r3, r4, push #16 205 movne r4, r4, pull #16 206 orrne r4, r4, r7, push #16 207 stmneia r0!, {r3 - r4} @ Shouldnt fault 208 tst ip, #4 209 movne r3, r7, pull #16 210 ldrne r7, [r1], #4 211 orrne r3, r3, r7, push #16 212 strnet r3, [r0], #4 @ Shouldnt fault 213 ands ip, ip, #3 214 beq .Lc2u_2fupi 215.Lc2u_2nowords: mov r3, r7, get_byte_2 216 teq ip, #0 217 beq .Lc2u_finished 218 cmp ip, #2 219USER( strbt r3, [r0], #1) @ May fault 220 movge r3, r7, get_byte_3 221USER( strgebt r3, [r0], #1) @ May fault 222 ldrgtb r3, [r1], #0 223USER( strgtbt r3, [r0], #1) @ May fault 224 b .Lc2u_finished 225 226.Lc2u_3fupi: subs r2, r2, #4 227 addmi ip, r2, #4 228 bmi .Lc2u_3nowords 229 mov r3, r7, pull #24 230 ldr r7, [r1], #4 231 orr r3, r3, r7, push #8 232USER( strt r3, [r0], #4) @ May fault 233 mov ip, r0, lsl #32 - PAGE_SHIFT 234 rsb ip, ip, #0 235 movs ip, ip, lsr #32 - PAGE_SHIFT 236 beq .Lc2u_3fupi 237 cmp r2, ip 238 movlt ip, r2 239 sub r2, r2, ip 240 subs ip, ip, #16 241 blt .Lc2u_3rem8lp 242 243.Lc2u_3cpy8lp: mov r3, r7, pull #24 244 ldmia r1!, {r4 - r7} 245 subs ip, ip, #16 246 orr r3, r3, r4, push #8 247 mov r4, r4, pull #24 248 orr r4, r4, r5, push #8 249 mov r5, r5, pull #24 250 orr r5, r5, r6, push #8 251 mov r6, r6, pull #24 252 orr r6, r6, r7, push #8 253 stmia r0!, {r3 - r6} @ Shouldnt fault 254 bpl .Lc2u_3cpy8lp 255 256.Lc2u_3rem8lp: tst ip, #8 257 movne r3, r7, pull #24 258 ldmneia r1!, {r4, r7} 259 orrne r3, r3, r4, push #8 260 movne r4, r4, pull #24 261 orrne r4, r4, r7, push #8 262 stmneia r0!, {r3 - r4} @ Shouldnt fault 263 tst ip, #4 264 movne r3, r7, pull #24 265 ldrne r7, [r1], #4 266 orrne r3, r3, r7, push #8 267 strnet r3, [r0], #4 @ Shouldnt fault 268 ands ip, ip, #3 269 beq .Lc2u_3fupi 270.Lc2u_3nowords: mov r3, r7, get_byte_3 271 teq ip, #0 272 beq .Lc2u_finished 273 cmp ip, #2 274USER( strbt r3, [r0], #1) @ May fault 275 ldrgeb r3, [r1], #1 276USER( strgebt r3, [r0], #1) @ May fault 277 ldrgtb r3, [r1], #0 278USER( strgtbt r3, [r0], #1) @ May fault 279 b .Lc2u_finished 280ENDPROC(__copy_to_user) 281 282 .pushsection .fixup,"ax" 283 .align 0 2849001: ldmfd sp!, {r0, r4 - r7, pc} 285 .popsection 286 287/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n); 288 * Purpose : copy a block from user memory to kernel memory 289 * Params : to - kernel memory 290 * : from - user memory 291 * : n - number of bytes to copy 292 * Returns : Number of bytes NOT copied. 293 */ 294.Lcfu_dest_not_aligned: 295 rsb ip, ip, #4 296 cmp ip, #2 297USER( ldrbt r3, [r1], #1) @ May fault 298 strb r3, [r0], #1 299USER( ldrgebt r3, [r1], #1) @ May fault 300 strgeb r3, [r0], #1 301USER( ldrgtbt r3, [r1], #1) @ May fault 302 strgtb r3, [r0], #1 303 sub r2, r2, ip 304 b .Lcfu_dest_aligned 305 306ENTRY(__copy_from_user) 307 stmfd sp!, {r0, r2, r4 - r7, lr} 308 cmp r2, #4 309 blt .Lcfu_not_enough 310 ands ip, r0, #3 311 bne .Lcfu_dest_not_aligned 312.Lcfu_dest_aligned: 313 ands ip, r1, #3 314 bne .Lcfu_src_not_aligned 315 316/* 317 * Seeing as there has to be at least 8 bytes to copy, we can 318 * copy one word, and force a user-mode page fault... 319 */ 320 321.Lcfu_0fupi: subs r2, r2, #4 322 addmi ip, r2, #4 323 bmi .Lcfu_0nowords 324USER( ldrt r3, [r1], #4) 325 str r3, [r0], #4 326 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 327 rsb ip, ip, #0 328 movs ip, ip, lsr #32 - PAGE_SHIFT 329 beq .Lcfu_0fupi 330/* 331 * ip = max no. of bytes to copy before needing another "strt" insn 332 */ 333 cmp r2, ip 334 movlt ip, r2 335 sub r2, r2, ip 336 subs ip, ip, #32 337 blt .Lcfu_0rem8lp 338 339.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault 340 stmia r0!, {r3 - r6} 341 ldmia r1!, {r3 - r6} @ Shouldnt fault 342 subs ip, ip, #32 343 stmia r0!, {r3 - r6} 344 bpl .Lcfu_0cpy8lp 345 346.Lcfu_0rem8lp: cmn ip, #16 347 ldmgeia r1!, {r3 - r6} @ Shouldnt fault 348 stmgeia r0!, {r3 - r6} 349 tst ip, #8 350 ldmneia r1!, {r3 - r4} @ Shouldnt fault 351 stmneia r0!, {r3 - r4} 352 tst ip, #4 353 ldrnet r3, [r1], #4 @ Shouldnt fault 354 strne r3, [r0], #4 355 ands ip, ip, #3 356 beq .Lcfu_0fupi 357.Lcfu_0nowords: teq ip, #0 358 beq .Lcfu_finished 359.Lcfu_nowords: cmp ip, #2 360USER( ldrbt r3, [r1], #1) @ May fault 361 strb r3, [r0], #1 362USER( ldrgebt r3, [r1], #1) @ May fault 363 strgeb r3, [r0], #1 364USER( ldrgtbt r3, [r1], #1) @ May fault 365 strgtb r3, [r0], #1 366 b .Lcfu_finished 367 368.Lcfu_not_enough: 369 movs ip, r2 370 bne .Lcfu_nowords 371.Lcfu_finished: mov r0, #0 372 add sp, sp, #8 373 ldmfd sp!, {r4 - r7, pc} 374 375.Lcfu_src_not_aligned: 376 bic r1, r1, #3 377USER( ldrt r7, [r1], #4) @ May fault 378 cmp ip, #2 379 bgt .Lcfu_3fupi 380 beq .Lcfu_2fupi 381.Lcfu_1fupi: subs r2, r2, #4 382 addmi ip, r2, #4 383 bmi .Lcfu_1nowords 384 mov r3, r7, pull #8 385USER( ldrt r7, [r1], #4) @ May fault 386 orr r3, r3, r7, push #24 387 str r3, [r0], #4 388 mov ip, r1, lsl #32 - PAGE_SHIFT 389 rsb ip, ip, #0 390 movs ip, ip, lsr #32 - PAGE_SHIFT 391 beq .Lcfu_1fupi 392 cmp r2, ip 393 movlt ip, r2 394 sub r2, r2, ip 395 subs ip, ip, #16 396 blt .Lcfu_1rem8lp 397 398.Lcfu_1cpy8lp: mov r3, r7, pull #8 399 ldmia r1!, {r4 - r7} @ Shouldnt fault 400 subs ip, ip, #16 401 orr r3, r3, r4, push #24 402 mov r4, r4, pull #8 403 orr r4, r4, r5, push #24 404 mov r5, r5, pull #8 405 orr r5, r5, r6, push #24 406 mov r6, r6, pull #8 407 orr r6, r6, r7, push #24 408 stmia r0!, {r3 - r6} 409 bpl .Lcfu_1cpy8lp 410 411.Lcfu_1rem8lp: tst ip, #8 412 movne r3, r7, pull #8 413 ldmneia r1!, {r4, r7} @ Shouldnt fault 414 orrne r3, r3, r4, push #24 415 movne r4, r4, pull #8 416 orrne r4, r4, r7, push #24 417 stmneia r0!, {r3 - r4} 418 tst ip, #4 419 movne r3, r7, pull #8 420USER( ldrnet r7, [r1], #4) @ May fault 421 orrne r3, r3, r7, push #24 422 strne r3, [r0], #4 423 ands ip, ip, #3 424 beq .Lcfu_1fupi 425.Lcfu_1nowords: mov r3, r7, get_byte_1 426 teq ip, #0 427 beq .Lcfu_finished 428 cmp ip, #2 429 strb r3, [r0], #1 430 movge r3, r7, get_byte_2 431 strgeb r3, [r0], #1 432 movgt r3, r7, get_byte_3 433 strgtb r3, [r0], #1 434 b .Lcfu_finished 435 436.Lcfu_2fupi: subs r2, r2, #4 437 addmi ip, r2, #4 438 bmi .Lcfu_2nowords 439 mov r3, r7, pull #16 440USER( ldrt r7, [r1], #4) @ May fault 441 orr r3, r3, r7, push #16 442 str r3, [r0], #4 443 mov ip, r1, lsl #32 - PAGE_SHIFT 444 rsb ip, ip, #0 445 movs ip, ip, lsr #32 - PAGE_SHIFT 446 beq .Lcfu_2fupi 447 cmp r2, ip 448 movlt ip, r2 449 sub r2, r2, ip 450 subs ip, ip, #16 451 blt .Lcfu_2rem8lp 452 453 454.Lcfu_2cpy8lp: mov r3, r7, pull #16 455 ldmia r1!, {r4 - r7} @ Shouldnt fault 456 subs ip, ip, #16 457 orr r3, r3, r4, push #16 458 mov r4, r4, pull #16 459 orr r4, r4, r5, push #16 460 mov r5, r5, pull #16 461 orr r5, r5, r6, push #16 462 mov r6, r6, pull #16 463 orr r6, r6, r7, push #16 464 stmia r0!, {r3 - r6} 465 bpl .Lcfu_2cpy8lp 466 467.Lcfu_2rem8lp: tst ip, #8 468 movne r3, r7, pull #16 469 ldmneia r1!, {r4, r7} @ Shouldnt fault 470 orrne r3, r3, r4, push #16 471 movne r4, r4, pull #16 472 orrne r4, r4, r7, push #16 473 stmneia r0!, {r3 - r4} 474 tst ip, #4 475 movne r3, r7, pull #16 476USER( ldrnet r7, [r1], #4) @ May fault 477 orrne r3, r3, r7, push #16 478 strne r3, [r0], #4 479 ands ip, ip, #3 480 beq .Lcfu_2fupi 481.Lcfu_2nowords: mov r3, r7, get_byte_2 482 teq ip, #0 483 beq .Lcfu_finished 484 cmp ip, #2 485 strb r3, [r0], #1 486 movge r3, r7, get_byte_3 487 strgeb r3, [r0], #1 488USER( ldrgtbt r3, [r1], #0) @ May fault 489 strgtb r3, [r0], #1 490 b .Lcfu_finished 491 492.Lcfu_3fupi: subs r2, r2, #4 493 addmi ip, r2, #4 494 bmi .Lcfu_3nowords 495 mov r3, r7, pull #24 496USER( ldrt r7, [r1], #4) @ May fault 497 orr r3, r3, r7, push #8 498 str r3, [r0], #4 499 mov ip, r1, lsl #32 - PAGE_SHIFT 500 rsb ip, ip, #0 501 movs ip, ip, lsr #32 - PAGE_SHIFT 502 beq .Lcfu_3fupi 503 cmp r2, ip 504 movlt ip, r2 505 sub r2, r2, ip 506 subs ip, ip, #16 507 blt .Lcfu_3rem8lp 508 509.Lcfu_3cpy8lp: mov r3, r7, pull #24 510 ldmia r1!, {r4 - r7} @ Shouldnt fault 511 orr r3, r3, r4, push #8 512 mov r4, r4, pull #24 513 orr r4, r4, r5, push #8 514 mov r5, r5, pull #24 515 orr r5, r5, r6, push #8 516 mov r6, r6, pull #24 517 orr r6, r6, r7, push #8 518 stmia r0!, {r3 - r6} 519 subs ip, ip, #16 520 bpl .Lcfu_3cpy8lp 521 522.Lcfu_3rem8lp: tst ip, #8 523 movne r3, r7, pull #24 524 ldmneia r1!, {r4, r7} @ Shouldnt fault 525 orrne r3, r3, r4, push #8 526 movne r4, r4, pull #24 527 orrne r4, r4, r7, push #8 528 stmneia r0!, {r3 - r4} 529 tst ip, #4 530 movne r3, r7, pull #24 531USER( ldrnet r7, [r1], #4) @ May fault 532 orrne r3, r3, r7, push #8 533 strne r3, [r0], #4 534 ands ip, ip, #3 535 beq .Lcfu_3fupi 536.Lcfu_3nowords: mov r3, r7, get_byte_3 537 teq ip, #0 538 beq .Lcfu_finished 539 cmp ip, #2 540 strb r3, [r0], #1 541USER( ldrgebt r3, [r1], #1) @ May fault 542 strgeb r3, [r0], #1 543USER( ldrgtbt r3, [r1], #1) @ May fault 544 strgtb r3, [r0], #1 545 b .Lcfu_finished 546ENDPROC(__copy_from_user) 547 548 .pushsection .fixup,"ax" 549 .align 0 550 /* 551 * We took an exception. r0 contains a pointer to 552 * the byte not copied. 553 */ 5549001: ldr r2, [sp], #4 @ void *to 555 sub r2, r0, r2 @ bytes copied 556 ldr r1, [sp], #4 @ unsigned long count 557 subs r4, r1, r2 @ bytes left to copy 558 movne r1, r4 559 blne __memzero 560 mov r0, r4 561 ldmfd sp!, {r4 - r7, pc} 562 .popsection 563