1/* 2 * linux/arch/arm/lib/uaccess.S 3 * 4 * Copyright (C) 1995, 1996,1997,1998 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * Routines to block copy data to/from user memory 11 * These are highly optimised both for the 4k page size 12 * and for various alignments. 13 */ 14#include <linux/linkage.h> 15#include <asm/assembler.h> 16#include <asm/errno.h> 17 18 .text 19 20#define PAGE_SHIFT 12 21 22/* Prototype: int __copy_to_user(void *to, const char *from, size_t n) 23 * Purpose : copy a block to user memory from kernel memory 24 * Params : to - user memory 25 * : from - kernel memory 26 * : n - number of bytes to copy 27 * Returns : Number of bytes NOT copied. 28 */ 29 30.Lc2u_dest_not_aligned: 31 rsb ip, ip, #4 32 cmp ip, #2 33 ldrb r3, [r1], #1 34USER( strbt r3, [r0], #1) @ May fault 35 ldrgeb r3, [r1], #1 36USER( strgebt r3, [r0], #1) @ May fault 37 ldrgtb r3, [r1], #1 38USER( strgtbt r3, [r0], #1) @ May fault 39 sub r2, r2, ip 40 b .Lc2u_dest_aligned 41 42ENTRY(__copy_to_user) 43 stmfd sp!, {r2, r4 - r7, lr} 44 cmp r2, #4 45 blt .Lc2u_not_enough 46 ands ip, r0, #3 47 bne .Lc2u_dest_not_aligned 48.Lc2u_dest_aligned: 49 50 ands ip, r1, #3 51 bne .Lc2u_src_not_aligned 52/* 53 * Seeing as there has to be at least 8 bytes to copy, we can 54 * copy one word, and force a user-mode page fault... 55 */ 56 57.Lc2u_0fupi: subs r2, r2, #4 58 addmi ip, r2, #4 59 bmi .Lc2u_0nowords 60 ldr r3, [r1], #4 61USER( strt r3, [r0], #4) @ May fault 62 mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 63 rsb ip, ip, #0 64 movs ip, ip, lsr #32 - PAGE_SHIFT 65 beq .Lc2u_0fupi 66/* 67 * ip = max no. of bytes to copy before needing another "strt" insn 68 */ 69 cmp r2, ip 70 movlt ip, r2 71 sub r2, r2, ip 72 subs ip, ip, #32 73 blt .Lc2u_0rem8lp 74 75.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6} 76 stmia r0!, {r3 - r6} @ Shouldnt fault 77 ldmia r1!, {r3 - r6} 78 subs ip, ip, #32 79 stmia r0!, {r3 - r6} @ Shouldnt fault 80 bpl .Lc2u_0cpy8lp 81 82.Lc2u_0rem8lp: cmn ip, #16 83 ldmgeia r1!, {r3 - r6} 84 stmgeia r0!, {r3 - r6} @ Shouldnt fault 85 tst ip, #8 86 ldmneia r1!, {r3 - r4} 87 stmneia r0!, {r3 - r4} @ Shouldnt fault 88 tst ip, #4 89 ldrne r3, [r1], #4 90 strnet r3, [r0], #4 @ Shouldnt fault 91 ands ip, ip, #3 92 beq .Lc2u_0fupi 93.Lc2u_0nowords: teq ip, #0 94 beq .Lc2u_finished 95.Lc2u_nowords: cmp ip, #2 96 ldrb r3, [r1], #1 97USER( strbt r3, [r0], #1) @ May fault 98 ldrgeb r3, [r1], #1 99USER( strgebt r3, [r0], #1) @ May fault 100 ldrgtb r3, [r1], #1 101USER( strgtbt r3, [r0], #1) @ May fault 102 b .Lc2u_finished 103 104.Lc2u_not_enough: 105 movs ip, r2 106 bne .Lc2u_nowords 107.Lc2u_finished: mov r0, #0 108 ldmfd sp!, {r2, r4 - r7, pc} 109 110.Lc2u_src_not_aligned: 111 bic r1, r1, #3 112 ldr r7, [r1], #4 113 cmp ip, #2 114 bgt .Lc2u_3fupi 115 beq .Lc2u_2fupi 116.Lc2u_1fupi: subs r2, r2, #4 117 addmi ip, r2, #4 118 bmi .Lc2u_1nowords 119 mov r3, r7, pull #8 120 ldr r7, [r1], #4 121 orr r3, r3, r7, push #24 122USER( strt r3, [r0], #4) @ May fault 123 mov ip, r0, lsl #32 - PAGE_SHIFT 124 rsb ip, ip, #0 125 movs ip, ip, lsr #32 - PAGE_SHIFT 126 beq .Lc2u_1fupi 127 cmp r2, ip 128 movlt ip, r2 129 sub r2, r2, ip 130 subs ip, ip, #16 131 blt .Lc2u_1rem8lp 132 133.Lc2u_1cpy8lp: mov r3, r7, pull #8 134 ldmia r1!, {r4 - r7} 135 subs ip, ip, #16 136 orr r3, r3, r4, push #24 137 mov r4, r4, pull #8 138 orr r4, r4, r5, push #24 139 mov r5, r5, pull #8 140 orr r5, r5, r6, push #24 141 mov r6, r6, pull #8 142 orr r6, r6, r7, push #24 143 stmia r0!, {r3 - r6} @ Shouldnt fault 144 bpl .Lc2u_1cpy8lp 145 146.Lc2u_1rem8lp: tst ip, #8 147 movne r3, r7, pull #8 148 ldmneia r1!, {r4, r7} 149 orrne r3, r3, r4, push #24 150 movne r4, r4, pull #8 151 orrne r4, r4, r7, push #24 152 stmneia r0!, {r3 - r4} @ Shouldnt fault 153 tst ip, #4 154 movne r3, r7, pull #8 155 ldrne r7, [r1], #4 156 orrne r3, r3, r7, push #24 157 strnet r3, [r0], #4 @ Shouldnt fault 158 ands ip, ip, #3 159 beq .Lc2u_1fupi 160.Lc2u_1nowords: mov r3, r7, get_byte_1 161 teq ip, #0 162 beq .Lc2u_finished 163 cmp ip, #2 164USER( strbt r3, [r0], #1) @ May fault 165 movge r3, r7, get_byte_2 166USER( strgebt r3, [r0], #1) @ May fault 167 movgt r3, r7, get_byte_3 168USER( strgtbt r3, [r0], #1) @ May fault 169 b .Lc2u_finished 170 171.Lc2u_2fupi: subs r2, r2, #4 172 addmi ip, r2, #4 173 bmi .Lc2u_2nowords 174 mov r3, r7, pull #16 175 ldr r7, [r1], #4 176 orr r3, r3, r7, push #16 177USER( strt r3, [r0], #4) @ May fault 178 mov ip, r0, lsl #32 - PAGE_SHIFT 179 rsb ip, ip, #0 180 movs ip, ip, lsr #32 - PAGE_SHIFT 181 beq .Lc2u_2fupi 182 cmp r2, ip 183 movlt ip, r2 184 sub r2, r2, ip 185 subs ip, ip, #16 186 blt .Lc2u_2rem8lp 187 188.Lc2u_2cpy8lp: mov r3, r7, pull #16 189 ldmia r1!, {r4 - r7} 190 subs ip, ip, #16 191 orr r3, r3, r4, push #16 192 mov r4, r4, pull #16 193 orr r4, r4, r5, push #16 194 mov r5, r5, pull #16 195 orr r5, r5, r6, push #16 196 mov r6, r6, pull #16 197 orr r6, r6, r7, push #16 198 stmia r0!, {r3 - r6} @ Shouldnt fault 199 bpl .Lc2u_2cpy8lp 200 201.Lc2u_2rem8lp: tst ip, #8 202 movne r3, r7, pull #16 203 ldmneia r1!, {r4, r7} 204 orrne r3, r3, r4, push #16 205 movne r4, r4, pull #16 206 orrne r4, r4, r7, push #16 207 stmneia r0!, {r3 - r4} @ Shouldnt fault 208 tst ip, #4 209 movne r3, r7, pull #16 210 ldrne r7, [r1], #4 211 orrne r3, r3, r7, push #16 212 strnet r3, [r0], #4 @ Shouldnt fault 213 ands ip, ip, #3 214 beq .Lc2u_2fupi 215.Lc2u_2nowords: mov r3, r7, get_byte_2 216 teq ip, #0 217 beq .Lc2u_finished 218 cmp ip, #2 219USER( strbt r3, [r0], #1) @ May fault 220 movge r3, r7, get_byte_3 221USER( strgebt r3, [r0], #1) @ May fault 222 ldrgtb r3, [r1], #0 223USER( strgtbt r3, [r0], #1) @ May fault 224 b .Lc2u_finished 225 226.Lc2u_3fupi: subs r2, r2, #4 227 addmi ip, r2, #4 228 bmi .Lc2u_3nowords 229 mov r3, r7, pull #24 230 ldr r7, [r1], #4 231 orr r3, r3, r7, push #8 232USER( strt r3, [r0], #4) @ May fault 233 mov ip, r0, lsl #32 - PAGE_SHIFT 234 rsb ip, ip, #0 235 movs ip, ip, lsr #32 - PAGE_SHIFT 236 beq .Lc2u_3fupi 237 cmp r2, ip 238 movlt ip, r2 239 sub r2, r2, ip 240 subs ip, ip, #16 241 blt .Lc2u_3rem8lp 242 243.Lc2u_3cpy8lp: mov r3, r7, pull #24 244 ldmia r1!, {r4 - r7} 245 subs ip, ip, #16 246 orr r3, r3, r4, push #8 247 mov r4, r4, pull #24 248 orr r4, r4, r5, push #8 249 mov r5, r5, pull #24 250 orr r5, r5, r6, push #8 251 mov r6, r6, pull #24 252 orr r6, r6, r7, push #8 253 stmia r0!, {r3 - r6} @ Shouldnt fault 254 bpl .Lc2u_3cpy8lp 255 256.Lc2u_3rem8lp: tst ip, #8 257 movne r3, r7, pull #24 258 ldmneia r1!, {r4, r7} 259 orrne r3, r3, r4, push #8 260 movne r4, r4, pull #24 261 orrne r4, r4, r7, push #8 262 stmneia r0!, {r3 - r4} @ Shouldnt fault 263 tst ip, #4 264 movne r3, r7, pull #24 265 ldrne r7, [r1], #4 266 orrne r3, r3, r7, push #8 267 strnet r3, [r0], #4 @ Shouldnt fault 268 ands ip, ip, #3 269 beq .Lc2u_3fupi 270.Lc2u_3nowords: mov r3, r7, get_byte_3 271 teq ip, #0 272 beq .Lc2u_finished 273 cmp ip, #2 274USER( strbt r3, [r0], #1) @ May fault 275 ldrgeb r3, [r1], #1 276USER( strgebt r3, [r0], #1) @ May fault 277 ldrgtb r3, [r1], #0 278USER( strgtbt r3, [r0], #1) @ May fault 279 b .Lc2u_finished 280 281 .section .fixup,"ax" 282 .align 0 2839001: ldmfd sp!, {r0, r4 - r7, pc} 284 .previous 285 286/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n); 287 * Purpose : copy a block from user memory to kernel memory 288 * Params : to - kernel memory 289 * : from - user memory 290 * : n - number of bytes to copy 291 * Returns : Number of bytes NOT copied. 292 */ 293.Lcfu_dest_not_aligned: 294 rsb ip, ip, #4 295 cmp ip, #2 296USER( ldrbt r3, [r1], #1) @ May fault 297 strb r3, [r0], #1 298USER( ldrgebt r3, [r1], #1) @ May fault 299 strgeb r3, [r0], #1 300USER( ldrgtbt r3, [r1], #1) @ May fault 301 strgtb r3, [r0], #1 302 sub r2, r2, ip 303 b .Lcfu_dest_aligned 304 305ENTRY(__copy_from_user) 306 stmfd sp!, {r0, r2, r4 - r7, lr} 307 cmp r2, #4 308 blt .Lcfu_not_enough 309 ands ip, r0, #3 310 bne .Lcfu_dest_not_aligned 311.Lcfu_dest_aligned: 312 ands ip, r1, #3 313 bne .Lcfu_src_not_aligned 314 315/* 316 * Seeing as there has to be at least 8 bytes to copy, we can 317 * copy one word, and force a user-mode page fault... 318 */ 319 320.Lcfu_0fupi: subs r2, r2, #4 321 addmi ip, r2, #4 322 bmi .Lcfu_0nowords 323USER( ldrt r3, [r1], #4) 324 str r3, [r0], #4 325 mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction 326 rsb ip, ip, #0 327 movs ip, ip, lsr #32 - PAGE_SHIFT 328 beq .Lcfu_0fupi 329/* 330 * ip = max no. of bytes to copy before needing another "strt" insn 331 */ 332 cmp r2, ip 333 movlt ip, r2 334 sub r2, r2, ip 335 subs ip, ip, #32 336 blt .Lcfu_0rem8lp 337 338.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault 339 stmia r0!, {r3 - r6} 340 ldmia r1!, {r3 - r6} @ Shouldnt fault 341 subs ip, ip, #32 342 stmia r0!, {r3 - r6} 343 bpl .Lcfu_0cpy8lp 344 345.Lcfu_0rem8lp: cmn ip, #16 346 ldmgeia r1!, {r3 - r6} @ Shouldnt fault 347 stmgeia r0!, {r3 - r6} 348 tst ip, #8 349 ldmneia r1!, {r3 - r4} @ Shouldnt fault 350 stmneia r0!, {r3 - r4} 351 tst ip, #4 352 ldrnet r3, [r1], #4 @ Shouldnt fault 353 strne r3, [r0], #4 354 ands ip, ip, #3 355 beq .Lcfu_0fupi 356.Lcfu_0nowords: teq ip, #0 357 beq .Lcfu_finished 358.Lcfu_nowords: cmp ip, #2 359USER( ldrbt r3, [r1], #1) @ May fault 360 strb r3, [r0], #1 361USER( ldrgebt r3, [r1], #1) @ May fault 362 strgeb r3, [r0], #1 363USER( ldrgtbt r3, [r1], #1) @ May fault 364 strgtb r3, [r0], #1 365 b .Lcfu_finished 366 367.Lcfu_not_enough: 368 movs ip, r2 369 bne .Lcfu_nowords 370.Lcfu_finished: mov r0, #0 371 add sp, sp, #8 372 ldmfd sp!, {r4 - r7, pc} 373 374.Lcfu_src_not_aligned: 375 bic r1, r1, #3 376USER( ldrt r7, [r1], #4) @ May fault 377 cmp ip, #2 378 bgt .Lcfu_3fupi 379 beq .Lcfu_2fupi 380.Lcfu_1fupi: subs r2, r2, #4 381 addmi ip, r2, #4 382 bmi .Lcfu_1nowords 383 mov r3, r7, pull #8 384USER( ldrt r7, [r1], #4) @ May fault 385 orr r3, r3, r7, push #24 386 str r3, [r0], #4 387 mov ip, r1, lsl #32 - PAGE_SHIFT 388 rsb ip, ip, #0 389 movs ip, ip, lsr #32 - PAGE_SHIFT 390 beq .Lcfu_1fupi 391 cmp r2, ip 392 movlt ip, r2 393 sub r2, r2, ip 394 subs ip, ip, #16 395 blt .Lcfu_1rem8lp 396 397.Lcfu_1cpy8lp: mov r3, r7, pull #8 398 ldmia r1!, {r4 - r7} @ Shouldnt fault 399 subs ip, ip, #16 400 orr r3, r3, r4, push #24 401 mov r4, r4, pull #8 402 orr r4, r4, r5, push #24 403 mov r5, r5, pull #8 404 orr r5, r5, r6, push #24 405 mov r6, r6, pull #8 406 orr r6, r6, r7, push #24 407 stmia r0!, {r3 - r6} 408 bpl .Lcfu_1cpy8lp 409 410.Lcfu_1rem8lp: tst ip, #8 411 movne r3, r7, pull #8 412 ldmneia r1!, {r4, r7} @ Shouldnt fault 413 orrne r3, r3, r4, push #24 414 movne r4, r4, pull #8 415 orrne r4, r4, r7, push #24 416 stmneia r0!, {r3 - r4} 417 tst ip, #4 418 movne r3, r7, pull #8 419USER( ldrnet r7, [r1], #4) @ May fault 420 orrne r3, r3, r7, push #24 421 strne r3, [r0], #4 422 ands ip, ip, #3 423 beq .Lcfu_1fupi 424.Lcfu_1nowords: mov r3, r7, get_byte_1 425 teq ip, #0 426 beq .Lcfu_finished 427 cmp ip, #2 428 strb r3, [r0], #1 429 movge r3, r7, get_byte_2 430 strgeb r3, [r0], #1 431 movgt r3, r7, get_byte_3 432 strgtb r3, [r0], #1 433 b .Lcfu_finished 434 435.Lcfu_2fupi: subs r2, r2, #4 436 addmi ip, r2, #4 437 bmi .Lcfu_2nowords 438 mov r3, r7, pull #16 439USER( ldrt r7, [r1], #4) @ May fault 440 orr r3, r3, r7, push #16 441 str r3, [r0], #4 442 mov ip, r1, lsl #32 - PAGE_SHIFT 443 rsb ip, ip, #0 444 movs ip, ip, lsr #32 - PAGE_SHIFT 445 beq .Lcfu_2fupi 446 cmp r2, ip 447 movlt ip, r2 448 sub r2, r2, ip 449 subs ip, ip, #16 450 blt .Lcfu_2rem8lp 451 452 453.Lcfu_2cpy8lp: mov r3, r7, pull #16 454 ldmia r1!, {r4 - r7} @ Shouldnt fault 455 subs ip, ip, #16 456 orr r3, r3, r4, push #16 457 mov r4, r4, pull #16 458 orr r4, r4, r5, push #16 459 mov r5, r5, pull #16 460 orr r5, r5, r6, push #16 461 mov r6, r6, pull #16 462 orr r6, r6, r7, push #16 463 stmia r0!, {r3 - r6} 464 bpl .Lcfu_2cpy8lp 465 466.Lcfu_2rem8lp: tst ip, #8 467 movne r3, r7, pull #16 468 ldmneia r1!, {r4, r7} @ Shouldnt fault 469 orrne r3, r3, r4, push #16 470 movne r4, r4, pull #16 471 orrne r4, r4, r7, push #16 472 stmneia r0!, {r3 - r4} 473 tst ip, #4 474 movne r3, r7, pull #16 475USER( ldrnet r7, [r1], #4) @ May fault 476 orrne r3, r3, r7, push #16 477 strne r3, [r0], #4 478 ands ip, ip, #3 479 beq .Lcfu_2fupi 480.Lcfu_2nowords: mov r3, r7, get_byte_2 481 teq ip, #0 482 beq .Lcfu_finished 483 cmp ip, #2 484 strb r3, [r0], #1 485 movge r3, r7, get_byte_3 486 strgeb r3, [r0], #1 487USER( ldrgtbt r3, [r1], #0) @ May fault 488 strgtb r3, [r0], #1 489 b .Lcfu_finished 490 491.Lcfu_3fupi: subs r2, r2, #4 492 addmi ip, r2, #4 493 bmi .Lcfu_3nowords 494 mov r3, r7, pull #24 495USER( ldrt r7, [r1], #4) @ May fault 496 orr r3, r3, r7, push #8 497 str r3, [r0], #4 498 mov ip, r1, lsl #32 - PAGE_SHIFT 499 rsb ip, ip, #0 500 movs ip, ip, lsr #32 - PAGE_SHIFT 501 beq .Lcfu_3fupi 502 cmp r2, ip 503 movlt ip, r2 504 sub r2, r2, ip 505 subs ip, ip, #16 506 blt .Lcfu_3rem8lp 507 508.Lcfu_3cpy8lp: mov r3, r7, pull #24 509 ldmia r1!, {r4 - r7} @ Shouldnt fault 510 orr r3, r3, r4, push #8 511 mov r4, r4, pull #24 512 orr r4, r4, r5, push #8 513 mov r5, r5, pull #24 514 orr r5, r5, r6, push #8 515 mov r6, r6, pull #24 516 orr r6, r6, r7, push #8 517 stmia r0!, {r3 - r6} 518 subs ip, ip, #16 519 bpl .Lcfu_3cpy8lp 520 521.Lcfu_3rem8lp: tst ip, #8 522 movne r3, r7, pull #24 523 ldmneia r1!, {r4, r7} @ Shouldnt fault 524 orrne r3, r3, r4, push #8 525 movne r4, r4, pull #24 526 orrne r4, r4, r7, push #8 527 stmneia r0!, {r3 - r4} 528 tst ip, #4 529 movne r3, r7, pull #24 530USER( ldrnet r7, [r1], #4) @ May fault 531 orrne r3, r3, r7, push #8 532 strne r3, [r0], #4 533 ands ip, ip, #3 534 beq .Lcfu_3fupi 535.Lcfu_3nowords: mov r3, r7, get_byte_3 536 teq ip, #0 537 beq .Lcfu_finished 538 cmp ip, #2 539 strb r3, [r0], #1 540USER( ldrgebt r3, [r1], #1) @ May fault 541 strgeb r3, [r0], #1 542USER( ldrgtbt r3, [r1], #1) @ May fault 543 strgtb r3, [r0], #1 544 b .Lcfu_finished 545 546 .section .fixup,"ax" 547 .align 0 548 /* 549 * We took an exception. r0 contains a pointer to 550 * the byte not copied. 551 */ 5529001: ldr r2, [sp], #4 @ void *to 553 sub r2, r0, r2 @ bytes copied 554 ldr r1, [sp], #4 @ unsigned long count 555 subs r4, r1, r2 @ bytes left to copy 556 movne r1, r4 557 blne __memzero 558 mov r0, r4 559 ldmfd sp!, {r4 - r7, pc} 560 .previous 561