1/* $NetBSD: bcopyinout_xscale.S,v 1.3 2003/12/15 09:27:18 scw Exp $ */ 2 3/*- 4 * Copyright 2003 Wasabi Systems, Inc. 5 * All rights reserved. 6 * 7 * Written by Steve C. Woodford for Wasabi Systems, Inc. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. All advertising materials mentioning features or use of this software 18 * must display the following acknowledgement: 19 * This product includes software developed for the NetBSD Project by 20 * Wasabi Systems, Inc. 21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 22 * or promote products derived from this software without specific prior 23 * written permission. 24 * 25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 35 * POSSIBILITY OF SUCH DAMAGE. 36 */ 37 38#include <arm/asm_help.h> 39#include <assym.s> 40#include <mach/arm/asm.h> 41 42#define ENAMETOOLONG 0x3f 43#define SAVE_REGS stmfd sp!, {r4-r6} 44#define RESTORE_REGS ldmfd sp!, {r4-r6} 45 46.code 32 47.arm 48 49/* 50 * r0 = user space address 51 * r1 = kernel space address 52 * r2 = length 53 * 54 * Copies bytes from user space to kernel space 55 */ 56EnterARM(copyin) 57EnterARM(copyinmsg) 58 cmp r2, #0 59 moveq r0, #0 60 bxeq lr 61 cmp r0, #0x80000000 62 bcs copyio_kernel 63 stmfd sp!,{r10,r11,lr} 64 mov r3, #0 65 LoadThreadRegister(r10) 66 adr r12, .Lcopyin_fault 67 str r12, [r10, TH_RECOVER] 68 bl .Lcopyin_guts 69 mov r11, #0 70 str r11, [r10, TH_RECOVER] 71 mov r0, #0 72 ldmfd sp!,{r10,r11,pc} 73 74.Lcopyin_fault: 75 mov r11, #0 76 str r11, [r10, TH_RECOVER] 77 mov r0, #0xE /* EFAULT */ 78 cmp r3, #0x00 79 ldmgtfd sp!, {r4-r7} /* r3 > 0 Restore r4-r7 */ 80 ldmltfd sp!, {r4-r9} /* r3 < 0 Restore r4-r9 */ 81 ldmfd sp!, {r10-r11, pc} 82 83.Lcopyin_guts: 84 pld [r0] 85 /* Word-align the destination buffer */ 86 ands ip, r1, #0x03 /* Already word aligned? */ 87 beq .Lcopyin_wordaligned /* Yup */ 88 rsb ip, ip, #0x04 89 cmp r2, ip /* Enough bytes left to align it? */ 90 blt .Lcopyin_l4_2 /* Nope. Just copy bytewise */ 91 sub r2, r2, ip 92 rsbs ip, ip, #0x03 93 addne pc, pc, ip, lsl #3 94 nop 95 ldrbt ip, [r0], #0x01 96 strb ip, [r1], #0x01 97 ldrbt ip, [r0], #0x01 98 strb ip, [r1], #0x01 99 ldrbt ip, [r0], #0x01 100 strb ip, [r1], #0x01 101 cmp r2, #0x00 /* All done? */ 102 RETeq 103 104 /* Destination buffer is now word aligned */ 105.Lcopyin_wordaligned: 106 ands ip, r0, #0x03 /* Is src also word-aligned? */ 107 bne .Lcopyin_bad_align /* Nope. Things just got bad */ 108 cmp r2, #0x08 /* Less than 8 bytes remaining? */ 109 blt .Lcopyin_w_less_than8 110 111 /* Quad-align the destination buffer */ 112 tst r1, #0x07 /* Already quad aligned? */ 113 ldrnet ip, [r0], #0x04 114 strne ip, [r1], #0x04 115 subne r2, r2, #0x04 116 stmfd sp!, {r4-r9} /* Free up some registers */ 117 mov r3, #-1 /* Signal restore r4-r9 */ 118 119 /* Destination buffer quad aligned, source is word aligned */ 120 subs r2, r2, #0x80 121 blt .Lcopyin_w_lessthan128 122 123 /* Copy 128 bytes at a time */ 124.Lcopyin_w_loop128: 125 ldrt r4, [r0], #0x04 /* LD:00-03 */ 126 ldrt r5, [r0], #0x04 /* LD:04-07 */ 127 pld [r0, #0x18] /* Prefetch 0x20 */ 128 ldrt r6, [r0], #0x04 /* LD:08-0b */ 129 ldrt r7, [r0], #0x04 /* LD:0c-0f */ 130 ldrt r8, [r0], #0x04 /* LD:10-13 */ 131 ldrt r9, [r0], #0x04 /* LD:14-17 */ 132 strd r4, [r1], #0x08 /* ST:00-07 */ 133 ldrt r4, [r0], #0x04 /* LD:18-1b */ 134 ldrt r5, [r0], #0x04 /* LD:1c-1f */ 135 strd r6, [r1], #0x08 /* ST:08-0f */ 136 ldrt r6, [r0], #0x04 /* LD:20-23 */ 137 ldrt r7, [r0], #0x04 /* LD:24-27 */ 138 pld [r0, #0x18] /* Prefetch 0x40 */ 139 strd r8, [r1], #0x08 /* ST:10-17 */ 140 ldrt r8, [r0], #0x04 /* LD:28-2b */ 141 ldrt r9, [r0], #0x04 /* LD:2c-2f */ 142 strd r4, [r1], #0x08 /* ST:18-1f */ 143 ldrt r4, [r0], #0x04 /* LD:30-33 */ 144 ldrt r5, [r0], #0x04 /* LD:34-37 */ 145 strd r6, [r1], #0x08 /* ST:20-27 */ 146 ldrt r6, [r0], #0x04 /* LD:38-3b */ 147 ldrt r7, [r0], #0x04 /* LD:3c-3f */ 148 strd r8, [r1], #0x08 /* ST:28-2f */ 149 ldrt r8, [r0], #0x04 /* LD:40-43 */ 150 ldrt r9, [r0], #0x04 /* LD:44-47 */ 151 pld [r0, #0x18] /* Prefetch 0x60 */ 152 strd r4, [r1], #0x08 /* ST:30-37 */ 153 ldrt r4, [r0], #0x04 /* LD:48-4b */ 154 ldrt r5, [r0], #0x04 /* LD:4c-4f */ 155 strd r6, [r1], #0x08 /* ST:38-3f */ 156 ldrt r6, [r0], #0x04 /* LD:50-53 */ 157 ldrt r7, [r0], #0x04 /* LD:54-57 */ 158 strd r8, [r1], #0x08 /* ST:40-47 */ 159 ldrt r8, [r0], #0x04 /* LD:58-5b */ 160 ldrt r9, [r0], #0x04 /* LD:5c-5f */ 161 strd r4, [r1], #0x08 /* ST:48-4f */ 162 ldrt r4, [r0], #0x04 /* LD:60-63 */ 163 ldrt r5, [r0], #0x04 /* LD:64-67 */ 164 pld [r0, #0x18] /* Prefetch 0x80 */ 165 strd r6, [r1], #0x08 /* ST:50-57 */ 166 ldrt r6, [r0], #0x04 /* LD:68-6b */ 167 ldrt r7, [r0], #0x04 /* LD:6c-6f */ 168 strd r8, [r1], #0x08 /* ST:58-5f */ 169 ldrt r8, [r0], #0x04 /* LD:70-73 */ 170 ldrt r9, [r0], #0x04 /* LD:74-77 */ 171 strd r4, [r1], #0x08 /* ST:60-67 */ 172 ldrt r4, [r0], #0x04 /* LD:78-7b */ 173 ldrt r5, [r0], #0x04 /* LD:7c-7f */ 174 strd r6, [r1], #0x08 /* ST:68-6f */ 175 strd r8, [r1], #0x08 /* ST:70-77 */ 176 subs r2, r2, #0x80 177 strd r4, [r1], #0x08 /* ST:78-7f */ 178 bge .Lcopyin_w_loop128 179 180.Lcopyin_w_lessthan128: 181 adds r2, r2, #0x80 /* Adjust for extra sub */ 182 ldmeqfd sp!, {r4-r9} 183 RETeq 184 subs r2, r2, #0x20 185 blt .Lcopyin_w_lessthan32 186 187 /* Copy 32 bytes at a time */ 188.Lcopyin_w_loop32: 189 ldrt r4, [r0], #0x04 190 ldrt r5, [r0], #0x04 191 pld [r0, #0x18] 192 ldrt r6, [r0], #0x04 193 ldrt r7, [r0], #0x04 194 ldrt r8, [r0], #0x04 195 ldrt r9, [r0], #0x04 196 strd r4, [r1], #0x08 197 ldrt r4, [r0], #0x04 198 ldrt r5, [r0], #0x04 199 strd r6, [r1], #0x08 200 strd r8, [r1], #0x08 201 subs r2, r2, #0x20 202 strd r4, [r1], #0x08 203 bge .Lcopyin_w_loop32 204 205.Lcopyin_w_lessthan32: 206 adds r2, r2, #0x20 /* Adjust for extra sub */ 207 ldmeqfd sp!, {r4-r9} 208 RETeq /* Return now if done */ 209 210 and r4, r2, #0x18 211 rsb r5, r4, #0x18 212 subs r2, r2, r4 213 add pc, pc, r5, lsl #1 214 nop 215 216 /* At least 24 bytes remaining */ 217 ldrt r4, [r0], #0x04 218 ldrt r5, [r0], #0x04 219 nop 220 strd r4, [r1], #0x08 221 222 /* At least 16 bytes remaining */ 223 ldrt r4, [r0], #0x04 224 ldrt r5, [r0], #0x04 225 nop 226 strd r4, [r1], #0x08 227 228 /* At least 8 bytes remaining */ 229 ldrt r4, [r0], #0x04 230 ldrt r5, [r0], #0x04 231 nop 232 strd r4, [r1], #0x08 233 234 /* Less than 8 bytes remaining */ 235 ldmfd sp!, {r4-r9} 236 RETeq /* Return now if done */ 237 mov r3, #0x00 238 239.Lcopyin_w_less_than8: 240 subs r2, r2, #0x04 241 ldrget ip, [r0], #0x04 242 strge ip, [r1], #0x04 243 RETeq /* Return now if done */ 244 addlt r2, r2, #0x04 245 ldrbt ip, [r0], #0x01 246 cmp r2, #0x02 247 ldrgebt r2, [r0], #0x01 248 strb ip, [r1], #0x01 249 ldrgtbt ip, [r0] 250 strgeb r2, [r1], #0x01 251 strgtb ip, [r1] 252 RET 253 254/* 255 * At this point, it has not been possible to word align both buffers. 256 * The destination buffer (r1) is word aligned, but the source buffer 257 * (r0) is not. 258 */ 259.Lcopyin_bad_align: 260 stmfd sp!, {r4-r7} 261 mov r3, #0x01 262 bic r0, r0, #0x03 263 cmp ip, #2 264 ldrt ip, [r0], #0x04 265 bgt .Lcopyin_bad3 266 beq .Lcopyin_bad2 267 b .Lcopyin_bad1 268 269.Lcopyin_bad1_loop16: 270#ifdef __ARMEB__ 271 mov r4, ip, lsl #8 272#else 273 mov r4, ip, lsr #8 274#endif 275 ldrt r5, [r0], #0x04 276 pld [r0, #0x018] 277 ldrt r6, [r0], #0x04 278 ldrt r7, [r0], #0x04 279 ldrt ip, [r0], #0x04 280#ifdef __ARMEB__ 281 orr r4, r4, r5, lsr #24 282 mov r5, r5, lsl #8 283 orr r5, r5, r6, lsr #24 284 mov r6, r6, lsl #8 285 orr r6, r6, r7, lsr #24 286 mov r7, r7, lsl #8 287 orr r7, r7, ip, lsr #24 288#else 289 orr r4, r4, r5, lsl #24 290 mov r5, r5, lsr #8 291 orr r5, r5, r6, lsl #24 292 mov r6, r6, lsr #8 293 orr r6, r6, r7, lsl #24 294 mov r7, r7, lsr #8 295 orr r7, r7, ip, lsl #24 296#endif 297 str r4, [r1], #0x04 298 str r5, [r1], #0x04 299 str r6, [r1], #0x04 300 str r7, [r1], #0x04 301.Lcopyin_bad1: 302 subs r2, r2, #0x10 303 bge .Lcopyin_bad1_loop16 304 305 adds r2, r2, #0x10 306 ldmeqfd sp!, {r4-r7} 307 RETeq /* Return now if done */ 308 subs r2, r2, #0x04 309 sublt r0, r0, #0x03 310 blt .Lcopyin_l4 311 312.Lcopyin_bad1_loop4: 313#ifdef __ARMEB__ 314 mov r4, ip, lsl #8 315#else 316 mov r4, ip, lsr #8 317#endif 318 ldrt ip, [r0], #0x04 319 subs r2, r2, #0x04 320#ifdef __ARMEB__ 321 orr r4, r4, ip, lsr #24 322#else 323 orr r4, r4, ip, lsl #24 324#endif 325 str r4, [r1], #0x04 326 bge .Lcopyin_bad1_loop4 327 sub r0, r0, #0x03 328 b .Lcopyin_l4 329 330.Lcopyin_bad2_loop16: 331#ifdef __ARMEB__ 332 mov r4, ip, lsl #16 333#else 334 mov r4, ip, lsr #16 335#endif 336 ldrt r5, [r0], #0x04 337 pld [r0, #0x018] 338 ldrt r6, [r0], #0x04 339 ldrt r7, [r0], #0x04 340 ldrt ip, [r0], #0x04 341#ifdef __ARMEB__ 342 orr r4, r4, r5, lsr #16 343 mov r5, r5, lsl #16 344 orr r5, r5, r6, lsr #16 345 mov r6, r6, lsl #16 346 orr r6, r6, r7, lsr #16 347 mov r7, r7, lsl #16 348 orr r7, r7, ip, lsr #16 349#else 350 orr r4, r4, r5, lsl #16 351 mov r5, r5, lsr #16 352 orr r5, r5, r6, lsl #16 353 mov r6, r6, lsr #16 354 orr r6, r6, r7, lsl #16 355 mov r7, r7, lsr #16 356 orr r7, r7, ip, lsl #16 357#endif 358 str r4, [r1], #0x04 359 str r5, [r1], #0x04 360 str r6, [r1], #0x04 361 str r7, [r1], #0x04 362.Lcopyin_bad2: 363 subs r2, r2, #0x10 364 bge .Lcopyin_bad2_loop16 365 366 adds r2, r2, #0x10 367 ldmeqfd sp!, {r4-r7} 368 RETeq /* Return now if done */ 369 subs r2, r2, #0x04 370 sublt r0, r0, #0x02 371 blt .Lcopyin_l4 372 373.Lcopyin_bad2_loop4: 374#ifdef __ARMEB__ 375 mov r4, ip, lsl #16 376#else 377 mov r4, ip, lsr #16 378#endif 379 ldrt ip, [r0], #0x04 380 subs r2, r2, #0x04 381#ifdef __ARMEB__ 382 orr r4, r4, ip, lsr #16 383#else 384 orr r4, r4, ip, lsl #16 385#endif 386 str r4, [r1], #0x04 387 bge .Lcopyin_bad2_loop4 388 sub r0, r0, #0x02 389 b .Lcopyin_l4 390 391.Lcopyin_bad3_loop16: 392#ifdef __ARMEB__ 393 mov r4, ip, lsl #24 394#else 395 mov r4, ip, lsr #24 396#endif 397 ldrt r5, [r0], #0x04 398 pld [r0, #0x018] 399 ldrt r6, [r0], #0x04 400 ldrt r7, [r0], #0x04 401 ldrt ip, [r0], #0x04 402#ifdef __ARMEB__ 403 orr r4, r4, r5, lsr #8 404 mov r5, r5, lsl #24 405 orr r5, r5, r6, lsr #8 406 mov r6, r6, lsl #24 407 orr r6, r6, r7, lsr #8 408 mov r7, r7, lsl #24 409 orr r7, r7, ip, lsr #8 410#else 411 orr r4, r4, r5, lsl #8 412 mov r5, r5, lsr #24 413 orr r5, r5, r6, lsl #8 414 mov r6, r6, lsr #24 415 orr r6, r6, r7, lsl #8 416 mov r7, r7, lsr #24 417 orr r7, r7, ip, lsl #8 418#endif 419 str r4, [r1], #0x04 420 str r5, [r1], #0x04 421 str r6, [r1], #0x04 422 str r7, [r1], #0x04 423.Lcopyin_bad3: 424 subs r2, r2, #0x10 425 bge .Lcopyin_bad3_loop16 426 427 adds r2, r2, #0x10 428 ldmeqfd sp!, {r4-r7} 429 RETeq /* Return now if done */ 430 subs r2, r2, #0x04 431 sublt r0, r0, #0x01 432 blt .Lcopyin_l4 433 434.Lcopyin_bad3_loop4: 435#ifdef __ARMEB__ 436 mov r4, ip, lsl #24 437#else 438 mov r4, ip, lsr #24 439#endif 440 ldrt ip, [r0], #0x04 441 subs r2, r2, #0x04 442#ifdef __ARMEB__ 443 orr r4, r4, ip, lsr #8 444#else 445 orr r4, r4, ip, lsl #8 446#endif 447 str r4, [r1], #0x04 448 bge .Lcopyin_bad3_loop4 449 sub r0, r0, #0x01 450 451.Lcopyin_l4: 452 ldmfd sp!, {r4-r7} 453 mov r3, #0x00 454 adds r2, r2, #0x04 455 RETeq 456.Lcopyin_l4_2: 457 rsbs r2, r2, #0x03 458 addne pc, pc, r2, lsl #3 459 nop 460 ldrbt ip, [r0], #0x01 461 strb ip, [r1], #0x01 462 ldrbt ip, [r0], #0x01 463 strb ip, [r1], #0x01 464 ldrbt ip, [r0] 465 strb ip, [r1] 466 RET 467 468/* 469 * r0 = kernel space address 470 * r1 = user space address 471 * r2 = length 472 * 473 * Copies bytes from kernel space to user space 474 */ 475EnterARM(copyout) 476EnterARM(copyoutmsg) 477 cmp r2, #0 478 moveq r0, #0 479 bxeq lr 480 cmp r1, #0x80000000 481 bcs copyio_kernel 482 stmfd sp!,{r10,r11,lr} 483 LoadThreadRegister(r10) 484 mov r3, #0 485 adr r12, .Lcopyout_fault 486 ldr r11, [r10, TH_RECOVER] 487 str r12, [r10, TH_RECOVER] 488 bl .Lcopyout_guts 489 mov r11, #0 490 str r11, [r10, TH_RECOVER] 491 mov r0, #0 492 ldmfd sp!,{r10,r11,pc} 493 494.Lcopyout_fault: 495 mov r11, #0 496 str r11, [r10, TH_RECOVER] 497 mov r0, #0x0e 498 cmp r3, #0x00 499 ldmgtfd sp!, {r4-r7} /* r3 > 0 Restore r4-r7 */ 500 ldmltfd sp!, {r4-r9} /* r3 < 0 Restore r4-r9 */ 501 ldmfd sp!, {r10-r11, pc} 502 503.Lcopyout_guts: 504 pld [r0] 505 /* Word-align the destination buffer */ 506 ands ip, r1, #0x03 /* Already word aligned? */ 507 beq .Lcopyout_wordaligned /* Yup */ 508 rsb ip, ip, #0x04 509 cmp r2, ip /* Enough bytes left to align it? */ 510 blt .Lcopyout_l4_2 /* Nope. Just copy bytewise */ 511 sub r2, r2, ip 512 rsbs ip, ip, #0x03 513 addne pc, pc, ip, lsl #3 514 nop 515 ldrb ip, [r0], #0x01 516 strbt ip, [r1], #0x01 517 ldrb ip, [r0], #0x01 518 strbt ip, [r1], #0x01 519 ldrb ip, [r0], #0x01 520 strbt ip, [r1], #0x01 521 cmp r2, #0x00 /* All done? */ 522 RETeq 523 524 /* Destination buffer is now word aligned */ 525.Lcopyout_wordaligned: 526 ands ip, r0, #0x03 /* Is src also word-aligned? */ 527 bne .Lcopyout_bad_align /* Nope. Things just got bad */ 528 cmp r2, #0x08 /* Less than 8 bytes remaining? */ 529 blt .Lcopyout_w_less_than8 530 531 /* Quad-align the destination buffer */ 532 tst r0, #0x07 /* Already quad aligned? */ 533 ldrne ip, [r0], #0x04 534 subne r2, r2, #0x04 535 strnet ip, [r1], #0x04 536 537 stmfd sp!, {r4-r9} /* Free up some registers */ 538 mov r3, #-1 /* Signal restore r4-r9 */ 539 540 /* Destination buffer word aligned, source is quad aligned */ 541 subs r2, r2, #0x80 542 blt .Lcopyout_w_lessthan128 543 544 /* Copy 128 bytes at a time */ 545.Lcopyout_w_loop128: 546 ldrd r4, [r0], #0x08 /* LD:00-07 */ 547 pld [r0, #0x18] /* Prefetch 0x20 */ 548 ldrd r6, [r0], #0x08 /* LD:08-0f */ 549 ldrd r8, [r0], #0x08 /* LD:10-17 */ 550 strt r4, [r1], #0x04 /* ST:00-03 */ 551 strt r5, [r1], #0x04 /* ST:04-07 */ 552 ldrd r4, [r0], #0x08 /* LD:18-1f */ 553 strt r6, [r1], #0x04 /* ST:08-0b */ 554 strt r7, [r1], #0x04 /* ST:0c-0f */ 555 ldrd r6, [r0], #0x08 /* LD:20-27 */ 556 pld [r0, #0x18] /* Prefetch 0x40 */ 557 strt r8, [r1], #0x04 /* ST:10-13 */ 558 strt r9, [r1], #0x04 /* ST:14-17 */ 559 ldrd r8, [r0], #0x08 /* LD:28-2f */ 560 strt r4, [r1], #0x04 /* ST:18-1b */ 561 strt r5, [r1], #0x04 /* ST:1c-1f */ 562 ldrd r4, [r0], #0x08 /* LD:30-37 */ 563 strt r6, [r1], #0x04 /* ST:20-23 */ 564 strt r7, [r1], #0x04 /* ST:24-27 */ 565 ldrd r6, [r0], #0x08 /* LD:38-3f */ 566 strt r8, [r1], #0x04 /* ST:28-2b */ 567 strt r9, [r1], #0x04 /* ST:2c-2f */ 568 ldrd r8, [r0], #0x08 /* LD:40-47 */ 569 pld [r0, #0x18] /* Prefetch 0x60 */ 570 strt r4, [r1], #0x04 /* ST:30-33 */ 571 strt r5, [r1], #0x04 /* ST:34-37 */ 572 ldrd r4, [r0], #0x08 /* LD:48-4f */ 573 strt r6, [r1], #0x04 /* ST:38-3b */ 574 strt r7, [r1], #0x04 /* ST:3c-3f */ 575 ldrd r6, [r0], #0x08 /* LD:50-57 */ 576 strt r8, [r1], #0x04 /* ST:40-43 */ 577 strt r9, [r1], #0x04 /* ST:44-47 */ 578 ldrd r8, [r0], #0x08 /* LD:58-4f */ 579 strt r4, [r1], #0x04 /* ST:48-4b */ 580 strt r5, [r1], #0x04 /* ST:4c-4f */ 581 ldrd r4, [r0], #0x08 /* LD:60-67 */ 582 pld [r0, #0x18] /* Prefetch 0x80 */ 583 strt r6, [r1], #0x04 /* ST:50-53 */ 584 strt r7, [r1], #0x04 /* ST:54-57 */ 585 ldrd r6, [r0], #0x08 /* LD:68-6f */ 586 strt r8, [r1], #0x04 /* ST:58-5b */ 587 strt r9, [r1], #0x04 /* ST:5c-5f */ 588 ldrd r8, [r0], #0x08 /* LD:70-77 */ 589 strt r4, [r1], #0x04 /* ST:60-63 */ 590 strt r5, [r1], #0x04 /* ST:64-67 */ 591 ldrd r4, [r0], #0x08 /* LD:78-7f */ 592 strt r6, [r1], #0x04 /* ST:68-6b */ 593 strt r7, [r1], #0x04 /* ST:6c-6f */ 594 strt r8, [r1], #0x04 /* ST:70-73 */ 595 strt r9, [r1], #0x04 /* ST:74-77 */ 596 subs r2, r2, #0x80 597 strt r4, [r1], #0x04 /* ST:78-7b */ 598 strt r5, [r1], #0x04 /* ST:7c-7f */ 599 bge .Lcopyout_w_loop128 600 601.Lcopyout_w_lessthan128: 602 adds r2, r2, #0x80 /* Adjust for extra sub */ 603 ldmeqfd sp!, {r4-r9} 604 RETeq /* Return now if done */ 605 subs r2, r2, #0x20 606 blt .Lcopyout_w_lessthan32 607 608 /* Copy 32 bytes at a time */ 609.Lcopyout_w_loop32: 610 ldrd r4, [r0], #0x08 611 pld [r0, #0x18] 612 ldrd r6, [r0], #0x08 613 ldrd r8, [r0], #0x08 614 strt r4, [r1], #0x04 615 strt r5, [r1], #0x04 616 ldrd r4, [r0], #0x08 617 strt r6, [r1], #0x04 618 strt r7, [r1], #0x04 619 strt r8, [r1], #0x04 620 strt r9, [r1], #0x04 621 subs r2, r2, #0x20 622 strt r4, [r1], #0x04 623 strt r5, [r1], #0x04 624 bge .Lcopyout_w_loop32 625 626.Lcopyout_w_lessthan32: 627 adds r2, r2, #0x20 /* Adjust for extra sub */ 628 ldmeqfd sp!, {r4-r9} 629 RETeq /* Return now if done */ 630 631 and r4, r2, #0x18 632 rsb r5, r4, #0x18 633 subs r2, r2, r4 634 add pc, pc, r5, lsl #1 635 nop 636 637 /* At least 24 bytes remaining */ 638 ldrd r4, [r0], #0x08 639 strt r4, [r1], #0x04 640 strt r5, [r1], #0x04 641 nop 642 643 /* At least 16 bytes remaining */ 644 ldrd r4, [r0], #0x08 645 strt r4, [r1], #0x04 646 strt r5, [r1], #0x04 647 nop 648 649 /* At least 8 bytes remaining */ 650 ldrd r4, [r0], #0x08 651 strt r4, [r1], #0x04 652 strt r5, [r1], #0x04 653 nop 654 655 /* Less than 8 bytes remaining */ 656 ldmfd sp!, {r4-r9} 657 RETeq /* Return now if done */ 658 mov r3, #0x00 659 660.Lcopyout_w_less_than8: 661 subs r2, r2, #0x04 662 ldrge ip, [r0], #0x04 663 strget ip, [r1], #0x04 664 RETeq /* Return now if done */ 665 addlt r2, r2, #0x04 666 ldrb ip, [r0], #0x01 667 cmp r2, #0x02 668 ldrgeb r2, [r0], #0x01 669 strbt ip, [r1], #0x01 670 ldrgtb ip, [r0] 671 strgebt r2, [r1], #0x01 672 strgtbt ip, [r1] 673 RET 674 675/* 676 * At this point, it has not been possible to word align both buffers. 677 * The destination buffer (r1) is word aligned, but the source buffer 678 * (r0) is not. 679 */ 680.Lcopyout_bad_align: 681 stmfd sp!, {r4-r7} 682 mov r3, #0x01 683 bic r0, r0, #0x03 684 cmp ip, #2 685 ldr ip, [r0], #0x04 686 bgt .Lcopyout_bad3 687 beq .Lcopyout_bad2 688 b .Lcopyout_bad1 689 690.Lcopyout_bad1_loop16: 691#ifdef __ARMEB__ 692 mov r4, ip, lsl #8 693#else 694 mov r4, ip, lsr #8 695#endif 696 ldr r5, [r0], #0x04 697 pld [r0, #0x018] 698 ldr r6, [r0], #0x04 699 ldr r7, [r0], #0x04 700 ldr ip, [r0], #0x04 701#ifdef __ARMEB__ 702 orr r4, r4, r5, lsr #24 703 mov r5, r5, lsl #8 704 orr r5, r5, r6, lsr #24 705 mov r6, r6, lsl #8 706 orr r6, r6, r7, lsr #24 707 mov r7, r7, lsl #8 708 orr r7, r7, ip, lsr #24 709#else 710 orr r4, r4, r5, lsl #24 711 mov r5, r5, lsr #8 712 orr r5, r5, r6, lsl #24 713 mov r6, r6, lsr #8 714 orr r6, r6, r7, lsl #24 715 mov r7, r7, lsr #8 716 orr r7, r7, ip, lsl #24 717#endif 718 strt r4, [r1], #0x04 719 strt r5, [r1], #0x04 720 strt r6, [r1], #0x04 721 strt r7, [r1], #0x04 722.Lcopyout_bad1: 723 subs r2, r2, #0x10 724 bge .Lcopyout_bad1_loop16 725 726 adds r2, r2, #0x10 727 ldmeqfd sp!, {r4-r7} 728 RETeq /* Return now if done */ 729 subs r2, r2, #0x04 730 sublt r0, r0, #0x03 731 blt .Lcopyout_l4 732 733.Lcopyout_bad1_loop4: 734#ifdef __ARMEB__ 735 mov r4, ip, lsl #8 736#else 737 mov r4, ip, lsr #8 738#endif 739 ldr ip, [r0], #0x04 740 subs r2, r2, #0x04 741#ifdef __ARMEB__ 742 orr r4, r4, ip, lsr #24 743#else 744 orr r4, r4, ip, lsl #24 745#endif 746 strt r4, [r1], #0x04 747 bge .Lcopyout_bad1_loop4 748 sub r0, r0, #0x03 749 b .Lcopyout_l4 750 751.Lcopyout_bad2_loop16: 752#ifdef __ARMEB__ 753 mov r4, ip, lsl #16 754#else 755 mov r4, ip, lsr #16 756#endif 757 ldr r5, [r0], #0x04 758 pld [r0, #0x018] 759 ldr r6, [r0], #0x04 760 ldr r7, [r0], #0x04 761 ldr ip, [r0], #0x04 762#ifdef __ARMEB__ 763 orr r4, r4, r5, lsr #16 764 mov r5, r5, lsl #16 765 orr r5, r5, r6, lsr #16 766 mov r6, r6, lsl #16 767 orr r6, r6, r7, lsr #16 768 mov r7, r7, lsl #16 769 orr r7, r7, ip, lsr #16 770#else 771 orr r4, r4, r5, lsl #16 772 mov r5, r5, lsr #16 773 orr r5, r5, r6, lsl #16 774 mov r6, r6, lsr #16 775 orr r6, r6, r7, lsl #16 776 mov r7, r7, lsr #16 777 orr r7, r7, ip, lsl #16 778#endif 779 strt r4, [r1], #0x04 780 strt r5, [r1], #0x04 781 strt r6, [r1], #0x04 782 strt r7, [r1], #0x04 783.Lcopyout_bad2: 784 subs r2, r2, #0x10 785 bge .Lcopyout_bad2_loop16 786 787 adds r2, r2, #0x10 788 ldmeqfd sp!, {r4-r7} 789 RETeq /* Return now if done */ 790 subs r2, r2, #0x04 791 sublt r0, r0, #0x02 792 blt .Lcopyout_l4 793 794.Lcopyout_bad2_loop4: 795#ifdef __ARMEB__ 796 mov r4, ip, lsl #16 797#else 798 mov r4, ip, lsr #16 799#endif 800 ldr ip, [r0], #0x04 801 subs r2, r2, #0x04 802#ifdef __ARMEB__ 803 orr r4, r4, ip, lsr #16 804#else 805 orr r4, r4, ip, lsl #16 806#endif 807 strt r4, [r1], #0x04 808 bge .Lcopyout_bad2_loop4 809 sub r0, r0, #0x02 810 b .Lcopyout_l4 811 812.Lcopyout_bad3_loop16: 813#ifdef __ARMEB__ 814 mov r4, ip, lsl #24 815#else 816 mov r4, ip, lsr #24 817#endif 818 ldr r5, [r0], #0x04 819 pld [r0, #0x018] 820 ldr r6, [r0], #0x04 821 ldr r7, [r0], #0x04 822 ldr ip, [r0], #0x04 823#ifdef __ARMEB__ 824 orr r4, r4, r5, lsr #8 825 mov r5, r5, lsl #24 826 orr r5, r5, r6, lsr #8 827 mov r6, r6, lsl #24 828 orr r6, r6, r7, lsr #8 829 mov r7, r7, lsl #24 830 orr r7, r7, ip, lsr #8 831#else 832 orr r4, r4, r5, lsl #8 833 mov r5, r5, lsr #24 834 orr r5, r5, r6, lsl #8 835 mov r6, r6, lsr #24 836 orr r6, r6, r7, lsl #8 837 mov r7, r7, lsr #24 838 orr r7, r7, ip, lsl #8 839#endif 840 strt r4, [r1], #0x04 841 strt r5, [r1], #0x04 842 strt r6, [r1], #0x04 843 strt r7, [r1], #0x04 844.Lcopyout_bad3: 845 subs r2, r2, #0x10 846 bge .Lcopyout_bad3_loop16 847 848 adds r2, r2, #0x10 849 ldmeqfd sp!, {r4-r7} 850 RETeq /* Return now if done */ 851 subs r2, r2, #0x04 852 sublt r0, r0, #0x01 853 blt .Lcopyout_l4 854 855.Lcopyout_bad3_loop4: 856#ifdef __ARMEB__ 857 mov r4, ip, lsl #24 858#else 859 mov r4, ip, lsr #24 860#endif 861 ldr ip, [r0], #0x04 862 subs r2, r2, #0x04 863#ifdef __ARMEB__ 864 orr r4, r4, ip, lsr #8 865#else 866 orr r4, r4, ip, lsl #8 867#endif 868 strt r4, [r1], #0x04 869 bge .Lcopyout_bad3_loop4 870 sub r0, r0, #0x01 871 872.Lcopyout_l4: 873 ldmfd sp!, {r4-r7} 874 mov r3, #0x00 875 adds r2, r2, #0x04 876 RETeq 877.Lcopyout_l4_2: 878 rsbs r2, r2, #0x03 879 addne pc, pc, r2, lsl #3 880 nop 881 ldrb ip, [r0], #0x01 882 strbt ip, [r1], #0x01 883 ldrb ip, [r0], #0x01 884 strbt ip, [r1], #0x01 885 ldrb ip, [r0] 886 strbt ip, [r1] 887 RET 888 889/* 890 * r0 - user space address 891 * r1 - kernel space address 892 * r2 - maxlens 893 * r3 - lencopied 894 * 895 * Copy string from user space to kernel space 896 */ 897 898/* 899 * r0 - from 900 * r1 - to 901 * r2 - maxlens 902 * r3 - lencopied 903 * 904 * Copy string from r0 to r1 905 */ 906EnterARM(copystr) 907 stmfd sp!, {r4-r5} /* stack is 8 byte aligned */ 908 teq r2, #0x00000000 909 mov r5, #0x00000000 910 moveq r0, #ENAMETOOLONG 911 beq 2f 912 9131: ldrb r4, [r0], #0x0001 914 add r5, r5, #0x00000001 915 teq r4, #0x00000000 916 strb r4, [r1], #0x0001 917 teqne r5, r2 918 bne 1b 919 920 teq r4, #0x00000000 921 moveq r0, #0x00000000 922 movne r0, #ENAMETOOLONG 923 9242: teq r3, #0x00000000 925 strne r5, [r3] 926 927 ldmfd sp!, {r4-r5} /* stack is 8 byte aligned */ 928 mov pc, lr 929 930 931/* 932 * r0 - user space address 933 * r1 - kernel space address 934 * r2 - maxlens 935 * r3 - lencopied 936 * 937 * Copy string from user space to kernel space 938 */ 939EnterARM(copyinstr) 940 SAVE_REGS 941 942 teq r2, #0x00000000 943 mov r6, #0x00000000 944 moveq r0, #ENAMETOOLONG 945 beq 2f 946 947 LoadThreadRegister(r4) 948 949 adr r5, .Lcopystrfault 950 str r5, [r4, TH_RECOVER] 951 9521: ldrbt r5, [r0], #0x0001 953 add r6, r6, #0x00000001 954 teq r5, #0x00000000 955 strb r5, [r1], #0x0001 956 teqne r6, r2 957 bne 1b 958 959 mov r0, #0x00000000 960 str r0, [r4, TH_RECOVER] 961 962 teq r5, #0x00000000 963 moveq r0, #0x00000000 964 movne r0, #ENAMETOOLONG 965 9662: teq r3, #0x00000000 967 strne r6, [r3] 968 969 RESTORE_REGS 970 bx lr 971 972/* 973 * r0 - kernel space address 974 * r1 - user space address 975 * r2 - maxlens 976 * r3 - lencopied 977 * 978 * Copy string from kernel space to user space 979 */ 980EnterARM(copyoutstr) 981 SAVE_REGS 982 983 teq r2, #0x00000000 984 mov r6, #0x00000000 985 moveq r0, #ENAMETOOLONG 986 beq 2f 987 988 LoadThreadRegister(r4) 989 990 adr r5, .Lcopystrfault 991 str r5, [r4, TH_RECOVER] 992 9931: ldrb r5, [r0], #0x0001 994 add r6, r6, #0x00000001 995 teq r5, #0x00000000 996 strbt r5, [r1], #0x0001 997 teqne r6, r2 998 bne 1b 999 1000 mov r0, #0x00000000 1001 str r0, [r4, TH_RECOVER] 1002 1003 teq r5, #0x00000000 1004 moveq r0, #0x00000000 1005 movne r0, #ENAMETOOLONG 1006 10072: teq r3, #0x00000000 1008 strne r6, [r3] 1009 1010 RESTORE_REGS 1011 bx lr 1012 1013/* A fault occurred during the copy */ 1014.Lcopystrfault: 1015 mov r1, #0x00000000 1016 str r1, [r4, TH_RECOVER] 1017 RESTORE_REGS 1018 bx lr 1019 1020copyio_kernel: 1021 /* We're open to attack here since we don't really verify the addresses being copied... */ 1022 cmp r2, #0x10 1023 blt .Lcopyio_copy2 1024 orr r3, r0, r1 1025 tst r3, #3 1026 bne .Lcopyio_copy2 1027 sub r2, r2, #8 1028.Lcopyio_copy8_loop: 1029 ldr r3, [r0], #4 1030 ldr r12, [r0], #4 1031 str r3, [r1], #4 1032 str r12, [r1], #4 1033 subs r2, r2, #8 1034 bge .Lcopyio_copy8_loop 1035 adds r2, r2, #8 1036 beq .Lcopyio_exit 1037.Lcopyio_copy2: 1038 subs r2, r2, #2 1039 ldrb r3, [r0], #1 1040 ldrplb r12, [r0], #1 1041 strb r3, [r0], #1 1042 strplb r12, [r1], #1 1043 bhi .Lcopyio_copy2 1044.Lcopyio_exit: 1045 movs r0, #0 1046 bx lr 1047copyio_no_perms: 1048 mov r0, #0xE 1049 bx lr 1050 1051 1052/** 1053 * copyinframe 1054 * 1055 * Safely copy in a fp and lr. 1056 */ 1057EnterARM(copyinframe) 1058 stmfd sp!,{r4} 1059 adr r3, _copyinframe_error 1060 LoadThreadRegister(r12) 1061 ldr r4, [r12, TH_RECOVER] 1062 str r3, [r12, TH_RECOVER] 1063 ldmia r0, {r2,r3} 1064 stmia r1, {r2,r3} 1065_copyinframe_success: 1066 mov r0, #0 1067 LoadThreadRegister(r12) 1068 ldmfd sp!, {r4} 1069 bx lr 1070_copyinframe_error: 1071 mov r0, #0xE 1072 bx lr 1073 1074LOAD_ADDR_GEN_DEF(kernel_pmap) 1075 1076