1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD AND BSD-4-Clause 3 * 4 * Copyright (C) 2002 Benno Rice 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26*/ 27/*- 28 * Copyright (C) 1993 Wolfgang Solfrank. 29 * Copyright (C) 1993 TooLs GmbH. 30 * All rights reserved. 31 * 32 * Redistribution and use in source and binary forms, with or without 33 * modification, are permitted provided that the following conditions 34 * are met: 35 * 1. Redistributions of source code must retain the above copyright 36 * notice, this list of conditions and the following disclaimer. 37 * 2. Redistributions in binary form must reproduce the above copyright 38 * notice, this list of conditions and the following disclaimer in the 39 * documentation and/or other materials provided with the distribution. 40 * 3. All advertising materials mentioning features or use of this software 41 * must display the following acknowledgement: 42 * This product includes software developed by TooLs GmbH. 43 * 4. The name of TooLs GmbH may not be used to endorse or promote products 44 * derived from this software without specific prior written permission. 45 * 46 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR 47 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 48 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 49 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 50 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 51 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 52 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 53 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 54 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 55 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 56 */ 57 58#include <sys/cdefs.h> 59__FBSDID("$FreeBSD$"); 60 61#include <sys/param.h> 62#include <sys/lock.h> 63#include <sys/mutex.h> 64#include <sys/systm.h> 65#include <sys/proc.h> 66 67#include <vm/vm.h> 68#include <vm/pmap.h> 69#include <vm/vm_extern.h> 70#include <vm/vm_map.h> 71 72#include <machine/mmuvar.h> 73#include <machine/pcb.h> 74#include <machine/vmparam.h> 75#include <machine/ifunc.h> 76 77/* 78 * On powerpc64 (AIM only) the copy functions are IFUNCs, selecting the best 79 * option based on the PMAP in use. 80 * 81 * There are two options for copy functions on powerpc64: 82 * - 'remap' copies, which remap userspace segments into kernel space for 83 * copying. This is used by the 'oea64' pmap. 84 * - 'direct' copies, which copy directly from userspace. This does not require 85 * remapping user segments into kernel. This is used by the 'radix' pmap for 86 * performance. 87 * 88 * Book-E does not use the C 'remap' functions, opting instead to use the 89 * 'direct' copies, directly, avoiding the IFUNC overhead. 90 * 91 * On 32-bit AIM these functions bypass the IFUNC machinery for performance. 92 */ 93#ifdef __powerpc64__ 94int subyte_remap(volatile void *addr, int byte); 95int subyte_direct(volatile void *addr, int byte); 96int copyinstr_remap(const void *udaddr, void *kaddr, size_t len, size_t *done); 97int copyinstr_direct(const void *udaddr, void *kaddr, size_t len, size_t *done); 98int copyout_remap(const void *kaddr, void *udaddr, size_t len); 99int copyout_direct(const void *kaddr, void *udaddr, size_t len); 100int copyin_remap(const void *uaddr, void *kaddr, size_t len); 101int copyin_direct(const void *uaddr, void *kaddr, size_t len); 102int suword32_remap(volatile void *addr, int word); 103int suword32_direct(volatile void *addr, int word); 104int suword_remap(volatile void *addr, long word); 105int suword_direct(volatile void *addr, long word); 106int suword64_remap(volatile void *addr, int64_t word); 107int suword64_direct(volatile void *addr, int64_t word); 108int fubyte_remap(volatile const void *addr); 109int fubyte_direct(volatile const void *addr); 110int fuword16_remap(volatile const void *addr); 111int fuword16_direct(volatile const void *addr); 112int fueword32_remap(volatile const void *addr, int32_t *val); 113int fueword32_direct(volatile const void *addr, int32_t *val); 114int fueword64_remap(volatile const void *addr, int64_t *val); 115int fueword64_direct(volatile const void *addr, int64_t *val); 116int fueword_remap(volatile const void *addr, long *val); 117int fueword_direct(volatile const void *addr, long *val); 118int casueword32_remap(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 119 uint32_t new); 120int casueword32_direct(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 121 uint32_t new); 122int casueword_remap(volatile u_long *addr, u_long old, u_long *oldvalp, 123 u_long new); 124int casueword_direct(volatile u_long *addr, u_long old, u_long *oldvalp, 125 u_long new); 126 127/* 128 * The IFUNC resolver determines the copy based on whether the PMAP 129 * implementation includes a pmap_map_user_ptr function. 130 */ 131#define DEFINE_COPY_FUNC(ret, func, args) \ 132 DEFINE_IFUNC(, ret, func, args) \ 133 { \ 134 return (PMAP_RESOLVE_FUNC(map_user_ptr) ? \ 135 func##_remap : func##_direct); \ 136 } 137DEFINE_COPY_FUNC(int, subyte, (volatile void *, int)) 138DEFINE_COPY_FUNC(int, copyinstr, (const void *, void *, size_t, size_t *)) 139DEFINE_COPY_FUNC(int, copyin, (const void *, void *, size_t)) 140DEFINE_COPY_FUNC(int, copyout, (const void *, void *, size_t)) 141DEFINE_COPY_FUNC(int, suword, (volatile void *, long)) 142DEFINE_COPY_FUNC(int, suword32, (volatile void *, int)) 143DEFINE_COPY_FUNC(int, suword64, (volatile void *, int64_t)) 144DEFINE_COPY_FUNC(int, fubyte, (volatile const void *)) 145DEFINE_COPY_FUNC(int, fuword16, (volatile const void *)) 146DEFINE_COPY_FUNC(int, fueword32, (volatile const void *, int32_t *)) 147DEFINE_COPY_FUNC(int, fueword64, (volatile const void *, int64_t *)) 148DEFINE_COPY_FUNC(int, fueword, (volatile const void *, long *)) 149DEFINE_COPY_FUNC(int, casueword32, 150 (volatile uint32_t *, uint32_t, uint32_t *, uint32_t)) 151DEFINE_COPY_FUNC(int, casueword, (volatile u_long *, u_long, u_long *, u_long)) 152 153#define REMAP(x) x##_remap 154#else 155#define REMAP(x) x 156#endif 157 158int 159REMAP(copyout)(const void *kaddr, void *udaddr, size_t len) 160{ 161 struct thread *td; 162 pmap_t pm; 163 jmp_buf env; 164 const char *kp; 165 char *up, *p; 166 size_t l; 167 168 td = curthread; 169 pm = &td->td_proc->p_vmspace->vm_pmap; 170 171 td->td_pcb->pcb_onfault = &env; 172 if (setjmp(env)) { 173 td->td_pcb->pcb_onfault = NULL; 174 return (EFAULT); 175 } 176 177 kp = kaddr; 178 up = udaddr; 179 180 while (len > 0) { 181 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 182 td->td_pcb->pcb_onfault = NULL; 183 return (EFAULT); 184 } 185 186 bcopy(kp, p, l); 187 188 up += l; 189 kp += l; 190 len -= l; 191 } 192 193 td->td_pcb->pcb_onfault = NULL; 194 return (0); 195} 196 197int 198REMAP(copyin)(const void *udaddr, void *kaddr, size_t len) 199{ 200 struct thread *td; 201 pmap_t pm; 202 jmp_buf env; 203 const char *up; 204 char *kp, *p; 205 size_t l; 206 207 td = curthread; 208 pm = &td->td_proc->p_vmspace->vm_pmap; 209 210 td->td_pcb->pcb_onfault = &env; 211 if (setjmp(env)) { 212 td->td_pcb->pcb_onfault = NULL; 213 return (EFAULT); 214 } 215 216 kp = kaddr; 217 up = udaddr; 218 219 while (len > 0) { 220 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 221 td->td_pcb->pcb_onfault = NULL; 222 return (EFAULT); 223 } 224 225 bcopy(p, kp, l); 226 227 up += l; 228 kp += l; 229 len -= l; 230 } 231 232 td->td_pcb->pcb_onfault = NULL; 233 return (0); 234} 235 236int 237REMAP(copyinstr)(const void *udaddr, void *kaddr, size_t len, size_t *done) 238{ 239 struct thread *td; 240 pmap_t pm; 241 jmp_buf env; 242 const char *up; 243 char *kp, *p; 244 size_t i, l, t; 245 int rv; 246 247 td = curthread; 248 pm = &td->td_proc->p_vmspace->vm_pmap; 249 250 t = 0; 251 rv = ENAMETOOLONG; 252 253 td->td_pcb->pcb_onfault = &env; 254 if (setjmp(env)) { 255 rv = EFAULT; 256 goto done; 257 } 258 259 kp = kaddr; 260 up = udaddr; 261 262 while (len > 0) { 263 if (pmap_map_user_ptr(pm, up, (void **)&p, len, &l)) { 264 rv = EFAULT; 265 goto done; 266 } 267 268 for (i = 0; len > 0 && i < l; i++, t++, len--) { 269 if ((*kp++ = *p++) == 0) { 270 i++, t++; 271 rv = 0; 272 goto done; 273 } 274 } 275 276 up += l; 277 } 278 279done: 280 td->td_pcb->pcb_onfault = NULL; 281 282 if (done != NULL) { 283 *done = t; 284 } 285 286 return (rv); 287} 288 289int 290REMAP(subyte)(volatile void *addr, int byte) 291{ 292 struct thread *td; 293 pmap_t pm; 294 jmp_buf env; 295 char *p; 296 297 td = curthread; 298 pm = &td->td_proc->p_vmspace->vm_pmap; 299 300 td->td_pcb->pcb_onfault = &env; 301 if (setjmp(env)) { 302 td->td_pcb->pcb_onfault = NULL; 303 return (-1); 304 } 305 306 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 307 td->td_pcb->pcb_onfault = NULL; 308 return (-1); 309 } 310 311 *p = (char)byte; 312 313 td->td_pcb->pcb_onfault = NULL; 314 return (0); 315} 316 317#ifdef __powerpc64__ 318int 319REMAP(suword32)(volatile void *addr, int word) 320{ 321 struct thread *td; 322 pmap_t pm; 323 jmp_buf env; 324 int *p; 325 326 td = curthread; 327 pm = &td->td_proc->p_vmspace->vm_pmap; 328 329 td->td_pcb->pcb_onfault = &env; 330 if (setjmp(env)) { 331 td->td_pcb->pcb_onfault = NULL; 332 return (-1); 333 } 334 335 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 336 td->td_pcb->pcb_onfault = NULL; 337 return (-1); 338 } 339 340 *p = word; 341 342 td->td_pcb->pcb_onfault = NULL; 343 return (0); 344} 345#else 346int 347REMAP(suword32)(volatile void *addr, int32_t word) 348{ 349REMAP( return (suword)(addr, (long)word)); 350} 351#endif 352 353int 354REMAP(suword)(volatile void *addr, long word) 355{ 356 struct thread *td; 357 pmap_t pm; 358 jmp_buf env; 359 long *p; 360 361 td = curthread; 362 pm = &td->td_proc->p_vmspace->vm_pmap; 363 364 td->td_pcb->pcb_onfault = &env; 365 if (setjmp(env)) { 366 td->td_pcb->pcb_onfault = NULL; 367 return (-1); 368 } 369 370 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 371 td->td_pcb->pcb_onfault = NULL; 372 return (-1); 373 } 374 375 *p = word; 376 377 td->td_pcb->pcb_onfault = NULL; 378 return (0); 379} 380 381#ifdef __powerpc64__ 382int 383REMAP(suword64)(volatile void *addr, int64_t word) 384{ 385 return (REMAP(suword)(addr, (long)word)); 386} 387#endif 388 389int 390REMAP(fubyte)(volatile const void *addr) 391{ 392 struct thread *td; 393 pmap_t pm; 394 jmp_buf env; 395 u_char *p; 396 int val; 397 398 td = curthread; 399 pm = &td->td_proc->p_vmspace->vm_pmap; 400 401 td->td_pcb->pcb_onfault = &env; 402 if (setjmp(env)) { 403 td->td_pcb->pcb_onfault = NULL; 404 return (-1); 405 } 406 407 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 408 td->td_pcb->pcb_onfault = NULL; 409 return (-1); 410 } 411 412 val = *p; 413 414 td->td_pcb->pcb_onfault = NULL; 415 return (val); 416} 417 418int 419REMAP(fuword16)(volatile const void *addr) 420{ 421 struct thread *td; 422 pmap_t pm; 423 jmp_buf env; 424 uint16_t *p, val; 425 426 td = curthread; 427 pm = &td->td_proc->p_vmspace->vm_pmap; 428 429 td->td_pcb->pcb_onfault = &env; 430 if (setjmp(env)) { 431 td->td_pcb->pcb_onfault = NULL; 432 return (-1); 433 } 434 435 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 436 td->td_pcb->pcb_onfault = NULL; 437 return (-1); 438 } 439 440 val = *p; 441 442 td->td_pcb->pcb_onfault = NULL; 443 return (val); 444} 445 446int 447REMAP(fueword32)(volatile const void *addr, int32_t *val) 448{ 449 struct thread *td; 450 pmap_t pm; 451 jmp_buf env; 452 int32_t *p; 453 454 td = curthread; 455 pm = &td->td_proc->p_vmspace->vm_pmap; 456 457 td->td_pcb->pcb_onfault = &env; 458 if (setjmp(env)) { 459 td->td_pcb->pcb_onfault = NULL; 460 return (-1); 461 } 462 463 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 464 td->td_pcb->pcb_onfault = NULL; 465 return (-1); 466 } 467 468 *val = *p; 469 470 td->td_pcb->pcb_onfault = NULL; 471 return (0); 472} 473 474#ifdef __powerpc64__ 475int 476REMAP(fueword64)(volatile const void *addr, int64_t *val) 477{ 478 struct thread *td; 479 pmap_t pm; 480 jmp_buf env; 481 int64_t *p; 482 483 td = curthread; 484 pm = &td->td_proc->p_vmspace->vm_pmap; 485 486 td->td_pcb->pcb_onfault = &env; 487 if (setjmp(env)) { 488 td->td_pcb->pcb_onfault = NULL; 489 return (-1); 490 } 491 492 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 493 td->td_pcb->pcb_onfault = NULL; 494 return (-1); 495 } 496 497 *val = *p; 498 499 td->td_pcb->pcb_onfault = NULL; 500 return (0); 501} 502#endif 503 504int 505REMAP(fueword)(volatile const void *addr, long *val) 506{ 507 struct thread *td; 508 pmap_t pm; 509 jmp_buf env; 510 long *p; 511 512 td = curthread; 513 pm = &td->td_proc->p_vmspace->vm_pmap; 514 515 td->td_pcb->pcb_onfault = &env; 516 if (setjmp(env)) { 517 td->td_pcb->pcb_onfault = NULL; 518 return (-1); 519 } 520 521 if (pmap_map_user_ptr(pm, addr, (void **)&p, sizeof(*p), NULL)) { 522 td->td_pcb->pcb_onfault = NULL; 523 return (-1); 524 } 525 526 *val = *p; 527 528 td->td_pcb->pcb_onfault = NULL; 529 return (0); 530} 531 532int 533REMAP(casueword32)(volatile uint32_t *addr, uint32_t old, uint32_t *oldvalp, 534 uint32_t new) 535{ 536 struct thread *td; 537 pmap_t pm; 538 jmp_buf env; 539 uint32_t *p, val; 540 int res; 541 542 td = curthread; 543 pm = &td->td_proc->p_vmspace->vm_pmap; 544 545 td->td_pcb->pcb_onfault = &env; 546 if (setjmp(env)) { 547 td->td_pcb->pcb_onfault = NULL; 548 return (-1); 549 } 550 551 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 552 sizeof(*p), NULL)) { 553 td->td_pcb->pcb_onfault = NULL; 554 return (-1); 555 } 556 557 res = 0; 558 __asm __volatile ( 559 "lwarx %0, 0, %3\n\t" /* load old value */ 560 "cmplw %4, %0\n\t" /* compare */ 561 "bne 1f\n\t" /* exit if not equal */ 562 "stwcx. %5, 0, %3\n\t" /* attempt to store */ 563 "bne- 2f\n\t" /* if failed */ 564 "b 3f\n\t" /* we've succeeded */ 565 "1:\n\t" 566 "stwcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 567 "2:li %2, 1\n\t" 568 "3:\n\t" 569 : "=&r" (val), "=m" (*p), "+&r" (res) 570 : "r" (p), "r" (old), "r" (new), "m" (*p) 571 : "cr0", "memory"); 572 573 td->td_pcb->pcb_onfault = NULL; 574 575 *oldvalp = val; 576 return (res); 577} 578 579#ifndef __powerpc64__ 580int 581REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 582{ 583 584 return (casueword32((volatile uint32_t *)addr, old, 585 (uint32_t *)oldvalp, new)); 586} 587#else 588int 589REMAP(casueword)(volatile u_long *addr, u_long old, u_long *oldvalp, u_long new) 590{ 591 struct thread *td; 592 pmap_t pm; 593 jmp_buf env; 594 u_long *p, val; 595 int res; 596 597 td = curthread; 598 pm = &td->td_proc->p_vmspace->vm_pmap; 599 600 td->td_pcb->pcb_onfault = &env; 601 if (setjmp(env)) { 602 td->td_pcb->pcb_onfault = NULL; 603 return (-1); 604 } 605 606 if (pmap_map_user_ptr(pm, (void *)(uintptr_t)addr, (void **)&p, 607 sizeof(*p), NULL)) { 608 td->td_pcb->pcb_onfault = NULL; 609 return (-1); 610 } 611 612 res = 0; 613 __asm __volatile ( 614 "ldarx %0, 0, %3\n\t" /* load old value */ 615 "cmpld %4, %0\n\t" /* compare */ 616 "bne 1f\n\t" /* exit if not equal */ 617 "stdcx. %5, 0, %3\n\t" /* attempt to store */ 618 "bne- 2f\n\t" /* if failed */ 619 "b 3f\n\t" /* we've succeeded */ 620 "1:\n\t" 621 "stdcx. %0, 0, %3\n\t" /* clear reservation (74xx) */ 622 "2:li %2, 1\n\t" 623 "3:\n\t" 624 : "=&r" (val), "=m" (*p), "+&r" (res) 625 : "r" (p), "r" (old), "r" (new), "m" (*p) 626 : "cr0", "memory"); 627 628 td->td_pcb->pcb_onfault = NULL; 629 630 *oldvalp = val; 631 return (res); 632} 633#endif 634