1/*	$NetBSD: trap.c,v 1.102 2023/10/05 19:41:05 ad Exp $	*/
2
3/*
4 * Copyright 2001 Wasabi Systems, Inc.
5 * All rights reserved.
6 *
7 * Written by Eduardo Horvath and Simon Burge for Wasabi Systems, Inc.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 *    notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 *    notice, this list of conditions and the following disclaimer in the
16 *    documentation and/or other materials provided with the distribution.
17 * 3. All advertising materials mentioning features or use of this software
18 *    must display the following acknowledgement:
19 *      This product includes software developed for the NetBSD Project by
20 *      Wasabi Systems, Inc.
21 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
22 *    or promote products derived from this software without specific prior
23 *    written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
27 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL WASABI SYSTEMS, INC
29 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
30 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
31 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
32 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
33 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
34 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
35 * POSSIBILITY OF SUCH DAMAGE.
36 */
37
38/*
39 * Copyright (C) 1995, 1996 Wolfgang Solfrank.
40 * Copyright (C) 1995, 1996 TooLs GmbH.
41 * All rights reserved.
42 *
43 * Redistribution and use in source and binary forms, with or without
44 * modification, are permitted provided that the following conditions
45 * are met:
46 * 1. Redistributions of source code must retain the above copyright
47 *    notice, this list of conditions and the following disclaimer.
48 * 2. Redistributions in binary form must reproduce the above copyright
49 *    notice, this list of conditions and the following disclaimer in the
50 *    documentation and/or other materials provided with the distribution.
51 * 3. All advertising materials mentioning features or use of this software
52 *    must display the following acknowledgement:
53 *	This product includes software developed by TooLs GmbH.
54 * 4. The name of TooLs GmbH may not be used to endorse or promote products
55 *    derived from this software without specific prior written permission.
56 *
57 * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
58 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
59 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
60 * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
61 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
62 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
63 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
64 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
65 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
66 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
67 */
68
69#define	__UFETCHSTORE_PRIVATE
70
71#include <sys/cdefs.h>
72__KERNEL_RCSID(0, "$NetBSD: trap.c,v 1.102 2023/10/05 19:41:05 ad Exp $");
73
74#ifdef _KERNEL_OPT
75#include "opt_ddb.h"
76#include "opt_kgdb.h"
77#include "opt_ppcarch.h"
78#include "opt_ppcopts.h"
79#endif
80
81#include <sys/param.h>
82#include <sys/cpu.h>
83#include <sys/kauth.h>
84#include <sys/proc.h>
85#include <sys/ptrace.h>
86#include <sys/reboot.h>
87#include <sys/syscall.h>
88#include <sys/systm.h>
89
90#if defined(KGDB)
91#include <sys/kgdb.h>
92#endif
93
94#include <uvm/uvm_extern.h>
95
96#include <dev/cons.h>
97
98#include <machine/fpu.h>
99#include <machine/frame.h>
100#include <machine/pcb.h>
101#include <machine/psl.h>
102#include <machine/trap.h>
103
104#include <powerpc/db_machdep.h>
105#include <powerpc/spr.h>
106#include <powerpc/userret.h>
107
108#include <powerpc/ibm4xx/cpu.h>
109#include <powerpc/ibm4xx/pmap.h>
110#include <powerpc/ibm4xx/spr.h>
111#include <powerpc/ibm4xx/tlb.h>
112
113#include <powerpc/fpu/fpu_extern.h>
114
115/* These definitions should probably be somewhere else			XXX */
116#define	FIRSTARG	3		/* first argument is in reg 3 */
117#define	NARGREG		8		/* 8 args are in registers */
118#define	MOREARGS(sp)	((void *)((int)(sp) + 8)) /* more args go here */
119
120void trap(struct trapframe *);	/* Called from locore / trap_subr */
121#if 0
122/* Not currently used nor exposed externally in any header file */
123int badaddr(void *, size_t);
124int badaddr_read(void *, size_t, int *);
125#endif
126int ctx_setup(int, int);
127
128#ifndef PPC_NO_UNALIGNED
129static bool fix_unaligned(struct trapframe *, ksiginfo_t *);
130#endif
131
132#ifdef DEBUG
133#define TDB_ALL	0x1
134int trapdebug = /* TDB_ALL */ 0;
135#define	DBPRINTF(x, y)	if (trapdebug & (x)) printf y
136#else
137#define DBPRINTF(x, y)
138#endif
139
140void
141trap(struct trapframe *tf)
142{
143	struct lwp *l = curlwp;
144	struct proc *p = l->l_proc;
145	struct pcb *pcb;
146	int type = tf->tf_exc;
147	int ftype, rv;
148	ksiginfo_t ksi;
149
150	KASSERT(l->l_stat == LSONPROC);
151
152	if (tf->tf_srr1 & PSL_PR) {
153		type |= EXC_USER;
154	}
155
156	ftype = VM_PROT_READ;
157
158	DBPRINTF(TDB_ALL, ("trap(%x) at %lx from frame %p &frame %p\n",
159	    type, tf->tf_srr0, tf, &tf));
160
161	switch (type) {
162	case EXC_DEBUG|EXC_USER:
163		/* We don't use hardware breakpoints for userland. */
164		goto brain_damage;
165
166	case EXC_TRC|EXC_USER:
167		KSI_INIT_TRAP(&ksi);
168		ksi.ksi_signo = SIGTRAP;
169		ksi.ksi_trap = EXC_TRC;
170		ksi.ksi_addr = (void *)tf->tf_srr0;
171		trapsignal(l, &ksi);
172		break;
173
174	case EXC_DSI:
175		/* FALLTHROUGH */
176	case EXC_DTMISS:
177		{
178			struct vm_map *map;
179			vaddr_t va;
180			struct faultbuf *fb;
181
182			pcb = lwp_getpcb(l);
183			fb = pcb->pcb_onfault;
184
185			if (curcpu()->ci_idepth >= 0) {
186				rv = EFAULT;
187				goto out;
188			}
189
190			va = tf->tf_dear;
191			if (tf->tf_pid == KERNEL_PID) {
192				map = kernel_map;
193			} else {
194				map = &p->p_vmspace->vm_map;
195			}
196
197			if (tf->tf_esr & (ESR_DST|ESR_DIZ))
198				ftype = VM_PROT_WRITE;
199
200			DBPRINTF(TDB_ALL,
201			    ("trap(EXC_DSI) at %lx %s fault on %p esr %x\n",
202			    tf->tf_srr0,
203			    (ftype & VM_PROT_WRITE) ? "write" : "read",
204			    (void *)va, tf->tf_esr));
205
206			pcb->pcb_onfault = NULL;
207			rv = uvm_fault(map, trunc_page(va), ftype);
208			pcb->pcb_onfault = fb;
209			if (rv == 0)
210				return;
211out:
212			if (fb != NULL) {
213				tf->tf_pid = KERNEL_PID;
214				tf->tf_srr0 = fb->fb_pc;
215				tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */
216				tf->tf_cr = fb->fb_cr;
217				tf->tf_fixreg[1] = fb->fb_sp;
218				tf->tf_fixreg[2] = fb->fb_r2;
219				tf->tf_fixreg[3] = rv;
220				memcpy(&tf->tf_fixreg[13], fb->fb_fixreg,
221				    sizeof(fb->fb_fixreg));
222				return;
223			}
224		}
225		goto brain_damage;
226
227	case EXC_DSI|EXC_USER:
228		/* FALLTHROUGH */
229	case EXC_DTMISS|EXC_USER:
230		if (tf->tf_esr & (ESR_DST|ESR_DIZ))
231			ftype = VM_PROT_WRITE;
232
233		DBPRINTF(TDB_ALL,
234		    ("trap(EXC_DSI|EXC_USER) at %lx %s fault on %lx %x\n",
235		    tf->tf_srr0, (ftype & VM_PROT_WRITE) ? "write" : "read",
236		    tf->tf_dear, tf->tf_esr));
237		KASSERT(l == curlwp && (l->l_stat == LSONPROC));
238//		KASSERT(curpcb->pcb_onfault == NULL);
239		rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_dear),
240		    ftype);
241		if (rv == 0) {
242			break;
243		}
244		KSI_INIT_TRAP(&ksi);
245		ksi.ksi_trap = EXC_DSI;
246		ksi.ksi_addr = (void *)tf->tf_dear;
247vm_signal:
248		switch (rv) {
249		case EINVAL:
250			ksi.ksi_signo = SIGBUS;
251			ksi.ksi_code = BUS_ADRERR;
252			break;
253		case EACCES:
254			ksi.ksi_signo = SIGSEGV;
255			ksi.ksi_code = SEGV_ACCERR;
256			break;
257		case ENOMEM:
258			ksi.ksi_signo = SIGKILL;
259			printf("UVM: pid %d.%d (%s), uid %d killed: "
260			       "out of swap\n", p->p_pid, l->l_lid, p->p_comm,
261			       l->l_cred ? kauth_cred_geteuid(l->l_cred) : -1);
262			break;
263		default:
264			ksi.ksi_signo = SIGSEGV;
265			ksi.ksi_code = SEGV_MAPERR;
266			break;
267		}
268		trapsignal(l, &ksi);
269		break;
270
271	case EXC_ITMISS|EXC_USER:
272	case EXC_ISI|EXC_USER:
273		ftype = VM_PROT_EXECUTE;
274		DBPRINTF(TDB_ALL,
275		    ("trap(EXC_ISI|EXC_USER) at %lx execute fault tf %p\n",
276		    tf->tf_srr0, tf));
277//		KASSERT(curpcb->pcb_onfault == NULL);
278		rv = uvm_fault(&p->p_vmspace->vm_map, trunc_page(tf->tf_srr0),
279		    ftype);
280		if (rv == 0) {
281			break;
282		}
283isi:
284		KSI_INIT_TRAP(&ksi);
285		ksi.ksi_trap = EXC_ISI;
286		ksi.ksi_addr = (void *)tf->tf_srr0;
287		goto vm_signal;
288		break;
289
290	case EXC_AST|EXC_USER:
291		cpu_ast(l, curcpu());
292		break;
293
294	case EXC_ALI|EXC_USER:
295		if (fix_unaligned(tf, &ksi))
296			trapsignal(l, &ksi);
297		break;
298
299	case EXC_PGM|EXC_USER:
300		curcpu()->ci_data.cpu_ntrap++;
301
302		KSI_INIT_TRAP(&ksi);
303		ksi.ksi_trap = EXC_PGM;
304		ksi.ksi_addr = (void *)tf->tf_srr0;
305
306		if (tf->tf_esr & ESR_PTR) {
307			vaddr_t va;
308sigtrap:
309			va = (vaddr_t)tf->tf_srr0;
310			/*
311		 	 * Restore original instruction and clear BP.
312		 	 */
313			if (p->p_md.md_ss_addr[0] == va ||
314			    p->p_md.md_ss_addr[1] == va) {
315				rv = ppc_sstep(l, 0);
316				if (rv != 0)
317					goto vm_signal;
318				ksi.ksi_code = TRAP_TRACE;
319			} else
320				ksi.ksi_code = TRAP_BRKPT;
321			if (p->p_raslist != NULL &&
322			    ras_lookup(p, (void *)va) != (void *)-1) {
323				tf->tf_srr0 += (ksi.ksi_code == TRAP_TRACE) ?
324				    0 : 4;
325				break;
326			}
327			ksi.ksi_signo = SIGTRAP;
328		} else if (tf->tf_esr & ESR_PPR) {
329			uint32_t opcode;
330
331			rv = copyin((void *)tf->tf_srr0, &opcode,
332			    sizeof(opcode));
333			if (rv)
334				goto isi;
335			if (emulate_mxmsr(l, tf, opcode)) {
336				tf->tf_srr0 += 4;
337				break;
338			}
339
340			ksi.ksi_code = ILL_PRVOPC;
341			ksi.ksi_signo = SIGILL;
342		} else {
343			pcb = lwp_getpcb(l);
344
345			if (__predict_false(!fpu_used_p(l))) {
346				memset(&pcb->pcb_fpu, 0, sizeof(pcb->pcb_fpu));
347				fpu_mark_used(l);
348			}
349
350			if (fpu_emulate(tf, &pcb->pcb_fpu, &ksi)) {
351				if (ksi.ksi_signo == 0)	/* was emulated */
352					break;
353				else if (ksi.ksi_signo == SIGTRAP)
354					goto sigtrap;	/* XXX H/W bug? */
355			} else {
356				ksi.ksi_code = ILL_ILLOPC;
357				ksi.ksi_signo = SIGILL;
358			}
359		}
360
361		trapsignal(l, &ksi);
362		break;
363
364	case EXC_MCHK:
365		{
366			struct faultbuf *fb;
367
368			pcb = lwp_getpcb(l);
369			if ((fb = pcb->pcb_onfault) != NULL) {
370				tf->tf_pid = KERNEL_PID;
371				tf->tf_srr0 = fb->fb_pc;
372				tf->tf_srr1 |= PSL_IR; /* Re-enable IMMU */
373				tf->tf_fixreg[1] = fb->fb_sp;
374				tf->tf_fixreg[2] = fb->fb_r2;
375				tf->tf_fixreg[3] = 1; /* Return TRUE */
376				tf->tf_cr = fb->fb_cr;
377				memcpy(&tf->tf_fixreg[13], fb->fb_fixreg,
378				    sizeof(fb->fb_fixreg));
379				return;
380			}
381		}
382		goto brain_damage;
383
384	default:
385brain_damage:
386		printf("trap type 0x%x at 0x%lx\n", type, tf->tf_srr0);
387#if defined(DDB) || defined(KGDB)
388		if (kdb_trap(type, tf))
389			return;
390#endif
391#ifdef TRAP_PANICWAIT
392		printf("Press a key to panic.\n");
393		cngetc();
394#endif
395		panic("trap");
396	}
397
398	/* Invoke powerpc userret code */
399	userret(l, tf);
400}
401
402int
403ctx_setup(int ctx, int srr1)
404{
405	volatile struct pmap *pm;
406
407	/* Update PID if we're returning to user mode. */
408	if (srr1 & PSL_PR) {
409		pm = curproc->p_vmspace->vm_map.pmap;
410		if (!pm->pm_ctx) {
411			ctx_alloc(__UNVOLATILE(pm));
412		}
413		ctx = pm->pm_ctx;
414	}
415	else if (!ctx) {
416		ctx = KERNEL_PID;
417	}
418	return (ctx);
419}
420
421/*
422 * Used by copyin()/copyout()
423 */
424extern vaddr_t vmaprange(struct proc *, vaddr_t, vsize_t, int);
425extern void vunmaprange(vaddr_t, vsize_t);
426static int bigcopyin(const void *, void *, size_t );
427static int bigcopyout(const void *, void *, size_t );
428
429#ifdef __clang__
430#pragma clang optimize off
431#endif
432int
433copyin(const void *uaddr, void *kaddr, size_t len)
434{
435	struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
436	int rv, msr, pid, tmp, ctx;
437	struct faultbuf env;
438
439	/* For bigger buffers use the faster copy */
440	if (len > 1024)
441		return (bigcopyin(uaddr, kaddr, len));
442
443	if ((rv = setfault(&env))) {
444		curpcb->pcb_onfault = NULL;
445		return rv;
446	}
447
448	if (!(ctx = pm->pm_ctx)) {
449		/* No context -- assign it one */
450		ctx_alloc(pm);
451		ctx = pm->pm_ctx;
452	}
453
454	__asm volatile (
455		"mfmsr	%[msr];"		/* Save MSR */
456		"li	%[tmp],0x20;"		/* Disable IMMU */
457		"andc	%[tmp],%[msr],%[tmp];"
458		"mtmsr	%[tmp];"
459		"isync;"
460		MFPID(%[pid])			/* Save old PID */
461
462		"srwi.	%[tmp],%[len],0x2;"	/* How many words? */
463		"beq-	2f;"			/* No words. Go do bytes */
464		"mtctr	%[tmp];"
465
466	"1:"	MTPID(%[ctx])
467		"isync;"
468#ifdef PPC_IBM403
469		"lswi	%[tmp],%[uaddr],4;"	/* Load user word */
470#else
471		"lwz	%[tmp],0(%[uaddr]);"
472#endif
473		"addi	%[uaddr],%[uaddr],0x4;"	/* next uaddr word */
474		"sync;"
475
476		MTPID(%[pid])
477		"isync;"
478#ifdef PPC_IBM403
479		"stswi	%[tmp],%[kaddr],4;"	/* Store kernel word */
480#else
481		"stw	%[tmp],0(%[kaddr]);"
482#endif
483		"addi	%[kaddr],%[kaddr],0x4;"	/* next kaddr word */
484		"sync;"
485		"bdnz	1b;"			/* repeat */
486
487	"2:"	"andi.	%[tmp],%[len],0x3;"	/* How many remaining bytes? */
488		"beq	10f;"
489		"mtxer	%[tmp];"
490
491		MTPID(%[ctx])
492		"isync;"
493		"lswx	%[tmp],0,%[uaddr];"	/* Load user bytes */
494		"sync;"
495
496		MTPID(%[pid])
497		"isync;"
498		"stswx	%[tmp],0,%[kaddr];"	/* Store kernel bytes */
499		"sync;"
500
501	"10:"	"mtmsr	%[msr];"		/* Restore MSR */
502		"isync;"
503
504		: [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
505		: [uaddr] "b" (uaddr), [kaddr] "b" (kaddr),
506		  [ctx] "r" (ctx), [len] "r" (len)
507		: "cr0", "ctr", "xer");
508
509	curpcb->pcb_onfault = NULL;
510	return 0;
511}
512#ifdef __clang__
513#pragma clang optimize on
514#endif
515
516static int
517bigcopyin(const void *uaddr, void *kaddr, size_t len)
518{
519	const char *up;
520	char *kp = kaddr;
521	struct lwp *l = curlwp;
522	struct proc *p;
523	struct faultbuf env;
524	int error;
525
526	p = l->l_proc;
527
528	/*
529	 * Stolen from physio():
530	 */
531	error = uvm_vslock(p->p_vmspace, __UNCONST(uaddr), len, VM_PROT_READ);
532	if (error) {
533		return error;
534	}
535	up = (char *)vmaprange(p, (vaddr_t)uaddr, len, VM_PROT_READ);
536
537	if ((error = setfault(&env)) == 0) {
538		memcpy(kp, up, len);
539	}
540
541	curpcb->pcb_onfault = NULL;
542	vunmaprange((vaddr_t)up, len);
543	uvm_vsunlock(p->p_vmspace, __UNCONST(uaddr), len);
544
545	return error;
546}
547
548#ifdef __clang__
549#pragma clang optimize off
550#endif
551int
552copyout(const void *kaddr, void *uaddr, size_t len)
553{
554	struct pmap *pm = curproc->p_vmspace->vm_map.pmap;
555	int rv, msr, pid, tmp, ctx;
556	struct faultbuf env;
557
558	/* For big copies use more efficient routine */
559	if (len > 1024)
560		return (bigcopyout(kaddr, uaddr, len));
561
562	if ((rv = setfault(&env))) {
563		curpcb->pcb_onfault = NULL;
564		return rv;
565	}
566
567	if (!(ctx = pm->pm_ctx)) {
568		/* No context -- assign it one */
569		ctx_alloc(pm);
570		ctx = pm->pm_ctx;
571	}
572
573	__asm volatile (
574		"mfmsr	%[msr];"		/* Save MSR */
575		"li	%[tmp],0x20;"		/* Disable IMMU */
576		"andc	%[tmp],%[msr],%[tmp];"
577		"mtmsr	%[tmp];"
578		"isync;"
579		MFPID(%[pid])			/* Save old PID */
580
581		"srwi.	%[tmp],%[len],0x2;"	/* How many words? */
582		"beq-	2f;"			/* No words. Go do bytes */
583		"mtctr	%[tmp];"
584
585	"1:"
586#ifdef PPC_IBM403
587		"lswi	%[tmp],%[kaddr],4;"	/* Load kernel word */
588#else
589		"lwz	%[tmp],0(%[kaddr]);"
590#endif
591		"addi	%[kaddr],%[kaddr],0x4;"	/* next kaddr word */
592		"sync;"
593
594		MTPID(%[ctx])
595		"isync;"
596#ifdef PPC_IBM403
597		"stswi	%[tmp],%[uaddr],4;"	/* Store user word */
598#else
599		"stw	%[tmp],0(%[uaddr]);"
600#endif
601		"addi	%[uaddr],%[uaddr],0x4;"	/* next uaddr word */
602		"sync;"
603
604		MTPID(%[pid])
605		"isync;"
606		"bdnz	1b;"			/* repeat */
607
608	"2:"	"andi.	%[tmp],%[len],0x3;"	/* How many remaining bytes? */
609		"beq	10f;"
610		"mtxer	%[tmp];"
611
612		"lswx	%[tmp],0,%[kaddr];"	/* Load kernel bytes */
613		"sync;"
614
615		MTPID(%[ctx])
616		"isync;"
617		"stswx	%[tmp],0,%[uaddr];"	/* Store user bytes */
618		"sync;"
619
620		MTPID(%[pid])			/* Restore PID and MSR */
621	"10:"	"mtmsr	%[msr];"
622		"isync;"
623
624		: [msr] "=&r" (msr), [pid] "=&r" (pid), [tmp] "=&r" (tmp)
625		: [uaddr] "b" (uaddr), [kaddr] "b" (kaddr),
626		  [ctx] "r" (ctx), [len] "r" (len)
627		: "cr0", "ctr", "xer");
628
629	curpcb->pcb_onfault = NULL;
630	return 0;
631}
632#ifdef __clang__
633#pragma clang optimize on
634#endif
635
636static int
637bigcopyout(const void *kaddr, void *uaddr, size_t len)
638{
639	char *up;
640	const char *kp = (const char *)kaddr;
641	struct lwp *l = curlwp;
642	struct proc *p;
643	struct faultbuf env;
644	int error;
645
646	p = l->l_proc;
647
648	/*
649	 * Stolen from physio():
650	 */
651	error = uvm_vslock(p->p_vmspace, uaddr, len, VM_PROT_WRITE);
652	if (error) {
653		return error;
654	}
655	up = (char *)vmaprange(p, (vaddr_t)uaddr, len,
656	    VM_PROT_READ | VM_PROT_WRITE);
657
658	if ((error = setfault(&env)) == 0) {
659		memcpy(up, kp, len);
660	}
661
662	curpcb->pcb_onfault = NULL;
663	vunmaprange((vaddr_t)up, len);
664	uvm_vsunlock(p->p_vmspace, uaddr, len);
665
666	return error;
667}
668
669/*
670 * kcopy(const void *src, void *dst, size_t len);
671 *
672 * Copy len bytes from src to dst, aborting if we encounter a fatal
673 * page fault.
674 *
675 * kcopy() _must_ save and restore the old fault handler since it is
676 * called by uiomove(), which may be in the path of servicing a non-fatal
677 * page fault.
678 */
679int
680kcopy(const void *src, void *dst, size_t len)
681{
682	struct faultbuf env, *oldfault;
683	int rv;
684
685	oldfault = curpcb->pcb_onfault;
686	if ((rv = setfault(&env))) {
687		curpcb->pcb_onfault = oldfault;
688		return rv;
689	}
690
691	memcpy(dst, src, len);
692
693	curpcb->pcb_onfault = oldfault;
694	return 0;
695}
696
697#if 0
698int
699badaddr(void *addr, size_t size)
700{
701
702	return badaddr_read(addr, size, NULL);
703}
704
705int
706badaddr_read(void *addr, size_t size, int *rptr)
707{
708	struct faultbuf env;
709	int x;
710
711	/* Get rid of any stale machine checks that have been waiting.  */
712	__asm volatile ("sync; isync");
713
714	if (setfault(&env)) {
715		curpcb->pcb_onfault = NULL;
716		__asm volatile ("sync");
717		return 1;
718	}
719
720	__asm volatile ("sync");
721
722	switch (size) {
723	case 1:
724		x = *(volatile int8_t *)addr;
725		break;
726	case 2:
727		x = *(volatile int16_t *)addr;
728		break;
729	case 4:
730		x = *(volatile int32_t *)addr;
731		break;
732	default:
733		panic("badaddr: invalid size (%d)", size);
734	}
735
736	/* Make sure we took the machine check, if we caused one. */
737	__asm volatile ("sync; isync");
738
739	curpcb->pcb_onfault = NULL;
740	__asm volatile ("sync");	/* To be sure. */
741
742	/* Use the value to avoid reorder. */
743	if (rptr)
744		*rptr = x;
745
746	return 0;
747}
748#endif
749
750#ifndef PPC_NO_UNALIGNED
751static bool
752fix_unaligned(struct trapframe *tf, ksiginfo_t *ksi)
753{
754
755	KSI_INIT_TRAP(ksi);
756	ksi->ksi_signo = SIGBUS;
757	ksi->ksi_trap = EXC_ALI;
758	ksi->ksi_addr = (void *)tf->tf_dear;
759	return true;
760}
761#endif
762
763/*
764 * XXX Extremely lame implementations of _ufetch_* / _ustore_*.  IBM 4xx
765 * experts should make versions that are good.
766 */
767
768#define UFETCH(sz)							\
769int									\
770_ufetch_ ## sz(const uint ## sz ## _t *uaddr, uint ## sz ## _t *valp)	\
771{									\
772	return copyin(uaddr, valp, sizeof(*valp));			\
773}
774
775UFETCH(8)
776UFETCH(16)
777UFETCH(32)
778
779#define USTORE(sz)							\
780int									\
781_ustore_ ## sz(uint ## sz ## _t *uaddr, uint ## sz ## _t val)		\
782{									\
783	return copyout(&val, uaddr, sizeof(val));			\
784}
785
786USTORE(8)
787USTORE(16)
788USTORE(32)
789