trap.c revision 113686
1116742Ssam/*- 2116904Ssam * Copyright (c) 2001, Jake Burkholder 3186904Ssam * Copyright (C) 1994, David Greenman 4116742Ssam * Copyright (c) 1990, 1993 5116742Ssam * The Regents of the University of California. All rights reserved. 6116742Ssam * 7116742Ssam * This code is derived from software contributed to Berkeley by 8116742Ssam * the University of Utah, and William Jolitz. 9116742Ssam * 10116904Ssam * Redistribution and use in source and binary forms, with or without 11116904Ssam * modification, are permitted provided that the following conditions 12116904Ssam * are met: 13116904Ssam * 1. Redistributions of source code must retain the above copyright 14116742Ssam * notice, this list of conditions and the following disclaimer. 15116904Ssam * 2. Redistributions in binary form must reproduce the above copyright 16116904Ssam * notice, this list of conditions and the following disclaimer in the 17116904Ssam * documentation and/or other materials provided with the distribution. 18116904Ssam * 3. All advertising materials mentioning features or use of this software 19116904Ssam * must display the following acknowledgement: 20116904Ssam * This product includes software developed by the University of 21116904Ssam * California, Berkeley and its contributors. 22116904Ssam * 4. Neither the name of the University nor the names of its contributors 23116904Ssam * may be used to endorse or promote products derived from this software 24116904Ssam * without specific prior written permission. 25116742Ssam * 26116742Ssam * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27116742Ssam * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28116742Ssam * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29116742Ssam * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30116742Ssam * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31116742Ssam * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32116742Ssam * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33178354Ssam * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34116742Ssam * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35116742Ssam * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36191746Sthompsa * SUCH DAMAGE. 37116742Ssam * 38182742Sbrooks * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 39116742Ssam * from: FreeBSD: src/sys/i386/i386/trap.c,v 1.197 2001/07/19 40116742Ssam * $FreeBSD: head/sys/sparc64/sparc64/trap.c 113686 2003-04-18 20:20:00Z jhb $ 41116742Ssam */ 42178354Ssam 43116742Ssam#include "opt_ddb.h" 44178354Ssam#include "opt_ktr.h" 45116742Ssam#include "opt_ktrace.h" 46116742Ssam 47116742Ssam#include <sys/param.h> 48178354Ssam#include <sys/kernel.h> 49190391Ssam#include <sys/bus.h> 50190391Ssam#include <sys/interrupt.h> 51190391Ssam#include <sys/ktr.h> 52116742Ssam#include <sys/lock.h> 53116742Ssam#include <sys/mutex.h> 54116742Ssam#include <sys/systm.h> 55178955Ssam#include <sys/pioctl.h> 56178955Ssam#include <sys/proc.h> 57178955Ssam#include <sys/smp.h> 58178955Ssam#include <sys/syscall.h> 59178955Ssam#include <sys/sysctl.h> 60178955Ssam#include <sys/sysent.h> 61178955Ssam#include <sys/user.h> 62178955Ssam#include <sys/vmmeter.h> 63178955Ssam#ifdef KTRACE 64188782Ssam#include <sys/uio.h> 65188782Ssam#include <sys/ktrace.h> 66178955Ssam#endif 67178955Ssam 68116742Ssam#include <vm/vm.h> 69178957Ssam#include <vm/pmap.h> 70178957Ssam#include <vm/vm_extern.h> 71178957Ssam#include <vm/vm_param.h> 72178957Ssam#include <vm/vm_kern.h> 73178957Ssam#include <vm/vm_map.h> 74178957Ssam#include <vm/vm_page.h> 75178957Ssam 76178957Ssam#include <machine/clock.h> 77195618Srpaulo#include <machine/cpu.h> 78195618Srpaulo#include <machine/frame.h> 79195618Srpaulo#include <machine/intr_machdep.h> 80178957Ssam#include <machine/pcb.h> 81178957Ssam#include <machine/smp.h> 82178354Ssam#include <machine/trap.h> 83178354Ssam#include <machine/tstate.h> 84116742Ssam#include <machine/tte.h> 85178354Ssam#include <machine/tlb.h> 86193655Ssam#include <machine/tsb.h> 87178354Ssam#include <machine/watch.h> 88178354Ssam 89178354Ssamvoid trap(struct trapframe *tf); 90178354Ssamvoid syscall(struct trapframe *tf); 91178354Ssam 92178354Ssamstatic int trap_pfault(struct thread *td, struct trapframe *tf); 93178354Ssam 94178354Ssamextern char copy_fault[]; 95178354Ssamextern char copy_nofault_begin[]; 96178354Ssamextern char copy_nofault_end[]; 97178354Ssam 98164645Ssamextern char fs_fault[]; 99164645Ssamextern char fs_nofault_begin[]; 100164645Ssamextern char fs_nofault_end[]; 101164645Ssamextern char fs_nofault_intr_begin[]; 102164645Ssamextern char fs_nofault_intr_end[]; 103164645Ssam 104165569Ssamextern char *syscallnames[]; 105165569Ssam 106165569Ssamconst char *trap_msg[] = { 107165569Ssam "reserved", 108164645Ssam "instruction access exception", 109164645Ssam "instruction access error", 110164645Ssam "instruction access protection", 111164645Ssam "illtrap instruction", 112164645Ssam "illegal instruction", 113164645Ssam "privileged opcode", 114164645Ssam "floating point disabled", 115140915Ssam "floating point exception ieee 754", 116165569Ssam "floating point exception other", 117165569Ssam "tag overflow", 118165569Ssam "division by zero", 119165569Ssam "data access exception", 120165569Ssam "data access error", 121165569Ssam "data access protection", 122116742Ssam "memory address not aligned", 123165569Ssam "privileged action", 124188782Ssam "async data error", 125165574Ssam "trap instruction 16", 126165569Ssam "trap instruction 17", 127116742Ssam "trap instruction 18", 128116742Ssam "trap instruction 19", 129116742Ssam "trap instruction 20", 130186107Ssam "trap instruction 21", 131170530Ssam "trap instruction 22", 132116742Ssam "trap instruction 23", 133178354Ssam "trap instruction 24", 134167468Ssam "trap instruction 25", 135170530Ssam "trap instruction 26", 136116742Ssam "trap instruction 27", 137170530Ssam "trap instruction 28", 138187796Ssam "trap instruction 29", 139187796Ssam "trap instruction 30", 140187796Ssam "trap instruction 31", 141187796Ssam "interrupt", 142187796Ssam "physical address watchpoint", 143187796Ssam "virtual address watchpoint", 144187796Ssam "corrected ecc error", 145187796Ssam "fast instruction access mmu miss", 146187796Ssam "fast data access mmu miss", 147187796Ssam "spill", 148187796Ssam "fill", 149187796Ssam "fill", 150187796Ssam "breakpoint", 151187796Ssam "clean window", 152187796Ssam "range check", 153170530Ssam "fix alignment", 154170530Ssam "integer overflow", 155170530Ssam "syscall", 156170530Ssam "restore physical watchpoint", 157170530Ssam "restore virtual watchpoint", 158170530Ssam "kernel stack fault", 159170530Ssam}; 160170530Ssam 161170530Ssamconst int trap_sig[] = { 162170530Ssam SIGILL, /* reserved */ 163170530Ssam SIGILL, /* instruction access exception */ 164170530Ssam SIGILL, /* instruction access error */ 165170530Ssam SIGILL, /* instruction access protection */ 166170530Ssam SIGILL, /* illtrap instruction */ 167170530Ssam SIGILL, /* illegal instruction */ 168170530Ssam SIGBUS, /* privileged opcode */ 169170530Ssam SIGFPE, /* floating point disabled */ 170170530Ssam SIGFPE, /* floating point exception ieee 754 */ 171188782Ssam SIGFPE, /* floating point exception other */ 172188782Ssam SIGEMT, /* tag overflow */ 173188782Ssam SIGFPE, /* division by zero */ 174188782Ssam SIGILL, /* data access exception */ 175170530Ssam SIGILL, /* data access error */ 176170530Ssam SIGBUS, /* data access protection */ 177170530Ssam SIGBUS, /* memory address not aligned */ 178170530Ssam SIGBUS, /* privileged action */ 179116742Ssam SIGBUS, /* async data error */ 180170530Ssam SIGILL, /* trap instruction 16 */ 181170530Ssam SIGILL, /* trap instruction 17 */ 182170530Ssam SIGILL, /* trap instruction 18 */ 183164645Ssam SIGILL, /* trap instruction 19 */ 184178354Ssam SIGILL, /* trap instruction 20 */ 185178354Ssam SIGILL, /* trap instruction 21 */ 186178354Ssam SIGILL, /* trap instruction 22 */ 187178354Ssam SIGILL, /* trap instruction 23 */ 188170530Ssam SIGILL, /* trap instruction 24 */ 189172233Ssam SIGILL, /* trap instruction 25 */ 190178354Ssam SIGILL, /* trap instruction 26 */ 191170530Ssam SIGILL, /* trap instruction 27 */ 192170530Ssam SIGILL, /* trap instruction 28 */ 193190532Ssam SIGILL, /* trap instruction 29 */ 194170530Ssam SIGILL, /* trap instruction 30 */ 195164645Ssam SIGILL, /* trap instruction 31 */ 196165569Ssam -1, /* interrupt */ 197165569Ssam -1, /* physical address watchpoint */ 198165569Ssam -1, /* virtual address watchpoint */ 199165569Ssam -1, /* corrected ecc error */ 200165569Ssam SIGSEGV, /* fast instruction access mmu miss */ 201187897Ssam SIGSEGV, /* fast data access mmu miss */ 202188782Ssam SIGILL, /* spill */ 203188782Ssam SIGILL, /* fill */ 204188774Ssam SIGILL, /* fill */ 205188774Ssam SIGTRAP, /* breakpoint */ 206165569Ssam SIGILL, /* clean window */ 207165569Ssam SIGILL, /* range check */ 208165569Ssam SIGILL, /* fix alignment */ 209165569Ssam SIGILL, /* integer overflow */ 210165569Ssam SIGSYS, /* syscall */ 211165569Ssam -1, /* restore physical watchpoint */ 212165569Ssam -1, /* restore virtual watchpoint */ 213165569Ssam -1, /* kernel stack fault */ 214178354Ssam}; 215178354Ssam 216178354SsamCTASSERT(sizeof(struct trapframe) == 256); 217178354Ssam 218178354Ssamint debugger_on_signal = 0; 219178354SsamSYSCTL_INT(_debug, OID_AUTO, debugger_on_signal, CTLFLAG_RW, 220178354Ssam &debugger_on_signal, 0, ""); 221178354Ssam 222178354Ssamvoid 223178354Ssamtrap(struct trapframe *tf) 224178354Ssam{ 225178354Ssam struct thread *td; 226178521Ssam struct proc *p; 227195846Ssam u_int sticks; 228195846Ssam int error; 229195846Ssam int sig; 230195846Ssam 231195846Ssam td = PCPU_GET(curthread); 232195846Ssam 233195846Ssam CTR4(KTR_TRAP, "trap: %p type=%s (%s) pil=%#lx", td, 234195846Ssam trap_msg[tf->tf_type & ~T_KERNEL], 235178521Ssam (TRAPF_USERMODE(tf) ? "user" : "kernel"), rdpr(pil)); 236191148Skmacy 237178521Ssam atomic_add_int(&cnt.v_trap, 1); 238178521Ssam 239195846Ssam if ((tf->tf_tstate & TSTATE_PRIV) == 0) { 240178521Ssam KASSERT(td != NULL, ("trap: curthread NULL")); 241178521Ssam KASSERT(td->td_proc != NULL, ("trap: curproc NULL")); 242178521Ssam 243178521Ssam p = td->td_proc; 244178521Ssam sticks = td->td_sticks; 245178521Ssam td->td_frame = tf; 246178521Ssam if (td->td_ucred != p->p_ucred) 247178521Ssam cred_update_thread(td); 248178521Ssam PROC_LOCK(p); 249178354Ssam if ((p->p_flag & P_WEXIT) && (p->p_singlethread != td)) { 250178354Ssam mtx_lock_spin(&sched_lock); 251178354Ssam thread_exit(); 252178354Ssam /* NOTREACHED */ 253165569Ssam } 254190526Ssam PROC_UNLOCK(p); 255190526Ssam 256165569Ssam switch (tf->tf_type) { 257165569Ssam case T_DATA_MISS: 258178354Ssam case T_DATA_PROTECTION: 259178354Ssam case T_INSTRUCTION_MISS: 260165569Ssam sig = trap_pfault(td, tf); 261178354Ssam break; 262165569Ssam case T_FILL: 263179388Ssam sig = rwindow_load(td, tf, 2); 264178354Ssam break; 265191746Sthompsa case T_FILL_RET: 266191746Sthompsa sig = rwindow_load(td, tf, 1); 267191746Sthompsa break; 268191746Sthompsa case T_SPILL: 269191746Sthompsa sig = rwindow_save(td); 270191746Sthompsa break; 271165569Ssam default: 272165569Ssam if (tf->tf_type < 0 || tf->tf_type >= T_MAX || 273165569Ssam trap_sig[tf->tf_type] == -1) 274165569Ssam panic("trap: bad trap type"); 275165569Ssam sig = trap_sig[tf->tf_type]; 276178354Ssam break; 277170530Ssam } 278178354Ssam 279178354Ssam if (sig != 0) { 280116742Ssam /* Translate fault for emulators. */ 281195379Ssam if (p->p_sysent->sv_transtrap != NULL) { 282155688Ssam sig = p->p_sysent->sv_transtrap(sig, 283155688Ssam tf->tf_type); 284138568Ssam } 285138568Ssam if (debugger_on_signal && 286170530Ssam (sig == 4 || sig == 10 || sig == 11)) 287138568Ssam Debugger("trapsig"); 288170530Ssam trapsignal(td, sig, tf->tf_type); 289138568Ssam } 290190391Ssam 291190391Ssam userret(td, tf, sticks); 292190391Ssam mtx_assert(&Giant, MA_NOTOWNED); 293170530Ssam#ifdef DIAGNOSTIC 294170530Ssam cred_free_thread(td); 295178354Ssam#endif 296193843Ssam } else { 297138568Ssam KASSERT((tf->tf_type & T_KERNEL) != 0, 298178354Ssam ("trap: kernel trap isn't")); 299138568Ssam 300178354Ssam switch (tf->tf_type & ~T_KERNEL) { 301178354Ssam#ifdef DDB 302178354Ssam case T_BREAKPOINT: 303178354Ssam case T_KSTACK_FAULT: 304178354Ssam error = (kdb_trap(tf) == 0); 305178521Ssam break; 306178521Ssam#ifdef notyet 307178521Ssam case T_PA_WATCHPOINT: 308140915Ssam case T_VA_WATCHPOINT: 309178354Ssam error = db_watch_trap(tf); 310178354Ssam break; 311178354Ssam#endif 312178354Ssam#endif 313178354Ssam case T_DATA_MISS: 314190526Ssam case T_DATA_PROTECTION: 315194760Srwatson case T_INSTRUCTION_MISS: 316116742Ssam error = trap_pfault(td, tf); 317116742Ssam break; 318178354Ssam case T_DATA_EXCEPTION: 319178354Ssam case T_MEM_ADDRESS_NOT_ALIGNED: 320178354Ssam if ((tf->tf_sfsr & MMU_SFSR_FV) != 0 && 321178354Ssam MMU_SFSR_GET_ASI(tf->tf_sfsr) == ASI_AIUP) { 322178354Ssam if (tf->tf_tpc >= (u_long)copy_nofault_begin && 323178354Ssam tf->tf_tpc <= (u_long)copy_nofault_end) { 324116742Ssam tf->tf_tpc = (u_long)copy_fault; 325138568Ssam tf->tf_tnpc = tf->tf_tpc + 4; 326116742Ssam error = 0; 327138568Ssam break; 328178354Ssam } 329116742Ssam if (tf->tf_tpc >= (u_long)fs_nofault_begin && 330193337Ssam tf->tf_tpc <= (u_long)fs_nofault_end) { 331193337Ssam tf->tf_tpc = (u_long)fs_fault; 332178354Ssam tf->tf_tnpc = tf->tf_tpc + 4; 333178354Ssam error = 0; 334188533Sthompsa break; 335138568Ssam } 336138568Ssam } 337193843Ssam error = 1; 338178354Ssam break; 339170530Ssam default: 340190391Ssam error = 1; 341190391Ssam break; 342190391Ssam } 343170530Ssam 344166012Ssam if (error != 0) 345138568Ssam panic("trap: %s", trap_msg[tf->tf_type & ~T_KERNEL]); 346138568Ssam } 347170530Ssam CTR1(KTR_TRAP, "trap: td=%p return", td); 348138568Ssam} 349193337Ssam 350116742Ssamstatic int 351191746Sthompsatrap_pfault(struct thread *td, struct trapframe *tf) 352170530Ssam{ 353178354Ssam struct vmspace *vm; 354138568Ssam struct pcb *pcb; 355178354Ssam struct proc *p; 356178354Ssam vm_offset_t va; 357178354Ssam vm_prot_t prot; 358178354Ssam u_long ctx; 359178354Ssam int flags; 360178354Ssam int type; 361178354Ssam int rv; 362178354Ssam 363178354Ssam if (td == NULL) 364178354Ssam return (-1); 365178354Ssam KASSERT(td->td_pcb != NULL, ("trap_pfault: pcb NULL")); 366178354Ssam KASSERT(td->td_proc != NULL, ("trap_pfault: curproc NULL")); 367178354Ssam KASSERT(td->td_proc->p_vmspace != NULL, ("trap_pfault: vmspace NULL")); 368178354Ssam 369178354Ssam p = td->td_proc; 370178354Ssam 371178354Ssam rv = KERN_SUCCESS; 372178354Ssam ctx = TLB_TAR_CTX(tf->tf_tar); 373178354Ssam pcb = td->td_pcb; 374178354Ssam type = tf->tf_type & ~T_KERNEL; 375178354Ssam va = TLB_TAR_VA(tf->tf_tar); 376178354Ssam 377178354Ssam CTR4(KTR_TRAP, "trap_pfault: td=%p pm_ctx=%#lx va=%#lx ctx=%#lx", 378178354Ssam td, p->p_vmspace->vm_pmap.pm_context[PCPU_GET(cpuid)], va, ctx); 379178354Ssam 380178354Ssam if (type == T_DATA_PROTECTION) { 381178354Ssam prot = VM_PROT_WRITE; 382178354Ssam flags = VM_FAULT_DIRTY; 383178354Ssam } else { 384178354Ssam if (type == T_DATA_MISS) 385178354Ssam prot = VM_PROT_READ; 386178354Ssam else 387178354Ssam prot = VM_PROT_READ | VM_PROT_EXECUTE; 388178354Ssam flags = VM_FAULT_NORMAL; 389178354Ssam } 390178354Ssam 391178354Ssam if (ctx != TLB_CTX_KERNEL) { 392178354Ssam if ((tf->tf_tstate & TSTATE_PRIV) != 0 && 393178354Ssam (tf->tf_tpc >= (u_long)fs_nofault_intr_begin && 394178354Ssam tf->tf_tpc <= (u_long)fs_nofault_intr_end)) { 395178354Ssam tf->tf_tpc = (u_long)fs_fault; 396178354Ssam tf->tf_tnpc = tf->tf_tpc + 4; 397178354Ssam return (0); 398178354Ssam } 399178354Ssam 400178354Ssam /* 401178354Ssam * This is a fault on non-kernel virtual memory. 402178354Ssam */ 403178354Ssam vm = p->p_vmspace; 404178354Ssam 405178354Ssam /* 406178354Ssam * Keep swapout from messing with us during this 407178354Ssam * critical time. 408178957Ssam */ 409178354Ssam PROC_LOCK(p); 410178354Ssam ++p->p_lock; 411178354Ssam PROC_UNLOCK(p); 412178354Ssam 413178354Ssam /* Fault in the user page. */ 414178354Ssam rv = vm_fault(&vm->vm_map, va, prot, flags); 415178354Ssam 416178354Ssam /* 417178354Ssam * Now the process can be swapped again. 418178354Ssam */ 419178354Ssam PROC_LOCK(p); 420178354Ssam --p->p_lock; 421178354Ssam PROC_UNLOCK(p); 422178354Ssam } else { 423178354Ssam /* 424186904Ssam * This is a fault on kernel virtual memory. Attempts to 425186904Ssam * access kernel memory from user mode cause privileged 426186904Ssam * action traps, not page fault. 427186904Ssam */ 428186904Ssam KASSERT(tf->tf_tstate & TSTATE_PRIV, 429186904Ssam ("trap_pfault: fault on nucleus context from user mode")); 430186904Ssam 431186904Ssam /* 432186904Ssam * Don't have to worry about process locking or stacks in the 433186904Ssam * kernel. 434186904Ssam */ 435186904Ssam rv = vm_fault(kernel_map, va, prot, VM_FAULT_NORMAL); 436186904Ssam } 437186904Ssam 438186904Ssam CTR3(KTR_TRAP, "trap_pfault: return td=%p va=%#lx rv=%d", 439178354Ssam td, va, rv); 440184278Ssam if (rv == KERN_SUCCESS) 441184278Ssam return (0); 442184278Ssam if (ctx != TLB_CTX_KERNEL && (tf->tf_tstate & TSTATE_PRIV) != 0) { 443202612Sthompsa if (tf->tf_tpc >= (u_long)fs_nofault_begin && 444202612Sthompsa tf->tf_tpc <= (u_long)fs_nofault_end) { 445202612Sthompsa tf->tf_tpc = (u_long)fs_fault; 446178354Ssam tf->tf_tnpc = tf->tf_tpc + 4; 447178354Ssam return (0); 448178354Ssam } 449178354Ssam if (tf->tf_tpc >= (u_long)copy_nofault_begin && 450178354Ssam tf->tf_tpc <= (u_long)copy_nofault_end) { 451178354Ssam tf->tf_tpc = (u_long)copy_fault; 452178354Ssam tf->tf_tnpc = tf->tf_tpc + 4; 453178354Ssam return (0); 454178354Ssam } 455178354Ssam } 456178354Ssam return ((rv == KERN_PROTECTION_FAILURE) ? SIGBUS : SIGSEGV); 457178354Ssam} 458178957Ssam 459178954Ssam/* Maximum number of arguments that can be passed via the out registers. */ 460178954Ssam#define REG_MAXARGS 6 461178954Ssam 462178354Ssam/* 463178354Ssam * Syscall handler. The arguments to the syscall are passed in the o registers 464178354Ssam * by the caller, and are saved in the trap frame. The syscall number is passed 465178354Ssam * in %g1 (and also saved in the trap frame). 466178354Ssam */ 467178354Ssamvoid 468178354Ssamsyscall(struct trapframe *tf) 469178354Ssam{ 470178354Ssam struct sysent *callp; 471178354Ssam struct thread *td; 472178354Ssam register_t args[8]; 473178354Ssam register_t *argp; 474178354Ssam struct proc *p; 475178354Ssam u_int sticks; 476178354Ssam u_long code; 477178354Ssam u_long tpc; 478178354Ssam int reg; 479178354Ssam int regcnt; 480190391Ssam int narg; 481190391Ssam int error; 482190391Ssam 483178354Ssam td = PCPU_GET(curthread); 484178354Ssam KASSERT(td != NULL, ("trap: curthread NULL")); 485178354Ssam KASSERT(td->td_proc != NULL, ("trap: curproc NULL")); 486192468Ssam 487178354Ssam p = td->td_proc; 488178354Ssam 489178354Ssam atomic_add_int(&cnt.v_syscall, 1); 490178354Ssam 491178354Ssam narg = 0; 492178354Ssam error = 0; 493178354Ssam reg = 0; 494178354Ssam regcnt = REG_MAXARGS; 495178354Ssam 496178354Ssam sticks = td->td_sticks; 497178354Ssam td->td_frame = tf; 498178354Ssam if (td->td_ucred != p->p_ucred) 499178354Ssam cred_update_thread(td); 500178354Ssam if (p->p_flag & P_THREADED) 501178354Ssam thread_user_enter(p, td); 502178354Ssam code = tf->tf_global[1]; 503178354Ssam 504178354Ssam /* 505178354Ssam * For syscalls, we don't want to retry the faulting instruction 506178354Ssam * (usually), instead we need to advance one instruction. 507178354Ssam */ 508178354Ssam tpc = tf->tf_tpc; 509178354Ssam TF_DONE(tf); 510178354Ssam 511178354Ssam if (p->p_sysent->sv_prepsyscall) { 512178354Ssam /* 513178354Ssam * The prep code is MP aware. 514178354Ssam */ 515178354Ssam#if 0 516178354Ssam (*p->p_sysent->sv_prepsyscall)(tf, args, &code, ¶ms); 517178354Ssam#endif 518178354Ssam } else if (code == SYS_syscall || code == SYS___syscall) { 519178354Ssam code = tf->tf_out[reg++]; 520178354Ssam regcnt--; 521188106Ssam } 522188106Ssam 523178354Ssam if (p->p_sysent->sv_mask) 524178354Ssam code &= p->p_sysent->sv_mask; 525178354Ssam 526178354Ssam if (code >= p->p_sysent->sv_size) 527195846Ssam callp = &p->p_sysent->sv_table[0]; 528195846Ssam else 529195846Ssam callp = &p->p_sysent->sv_table[code]; 530195846Ssam 531195846Ssam narg = callp->sy_narg & SYF_ARGMASK; 532195846Ssam 533195846Ssam if (narg <= regcnt) { 534195846Ssam argp = &tf->tf_out[reg]; 535195846Ssam error = 0; 536178354Ssam } else { 537178354Ssam KASSERT(narg <= sizeof(args) / sizeof(args[0]), 538178354Ssam ("Too many syscall arguments!")); 539178354Ssam argp = args; 540178354Ssam bcopy(&tf->tf_out[reg], args, sizeof(args[0]) * regcnt); 541190391Ssam error = copyin((void *)(tf->tf_out[6] + SPOFF + 542178354Ssam offsetof(struct frame, fr_pad[6])), 543190391Ssam &args[regcnt], (narg - regcnt) * sizeof(args[0])); 544178354Ssam } 545178354Ssam 546193655Ssam CTR5(KTR_SYSC, "syscall: td=%p %s(%#lx, %#lx, %#lx)", td, 547193655Ssam syscallnames[code], argp[0], argp[1], argp[2]); 548178354Ssam 549178354Ssam /* 550178354Ssam * Try to run the syscall without the MP lock if the syscall 551178354Ssam * is MP safe. 552178354Ssam */ 553178354Ssam if ((callp->sy_narg & SYF_MPSAFE) == 0) 554178354Ssam mtx_lock(&Giant); 555178354Ssam 556178354Ssam#ifdef KTRACE 557178354Ssam if (KTRPOINT(td, KTR_SYSCALL)) 558178354Ssam ktrsyscall(code, narg, argp); 559178354Ssam#endif 560178354Ssam if (error == 0) { 561178354Ssam td->td_retval[0] = 0; 562178354Ssam td->td_retval[1] = 0; 563178354Ssam 564178354Ssam STOPEVENT(p, S_SCE, narg); /* MP aware */ 565178354Ssam 566178354Ssam error = (*callp->sy_call)(td, argp); 567178354Ssam 568178354Ssam CTR5(KTR_SYSC, "syscall: p=%p error=%d %s return %#lx %#lx ", p, 569178354Ssam error, syscallnames[code], td->td_retval[0], 570178354Ssam td->td_retval[1]); 571193312Ssam } 572193312Ssam 573178354Ssam /* 574193312Ssam * MP SAFE (we may or may not have the MP lock at this point) 575193312Ssam */ 576191746Sthompsa switch (error) { 577191746Sthompsa case 0: 578191746Sthompsa tf->tf_out[0] = td->td_retval[0]; 579191746Sthompsa tf->tf_out[1] = td->td_retval[1]; 580191746Sthompsa tf->tf_tstate &= ~TSTATE_XCC_C; 581191746Sthompsa break; 582196159Ssam 583196159Ssam case ERESTART: 584196159Ssam /* 585191746Sthompsa * Undo the tpc advancement we have done above, we want to 586191746Sthompsa * reexecute the system call. 587178354Ssam */ 588178354Ssam tf->tf_tpc = tpc; 589190391Ssam tf->tf_tnpc -= 4; 590178354Ssam break; 591190391Ssam 592178354Ssam case EJUSTRETURN: 593178354Ssam break; 594193655Ssam 595193655Ssam default: 596192468Ssam if (p->p_sysent->sv_errsize) { 597192468Ssam if (error >= p->p_sysent->sv_errsize) 598178354Ssam error = -1; /* XXX */ 599178354Ssam else 600178354Ssam error = p->p_sysent->sv_errtbl[error]; 601178354Ssam } 602178354Ssam tf->tf_out[0] = error; 603178354Ssam tf->tf_tstate |= TSTATE_XCC_C; 604192468Ssam break; 605178354Ssam } 606178354Ssam 607190391Ssam /* 608190391Ssam * Release Giant if we had to get it. Don't use mtx_owned(), 609190391Ssam * we want to catch broken syscalls. 610178354Ssam */ 611178354Ssam if ((callp->sy_narg & SYF_MPSAFE) == 0) 612178354Ssam mtx_unlock(&Giant); 613178354Ssam 614178354Ssam /* 615178354Ssam * Handle reschedule and other end-of-syscall issues 616178354Ssam */ 617182674Sweongyo userret(td, tf, sticks); 618182674Sweongyo 619116742Ssam#ifdef KTRACE 620116742Ssam if (KTRPOINT(td, KTR_SYSRET)) 621178354Ssam ktrsysret(code, error, td->td_retval[0]); 622178354Ssam#endif 623178354Ssam /* 624178354Ssam * This works because errno is findable through the 625178354Ssam * register set. If we ever support an emulation where this 626178354Ssam * is not the case, this code will need to be revisited. 627178354Ssam */ 628178354Ssam STOPEVENT(p, S_SCX, code); 629178354Ssam 630178354Ssam#ifdef DIAGNOSTIC 631178354Ssam cred_free_thread(td); 632178354Ssam#endif 633178354Ssam WITNESS_WARN(WARN_PANIC, NULL, "System call %s returning", 634178354Ssam syscallnames[code]); 635178354Ssam mtx_assert(&sched_lock, MA_NOTOWNED); 636178354Ssam mtx_assert(&Giant, MA_NOTOWNED); 637178354Ssam} 638178354Ssam