machdep.c revision 9160:1517e6edbc6f
1/* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22/* 23 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27#include <sys/types.h> 28#include <sys/t_lock.h> 29#include <sys/param.h> 30#include <sys/segments.h> 31#include <sys/sysmacros.h> 32#include <sys/signal.h> 33#include <sys/systm.h> 34#include <sys/user.h> 35#include <sys/mman.h> 36#include <sys/vm.h> 37 38#include <sys/disp.h> 39#include <sys/class.h> 40 41#include <sys/proc.h> 42#include <sys/buf.h> 43#include <sys/kmem.h> 44 45#include <sys/reboot.h> 46#include <sys/uadmin.h> 47#include <sys/callb.h> 48 49#include <sys/cred.h> 50#include <sys/vnode.h> 51#include <sys/file.h> 52 53#include <sys/procfs.h> 54#include <sys/acct.h> 55 56#include <sys/vfs.h> 57#include <sys/dnlc.h> 58#include <sys/var.h> 59#include <sys/cmn_err.h> 60#include <sys/utsname.h> 61#include <sys/debug.h> 62 63#include <sys/dumphdr.h> 64#include <sys/bootconf.h> 65#include <sys/varargs.h> 66#include <sys/promif.h> 67#include <sys/modctl.h> 68 69#include <sys/consdev.h> 70#include <sys/frame.h> 71 72#include <sys/sunddi.h> 73#include <sys/ddidmareq.h> 74#include <sys/psw.h> 75#include <sys/regset.h> 76#include <sys/privregs.h> 77#include <sys/clock.h> 78#include <sys/tss.h> 79#include <sys/cpu.h> 80#include <sys/stack.h> 81#include <sys/trap.h> 82#include <sys/pic.h> 83#include <vm/hat.h> 84#include <vm/anon.h> 85#include <vm/as.h> 86#include <vm/page.h> 87#include <vm/seg.h> 88#include <vm/seg_kmem.h> 89#include <vm/seg_map.h> 90#include <vm/seg_vn.h> 91#include <vm/seg_kp.h> 92#include <vm/hat_i86.h> 93#include <sys/swap.h> 94#include <sys/thread.h> 95#include <sys/sysconf.h> 96#include <sys/vm_machparam.h> 97#include <sys/archsystm.h> 98#include <sys/machsystm.h> 99#include <sys/machlock.h> 100#include <sys/x_call.h> 101#include <sys/instance.h> 102 103#include <sys/time.h> 104#include <sys/smp_impldefs.h> 105#include <sys/psm_types.h> 106#include <sys/atomic.h> 107#include <sys/panic.h> 108#include <sys/cpuvar.h> 109#include <sys/dtrace.h> 110#include <sys/bl.h> 111#include <sys/nvpair.h> 112#include <sys/x86_archext.h> 113#include <sys/pool_pset.h> 114#include <sys/autoconf.h> 115#include <sys/mem.h> 116#include <sys/dumphdr.h> 117#include <sys/compress.h> 118#include <sys/cpu_module.h> 119#if defined(__xpv) 120#include <sys/hypervisor.h> 121#include <sys/xpv_panic.h> 122#endif 123 124#include <sys/fastboot.h> 125#include <sys/machelf.h> 126#include <sys/kobj.h> 127#include <sys/multiboot.h> 128 129#ifdef TRAPTRACE 130#include <sys/traptrace.h> 131#endif /* TRAPTRACE */ 132 133extern void audit_enterprom(int); 134extern void audit_exitprom(int); 135 136/* 137 * Occassionally the kernel knows better whether to power-off or reboot. 138 */ 139int force_shutdown_method = AD_UNKNOWN; 140 141/* 142 * The panicbuf array is used to record messages and state: 143 */ 144char panicbuf[PANICBUFSIZE]; 145 146/* 147 * maxphys - used during physio 148 * klustsize - used for klustering by swapfs and specfs 149 */ 150int maxphys = 56 * 1024; /* XXX See vm_subr.c - max b_count in physio */ 151int klustsize = 56 * 1024; 152 153caddr_t p0_va; /* Virtual address for accessing physical page 0 */ 154 155/* 156 * defined here, though unused on x86, 157 * to make kstat_fr.c happy. 158 */ 159int vac; 160 161void stop_other_cpus(); 162void debug_enter(char *); 163 164extern void pm_cfb_check_and_powerup(void); 165extern void pm_cfb_rele(void); 166 167extern fastboot_info_t newkernel; 168 169/* 170 * Machine dependent code to reboot. 171 * "mdep" is interpreted as a character pointer; if non-null, it is a pointer 172 * to a string to be used as the argument string when rebooting. 173 * 174 * "invoke_cb" is a boolean. It is set to true when mdboot() can safely 175 * invoke CB_CL_MDBOOT callbacks before shutting the system down, i.e. when 176 * we are in a normal shutdown sequence (interrupts are not blocked, the 177 * system is not panic'ing or being suspended). 178 */ 179/*ARGSUSED*/ 180void 181mdboot(int cmd, int fcn, char *mdep, boolean_t invoke_cb) 182{ 183 processorid_t bootcpuid = 0; 184 static int is_first_quiesce = 1; 185 static int is_first_reset = 1; 186 int reset_status = 0; 187 static char fallback_str[] = "Falling back to regular reboot.\n"; 188 189 if (fcn == AD_FASTREBOOT && !newkernel.fi_valid) 190 fcn = AD_BOOT; 191 192 if (!panicstr) { 193 kpreempt_disable(); 194 if (fcn == AD_FASTREBOOT) { 195 mutex_enter(&cpu_lock); 196 if (CPU_ACTIVE(cpu_get(bootcpuid))) { 197 affinity_set(bootcpuid); 198 } 199 mutex_exit(&cpu_lock); 200 } else { 201 affinity_set(CPU_CURRENT); 202 } 203 } 204 205 if (force_shutdown_method != AD_UNKNOWN) 206 fcn = force_shutdown_method; 207 208 /* 209 * XXX - rconsvp is set to NULL to ensure that output messages 210 * are sent to the underlying "hardware" device using the 211 * monitor's printf routine since we are in the process of 212 * either rebooting or halting the machine. 213 */ 214 rconsvp = NULL; 215 216 /* 217 * Print the reboot message now, before pausing other cpus. 218 * There is a race condition in the printing support that 219 * can deadlock multiprocessor machines. 220 */ 221 if (!(fcn == AD_HALT || fcn == AD_POWEROFF)) 222 prom_printf("rebooting...\n"); 223 224 if (IN_XPV_PANIC()) 225 reset(); 226 227 /* 228 * We can't bring up the console from above lock level, so do it now 229 */ 230 pm_cfb_check_and_powerup(); 231 232 /* make sure there are no more changes to the device tree */ 233 devtree_freeze(); 234 235 if (invoke_cb) 236 (void) callb_execute_class(CB_CL_MDBOOT, NULL); 237 238 /* 239 * Clear any unresolved UEs from memory. 240 */ 241 page_retire_mdboot(); 242 243#if defined(__xpv) 244 /* 245 * XXPV Should probably think some more about how we deal 246 * with panicing before it's really safe to panic. 247 * On hypervisors, we reboot very quickly.. Perhaps panic 248 * should only attempt to recover by rebooting if, 249 * say, we were able to mount the root filesystem, 250 * or if we successfully launched init(1m). 251 */ 252 if (panicstr && proc_init == NULL) 253 (void) HYPERVISOR_shutdown(SHUTDOWN_poweroff); 254#endif 255 /* 256 * stop other cpus and raise our priority. since there is only 257 * one active cpu after this, and our priority will be too high 258 * for us to be preempted, we're essentially single threaded 259 * from here on out. 260 */ 261 (void) spl6(); 262 if (!panicstr) { 263 mutex_enter(&cpu_lock); 264 pause_cpus(NULL); 265 mutex_exit(&cpu_lock); 266 } 267 268 /* 269 * If the system is panicking, the preloaded kernel is valid, 270 * and fastreboot_onpanic has been set, choose Fast Reboot. 271 */ 272 if (fcn == AD_BOOT && panicstr && newkernel.fi_valid && 273 fastreboot_onpanic) 274 fcn = AD_FASTREBOOT; 275 276 /* 277 * Try to quiesce devices. 278 */ 279 if (is_first_quiesce) { 280 /* 281 * Clear is_first_quiesce before calling quiesce_devices() 282 * so that if quiesce_devices() causes panics, it will not 283 * be invoked again. 284 */ 285 is_first_quiesce = 0; 286 287 quiesce_active = 1; 288 quiesce_devices(ddi_root_node(), &reset_status); 289 if (reset_status == -1) { 290 if (fcn == AD_FASTREBOOT && !force_fastreboot) { 291 prom_printf("Driver(s) not capable of fast " 292 "reboot.\n"); 293 prom_printf(fallback_str); 294 fastreboot_capable = 0; 295 fcn = AD_BOOT; 296 } else if (fcn != AD_FASTREBOOT) 297 fastreboot_capable = 0; 298 } 299 quiesce_active = 0; 300 } 301 302 /* 303 * Try to reset devices. reset_leaves() should only be called 304 * a) when there are no other threads that could be accessing devices, 305 * and 306 * b) on a system that's not capable of fast reboot (fastreboot_capable 307 * being 0), or on a system where quiesce_devices() failed to 308 * complete (quiesce_active being 1). 309 */ 310 if (is_first_reset && (!fastreboot_capable || quiesce_active)) { 311 /* 312 * Clear is_first_reset before calling reset_devices() 313 * so that if reset_devices() causes panics, it will not 314 * be invoked again. 315 */ 316 is_first_reset = 0; 317 reset_leaves(); 318 } 319 320 /* Verify newkernel checksum */ 321 if (fastreboot_capable && fcn == AD_FASTREBOOT && 322 fastboot_cksum_verify(&newkernel) != 0) { 323 fastreboot_capable = 0; 324 prom_printf("Fast reboot: checksum failed for the new " 325 "kernel.\n"); 326 prom_printf(fallback_str); 327 } 328 329 (void) spl8(); 330 331 if (fastreboot_capable && fcn == AD_FASTREBOOT) { 332 /* 333 * psm_shutdown is called within fast_reboot() 334 */ 335 fast_reboot(); 336 } else { 337 (*psm_shutdownf)(cmd, fcn); 338 339 if (fcn == AD_HALT || fcn == AD_POWEROFF) 340 halt((char *)NULL); 341 else 342 prom_reboot(""); 343 } 344 /*NOTREACHED*/ 345} 346 347/* mdpreboot - may be called prior to mdboot while root fs still mounted */ 348/*ARGSUSED*/ 349void 350mdpreboot(int cmd, int fcn, char *mdep) 351{ 352 if (fcn == AD_FASTREBOOT && !fastreboot_capable) { 353 fcn = AD_BOOT; 354#ifdef __xpv 355 cmn_err(CE_WARN, "Fast reboot is not supported on xVM"); 356#else 357 cmn_err(CE_WARN, 358 "Fast reboot is not supported on this platform"); 359#endif 360 } 361 362 if (fcn == AD_FASTREBOOT) { 363 fastboot_load_kernel(mdep); 364 if (!newkernel.fi_valid) 365 fcn = AD_BOOT; 366 } 367 368 (*psm_preshutdownf)(cmd, fcn); 369} 370 371void 372idle_other_cpus() 373{ 374 int cpuid = CPU->cpu_id; 375 cpuset_t xcset; 376 377 ASSERT(cpuid < NCPU); 378 CPUSET_ALL_BUT(xcset, cpuid); 379 xc_capture_cpus(xcset); 380} 381 382void 383resume_other_cpus() 384{ 385 ASSERT(CPU->cpu_id < NCPU); 386 387 xc_release_cpus(); 388} 389 390void 391stop_other_cpus() 392{ 393 int cpuid = CPU->cpu_id; 394 cpuset_t xcset; 395 396 ASSERT(cpuid < NCPU); 397 398 /* 399 * xc_trycall will attempt to make all other CPUs execute mach_cpu_halt, 400 * and will return immediately regardless of whether or not it was 401 * able to make them do it. 402 */ 403 CPUSET_ALL_BUT(xcset, cpuid); 404 xc_trycall(NULL, NULL, NULL, xcset, (int (*)())mach_cpu_halt); 405} 406 407/* 408 * Machine dependent abort sequence handling 409 */ 410void 411abort_sequence_enter(char *msg) 412{ 413 if (abort_enable == 0) { 414 if (audit_active) 415 audit_enterprom(0); 416 return; 417 } 418 if (audit_active) 419 audit_enterprom(1); 420 debug_enter(msg); 421 if (audit_active) 422 audit_exitprom(1); 423} 424 425/* 426 * Enter debugger. Called when the user types ctrl-alt-d or whenever 427 * code wants to enter the debugger and possibly resume later. 428 */ 429void 430debug_enter( 431 char *msg) /* message to print, possibly NULL */ 432{ 433 if (dtrace_debugger_init != NULL) 434 (*dtrace_debugger_init)(); 435 436 if (msg) 437 prom_printf("%s\n", msg); 438 439 if (boothowto & RB_DEBUG) 440 kmdb_enter(); 441 442 if (dtrace_debugger_fini != NULL) 443 (*dtrace_debugger_fini)(); 444} 445 446void 447reset(void) 448{ 449#if !defined(__xpv) 450 ushort_t *bios_memchk; 451 452 /* 453 * Can't use psm_map_phys before the hat is initialized. 454 */ 455 if (khat_running) { 456 bios_memchk = (ushort_t *)psm_map_phys(0x472, 457 sizeof (ushort_t), PROT_READ | PROT_WRITE); 458 if (bios_memchk) 459 *bios_memchk = 0x1234; /* bios memory check disable */ 460 } 461 462 if (ddi_prop_exists(DDI_DEV_T_ANY, ddi_root_node(), 0, "efi-systab")) 463 efi_reset(); 464 pc_reset(); 465#else 466 if (IN_XPV_PANIC()) 467 pc_reset(); 468 (void) HYPERVISOR_shutdown(SHUTDOWN_reboot); 469 panic("HYPERVISOR_shutdown() failed"); 470#endif 471 /*NOTREACHED*/ 472} 473 474/* 475 * Halt the machine and return to the monitor 476 */ 477void 478halt(char *s) 479{ 480 stop_other_cpus(); /* send stop signal to other CPUs */ 481 if (s) 482 prom_printf("(%s) \n", s); 483 prom_exit_to_mon(); 484 /*NOTREACHED*/ 485} 486 487/* 488 * Initiate interrupt redistribution. 489 */ 490void 491i_ddi_intr_redist_all_cpus() 492{ 493} 494 495/* 496 * XXX These probably ought to live somewhere else 497 * XXX They are called from mem.c 498 */ 499 500/* 501 * Convert page frame number to an OBMEM page frame number 502 * (i.e. put in the type bits -- zero for this implementation) 503 */ 504pfn_t 505impl_obmem_pfnum(pfn_t pf) 506{ 507 return (pf); 508} 509 510#ifdef NM_DEBUG 511int nmi_test = 0; /* checked in intentry.s during clock int */ 512int nmtest = -1; 513nmfunc1(arg, rp) 514int arg; 515struct regs *rp; 516{ 517 printf("nmi called with arg = %x, regs = %x\n", arg, rp); 518 nmtest += 50; 519 if (arg == nmtest) { 520 printf("ip = %x\n", rp->r_pc); 521 return (1); 522 } 523 return (0); 524} 525 526#endif 527 528#include <sys/bootsvcs.h> 529 530/* Hacked up initialization for initial kernel check out is HERE. */ 531/* The basic steps are: */ 532/* kernel bootfuncs definition/initialization for KADB */ 533/* kadb bootfuncs pointer initialization */ 534/* putchar/getchar (interrupts disabled) */ 535 536/* kadb bootfuncs pointer initialization */ 537 538int 539sysp_getchar() 540{ 541 int i; 542 ulong_t s; 543 544 if (cons_polledio == NULL) { 545 /* Uh oh */ 546 prom_printf("getchar called with no console\n"); 547 for (;;) 548 /* LOOP FOREVER */; 549 } 550 551 s = clear_int_flag(); 552 i = cons_polledio->cons_polledio_getchar( 553 cons_polledio->cons_polledio_argument); 554 restore_int_flag(s); 555 return (i); 556} 557 558void 559sysp_putchar(int c) 560{ 561 ulong_t s; 562 563 /* 564 * We have no alternative but to drop the output on the floor. 565 */ 566 if (cons_polledio == NULL || 567 cons_polledio->cons_polledio_putchar == NULL) 568 return; 569 570 s = clear_int_flag(); 571 cons_polledio->cons_polledio_putchar( 572 cons_polledio->cons_polledio_argument, c); 573 restore_int_flag(s); 574} 575 576int 577sysp_ischar() 578{ 579 int i; 580 ulong_t s; 581 582 if (cons_polledio == NULL || 583 cons_polledio->cons_polledio_ischar == NULL) 584 return (0); 585 586 s = clear_int_flag(); 587 i = cons_polledio->cons_polledio_ischar( 588 cons_polledio->cons_polledio_argument); 589 restore_int_flag(s); 590 return (i); 591} 592 593int 594goany(void) 595{ 596 prom_printf("Type any key to continue "); 597 (void) prom_getchar(); 598 prom_printf("\n"); 599 return (1); 600} 601 602static struct boot_syscalls kern_sysp = { 603 sysp_getchar, /* unchar (*getchar)(); 7 */ 604 sysp_putchar, /* int (*putchar)(); 8 */ 605 sysp_ischar, /* int (*ischar)(); 9 */ 606}; 607 608#if defined(__xpv) 609int using_kern_polledio; 610#endif 611 612void 613kadb_uses_kernel() 614{ 615 /* 616 * This routine is now totally misnamed, since it does not in fact 617 * control kadb's I/O; it only controls the kernel's prom_* I/O. 618 */ 619 sysp = &kern_sysp; 620#if defined(__xpv) 621 using_kern_polledio = 1; 622#endif 623} 624 625/* 626 * the interface to the outside world 627 */ 628 629/* 630 * poll_port -- wait for a register to achieve a 631 * specific state. Arguments are a mask of bits we care about, 632 * and two sub-masks. To return normally, all the bits in the 633 * first sub-mask must be ON, all the bits in the second sub- 634 * mask must be OFF. If about seconds pass without the register 635 * achieving the desired bit configuration, we return 1, else 636 * 0. 637 */ 638int 639poll_port(ushort_t port, ushort_t mask, ushort_t onbits, ushort_t offbits) 640{ 641 int i; 642 ushort_t maskval; 643 644 for (i = 500000; i; i--) { 645 maskval = inb(port) & mask; 646 if (((maskval & onbits) == onbits) && 647 ((maskval & offbits) == 0)) 648 return (0); 649 drv_usecwait(10); 650 } 651 return (1); 652} 653 654/* 655 * set_idle_cpu is called from idle() when a CPU becomes idle. 656 */ 657/*LINTED: static unused */ 658static uint_t last_idle_cpu; 659 660/*ARGSUSED*/ 661void 662set_idle_cpu(int cpun) 663{ 664 last_idle_cpu = cpun; 665 (*psm_set_idle_cpuf)(cpun); 666} 667 668/* 669 * unset_idle_cpu is called from idle() when a CPU is no longer idle. 670 */ 671/*ARGSUSED*/ 672void 673unset_idle_cpu(int cpun) 674{ 675 (*psm_unset_idle_cpuf)(cpun); 676} 677 678/* 679 * This routine is almost correct now, but not quite. It still needs the 680 * equivalent concept of "hres_last_tick", just like on the sparc side. 681 * The idea is to take a snapshot of the hi-res timer while doing the 682 * hrestime_adj updates under hres_lock in locore, so that the small 683 * interval between interrupt assertion and interrupt processing is 684 * accounted for correctly. Once we have this, the code below should 685 * be modified to subtract off hres_last_tick rather than hrtime_base. 686 * 687 * I'd have done this myself, but I don't have source to all of the 688 * vendor-specific hi-res timer routines (grrr...). The generic hook I 689 * need is something like "gethrtime_unlocked()", which would be just like 690 * gethrtime() but would assume that you're already holding CLOCK_LOCK(). 691 * This is what the GET_HRTIME() macro is for on sparc (although it also 692 * serves the function of making time available without a function call 693 * so you don't take a register window overflow while traps are disabled). 694 */ 695void 696pc_gethrestime(timestruc_t *tp) 697{ 698 int lock_prev; 699 timestruc_t now; 700 int nslt; /* nsec since last tick */ 701 int adj; /* amount of adjustment to apply */ 702 703loop: 704 lock_prev = hres_lock; 705 now = hrestime; 706 nslt = (int)(gethrtime() - hres_last_tick); 707 if (nslt < 0) { 708 /* 709 * nslt < 0 means a tick came between sampling 710 * gethrtime() and hres_last_tick; restart the loop 711 */ 712 713 goto loop; 714 } 715 now.tv_nsec += nslt; 716 if (hrestime_adj != 0) { 717 if (hrestime_adj > 0) { 718 adj = (nslt >> ADJ_SHIFT); 719 if (adj > hrestime_adj) 720 adj = (int)hrestime_adj; 721 } else { 722 adj = -(nslt >> ADJ_SHIFT); 723 if (adj < hrestime_adj) 724 adj = (int)hrestime_adj; 725 } 726 now.tv_nsec += adj; 727 } 728 while ((unsigned long)now.tv_nsec >= NANOSEC) { 729 730 /* 731 * We might have a large adjustment or have been in the 732 * debugger for a long time; take care of (at most) four 733 * of those missed seconds (tv_nsec is 32 bits, so 734 * anything >4s will be wrapping around). However, 735 * anything more than 2 seconds out of sync will trigger 736 * timedelta from clock() to go correct the time anyway, 737 * so do what we can, and let the big crowbar do the 738 * rest. A similar correction while loop exists inside 739 * hres_tick(); in all cases we'd like tv_nsec to 740 * satisfy 0 <= tv_nsec < NANOSEC to avoid confusing 741 * user processes, but if tv_sec's a little behind for a 742 * little while, that's OK; time still monotonically 743 * increases. 744 */ 745 746 now.tv_nsec -= NANOSEC; 747 now.tv_sec++; 748 } 749 if ((hres_lock & ~1) != lock_prev) 750 goto loop; 751 752 *tp = now; 753} 754 755void 756gethrestime_lasttick(timespec_t *tp) 757{ 758 int s; 759 760 s = hr_clock_lock(); 761 *tp = hrestime; 762 hr_clock_unlock(s); 763} 764 765time_t 766gethrestime_sec(void) 767{ 768 timestruc_t now; 769 770 gethrestime(&now); 771 return (now.tv_sec); 772} 773 774/* 775 * Initialize a kernel thread's stack 776 */ 777 778caddr_t 779thread_stk_init(caddr_t stk) 780{ 781 ASSERT(((uintptr_t)stk & (STACK_ALIGN - 1)) == 0); 782 return (stk - SA(MINFRAME)); 783} 784 785/* 786 * Initialize lwp's kernel stack. 787 */ 788 789#ifdef TRAPTRACE 790/* 791 * There's a tricky interdependency here between use of sysenter and 792 * TRAPTRACE which needs recording to avoid future confusion (this is 793 * about the third time I've re-figured this out ..) 794 * 795 * Here's how debugging lcall works with TRAPTRACE. 796 * 797 * 1 We're in userland with a breakpoint on the lcall instruction. 798 * 2 We execute the instruction - the instruction pushes the userland 799 * %ss, %esp, %efl, %cs, %eip on the stack and zips into the kernel 800 * via the call gate. 801 * 3 The hardware raises a debug trap in kernel mode, the hardware 802 * pushes %efl, %cs, %eip and gets to dbgtrap via the idt. 803 * 4 dbgtrap pushes the error code and trapno and calls cmntrap 804 * 5 cmntrap finishes building a trap frame 805 * 6 The TRACE_REGS macros in cmntrap copy a REGSIZE worth chunk 806 * off the stack into the traptrace buffer. 807 * 808 * This means that the traptrace buffer contains the wrong values in 809 * %esp and %ss, but everything else in there is correct. 810 * 811 * Here's how debugging sysenter works with TRAPTRACE. 812 * 813 * a We're in userland with a breakpoint on the sysenter instruction. 814 * b We execute the instruction - the instruction pushes -nothing- 815 * on the stack, but sets %cs, %eip, %ss, %esp to prearranged 816 * values to take us to sys_sysenter, at the top of the lwp's 817 * stack. 818 * c goto 3 819 * 820 * At this point, because we got into the kernel without the requisite 821 * five pushes on the stack, if we didn't make extra room, we'd 822 * end up with the TRACE_REGS macro fetching the saved %ss and %esp 823 * values from negative (unmapped) stack addresses -- which really bites. 824 * That's why we do the '-= 8' below. 825 * 826 * XXX Note that reading "up" lwp0's stack works because t0 is declared 827 * right next to t0stack in locore.s 828 */ 829#endif 830 831caddr_t 832lwp_stk_init(klwp_t *lwp, caddr_t stk) 833{ 834 caddr_t oldstk; 835 struct pcb *pcb = &lwp->lwp_pcb; 836 837 oldstk = stk; 838 stk -= SA(sizeof (struct regs) + SA(MINFRAME)); 839#ifdef TRAPTRACE 840 stk -= 2 * sizeof (greg_t); /* space for phony %ss:%sp (see above) */ 841#endif 842 stk = (caddr_t)((uintptr_t)stk & ~(STACK_ALIGN - 1ul)); 843 bzero(stk, oldstk - stk); 844 lwp->lwp_regs = (void *)(stk + SA(MINFRAME)); 845 846 /* 847 * Arrange that the virtualized %fs and %gs GDT descriptors 848 * have a well-defined initial state (present, ring 3 849 * and of type data). 850 */ 851#if defined(__amd64) 852 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) 853 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 854 else 855 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_u32desc; 856#elif defined(__i386) 857 pcb->pcb_fsdesc = pcb->pcb_gsdesc = zero_udesc; 858#endif /* __i386 */ 859 lwp_installctx(lwp); 860 return (stk); 861} 862 863/*ARGSUSED*/ 864void 865lwp_stk_fini(klwp_t *lwp) 866{} 867 868/* 869 * If we're not the panic CPU, we wait in panic_idle for reboot. 870 */ 871static void 872panic_idle(void) 873{ 874 splx(ipltospl(CLOCK_LEVEL)); 875 (void) setjmp(&curthread->t_pcb); 876 877 for (;;) 878 ; 879} 880 881/* 882 * Stop the other CPUs by cross-calling them and forcing them to enter 883 * the panic_idle() loop above. 884 */ 885/*ARGSUSED*/ 886void 887panic_stopcpus(cpu_t *cp, kthread_t *t, int spl) 888{ 889 processorid_t i; 890 cpuset_t xcset; 891 892 /* 893 * In the case of a Xen panic, the hypervisor has already stopped 894 * all of the CPUs. 895 */ 896 if (!IN_XPV_PANIC()) { 897 (void) splzs(); 898 899 CPUSET_ALL_BUT(xcset, cp->cpu_id); 900 xc_trycall(NULL, NULL, NULL, xcset, (int (*)())panic_idle); 901 } 902 903 for (i = 0; i < NCPU; i++) { 904 if (i != cp->cpu_id && cpu[i] != NULL && 905 (cpu[i]->cpu_flags & CPU_EXISTS)) 906 cpu[i]->cpu_flags |= CPU_QUIESCED; 907 } 908} 909 910/* 911 * Platform callback following each entry to panicsys(). 912 */ 913/*ARGSUSED*/ 914void 915panic_enter_hw(int spl) 916{ 917 /* Nothing to do here */ 918} 919 920/* 921 * Platform-specific code to execute after panicstr is set: we invoke 922 * the PSM entry point to indicate that a panic has occurred. 923 */ 924/*ARGSUSED*/ 925void 926panic_quiesce_hw(panic_data_t *pdp) 927{ 928 psm_notifyf(PSM_PANIC_ENTER); 929 930 cmi_panic_callback(); 931 932#ifdef TRAPTRACE 933 /* 934 * Turn off TRAPTRACE 935 */ 936 TRAPTRACE_FREEZE; 937#endif /* TRAPTRACE */ 938} 939 940/* 941 * Platform callback prior to writing crash dump. 942 */ 943/*ARGSUSED*/ 944void 945panic_dump_hw(int spl) 946{ 947 /* Nothing to do here */ 948} 949 950void * 951plat_traceback(void *fpreg) 952{ 953#ifdef __xpv 954 if (IN_XPV_PANIC()) 955 return (xpv_traceback(fpreg)); 956#endif 957 return (fpreg); 958} 959 960/*ARGSUSED*/ 961void 962plat_tod_fault(enum tod_fault_type tod_bad) 963{} 964 965/*ARGSUSED*/ 966int 967blacklist(int cmd, const char *scheme, nvlist_t *fmri, const char *class) 968{ 969 return (ENOTSUP); 970} 971 972/* 973 * The underlying console output routines are protected by raising IPL in case 974 * we are still calling into the early boot services. Once we start calling 975 * the kernel console emulator, it will disable interrupts completely during 976 * character rendering (see sysp_putchar, for example). Refer to the comments 977 * and code in common/os/console.c for more information on these callbacks. 978 */ 979/*ARGSUSED*/ 980int 981console_enter(int busy) 982{ 983 return (splzs()); 984} 985 986/*ARGSUSED*/ 987void 988console_exit(int busy, int spl) 989{ 990 splx(spl); 991} 992 993/* 994 * Allocate a region of virtual address space, unmapped. 995 * Stubbed out except on sparc, at least for now. 996 */ 997/*ARGSUSED*/ 998void * 999boot_virt_alloc(void *addr, size_t size) 1000{ 1001 return (addr); 1002} 1003 1004volatile unsigned long tenmicrodata; 1005 1006void 1007tenmicrosec(void) 1008{ 1009 extern int gethrtime_hires; 1010 1011 if (gethrtime_hires) { 1012 hrtime_t start, end; 1013 start = end = gethrtime(); 1014 while ((end - start) < (10 * (NANOSEC / MICROSEC))) { 1015 SMT_PAUSE(); 1016 end = gethrtime(); 1017 } 1018 } else { 1019#if defined(__xpv) 1020 hrtime_t newtime; 1021 1022 newtime = xpv_gethrtime() + 10000; /* now + 10 us */ 1023 while (xpv_gethrtime() < newtime) 1024 SMT_PAUSE(); 1025#else /* __xpv */ 1026 int i; 1027 1028 /* 1029 * Artificial loop to induce delay. 1030 */ 1031 for (i = 0; i < microdata; i++) 1032 tenmicrodata = microdata; 1033#endif /* __xpv */ 1034 } 1035} 1036 1037/* 1038 * get_cpu_mstate() is passed an array of timestamps, NCMSTATES 1039 * long, and it fills in the array with the time spent on cpu in 1040 * each of the mstates, where time is returned in nsec. 1041 * 1042 * No guarantee is made that the returned values in times[] will 1043 * monotonically increase on sequential calls, although this will 1044 * be true in the long run. Any such guarantee must be handled by 1045 * the caller, if needed. This can happen if we fail to account 1046 * for elapsed time due to a generation counter conflict, yet we 1047 * did account for it on a prior call (see below). 1048 * 1049 * The complication is that the cpu in question may be updating 1050 * its microstate at the same time that we are reading it. 1051 * Because the microstate is only updated when the CPU's state 1052 * changes, the values in cpu_intracct[] can be indefinitely out 1053 * of date. To determine true current values, it is necessary to 1054 * compare the current time with cpu_mstate_start, and add the 1055 * difference to times[cpu_mstate]. 1056 * 1057 * This can be a problem if those values are changing out from 1058 * under us. Because the code path in new_cpu_mstate() is 1059 * performance critical, we have not added a lock to it. Instead, 1060 * we have added a generation counter. Before beginning 1061 * modifications, the counter is set to 0. After modifications, 1062 * it is set to the old value plus one. 1063 * 1064 * get_cpu_mstate() will not consider the values of cpu_mstate 1065 * and cpu_mstate_start to be usable unless the value of 1066 * cpu_mstate_gen is both non-zero and unchanged, both before and 1067 * after reading the mstate information. Note that we must 1068 * protect against out-of-order loads around accesses to the 1069 * generation counter. Also, this is a best effort approach in 1070 * that we do not retry should the counter be found to have 1071 * changed. 1072 * 1073 * cpu_intracct[] is used to identify time spent in each CPU 1074 * mstate while handling interrupts. Such time should be reported 1075 * against system time, and so is subtracted out from its 1076 * corresponding cpu_acct[] time and added to 1077 * cpu_acct[CMS_SYSTEM]. 1078 */ 1079 1080void 1081get_cpu_mstate(cpu_t *cpu, hrtime_t *times) 1082{ 1083 int i; 1084 hrtime_t now, start; 1085 uint16_t gen; 1086 uint16_t state; 1087 hrtime_t intracct[NCMSTATES]; 1088 1089 /* 1090 * Load all volatile state under the protection of membar. 1091 * cpu_acct[cpu_mstate] must be loaded to avoid double counting 1092 * of (now - cpu_mstate_start) by a change in CPU mstate that 1093 * arrives after we make our last check of cpu_mstate_gen. 1094 */ 1095 1096 now = gethrtime_unscaled(); 1097 gen = cpu->cpu_mstate_gen; 1098 1099 membar_consumer(); /* guarantee load ordering */ 1100 start = cpu->cpu_mstate_start; 1101 state = cpu->cpu_mstate; 1102 for (i = 0; i < NCMSTATES; i++) { 1103 intracct[i] = cpu->cpu_intracct[i]; 1104 times[i] = cpu->cpu_acct[i]; 1105 } 1106 membar_consumer(); /* guarantee load ordering */ 1107 1108 if (gen != 0 && gen == cpu->cpu_mstate_gen && now > start) 1109 times[state] += now - start; 1110 1111 for (i = 0; i < NCMSTATES; i++) { 1112 if (i == CMS_SYSTEM) 1113 continue; 1114 times[i] -= intracct[i]; 1115 if (times[i] < 0) { 1116 intracct[i] += times[i]; 1117 times[i] = 0; 1118 } 1119 times[CMS_SYSTEM] += intracct[i]; 1120 scalehrtime(×[i]); 1121 } 1122 scalehrtime(×[CMS_SYSTEM]); 1123} 1124 1125/* 1126 * This is a version of the rdmsr instruction that allows 1127 * an error code to be returned in the case of failure. 1128 */ 1129int 1130checked_rdmsr(uint_t msr, uint64_t *value) 1131{ 1132 if ((x86_feature & X86_MSR) == 0) 1133 return (ENOTSUP); 1134 *value = rdmsr(msr); 1135 return (0); 1136} 1137 1138/* 1139 * This is a version of the wrmsr instruction that allows 1140 * an error code to be returned in the case of failure. 1141 */ 1142int 1143checked_wrmsr(uint_t msr, uint64_t value) 1144{ 1145 if ((x86_feature & X86_MSR) == 0) 1146 return (ENOTSUP); 1147 wrmsr(msr, value); 1148 return (0); 1149} 1150 1151/* 1152 * The mem driver's usual method of using hat_devload() to establish a 1153 * temporary mapping will not work for foreign pages mapped into this 1154 * domain or for the special hypervisor-provided pages. For the foreign 1155 * pages, we often don't know which domain owns them, so we can't ask the 1156 * hypervisor to set up a new mapping. For the other pages, we don't have 1157 * a pfn, so we can't create a new PTE. For these special cases, we do a 1158 * direct uiomove() from the existing kernel virtual address. 1159 */ 1160/*ARGSUSED*/ 1161int 1162plat_mem_do_mmio(struct uio *uio, enum uio_rw rw) 1163{ 1164#if defined(__xpv) 1165 void *va = (void *)(uintptr_t)uio->uio_loffset; 1166 off_t pageoff = uio->uio_loffset & PAGEOFFSET; 1167 size_t nbytes = MIN((size_t)(PAGESIZE - pageoff), 1168 (size_t)uio->uio_iov->iov_len); 1169 1170 if ((rw == UIO_READ && 1171 (va == HYPERVISOR_shared_info || va == xen_info)) || 1172 (pfn_is_foreign(hat_getpfnum(kas.a_hat, va)))) 1173 return (uiomove(va, nbytes, rw, uio)); 1174#endif 1175 return (ENOTSUP); 1176} 1177 1178pgcnt_t 1179num_phys_pages() 1180{ 1181 pgcnt_t npages = 0; 1182 struct memlist *mp; 1183 1184#if defined(__xpv) 1185 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1186 xen_sysctl_t op; 1187 1188 op.cmd = XEN_SYSCTL_physinfo; 1189 op.interface_version = XEN_SYSCTL_INTERFACE_VERSION; 1190 if (HYPERVISOR_sysctl(&op) != 0) 1191 panic("physinfo op refused"); 1192 1193 return ((pgcnt_t)op.u.physinfo.total_pages); 1194 } 1195#endif /* __xpv */ 1196 1197 for (mp = phys_install; mp != NULL; mp = mp->next) 1198 npages += mp->size >> PAGESHIFT; 1199 1200 return (npages); 1201} 1202 1203int 1204dump_plat_addr() 1205{ 1206#ifdef __xpv 1207 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1208 mem_vtop_t mem_vtop; 1209 int cnt; 1210 1211 /* 1212 * On the hypervisor, we want to dump the page with shared_info on it. 1213 */ 1214 if (!IN_XPV_PANIC()) { 1215 mem_vtop.m_as = &kas; 1216 mem_vtop.m_va = HYPERVISOR_shared_info; 1217 mem_vtop.m_pfn = pfn; 1218 dumpvp_write(&mem_vtop, sizeof (mem_vtop_t)); 1219 cnt = 1; 1220 } else { 1221 cnt = dump_xpv_addr(); 1222 } 1223 return (cnt); 1224#else 1225 return (0); 1226#endif 1227} 1228 1229void 1230dump_plat_pfn() 1231{ 1232#ifdef __xpv 1233 pfn_t pfn = mmu_btop(xen_info->shared_info) | PFN_IS_FOREIGN_MFN; 1234 1235 if (!IN_XPV_PANIC()) 1236 dumpvp_write(&pfn, sizeof (pfn)); 1237 else 1238 dump_xpv_pfn(); 1239#endif 1240} 1241 1242/*ARGSUSED*/ 1243int 1244dump_plat_data(void *dump_cbuf) 1245{ 1246#ifdef __xpv 1247 uint32_t csize; 1248 int cnt; 1249 1250 if (!IN_XPV_PANIC()) { 1251 csize = (uint32_t)compress(HYPERVISOR_shared_info, dump_cbuf, 1252 PAGESIZE); 1253 dumpvp_write(&csize, sizeof (uint32_t)); 1254 dumpvp_write(dump_cbuf, csize); 1255 cnt = 1; 1256 } else { 1257 cnt = dump_xpv_data(dump_cbuf); 1258 } 1259 return (cnt); 1260#else 1261 return (0); 1262#endif 1263} 1264 1265/* 1266 * Calculates a linear address, given the CS selector and PC values 1267 * by looking up the %cs selector process's LDT or the CPU's GDT. 1268 * proc->p_ldtlock must be held across this call. 1269 */ 1270int 1271linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1272{ 1273 user_desc_t *descrp; 1274 caddr_t baseaddr; 1275 uint16_t idx = SELTOIDX(rp->r_cs); 1276 1277 ASSERT(rp->r_cs <= 0xFFFF); 1278 ASSERT(MUTEX_HELD(&p->p_ldtlock)); 1279 1280 if (SELISLDT(rp->r_cs)) { 1281 /* 1282 * Currently 64 bit processes cannot have private LDTs. 1283 */ 1284 ASSERT(p->p_model != DATAMODEL_LP64); 1285 1286 if (p->p_ldt == NULL) 1287 return (-1); 1288 1289 descrp = &p->p_ldt[idx]; 1290 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1291 1292 /* 1293 * Calculate the linear address (wraparound is not only ok, 1294 * it's expected behavior). The cast to uint32_t is because 1295 * LDT selectors are only allowed in 32-bit processes. 1296 */ 1297 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1298 rp->r_pc); 1299 } else { 1300#ifdef DEBUG 1301 descrp = &CPU->cpu_gdt[idx]; 1302 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1303 /* GDT-based descriptors' base addresses should always be 0 */ 1304 ASSERT(baseaddr == 0); 1305#endif 1306 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1307 } 1308 1309 return (0); 1310} 1311 1312/* 1313 * The implementation of dtrace_linear_pc is similar to the that of 1314 * linear_pc, above, but here we acquire p_ldtlock before accessing 1315 * p_ldt. This implementation is used by the pid provider; we prefix 1316 * it with "dtrace_" to avoid inducing spurious tracing events. 1317 */ 1318int 1319dtrace_linear_pc(struct regs *rp, proc_t *p, caddr_t *linearp) 1320{ 1321 user_desc_t *descrp; 1322 caddr_t baseaddr; 1323 uint16_t idx = SELTOIDX(rp->r_cs); 1324 1325 ASSERT(rp->r_cs <= 0xFFFF); 1326 1327 if (SELISLDT(rp->r_cs)) { 1328 /* 1329 * Currently 64 bit processes cannot have private LDTs. 1330 */ 1331 ASSERT(p->p_model != DATAMODEL_LP64); 1332 1333 mutex_enter(&p->p_ldtlock); 1334 if (p->p_ldt == NULL) { 1335 mutex_exit(&p->p_ldtlock); 1336 return (-1); 1337 } 1338 descrp = &p->p_ldt[idx]; 1339 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1340 mutex_exit(&p->p_ldtlock); 1341 1342 /* 1343 * Calculate the linear address (wraparound is not only ok, 1344 * it's expected behavior). The cast to uint32_t is because 1345 * LDT selectors are only allowed in 32-bit processes. 1346 */ 1347 *linearp = (caddr_t)(uintptr_t)(uint32_t)((uintptr_t)baseaddr + 1348 rp->r_pc); 1349 } else { 1350#ifdef DEBUG 1351 descrp = &CPU->cpu_gdt[idx]; 1352 baseaddr = (caddr_t)(uintptr_t)USEGD_GETBASE(descrp); 1353 /* GDT-based descriptors' base addresses should always be 0 */ 1354 ASSERT(baseaddr == 0); 1355#endif 1356 *linearp = (caddr_t)(uintptr_t)rp->r_pc; 1357 } 1358 1359 return (0); 1360} 1361