1/*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2001 Takanori Watanabe <takawata@jp.freebsd.org> 5 * Copyright (c) 2001-2012 Mitsuru IWASAKI <iwasaki@jp.freebsd.org> 6 * Copyright (c) 2003 Peter Wemm 7 * Copyright (c) 2008-2012 Jung-uk Kim <jkim@FreeBSD.org> 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32#include <sys/cdefs.h> 33__FBSDID("$FreeBSD$"); 34 35#if defined(__amd64__) 36#define DEV_APIC 37#else 38#include "opt_apic.h" 39#endif 40 41#include <sys/param.h> 42#include <sys/bus.h> 43#include <sys/eventhandler.h> 44#include <sys/kernel.h> 45#include <sys/malloc.h> 46#include <sys/memrange.h> 47#include <sys/smp.h> 48#include <sys/systm.h> 49#include <sys/cons.h> 50 51#include <vm/vm.h> 52#include <vm/pmap.h> 53 54#include <machine/clock.h> 55#include <machine/cpu.h> 56#include <machine/intr_machdep.h> 57#include <machine/md_var.h> 58#include <x86/mca.h> 59#include <machine/pcb.h> 60#include <machine/specialreg.h> 61#include <x86/ucode.h> 62 63#ifdef DEV_APIC 64#include <x86/apicreg.h> 65#include <x86/apicvar.h> 66#endif 67#ifdef SMP 68#include <machine/smp.h> 69#include <machine/vmparam.h> 70#endif 71 72#include <contrib/dev/acpica/include/acpi.h> 73 74#include <dev/acpica/acpivar.h> 75 76#include "acpi_wakecode.h" 77#include "acpi_wakedata.h" 78 79/* Make sure the code is less than a page and leave room for the stack. */ 80CTASSERT(sizeof(wakecode) < PAGE_SIZE - 1024); 81 82extern int acpi_resume_beep; 83extern int acpi_reset_video; 84extern int acpi_susp_bounce; 85 86#ifdef SMP 87extern struct susppcb **susppcbs; 88static cpuset_t suspcpus; 89#else 90static struct susppcb **susppcbs; 91#endif 92 93static void *acpi_alloc_wakeup_handler(void **); 94static void acpi_stop_beep(void *); 95 96#ifdef SMP 97static int acpi_wakeup_ap(struct acpi_softc *, int); 98static void acpi_wakeup_cpus(struct acpi_softc *); 99#endif 100 101#ifdef __amd64__ 102#define ACPI_WAKEPAGES 5 103#else 104#define ACPI_WAKEPAGES 1 105#endif 106 107#define WAKECODE_FIXUP(offset, type, val) do { \ 108 type *addr; \ 109 addr = (type *)(sc->acpi_wakeaddr + (offset)); \ 110 *addr = val; \ 111} while (0) 112 113static void 114acpi_stop_beep(void *arg) 115{ 116 117 if (acpi_resume_beep != 0) 118 timer_spkr_release(); 119} 120 121#ifdef SMP 122static int 123acpi_wakeup_ap(struct acpi_softc *sc, int cpu) 124{ 125 struct pcb *pcb; 126 int vector = (sc->acpi_wakephys >> 12) & 0xff; 127 int apic_id = cpu_apic_ids[cpu]; 128 int ms; 129 130 pcb = &susppcbs[cpu]->sp_pcb; 131 WAKECODE_FIXUP(wakeup_pcb, struct pcb *, pcb); 132 WAKECODE_FIXUP(wakeup_gdt, uint16_t, pcb->pcb_gdt.rd_limit); 133 WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, pcb->pcb_gdt.rd_base); 134 135 ipi_startup(apic_id, vector); 136 137 /* Wait up to 5 seconds for it to resume. */ 138 for (ms = 0; ms < 5000; ms++) { 139 if (!CPU_ISSET(cpu, &suspended_cpus)) 140 return (1); /* return SUCCESS */ 141 DELAY(1000); 142 } 143 return (0); /* return FAILURE */ 144} 145 146#define WARMBOOT_TARGET 0 147#ifdef __amd64__ 148#define WARMBOOT_OFF (KERNBASE + 0x0467) 149#define WARMBOOT_SEG (KERNBASE + 0x0469) 150#else /* __i386__ */ 151#define WARMBOOT_OFF (PMAP_MAP_LOW + 0x0467) 152#define WARMBOOT_SEG (PMAP_MAP_LOW + 0x0469) 153#endif 154 155#define CMOS_REG (0x70) 156#define CMOS_DATA (0x71) 157#define BIOS_RESET (0x0f) 158#define BIOS_WARM (0x0a) 159 160static void 161acpi_wakeup_cpus(struct acpi_softc *sc) 162{ 163 uint32_t mpbioswarmvec; 164 int cpu; 165 u_char mpbiosreason; 166 167 /* save the current value of the warm-start vector */ 168 mpbioswarmvec = *((uint32_t *)WARMBOOT_OFF); 169 outb(CMOS_REG, BIOS_RESET); 170 mpbiosreason = inb(CMOS_DATA); 171 172 /* setup a vector to our boot code */ 173 *((volatile u_short *)WARMBOOT_OFF) = WARMBOOT_TARGET; 174 *((volatile u_short *)WARMBOOT_SEG) = sc->acpi_wakephys >> 4; 175 outb(CMOS_REG, BIOS_RESET); 176 outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ 177 178 /* Wake up each AP. */ 179 for (cpu = 1; cpu < mp_ncpus; cpu++) { 180 if (!CPU_ISSET(cpu, &suspcpus)) 181 continue; 182 if (acpi_wakeup_ap(sc, cpu) == 0) { 183 /* restore the warmstart vector */ 184 *(uint32_t *)WARMBOOT_OFF = mpbioswarmvec; 185 panic("acpi_wakeup: failed to resume AP #%d (PHY #%d)", 186 cpu, cpu_apic_ids[cpu]); 187 } 188 } 189 190#ifdef __i386__ 191 /* 192 * Remove the identity mapping of low memory for all CPUs and sync 193 * the TLB for the BSP. The APs are now spinning in 194 * cpususpend_handler() and we will release them soon. Then each 195 * will invalidate its TLB. 196 */ 197 pmap_remap_lowptdi(false); 198#endif 199 200 /* restore the warmstart vector */ 201 *(uint32_t *)WARMBOOT_OFF = mpbioswarmvec; 202 203 outb(CMOS_REG, BIOS_RESET); 204 outb(CMOS_DATA, mpbiosreason); 205} 206#endif 207 208int 209acpi_sleep_machdep(struct acpi_softc *sc, int state) 210{ 211 ACPI_STATUS status; 212 struct pcb *pcb; 213#ifdef __amd64__ 214 struct pcpu *pc; 215 int i; 216#endif 217 218 if (sc->acpi_wakeaddr == 0ul) 219 return (-1); /* couldn't alloc wake memory */ 220 221#ifdef SMP 222 suspcpus = all_cpus; 223 CPU_CLR(PCPU_GET(cpuid), &suspcpus); 224#endif 225 226 if (acpi_resume_beep != 0) 227 timer_spkr_acquire(); 228 229 AcpiSetFirmwareWakingVector(sc->acpi_wakephys, 0); 230 231 intr_suspend(); 232 233 pcb = &susppcbs[0]->sp_pcb; 234 if (savectx(pcb)) { 235#ifdef __amd64__ 236 fpususpend(susppcbs[0]->sp_fpususpend); 237#else 238 npxsuspend(susppcbs[0]->sp_fpususpend); 239#endif 240#ifdef SMP 241 if (!CPU_EMPTY(&suspcpus) && suspend_cpus(suspcpus) == 0) { 242 device_printf(sc->acpi_dev, "Failed to suspend APs\n"); 243 return (0); /* couldn't sleep */ 244 } 245#endif 246#ifdef __amd64__ 247 hw_ibrs_ibpb_active = 0; 248 hw_ssb_active = 0; 249 cpu_stdext_feature3 = 0; 250 CPU_FOREACH(i) { 251 pc = pcpu_find(i); 252 pc->pc_ibpb_set = 0; 253 } 254#endif 255 256 WAKECODE_FIXUP(resume_beep, uint8_t, (acpi_resume_beep != 0)); 257 WAKECODE_FIXUP(reset_video, uint8_t, (acpi_reset_video != 0)); 258 259#ifdef __amd64__ 260 WAKECODE_FIXUP(wakeup_efer, uint64_t, rdmsr(MSR_EFER) & 261 ~(EFER_LMA)); 262#else 263 if ((amd_feature & AMDID_NX) != 0) 264 WAKECODE_FIXUP(wakeup_efer, uint64_t, rdmsr(MSR_EFER)); 265 WAKECODE_FIXUP(wakeup_cr4, register_t, pcb->pcb_cr4); 266#endif 267 WAKECODE_FIXUP(wakeup_pcb, struct pcb *, pcb); 268 WAKECODE_FIXUP(wakeup_gdt, uint16_t, pcb->pcb_gdt.rd_limit); 269 WAKECODE_FIXUP(wakeup_gdt + 2, uint64_t, pcb->pcb_gdt.rd_base); 270 271#ifdef __i386__ 272 /* 273 * Map some low memory with virt == phys for ACPI wakecode 274 * to use to jump to high memory after enabling paging. This 275 * is the same as for similar jump in locore, except the 276 * jump is a single instruction, and we know its address 277 * more precisely so only need a single PTD, and we have to 278 * be careful to use the kernel map (PTD[0] is for curthread 279 * which may be a user thread in deprecated APIs). 280 */ 281 pmap_remap_lowptdi(true); 282#endif 283 284 /* Call ACPICA to enter the desired sleep state */ 285 if (state == ACPI_STATE_S4 && sc->acpi_s4bios) 286 status = AcpiEnterSleepStateS4bios(); 287 else 288 status = AcpiEnterSleepState(state); 289 if (ACPI_FAILURE(status)) { 290 device_printf(sc->acpi_dev, 291 "AcpiEnterSleepState failed - %s\n", 292 AcpiFormatException(status)); 293 return (0); /* couldn't sleep */ 294 } 295 296 if (acpi_susp_bounce) 297 resumectx(pcb); 298 299 for (;;) 300 ia32_pause(); 301 } else { 302 /* 303 * Re-initialize console hardware as soon as possibe. 304 * No console output (e.g. printf) is allowed before 305 * this point. 306 */ 307 cnresume(); 308#ifdef __amd64__ 309 fpuresume(susppcbs[0]->sp_fpususpend); 310#else 311 npxresume(susppcbs[0]->sp_fpususpend); 312#endif 313 } 314 315 return (1); /* wakeup successfully */ 316} 317 318int 319acpi_wakeup_machdep(struct acpi_softc *sc, int state, int sleep_result, 320 int intr_enabled) 321{ 322 323 if (sleep_result == -1) 324 return (sleep_result); 325 326 if (!intr_enabled) { 327 /* Wakeup MD procedures in interrupt disabled context */ 328 if (sleep_result == 1) { 329 ucode_reload(); 330 pmap_init_pat(); 331 initializecpu(); 332 PCPU_SET(switchtime, 0); 333 PCPU_SET(switchticks, ticks); 334#ifdef DEV_APIC 335 lapic_xapic_mode(); 336#endif 337#ifdef SMP 338 if (!CPU_EMPTY(&suspcpus)) 339 acpi_wakeup_cpus(sc); 340#endif 341 } 342 343#ifdef SMP 344 if (!CPU_EMPTY(&suspcpus)) 345 resume_cpus(suspcpus); 346#endif 347 mca_resume(); 348#ifdef __amd64__ 349 if (vmm_resume_p != NULL) 350 vmm_resume_p(); 351#endif 352 intr_resume(/*suspend_cancelled*/false); 353 354 AcpiSetFirmwareWakingVector(0, 0); 355 } else { 356 /* Wakeup MD procedures in interrupt enabled context */ 357 if (sleep_result == 1 && mem_range_softc.mr_op != NULL && 358 mem_range_softc.mr_op->reinit != NULL) 359 mem_range_softc.mr_op->reinit(&mem_range_softc); 360 } 361 362 return (sleep_result); 363} 364 365static void * 366acpi_alloc_wakeup_handler(void *wakepages[ACPI_WAKEPAGES]) 367{ 368 int i; 369 370 memset(wakepages, 0, ACPI_WAKEPAGES * sizeof(*wakepages)); 371 372 /* 373 * Specify the region for our wakeup code. We want it in the low 1 MB 374 * region, excluding real mode IVT (0-0x3ff), BDA (0x400-0x4ff), EBDA 375 * (less than 128KB, below 0xa0000, must be excluded by SMAP and DSDT), 376 * and ROM area (0xa0000 and above). The temporary page tables must be 377 * page-aligned. 378 */ 379 for (i = 0; i < ACPI_WAKEPAGES; i++) { 380 wakepages[i] = contigmalloc(PAGE_SIZE, M_DEVBUF, 381 M_NOWAIT 382#ifdef __i386__ 383 | M_EXEC 384#endif 385 , 0x500, 0xa0000, PAGE_SIZE, 0ul); 386 if (wakepages[i] == NULL) { 387 printf("%s: can't alloc wake memory\n", __func__); 388 goto freepages; 389 } 390 } 391 if (EVENTHANDLER_REGISTER(power_resume, acpi_stop_beep, NULL, 392 EVENTHANDLER_PRI_LAST) == NULL) { 393 printf("%s: can't register event handler\n", __func__); 394 goto freepages; 395 } 396 susppcbs = malloc(mp_ncpus * sizeof(*susppcbs), M_DEVBUF, M_WAITOK); 397 for (i = 0; i < mp_ncpus; i++) { 398 susppcbs[i] = malloc(sizeof(**susppcbs), M_DEVBUF, M_WAITOK); 399 susppcbs[i]->sp_fpususpend = alloc_fpusave(M_WAITOK); 400 } 401 402 return (wakepages); 403 404freepages: 405 for (i = 0; i < ACPI_WAKEPAGES; i++) 406 if (wakepages[i] != NULL) 407 contigfree(wakepages[i], PAGE_SIZE, M_DEVBUF); 408 return (NULL); 409} 410 411void 412acpi_install_wakeup_handler(struct acpi_softc *sc) 413{ 414 static void *wakeaddr; 415 void *wakepages[ACPI_WAKEPAGES]; 416#ifdef __amd64__ 417 uint64_t *pt5, *pt4, *pt3, *pt2; 418 vm_paddr_t pt5pa, pt4pa, pt3pa, pt2pa; 419 int i; 420#endif 421 422 if (wakeaddr != NULL) 423 return; 424 425 if (acpi_alloc_wakeup_handler(wakepages) == NULL) 426 return; 427 428 wakeaddr = wakepages[0]; 429 sc->acpi_wakeaddr = (vm_offset_t)wakeaddr; 430 sc->acpi_wakephys = vtophys(wakeaddr); 431 432#ifdef __amd64__ 433 if (la57) { 434 pt5 = wakepages[4]; 435 pt5pa = vtophys(pt5); 436 } 437 pt4 = wakepages[1]; 438 pt3 = wakepages[2]; 439 pt2 = wakepages[3]; 440 pt4pa = vtophys(pt4); 441 pt3pa = vtophys(pt3); 442 pt2pa = vtophys(pt2); 443#endif 444 445 bcopy(wakecode, (void *)sc->acpi_wakeaddr, sizeof(wakecode)); 446 447 /* Patch GDT base address, ljmp targets. */ 448 WAKECODE_FIXUP((bootgdtdesc + 2), uint32_t, 449 sc->acpi_wakephys + bootgdt); 450 WAKECODE_FIXUP((wakeup_sw32 + 2), uint32_t, 451 sc->acpi_wakephys + wakeup_32); 452#ifdef __amd64__ 453 WAKECODE_FIXUP((wakeup_sw64 + 1), uint32_t, 454 sc->acpi_wakephys + wakeup_64); 455 WAKECODE_FIXUP(wakeup_pagetables, uint32_t, la57 ? (pt5pa | 0x1) : 456 pt4pa); 457#endif 458 459 /* Save pointers to some global data. */ 460 WAKECODE_FIXUP(wakeup_ret, void *, resumectx); 461#ifndef __amd64__ 462 WAKECODE_FIXUP(wakeup_cr3, register_t, pmap_get_kcr3()); 463#else /* __amd64__ */ 464 /* Create the initial 1GB replicated page tables */ 465 for (i = 0; i < NPTEPG; i++) { 466 if (la57) { 467 pt5[i] = (uint64_t)pt4pa; 468 pt5[i] |= PG_V | PG_RW | PG_U; 469 } 470 471 /* 472 * Each slot of the level 4 pages points 473 * to the same level 3 page 474 */ 475 pt4[i] = (uint64_t)pt3pa; 476 pt4[i] |= PG_V | PG_RW | PG_U; 477 478 /* 479 * Each slot of the level 3 pages points 480 * to the same level 2 page 481 */ 482 pt3[i] = (uint64_t)pt2pa; 483 pt3[i] |= PG_V | PG_RW | PG_U; 484 485 /* The level 2 page slots are mapped with 2MB pages for 1GB. */ 486 pt2[i] = i * NBPDR; 487 pt2[i] |= PG_V | PG_RW | PG_PS | PG_U; 488 } 489#endif /* !__amd64__ */ 490 491 if (bootverbose) 492 device_printf(sc->acpi_dev, "wakeup code va %#jx pa %#jx\n", 493 (uintmax_t)sc->acpi_wakeaddr, (uintmax_t)sc->acpi_wakephys); 494} 495