150472Speter/* $OpenBSD: machdep.c,v 1.89 2024/04/29 13:01:54 jsg Exp $ */ 247569Sache/* 347569Sache * Copyright (c) 2014 Patrick Wildt <patrick@blueri.se> 447569Sache * Copyright (c) 2021 Mark Kettenis <kettenis@openbsd.org> 547569Sache * 647569Sache * Permission to use, copy, modify, and distribute this software for any 747569Sache * purpose with or without fee is hereby granted, provided that the above 847569Sache * copyright notice and this permission notice appear in all copies. 947569Sache * 1047569Sache * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 1147569Sache * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 1247569Sache * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 1347569Sache * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 1447569Sache * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 1547569Sache * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 1647569Sache * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 1747569Sache */ 1847569Sache 19#include <sys/param.h> 20#include <sys/systm.h> 21#include <sys/sched.h> 22#include <sys/proc.h> 23#include <sys/sysctl.h> 24#include <sys/reboot.h> 25#include <sys/mount.h> 26#include <sys/exec.h> 27#include <sys/user.h> 28#include <sys/conf.h> 29#include <sys/kcore.h> 30#include <sys/core.h> 31#include <sys/msgbuf.h> 32#include <sys/buf.h> 33#include <sys/termios.h> 34#include <sys/sensors.h> 35#include <sys/malloc.h> 36 37#include <net/if.h> 38#include <uvm/uvm.h> 39#include <dev/cons.h> 40#include <dev/ofw/fdt.h> 41#include <dev/ofw/openfirm.h> 42#include <machine/param.h> 43#include <machine/kcore.h> 44#include <machine/bootconfig.h> 45#include <machine/bus.h> 46#include <machine/fpu.h> 47#include <arm64/arm64/arm64var.h> 48 49#include <machine/db_machdep.h> 50#include <ddb/db_extern.h> 51 52#include <dev/efi/efi.h> 53 54#include "softraid.h" 55#if NSOFTRAID > 0 56#include <dev/softraidvar.h> 57#endif 58 59extern vaddr_t virtual_avail; 60extern uint64_t esym; 61 62extern char _start[]; 63 64char *boot_args = NULL; 65uint8_t *bootmac = NULL; 66 67int stdout_node; 68int stdout_speed; 69 70void (*cpuresetfn)(void); 71void (*powerdownfn)(void); 72 73int cold = 1; 74int lid_action = 1; 75 76struct vm_map *exec_map = NULL; 77struct vm_map *phys_map = NULL; 78 79int physmem; 80 81struct consdev *cn_tab; 82 83caddr_t msgbufaddr; 84paddr_t msgbufphys; 85 86struct user *proc0paddr; 87 88struct uvm_constraint_range dma_constraint = { 0x0, (paddr_t)-1 }; 89struct uvm_constraint_range *uvm_md_constraints[] = { 90 &dma_constraint, 91 NULL, 92}; 93 94/* the following is used externally (sysctl_hw) */ 95char machine[] = MACHINE; /* from <machine/param.h> */ 96 97int safepri = 0; 98 99struct cpu_info cpu_info_primary; 100struct cpu_info *cpu_info[MAXCPUS] = { &cpu_info_primary }; 101 102struct fdt_reg memreg[VM_PHYSSEG_MAX]; 103int nmemreg; 104 105void memreg_add(const struct fdt_reg *); 106void memreg_remove(const struct fdt_reg *); 107 108static int 109atoi(const char *s) 110{ 111 int n, neg; 112 113 n = 0; 114 neg = 0; 115 116 while (*s == '-') { 117 s++; 118 neg = !neg; 119 } 120 121 while (*s != '\0') { 122 if (*s < '0' || *s > '9') 123 break; 124 125 n = (10 * n) + (*s - '0'); 126 s++; 127 } 128 129 return (neg ? -n : n); 130} 131 132void * 133fdt_find_cons(const char *name) 134{ 135 char *alias = "serial0"; 136 char buf[128]; 137 char *stdout = NULL; 138 char *p; 139 void *node; 140 141 /* First check if "stdout-path" is set. */ 142 node = fdt_find_node("/chosen"); 143 if (node) { 144 if (fdt_node_property(node, "stdout-path", &stdout) > 0) { 145 if (strchr(stdout, ':') != NULL) { 146 strlcpy(buf, stdout, sizeof(buf)); 147 if ((p = strchr(buf, ':')) != NULL) { 148 *p++ = '\0'; 149 stdout_speed = atoi(p); 150 } 151 stdout = buf; 152 } 153 if (stdout[0] != '/') { 154 /* It's an alias. */ 155 alias = stdout; 156 stdout = NULL; 157 } 158 } 159 } 160 161 /* Perform alias lookup if necessary. */ 162 if (stdout == NULL) { 163 node = fdt_find_node("/aliases"); 164 if (node) 165 fdt_node_property(node, alias, &stdout); 166 } 167 168 /* Lookup the physical address of the interface. */ 169 if (stdout) { 170 node = fdt_find_node(stdout); 171 if (node && fdt_is_compatible(node, name)) { 172 stdout_node = OF_finddevice(stdout); 173 return (node); 174 } 175 } 176 177 return (NULL); 178} 179 180void amluart_init_cons(void); 181void cduart_init_cons(void); 182void com_fdt_init_cons(void); 183void exuart_init_cons(void); 184void imxuart_init_cons(void); 185void mvuart_init_cons(void); 186void pluart_init_cons(void); 187void simplefb_init_cons(bus_space_tag_t); 188 189void 190consinit(void) 191{ 192 static int consinit_called = 0; 193 194 if (consinit_called != 0) 195 return; 196 197 consinit_called = 1; 198 199 amluart_init_cons(); 200 cduart_init_cons(); 201 com_fdt_init_cons(); 202 exuart_init_cons(); 203 imxuart_init_cons(); 204 mvuart_init_cons(); 205 pluart_init_cons(); 206 simplefb_init_cons(&arm64_bs_tag); 207} 208 209void 210cpu_idle_enter(void) 211{ 212 disable_irq_daif(); 213} 214 215void (*cpu_idle_cycle_fcn)(void) = cpu_wfi; 216 217void 218cpu_idle_cycle(void) 219{ 220 cpu_idle_cycle_fcn(); 221 enable_irq_daif(); 222 disable_irq_daif(); 223} 224 225void 226cpu_idle_leave(void) 227{ 228 enable_irq_daif(); 229} 230 231/* Dummy trapframe for proc0. */ 232struct trapframe proc0tf; 233 234void 235cpu_startup(void) 236{ 237 u_int loop; 238 paddr_t minaddr; 239 paddr_t maxaddr; 240 241 proc0.p_addr = proc0paddr; 242 243 /* 244 * Give pmap a chance to set up a few more things now the vm 245 * is initialised 246 */ 247 pmap_postinit(); 248 249 /* 250 * Initialize error message buffer (at end of core). 251 */ 252 253 /* msgbufphys was setup during the secondary boot strap */ 254 for (loop = 0; loop < atop(MSGBUFSIZE); ++loop) 255 pmap_kenter_pa((vaddr_t)msgbufaddr + loop * PAGE_SIZE, 256 msgbufphys + loop * PAGE_SIZE, PROT_READ | PROT_WRITE); 257 pmap_update(pmap_kernel()); 258 initmsgbuf(msgbufaddr, round_page(MSGBUFSIZE)); 259 260 /* 261 * Identify ourselves for the msgbuf (everything printed earlier will 262 * not be buffered). 263 */ 264 printf("%s", version); 265 266 printf("real mem = %lu (%luMB)\n", ptoa(physmem), 267 ptoa(physmem) / 1024 / 1024); 268 269 /* 270 * Allocate a submap for exec arguments. This map effectively 271 * limits the number of processes exec'ing at any time. 272 */ 273 minaddr = vm_map_min(kernel_map); 274 exec_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 275 16 * NCARGS, VM_MAP_PAGEABLE, FALSE, NULL); 276 277 /* 278 * Allocate a submap for physio 279 */ 280 phys_map = uvm_km_suballoc(kernel_map, &minaddr, &maxaddr, 281 VM_PHYS_SIZE, 0, FALSE, NULL); 282 283 /* 284 * Set up buffers, so they can be used to read disk labels. 285 */ 286 bufinit(); 287 288 printf("avail mem = %lu (%luMB)\n", ptoa(uvmexp.free), 289 ptoa(uvmexp.free) / 1024 / 1024); 290 291 curpcb = &proc0.p_addr->u_pcb; 292 curpcb->pcb_flags = 0; 293 curpcb->pcb_tf = &proc0tf; 294 295 if (boothowto & RB_CONFIG) { 296#ifdef BOOT_CONFIG 297 user_config(); 298#else 299 printf("kernel does not support -c; continuing..\n"); 300#endif 301 } 302} 303 304void cpu_switchto_asm(struct proc *, struct proc *); 305 306void 307cpu_switchto(struct proc *old, struct proc *new) 308{ 309 if (old) { 310 struct pcb *pcb = &old->p_addr->u_pcb; 311 312 if (pcb->pcb_flags & PCB_FPU) 313 fpu_save(old); 314 315 fpu_drop(); 316 } 317 318 cpu_switchto_asm(old, new); 319} 320 321/* 322 * machine dependent system variables. 323 */ 324 325const struct sysctl_bounded_args cpuctl_vars[] = { 326 { CPU_LIDACTION, &lid_action, 0, 2 }, 327}; 328 329int 330cpu_sysctl(int *name, u_int namelen, void *oldp, size_t *oldlenp, void *newp, 331 size_t newlen, struct proc *p) 332{ 333 char *compatible; 334 int node, len, error; 335 uint64_t value; 336 337 /* all sysctl names at this level are terminal */ 338 if (namelen != 1) 339 return (ENOTDIR); /* overloaded */ 340 341 switch (name[0]) { 342 case CPU_COMPATIBLE: 343 node = OF_finddevice("/"); 344 len = OF_getproplen(node, "compatible"); 345 if (len <= 0) 346 return (EOPNOTSUPP); 347 compatible = malloc(len, M_TEMP, M_WAITOK | M_ZERO); 348 OF_getprop(node, "compatible", compatible, len); 349 compatible[len - 1] = 0; 350 error = sysctl_rdstring(oldp, oldlenp, newp, compatible); 351 free(compatible, M_TEMP, len); 352 return error; 353 case CPU_ID_AA64ISAR0: 354 value = cpu_id_aa64isar0 & ID_AA64ISAR0_MASK; 355 value &= ~ID_AA64ISAR0_TLB_MASK; 356 return sysctl_rdquad(oldp, oldlenp, newp, value); 357 case CPU_ID_AA64ISAR1: 358 value = cpu_id_aa64isar1 & ID_AA64ISAR1_MASK; 359 value &= ~ID_AA64ISAR1_SPECRES_MASK; 360 return sysctl_rdquad(oldp, oldlenp, newp, value); 361 case CPU_ID_AA64PFR0: 362 value = 0; 363 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_FP_MASK; 364 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_ADV_SIMD_MASK; 365 value |= cpu_id_aa64pfr0 & ID_AA64PFR0_DIT_MASK; 366 return sysctl_rdquad(oldp, oldlenp, newp, value); 367 case CPU_ID_AA64PFR1: 368 value = 0; 369 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_BT_MASK; 370 value |= cpu_id_aa64pfr1 & ID_AA64PFR1_SSBS_MASK; 371 return sysctl_rdquad(oldp, oldlenp, newp, value); 372 case CPU_ID_AA64ISAR2: 373 case CPU_ID_AA64MMFR0: 374 case CPU_ID_AA64MMFR1: 375 case CPU_ID_AA64MMFR2: 376 case CPU_ID_AA64SMFR0: 377 case CPU_ID_AA64ZFR0: 378 return sysctl_rdquad(oldp, oldlenp, newp, 0); 379 default: 380 return (sysctl_bounded_arr(cpuctl_vars, nitems(cpuctl_vars), 381 name, namelen, oldp, oldlenp, newp, newlen)); 382 } 383 /* NOTREACHED */ 384} 385 386void dumpsys(void); 387 388int waittime = -1; 389 390__dead void 391boot(int howto) 392{ 393 if ((howto & RB_RESET) != 0) 394 goto doreset; 395 396 if (cold) { 397 if ((howto & RB_USERREQ) == 0) 398 howto |= RB_HALT; 399 goto haltsys; 400 } 401 402 boothowto = howto; 403 if ((howto & RB_NOSYNC) == 0 && waittime < 0) { 404 waittime = 0; 405 vfs_shutdown(curproc); 406 407 if ((howto & RB_TIMEBAD) == 0) { 408 resettodr(); 409 } else { 410 printf("WARNING: not updating battery clock\n"); 411 } 412 } 413 if_downall(); 414 415 uvm_shutdown(); 416 splhigh(); 417 cold = 1; 418 419 if ((howto & RB_DUMP) != 0) 420 dumpsys(); 421 422haltsys: 423 config_suspend_all(DVACT_POWERDOWN); 424 425 if ((howto & RB_HALT) != 0) { 426 if ((howto & RB_POWERDOWN) != 0) { 427 printf("\nAttempting to power down...\n"); 428 delay(500000); 429 if (powerdownfn) 430 (*powerdownfn)(); 431 } 432 433 printf("\n"); 434 printf("The operating system has halted.\n"); 435 printf("Please press any key to reboot.\n\n"); 436 cngetc(); 437 } 438 439doreset: 440 printf("rebooting...\n"); 441 delay(500000); 442 if (cpuresetfn) 443 (*cpuresetfn)(); 444 printf("reboot failed; spinning\n"); 445 for (;;) 446 continue; 447 /* NOTREACHED */ 448} 449 450void 451setregs(struct proc *p, struct exec_package *pack, u_long stack, 452 struct ps_strings *arginfo) 453{ 454 struct pmap *pm = p->p_vmspace->vm_map.pmap; 455 struct pcb *pcb = &p->p_addr->u_pcb; 456 struct trapframe *tf = pcb->pcb_tf; 457 458 if (pack->ep_flags & EXEC_NOBTCFI) 459 pm->pm_guarded = 0; 460 else 461 pm->pm_guarded = ATTR_GP; 462 463 arc4random_buf(&pm->pm_apiakey, sizeof(pm->pm_apiakey)); 464 arc4random_buf(&pm->pm_apdakey, sizeof(pm->pm_apdakey)); 465 arc4random_buf(&pm->pm_apibkey, sizeof(pm->pm_apibkey)); 466 arc4random_buf(&pm->pm_apdbkey, sizeof(pm->pm_apdbkey)); 467 arc4random_buf(&pm->pm_apgakey, sizeof(pm->pm_apgakey)); 468 pmap_setpauthkeys(pm); 469 470 /* If we were using the FPU, forget about it. */ 471 memset(&pcb->pcb_fpstate, 0, sizeof(pcb->pcb_fpstate)); 472 pcb->pcb_flags &= ~PCB_FPU; 473 fpu_drop(); 474 475 memset(tf, 0, sizeof *tf); 476 tf->tf_sp = stack; 477 tf->tf_lr = pack->ep_entry; 478 tf->tf_elr = pack->ep_entry; /* ??? */ 479 tf->tf_spsr = PSR_M_EL0t | PSR_DIT; 480} 481 482void 483need_resched(struct cpu_info *ci) 484{ 485 ci->ci_want_resched = 1; 486 487 /* There's a risk we'll be called before the idle threads start */ 488 if (ci->ci_curproc) { 489 aston(ci->ci_curproc); 490 cpu_kick(ci); 491 } 492} 493 494int cpu_dumpsize(void); 495u_long cpu_dump_mempagecnt(void); 496 497paddr_t dumpmem_paddr; 498vaddr_t dumpmem_vaddr; 499psize_t dumpmem_sz; 500 501/* 502 * These variables are needed by /sbin/savecore 503 */ 504u_long dumpmag = 0x8fca0101; /* magic number */ 505int dumpsize = 0; /* pages */ 506long dumplo = 0; /* blocks */ 507 508/* 509 * cpu_dump: dump the machine-dependent kernel core dump headers. 510 */ 511int 512cpu_dump(void) 513{ 514 int (*dump)(dev_t, daddr_t, caddr_t, size_t); 515 char buf[dbtob(1)]; 516 kcore_seg_t *segp; 517 cpu_kcore_hdr_t *cpuhdrp; 518 phys_ram_seg_t *memsegp; 519#if 0 520 caddr_t va; 521 int i; 522#endif 523 524 dump = bdevsw[major(dumpdev)].d_dump; 525 526 memset(buf, 0, sizeof buf); 527 segp = (kcore_seg_t *)buf; 528 cpuhdrp = (cpu_kcore_hdr_t *)&buf[ALIGN(sizeof(*segp))]; 529 memsegp = (phys_ram_seg_t *)&buf[ALIGN(sizeof(*segp)) + 530 ALIGN(sizeof(*cpuhdrp))]; 531 532 /* 533 * Generate a segment header. 534 */ 535 CORE_SETMAGIC(*segp, KCORE_MAGIC, MID_MACHINE, CORE_CPU); 536 segp->c_size = dbtob(1) - ALIGN(sizeof(*segp)); 537 538 /* 539 * Add the machine-dependent header info. 540 */ 541 cpuhdrp->kernelbase = KERNEL_BASE; 542 cpuhdrp->kerneloffs = 0; 543 cpuhdrp->staticsize = 0; 544 cpuhdrp->pmap_kernel_l1 = 0; 545 cpuhdrp->pmap_kernel_l2 = 0; 546 547#if 0 548 /* 549 * Fill in the memory segment descriptors. 550 */ 551 for (i = 0; i < mem_cluster_cnt; i++) { 552 memsegp[i].start = mem_clusters[i].start; 553 memsegp[i].size = mem_clusters[i].size & PMAP_PA_MASK; 554 } 555 556 /* 557 * If we have dump memory then assume the kernel stack is in high 558 * memory and bounce 559 */ 560 if (dumpmem_vaddr != 0) { 561 memcpy((char *)dumpmem_vaddr, buf, sizeof(buf)); 562 va = (caddr_t)dumpmem_vaddr; 563 } else { 564 va = (caddr_t)buf; 565 } 566 return (dump(dumpdev, dumplo, va, dbtob(1))); 567#else 568 return ENOSYS; 569#endif 570} 571 572/* 573 * This is called by main to set dumplo and dumpsize. 574 * Dumps always skip the first PAGE_SIZE of disk space 575 * in case there might be a disk label stored there. 576 * If there is extra space, put dump at the end to 577 * reduce the chance that swapping trashes it. 578 */ 579void 580dumpconf(void) 581{ 582 int nblks, dumpblks; /* size of dump area */ 583 584 if (dumpdev == NODEV || 585 (nblks = (bdevsw[major(dumpdev)].d_psize)(dumpdev)) == 0) 586 return; 587 if (nblks <= ctod(1)) 588 return; 589 590 dumpblks = cpu_dumpsize(); 591 if (dumpblks < 0) 592 return; 593 dumpblks += ctod(cpu_dump_mempagecnt()); 594 595 /* If dump won't fit (incl. room for possible label), punt. */ 596 if (dumpblks > (nblks - ctod(1))) 597 return; 598 599 /* Put dump at end of partition */ 600 dumplo = nblks - dumpblks; 601 602 /* dumpsize is in page units, and doesn't include headers. */ 603 dumpsize = cpu_dump_mempagecnt(); 604} 605 606/* 607 * Doadump comes here after turning off memory management and 608 * getting on the dump stack, either when called above, or by 609 * the auto-restart code. 610 */ 611#define BYTES_PER_DUMP MAXPHYS /* must be a multiple of pagesize */ 612 613void 614dumpsys(void) 615{ 616 u_long totalbytesleft, bytes, i, n, memseg; 617 u_long maddr; 618 daddr_t blkno; 619 void *va; 620 int (*dump)(dev_t, daddr_t, caddr_t, size_t); 621 int error; 622 623 if (dumpdev == NODEV) 624 return; 625 626 /* 627 * For dumps during autoconfiguration, 628 * if dump device has already configured... 629 */ 630 if (dumpsize == 0) 631 dumpconf(); 632 if (dumplo <= 0 || dumpsize == 0) { 633 printf("\ndump to dev %u,%u not possible\n", major(dumpdev), 634 minor(dumpdev)); 635 return; 636 } 637 printf("\ndumping to dev %u,%u offset %ld\n", major(dumpdev), 638 minor(dumpdev), dumplo); 639 640#ifdef UVM_SWAP_ENCRYPT 641 uvm_swap_finicrypt_all(); 642#endif 643 644 error = (*bdevsw[major(dumpdev)].d_psize)(dumpdev); 645 printf("dump "); 646 if (error == -1) { 647 printf("area unavailable\n"); 648 return; 649 } 650 651 if ((error = cpu_dump()) != 0) 652 goto err; 653 654 totalbytesleft = ptoa(cpu_dump_mempagecnt()); 655 blkno = dumplo + cpu_dumpsize(); 656 dump = bdevsw[major(dumpdev)].d_dump; 657 error = 0; 658 659 bytes = n = i = memseg = 0; 660 maddr = 0; 661 va = 0; 662#if 0 663 for (memseg = 0; memseg < mem_cluster_cnt; memseg++) { 664 maddr = mem_clusters[memseg].start; 665 bytes = mem_clusters[memseg].size; 666 667 for (i = 0; i < bytes; i += n, totalbytesleft -= n) { 668 /* Print out how many MBs we have left to go. */ 669 if ((totalbytesleft % (1024*1024)) < BYTES_PER_DUMP) 670 printf("%ld ", totalbytesleft / (1024 * 1024)); 671 672 /* Limit size for next transfer. */ 673 n = bytes - i; 674 if (n > BYTES_PER_DUMP) 675 n = BYTES_PER_DUMP; 676 if (maddr > 0xffffffff) { 677 va = (void *)dumpmem_vaddr; 678 if (n > dumpmem_sz) 679 n = dumpmem_sz; 680 memcpy(va, (void *)PMAP_DIRECT_MAP(maddr), n); 681 } else { 682 va = (void *)PMAP_DIRECT_MAP(maddr); 683 } 684 685 error = (*dump)(dumpdev, blkno, va, n); 686 if (error) 687 goto err; 688 maddr += n; 689 blkno += btodb(n); /* XXX? */ 690 691#if 0 /* XXX this doesn't work. grr. */ 692 /* operator aborting dump? */ 693 if (sget() != NULL) { 694 error = EINTR; 695 break; 696 } 697#endif 698 } 699 } 700#endif 701 702 err: 703 switch (error) { 704 705 case ENXIO: 706 printf("device bad\n"); 707 break; 708 709 case EFAULT: 710 printf("device not ready\n"); 711 break; 712 713 case EINVAL: 714 printf("area improper\n"); 715 break; 716 717 case EIO: 718 printf("i/o error\n"); 719 break; 720 721 case EINTR: 722 printf("aborted from console\n"); 723 break; 724 725 case 0: 726 printf("succeeded\n"); 727 break; 728 729 default: 730 printf("error %d\n", error); 731 break; 732 } 733 printf("\n\n"); 734 delay(5000000); /* 5 seconds */ 735} 736 737 738/* 739 * Size of memory segments, before any memory is stolen. 740 */ 741phys_ram_seg_t mem_clusters[VM_PHYSSEG_MAX]; 742int mem_cluster_cnt; 743 744/* 745 * cpu_dumpsize: calculate size of machine-dependent kernel core dump headers. 746 */ 747int 748cpu_dumpsize(void) 749{ 750 int size; 751 752 size = ALIGN(sizeof(kcore_seg_t)) + 753 ALIGN(mem_cluster_cnt * sizeof(phys_ram_seg_t)); 754 if (roundup(size, dbtob(1)) != dbtob(1)) 755 return (-1); 756 757 return (1); 758} 759 760u_long 761cpu_dump_mempagecnt(void) 762{ 763 return 0; 764} 765 766int64_t dcache_line_size; /* The minimum D cache line size */ 767int64_t icache_line_size; /* The minimum I cache line size */ 768int64_t idcache_line_size; /* The minimum cache line size */ 769int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 770 771void 772cache_setup(void) 773{ 774 int dcache_line_shift, icache_line_shift, dczva_line_shift; 775 uint32_t ctr_el0; 776 uint32_t dczid_el0; 777 778 ctr_el0 = READ_SPECIALREG(ctr_el0); 779 780 /* Read the log2 words in each D cache line */ 781 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); 782 /* Get the D cache line size */ 783 dcache_line_size = sizeof(int) << dcache_line_shift; 784 785 /* And the same for the I cache */ 786 icache_line_shift = CTR_ILINE_SIZE(ctr_el0); 787 icache_line_size = sizeof(int) << icache_line_shift; 788 789 idcache_line_size = MIN(dcache_line_size, icache_line_size); 790 791 dczid_el0 = READ_SPECIALREG(dczid_el0); 792 793 /* Check if dc zva is not prohibited */ 794 if (dczid_el0 & DCZID_DZP) 795 dczva_line_size = 0; 796 else { 797 /* Same as with above calculations */ 798 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 799 dczva_line_size = sizeof(int) << dczva_line_shift; 800 } 801} 802 803uint64_t mmap_start; 804uint32_t mmap_size; 805uint32_t mmap_desc_size; 806uint32_t mmap_desc_ver; 807 808EFI_MEMORY_DESCRIPTOR *mmap; 809 810void collect_kernel_args(const char *); 811void process_kernel_args(void); 812 813int pmap_bootstrap_bs_map(bus_space_tag_t, bus_addr_t, 814 bus_size_t, int, bus_space_handle_t *); 815 816void 817initarm(struct arm64_bootparams *abp) 818{ 819 long kernbase = (long)_start & ~PAGE_MASK; 820 long kvo = abp->kern_delta; 821 paddr_t memstart, memend; 822 paddr_t startpa, endpa, pa; 823 vaddr_t vstart, va; 824 struct fdt_head *fh; 825 void *config = abp->arg2; 826 void *fdt = NULL; 827 struct fdt_reg reg; 828 void *node; 829 EFI_PHYSICAL_ADDRESS system_table = 0; 830 int (*map_func_save)(bus_space_tag_t, bus_addr_t, bus_size_t, int, 831 bus_space_handle_t *); 832 int i; 833 834 /* 835 * Set the per-CPU pointer with a backup in tpidr_el1 to be 836 * loaded when entering the kernel from userland. 837 */ 838 __asm volatile("mov x18, %0\n" 839 "msr tpidr_el1, %0" :: "r"(&cpu_info_primary)); 840 841 cache_setup(); 842 843 /* The bootloader has loaded us into a 64MB block. */ 844 memstart = KERNBASE + kvo; 845 memend = memstart + 64 * 1024 * 1024; 846 847 /* Bootstrap enough of pmap to enter the kernel proper. */ 848 vstart = pmap_bootstrap(kvo, abp->kern_l1pt, 849 kernbase, esym, memstart, memend); 850 851 /* Map the FDT header to determine its size. */ 852 va = vstart; 853 startpa = trunc_page((paddr_t)config); 854 endpa = round_page((paddr_t)config + sizeof(struct fdt_head)); 855 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE) 856 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB); 857 fh = (void *)(vstart + ((paddr_t)config - startpa)); 858 if (betoh32(fh->fh_magic) != FDT_MAGIC || betoh32(fh->fh_size) == 0) 859 panic("%s: no FDT", __func__); 860 861 /* Map the remainder of the FDT. */ 862 endpa = round_page((paddr_t)config + betoh32(fh->fh_size)); 863 for (; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE) 864 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB); 865 config = (void *)(vstart + ((paddr_t)config - startpa)); 866 vstart = va; 867 868 if (!fdt_init(config)) 869 panic("%s: corrupt FDT", __func__); 870 871 node = fdt_find_node("/chosen"); 872 if (node != NULL) { 873 char *prop; 874 int len; 875 static uint8_t lladdr[6]; 876 877 len = fdt_node_property(node, "bootargs", &prop); 878 if (len > 0) 879 collect_kernel_args(prop); 880 881 len = fdt_node_property(node, "openbsd,boothowto", &prop); 882 if (len == sizeof(boothowto)) 883 boothowto = bemtoh32((uint32_t *)prop); 884 885 len = fdt_node_property(node, "openbsd,bootduid", &prop); 886 if (len == sizeof(bootduid)) 887 memcpy(bootduid, prop, sizeof(bootduid)); 888 889 len = fdt_node_property(node, "openbsd,bootmac", &prop); 890 if (len == sizeof(lladdr)) { 891 memcpy(lladdr, prop, sizeof(lladdr)); 892 bootmac = lladdr; 893 } 894 895 len = fdt_node_property(node, "openbsd,sr-bootuuid", &prop); 896#if NSOFTRAID > 0 897 if (len == sizeof(sr_bootuuid)) 898 memcpy(&sr_bootuuid, prop, sizeof(sr_bootuuid)); 899#endif 900 if (len > 0) 901 explicit_bzero(prop, len); 902 903 len = fdt_node_property(node, "openbsd,sr-bootkey", &prop); 904#if NSOFTRAID > 0 905 if (len == sizeof(sr_bootkey)) 906 memcpy(&sr_bootkey, prop, sizeof(sr_bootkey)); 907#endif 908 if (len > 0) 909 explicit_bzero(prop, len); 910 911 len = fdt_node_property(node, "openbsd,uefi-mmap-start", &prop); 912 if (len == sizeof(mmap_start)) 913 mmap_start = bemtoh64((uint64_t *)prop); 914 len = fdt_node_property(node, "openbsd,uefi-mmap-size", &prop); 915 if (len == sizeof(mmap_size)) 916 mmap_size = bemtoh32((uint32_t *)prop); 917 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-size", &prop); 918 if (len == sizeof(mmap_desc_size)) 919 mmap_desc_size = bemtoh32((uint32_t *)prop); 920 len = fdt_node_property(node, "openbsd,uefi-mmap-desc-ver", &prop); 921 if (len == sizeof(mmap_desc_ver)) 922 mmap_desc_ver = bemtoh32((uint32_t *)prop); 923 924 len = fdt_node_property(node, "openbsd,uefi-system-table", &prop); 925 if (len == sizeof(system_table)) 926 system_table = bemtoh64((uint64_t *)prop); 927 928 len = fdt_node_property(node, "openbsd,dma-constraint", &prop); 929 if (len == sizeof(dma_constraint)) { 930 dma_constraint.ucr_low = bemtoh64((uint64_t *)prop); 931 dma_constraint.ucr_high = bemtoh64((uint64_t *)prop + 1); 932 } 933 } 934 935 process_kernel_args(); 936 937 proc0paddr = (struct user *)abp->kern_stack; 938 939 msgbufaddr = (caddr_t)vstart; 940 msgbufphys = pmap_steal_avail(round_page(MSGBUFSIZE), PAGE_SIZE, NULL); 941 vstart += round_page(MSGBUFSIZE); 942 943 zero_page = vstart; 944 vstart += MAXCPUS * PAGE_SIZE; 945 copy_src_page = vstart; 946 vstart += MAXCPUS * PAGE_SIZE; 947 copy_dst_page = vstart; 948 vstart += MAXCPUS * PAGE_SIZE; 949 950 /* Relocate the FDT to safe memory. */ 951 if (fdt_get_size(config) != 0) { 952 uint32_t csize, size = round_page(fdt_get_size(config)); 953 paddr_t pa; 954 vaddr_t va; 955 956 pa = pmap_steal_avail(size, PAGE_SIZE, NULL); 957 memcpy((void *)pa, config, size); /* copy to physical */ 958 for (va = vstart, csize = size; csize > 0; 959 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE) 960 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB); 961 962 fdt = (void *)vstart; 963 vstart += size; 964 } 965 966 /* Relocate the EFI memory map too. */ 967 if (mmap_start != 0) { 968 uint32_t csize, size = round_page(mmap_size); 969 paddr_t pa, startpa, endpa; 970 vaddr_t va; 971 972 startpa = trunc_page(mmap_start); 973 endpa = round_page(mmap_start + mmap_size); 974 for (pa = startpa, va = vstart; pa < endpa; 975 pa += PAGE_SIZE, va += PAGE_SIZE) 976 pmap_kenter_cache(va, pa, PROT_READ, PMAP_CACHE_WB); 977 pa = pmap_steal_avail(size, PAGE_SIZE, NULL); 978 memcpy((void *)pa, (caddr_t)vstart + (mmap_start - startpa), 979 mmap_size); /* copy to physical */ 980 pmap_kremove(vstart, endpa - startpa); 981 982 for (va = vstart, csize = size; csize > 0; 983 csize -= PAGE_SIZE, va += PAGE_SIZE, pa += PAGE_SIZE) 984 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, PMAP_CACHE_WB); 985 986 mmap = (void *)vstart; 987 vstart += size; 988 } 989 990 /* No more KVA stealing after this point. */ 991 virtual_avail = vstart; 992 993 /* Now we can reinit the FDT, using the virtual address. */ 994 if (fdt) 995 fdt_init(fdt); 996 997 map_func_save = arm64_bs_tag._space_map; 998 arm64_bs_tag._space_map = pmap_bootstrap_bs_map; 999 1000 consinit(); 1001 1002 arm64_bs_tag._space_map = map_func_save; 1003 1004 pmap_avail_fixup(); 1005 1006 uvmexp.pagesize = PAGE_SIZE; 1007 uvm_setpagesize(); 1008 1009 /* Make what's left of the initial 64MB block available to UVM. */ 1010 pmap_physload_avail(); 1011 1012 /* Make all other physical memory available to UVM. */ 1013 if (mmap && mmap_desc_ver == EFI_MEMORY_DESCRIPTOR_VERSION) { 1014 EFI_MEMORY_DESCRIPTOR *desc = mmap; 1015 1016 /* 1017 * Load all memory marked as EfiConventionalMemory, 1018 * EfiBootServicesCode or EfiBootServicesData. 1019 * The initial 64MB memory block should be marked as 1020 * EfiLoaderData so it won't be added here. 1021 */ 1022 for (i = 0; i < mmap_size / mmap_desc_size; i++) { 1023#ifdef MMAP_DEBUG 1024 printf("type 0x%x pa 0x%llx va 0x%llx pages 0x%llx attr 0x%llx\n", 1025 desc->Type, desc->PhysicalStart, 1026 desc->VirtualStart, desc->NumberOfPages, 1027 desc->Attribute); 1028#endif 1029 if (desc->Type == EfiConventionalMemory || 1030 desc->Type == EfiBootServicesCode || 1031 desc->Type == EfiBootServicesData) { 1032 reg.addr = desc->PhysicalStart; 1033 reg.size = ptoa(desc->NumberOfPages); 1034 memreg_add(®); 1035 } 1036 desc = NextMemoryDescriptor(desc, mmap_desc_size); 1037 } 1038 } else { 1039 node = fdt_find_node("/memory"); 1040 if (node == NULL) 1041 panic("%s: no memory specified", __func__); 1042 1043 for (i = 0; nmemreg < nitems(memreg); i++) { 1044 if (fdt_get_reg(node, i, ®)) 1045 break; 1046 if (reg.size == 0) 1047 continue; 1048 memreg_add(®); 1049 } 1050 } 1051 1052 /* Remove reserved memory. */ 1053 node = fdt_find_node("/reserved-memory"); 1054 if (node) { 1055 for (node = fdt_child_node(node); node; 1056 node = fdt_next_node(node)) { 1057 char *no_map; 1058 if (fdt_node_property(node, "no-map", &no_map) < 0) 1059 continue; 1060 if (fdt_get_reg(node, 0, ®)) 1061 continue; 1062 if (reg.size == 0) 1063 continue; 1064 memreg_remove(®); 1065 } 1066 } 1067 1068 /* Remove the initial 64MB block. */ 1069 reg.addr = memstart; 1070 reg.size = memend - memstart; 1071 memreg_remove(®); 1072 1073 for (i = 0; i < nmemreg; i++) { 1074 paddr_t start = memreg[i].addr; 1075 paddr_t end = start + memreg[i].size; 1076 1077 uvm_page_physload(atop(start), atop(end), 1078 atop(start), atop(end), 0); 1079 physmem += atop(end - start); 1080 } 1081 1082 kmeminit_nkmempages(); 1083 1084 /* 1085 * Make sure that we have enough KVA to initialize UVM. In 1086 * particular, we need enough KVA to be able to allocate the 1087 * vm_page structures and nkmempages for malloc(9). 1088 */ 1089 pmap_growkernel(VM_MIN_KERNEL_ADDRESS + 1024 * 1024 * 1024 + 1090 physmem * sizeof(struct vm_page) + ptoa(nkmempages)); 1091 1092#ifdef DDB 1093 db_machine_init(); 1094 1095 /* Firmware doesn't load symbols. */ 1096 ddb_init(); 1097 1098 if (boothowto & RB_KDB) 1099 db_enter(); 1100#endif 1101 1102 softintr_init(); 1103 splraise(IPL_IPI); 1104} 1105 1106char bootargs[256]; 1107 1108void 1109collect_kernel_args(const char *args) 1110{ 1111 /* Make a local copy of the bootargs */ 1112 strlcpy(bootargs, args, sizeof(bootargs)); 1113} 1114 1115void 1116process_kernel_args(void) 1117{ 1118 char *cp = bootargs; 1119 1120 if (*cp == 0) 1121 return; 1122 1123 /* Skip the kernel image filename */ 1124 while (*cp != ' ' && *cp != 0) 1125 cp++; 1126 1127 if (*cp != 0) 1128 *cp++ = 0; 1129 1130 while (*cp == ' ') 1131 cp++; 1132 1133 boot_args = cp; 1134 1135 printf("bootargs: %s\n", boot_args); 1136 1137 /* Setup pointer to boot flags */ 1138 while (*cp != '-') 1139 if (*cp++ == '\0') 1140 return; 1141 1142 while (*cp != 0) { 1143 switch (*cp) { 1144 case 'a': 1145 boothowto |= RB_ASKNAME; 1146 break; 1147 case 'c': 1148 boothowto |= RB_CONFIG; 1149 break; 1150 case 'd': 1151 boothowto |= RB_KDB; 1152 break; 1153 case 's': 1154 boothowto |= RB_SINGLE; 1155 break; 1156 default: 1157 printf("unknown option `%c'\n", *cp); 1158 break; 1159 } 1160 cp++; 1161 } 1162} 1163 1164/* 1165 * Allow bootstrap to steal KVA after machdep has given it back to pmap. 1166 */ 1167int 1168pmap_bootstrap_bs_map(bus_space_tag_t t, bus_addr_t bpa, bus_size_t size, 1169 int flags, bus_space_handle_t *bshp) 1170{ 1171 u_long startpa, pa, endpa; 1172 vaddr_t va; 1173 1174 va = virtual_avail; /* steal memory from virtual avail. */ 1175 1176 startpa = trunc_page(bpa); 1177 endpa = round_page((bpa + size)); 1178 1179 *bshp = (bus_space_handle_t)(va + (bpa - startpa)); 1180 1181 for (pa = startpa; pa < endpa; pa += PAGE_SIZE, va += PAGE_SIZE) 1182 pmap_kenter_cache(va, pa, PROT_READ | PROT_WRITE, 1183 PMAP_CACHE_DEV_NGNRNE); 1184 1185 virtual_avail = va; 1186 1187 return 0; 1188} 1189 1190void 1191memreg_add(const struct fdt_reg *reg) 1192{ 1193 int i; 1194 1195 for (i = 0; i < nmemreg; i++) { 1196 if (reg->addr == memreg[i].addr + memreg[i].size) { 1197 memreg[i].size += reg->size; 1198 return; 1199 } 1200 if (reg->addr + reg->size == memreg[i].addr) { 1201 memreg[i].addr = reg->addr; 1202 memreg[i].size += reg->size; 1203 return; 1204 } 1205 } 1206 1207 if (nmemreg >= nitems(memreg)) 1208 return; 1209 1210 memreg[nmemreg++] = *reg; 1211} 1212 1213void 1214memreg_remove(const struct fdt_reg *reg) 1215{ 1216 uint64_t start = reg->addr; 1217 uint64_t end = reg->addr + reg->size; 1218 int i, j; 1219 1220 for (i = 0; i < nmemreg; i++) { 1221 uint64_t memstart = memreg[i].addr; 1222 uint64_t memend = memreg[i].addr + memreg[i].size; 1223 1224 if (end <= memstart) 1225 continue; 1226 if (start >= memend) 1227 continue; 1228 1229 if (start <= memstart) 1230 memstart = MIN(end, memend); 1231 if (end >= memend) 1232 memend = MAX(start, memstart); 1233 1234 if (start > memstart && end < memend) { 1235 if (nmemreg < nitems(memreg)) { 1236 memreg[nmemreg].addr = end; 1237 memreg[nmemreg].size = memend - end; 1238 nmemreg++; 1239 } 1240 memend = start; 1241 } 1242 memreg[i].addr = memstart; 1243 memreg[i].size = memend - memstart; 1244 } 1245 1246 /* Remove empty slots. */ 1247 for (i = nmemreg - 1; i >= 0; i--) { 1248 if (memreg[i].size == 0) { 1249 for (j = i; (j + 1) < nmemreg; j++) 1250 memreg[j] = memreg[j + 1]; 1251 nmemreg--; 1252 } 1253 } 1254} 1255