machdep.c revision 331022
1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28#include "opt_platform.h" 29#include "opt_ddb.h" 30 31#include <sys/cdefs.h> 32__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/machdep.c 331022 2018-03-15 19:56:44Z kevans $"); 33 34#include <sys/param.h> 35#include <sys/systm.h> 36#include <sys/buf.h> 37#include <sys/bus.h> 38#include <sys/cons.h> 39#include <sys/cpu.h> 40#include <sys/devmap.h> 41#include <sys/efi.h> 42#include <sys/exec.h> 43#include <sys/imgact.h> 44#include <sys/kdb.h> 45#include <sys/kernel.h> 46#include <sys/limits.h> 47#include <sys/linker.h> 48#include <sys/msgbuf.h> 49#include <sys/pcpu.h> 50#include <sys/proc.h> 51#include <sys/ptrace.h> 52#include <sys/reboot.h> 53#include <sys/rwlock.h> 54#include <sys/sched.h> 55#include <sys/signalvar.h> 56#include <sys/syscallsubr.h> 57#include <sys/sysent.h> 58#include <sys/sysproto.h> 59#include <sys/ucontext.h> 60#include <sys/vdso.h> 61 62#include <vm/vm.h> 63#include <vm/vm_kern.h> 64#include <vm/vm_object.h> 65#include <vm/vm_page.h> 66#include <vm/pmap.h> 67#include <vm/vm_map.h> 68#include <vm/vm_pager.h> 69 70#include <machine/armreg.h> 71#include <machine/cpu.h> 72#include <machine/debug_monitor.h> 73#include <machine/kdb.h> 74#include <machine/machdep.h> 75#include <machine/metadata.h> 76#include <machine/md_var.h> 77#include <machine/pcb.h> 78#include <machine/reg.h> 79#include <machine/vmparam.h> 80 81#ifdef VFP 82#include <machine/vfp.h> 83#endif 84 85#ifdef FDT 86#include <dev/fdt/fdt_common.h> 87#include <dev/ofw/openfirm.h> 88#endif 89 90struct pcpu __pcpu[MAXCPU]; 91 92static struct trapframe proc0_tf; 93 94vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2]; 95vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2]; 96 97int early_boot = 1; 98int cold = 1; 99long realmem = 0; 100long Maxmem = 0; 101 102#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) 103vm_paddr_t physmap[PHYSMAP_SIZE]; 104u_int physmap_idx; 105 106struct kva_md_info kmi; 107 108int64_t dcache_line_size; /* The minimum D cache line size */ 109int64_t icache_line_size; /* The minimum I cache line size */ 110int64_t idcache_line_size; /* The minimum cache line size */ 111int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 112int has_pan; 113 114/* 115 * Physical address of the EFI System Table. Stashed from the metadata hints 116 * passed into the kernel and used by the EFI code to call runtime services. 117 */ 118vm_paddr_t efi_systbl_phys; 119 120/* pagezero_* implementations are provided in support.S */ 121void pagezero_simple(void *); 122void pagezero_cache(void *); 123 124/* pagezero_simple is default pagezero */ 125void (*pagezero)(void *p) = pagezero_simple; 126 127static void 128pan_setup(void) 129{ 130 uint64_t id_aa64mfr1; 131 132 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 133 if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 134 has_pan = 1; 135} 136 137void 138pan_enable(void) 139{ 140 141 /* 142 * The LLVM integrated assembler doesn't understand the PAN 143 * PSTATE field. Because of this we need to manually create 144 * the instruction in an asm block. This is equivalent to: 145 * msr pan, #1 146 * 147 * This sets the PAN bit, stopping the kernel from accessing 148 * memory when userspace can also access it unless the kernel 149 * uses the userspace load/store instructions. 150 */ 151 if (has_pan) { 152 WRITE_SPECIALREG(sctlr_el1, 153 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 154 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 155 } 156} 157 158static void 159cpu_startup(void *dummy) 160{ 161 162 identify_cpu(); 163 164 vm_ksubmap_init(&kmi); 165 bufinit(); 166 vm_pager_bufferinit(); 167} 168 169SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 170 171int 172cpu_idle_wakeup(int cpu) 173{ 174 175 return (0); 176} 177 178int 179fill_regs(struct thread *td, struct reg *regs) 180{ 181 struct trapframe *frame; 182 183 frame = td->td_frame; 184 regs->sp = frame->tf_sp; 185 regs->lr = frame->tf_lr; 186 regs->elr = frame->tf_elr; 187 regs->spsr = frame->tf_spsr; 188 189 memcpy(regs->x, frame->tf_x, sizeof(regs->x)); 190 191 return (0); 192} 193 194int 195set_regs(struct thread *td, struct reg *regs) 196{ 197 struct trapframe *frame; 198 199 frame = td->td_frame; 200 frame->tf_sp = regs->sp; 201 frame->tf_lr = regs->lr; 202 frame->tf_elr = regs->elr; 203 frame->tf_spsr &= ~PSR_FLAGS; 204 frame->tf_spsr |= regs->spsr & PSR_FLAGS; 205 206 memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); 207 208 return (0); 209} 210 211int 212fill_fpregs(struct thread *td, struct fpreg *regs) 213{ 214#ifdef VFP 215 struct pcb *pcb; 216 217 pcb = td->td_pcb; 218 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 219 /* 220 * If we have just been running VFP instructions we will 221 * need to save the state to memcpy it below. 222 */ 223 if (td == curthread) 224 vfp_save_state(td, pcb); 225 226 memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q)); 227 regs->fp_cr = pcb->pcb_fpcr; 228 regs->fp_sr = pcb->pcb_fpsr; 229 } else 230#endif 231 memset(regs->fp_q, 0, sizeof(regs->fp_q)); 232 return (0); 233} 234 235int 236set_fpregs(struct thread *td, struct fpreg *regs) 237{ 238#ifdef VFP 239 struct pcb *pcb; 240 241 pcb = td->td_pcb; 242 memcpy(pcb->pcb_vfp, regs->fp_q, sizeof(regs->fp_q)); 243 pcb->pcb_fpcr = regs->fp_cr; 244 pcb->pcb_fpsr = regs->fp_sr; 245#endif 246 return (0); 247} 248 249int 250fill_dbregs(struct thread *td, struct dbreg *regs) 251{ 252 253 printf("ARM64TODO: fill_dbregs"); 254 return (EDOOFUS); 255} 256 257int 258set_dbregs(struct thread *td, struct dbreg *regs) 259{ 260 261 printf("ARM64TODO: set_dbregs"); 262 return (EDOOFUS); 263} 264 265int 266ptrace_set_pc(struct thread *td, u_long addr) 267{ 268 269 printf("ARM64TODO: ptrace_set_pc"); 270 return (EDOOFUS); 271} 272 273int 274ptrace_single_step(struct thread *td) 275{ 276 277 td->td_frame->tf_spsr |= PSR_SS; 278 td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; 279 return (0); 280} 281 282int 283ptrace_clear_single_step(struct thread *td) 284{ 285 286 td->td_frame->tf_spsr &= ~PSR_SS; 287 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 288 return (0); 289} 290 291void 292exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) 293{ 294 struct trapframe *tf = td->td_frame; 295 296 memset(tf, 0, sizeof(struct trapframe)); 297 298 /* 299 * We need to set x0 for init as it doesn't call 300 * cpu_set_syscall_retval to copy the value. We also 301 * need to set td_retval for the cases where we do. 302 */ 303 tf->tf_x[0] = td->td_retval[0] = stack; 304 tf->tf_sp = STACKALIGN(stack); 305 tf->tf_lr = imgp->entry_addr; 306 tf->tf_elr = imgp->entry_addr; 307} 308 309/* Sanity check these are the same size, they will be memcpy'd to and fro */ 310CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 311 sizeof((struct gpregs *)0)->gp_x); 312CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 313 sizeof((struct reg *)0)->x); 314 315int 316get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) 317{ 318 struct trapframe *tf = td->td_frame; 319 320 if (clear_ret & GET_MC_CLEAR_RET) { 321 mcp->mc_gpregs.gp_x[0] = 0; 322 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; 323 } else { 324 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; 325 mcp->mc_gpregs.gp_spsr = tf->tf_spsr; 326 } 327 328 memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], 329 sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); 330 331 mcp->mc_gpregs.gp_sp = tf->tf_sp; 332 mcp->mc_gpregs.gp_lr = tf->tf_lr; 333 mcp->mc_gpregs.gp_elr = tf->tf_elr; 334 335 return (0); 336} 337 338int 339set_mcontext(struct thread *td, mcontext_t *mcp) 340{ 341 struct trapframe *tf = td->td_frame; 342 uint32_t spsr; 343 344 spsr = mcp->mc_gpregs.gp_spsr; 345 if ((spsr & PSR_M_MASK) != PSR_M_EL0t || 346 (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0) 347 return (EINVAL); 348 349 memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); 350 351 tf->tf_sp = mcp->mc_gpregs.gp_sp; 352 tf->tf_lr = mcp->mc_gpregs.gp_lr; 353 tf->tf_elr = mcp->mc_gpregs.gp_elr; 354 tf->tf_spsr = mcp->mc_gpregs.gp_spsr; 355 356 return (0); 357} 358 359static void 360get_fpcontext(struct thread *td, mcontext_t *mcp) 361{ 362#ifdef VFP 363 struct pcb *curpcb; 364 365 critical_enter(); 366 367 curpcb = curthread->td_pcb; 368 369 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 370 /* 371 * If we have just been running VFP instructions we will 372 * need to save the state to memcpy it below. 373 */ 374 vfp_save_state(td, curpcb); 375 376 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_vfp, 377 sizeof(mcp->mc_fpregs)); 378 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpcr; 379 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpsr; 380 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; 381 mcp->mc_flags |= _MC_FP_VALID; 382 } 383 384 critical_exit(); 385#endif 386} 387 388static void 389set_fpcontext(struct thread *td, mcontext_t *mcp) 390{ 391#ifdef VFP 392 struct pcb *curpcb; 393 394 critical_enter(); 395 396 if ((mcp->mc_flags & _MC_FP_VALID) != 0) { 397 curpcb = curthread->td_pcb; 398 399 /* 400 * Discard any vfp state for the current thread, we 401 * are about to override it. 402 */ 403 vfp_discard(td); 404 405 memcpy(curpcb->pcb_vfp, mcp->mc_fpregs.fp_q, 406 sizeof(mcp->mc_fpregs)); 407 curpcb->pcb_fpcr = mcp->mc_fpregs.fp_cr; 408 curpcb->pcb_fpsr = mcp->mc_fpregs.fp_sr; 409 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags; 410 } 411 412 critical_exit(); 413#endif 414} 415 416void 417cpu_idle(int busy) 418{ 419 420 spinlock_enter(); 421 if (!busy) 422 cpu_idleclock(); 423 if (!sched_runnable()) 424 __asm __volatile( 425 "dsb sy \n" 426 "wfi \n"); 427 if (!busy) 428 cpu_activeclock(); 429 spinlock_exit(); 430} 431 432void 433cpu_halt(void) 434{ 435 436 /* We should have shutdown by now, if not enter a low power sleep */ 437 intr_disable(); 438 while (1) { 439 __asm __volatile("wfi"); 440 } 441} 442 443/* 444 * Flush the D-cache for non-DMA I/O so that the I-cache can 445 * be made coherent later. 446 */ 447void 448cpu_flush_dcache(void *ptr, size_t len) 449{ 450 451 /* ARM64TODO TBD */ 452} 453 454/* Get current clock frequency for the given CPU ID. */ 455int 456cpu_est_clockrate(int cpu_id, uint64_t *rate) 457{ 458 struct pcpu *pc; 459 460 pc = pcpu_find(cpu_id); 461 if (pc == NULL || rate == NULL) 462 return (EINVAL); 463 464 if (pc->pc_clock == 0) 465 return (EOPNOTSUPP); 466 467 *rate = pc->pc_clock; 468 return (0); 469} 470 471void 472cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 473{ 474 475 pcpu->pc_acpi_id = 0xffffffff; 476} 477 478void 479spinlock_enter(void) 480{ 481 struct thread *td; 482 register_t daif; 483 484 td = curthread; 485 if (td->td_md.md_spinlock_count == 0) { 486 daif = intr_disable(); 487 td->td_md.md_spinlock_count = 1; 488 td->td_md.md_saved_daif = daif; 489 } else 490 td->td_md.md_spinlock_count++; 491 critical_enter(); 492} 493 494void 495spinlock_exit(void) 496{ 497 struct thread *td; 498 register_t daif; 499 500 td = curthread; 501 critical_exit(); 502 daif = td->td_md.md_saved_daif; 503 td->td_md.md_spinlock_count--; 504 if (td->td_md.md_spinlock_count == 0) 505 intr_restore(daif); 506} 507 508#ifndef _SYS_SYSPROTO_H_ 509struct sigreturn_args { 510 ucontext_t *ucp; 511}; 512#endif 513 514int 515sys_sigreturn(struct thread *td, struct sigreturn_args *uap) 516{ 517 ucontext_t uc; 518 int error; 519 520 if (uap == NULL) 521 return (EFAULT); 522 if (copyin(uap->sigcntxp, &uc, sizeof(uc))) 523 return (EFAULT); 524 525 error = set_mcontext(td, &uc.uc_mcontext); 526 if (error != 0) 527 return (error); 528 set_fpcontext(td, &uc.uc_mcontext); 529 530 /* Restore signal mask. */ 531 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); 532 533 return (EJUSTRETURN); 534} 535 536/* 537 * Construct a PCB from a trapframe. This is called from kdb_trap() where 538 * we want to start a backtrace from the function that caused us to enter 539 * the debugger. We have the context in the trapframe, but base the trace 540 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 541 * enough for a backtrace. 542 */ 543void 544makectx(struct trapframe *tf, struct pcb *pcb) 545{ 546 int i; 547 548 for (i = 0; i < PCB_LR; i++) 549 pcb->pcb_x[i] = tf->tf_x[i]; 550 551 pcb->pcb_x[PCB_LR] = tf->tf_lr; 552 pcb->pcb_pc = tf->tf_elr; 553 pcb->pcb_sp = tf->tf_sp; 554} 555 556void 557sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) 558{ 559 struct thread *td; 560 struct proc *p; 561 struct trapframe *tf; 562 struct sigframe *fp, frame; 563 struct sigacts *psp; 564 struct sysentvec *sysent; 565 int code, onstack, sig; 566 567 td = curthread; 568 p = td->td_proc; 569 PROC_LOCK_ASSERT(p, MA_OWNED); 570 571 sig = ksi->ksi_signo; 572 code = ksi->ksi_code; 573 psp = p->p_sigacts; 574 mtx_assert(&psp->ps_mtx, MA_OWNED); 575 576 tf = td->td_frame; 577 onstack = sigonstack(tf->tf_sp); 578 579 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, 580 catcher, sig); 581 582 /* Allocate and validate space for the signal handler context. */ 583 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && 584 SIGISMEMBER(psp->ps_sigonstack, sig)) { 585 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + 586 td->td_sigstk.ss_size); 587#if defined(COMPAT_43) 588 td->td_sigstk.ss_flags |= SS_ONSTACK; 589#endif 590 } else { 591 fp = (struct sigframe *)td->td_frame->tf_sp; 592 } 593 594 /* Make room, keeping the stack aligned */ 595 fp--; 596 fp = (struct sigframe *)STACKALIGN(fp); 597 598 /* Fill in the frame to copy out */ 599 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); 600 get_fpcontext(td, &frame.sf_uc.uc_mcontext); 601 frame.sf_si = ksi->ksi_info; 602 frame.sf_uc.uc_sigmask = *mask; 603 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? 604 ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; 605 frame.sf_uc.uc_stack = td->td_sigstk; 606 mtx_unlock(&psp->ps_mtx); 607 PROC_UNLOCK(td->td_proc); 608 609 /* Copy the sigframe out to the user's stack. */ 610 if (copyout(&frame, fp, sizeof(*fp)) != 0) { 611 /* Process has trashed its stack. Kill it. */ 612 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); 613 PROC_LOCK(p); 614 sigexit(td, SIGILL); 615 } 616 617 tf->tf_x[0]= sig; 618 tf->tf_x[1] = (register_t)&fp->sf_si; 619 tf->tf_x[2] = (register_t)&fp->sf_uc; 620 621 tf->tf_elr = (register_t)catcher; 622 tf->tf_sp = (register_t)fp; 623 sysent = p->p_sysent; 624 if (sysent->sv_sigcode_base != 0) 625 tf->tf_lr = (register_t)sysent->sv_sigcode_base; 626 else 627 tf->tf_lr = (register_t)(sysent->sv_psstrings - 628 *(sysent->sv_szsigcode)); 629 630 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, 631 tf->tf_sp); 632 633 PROC_LOCK(p); 634 mtx_lock(&psp->ps_mtx); 635} 636 637static void 638init_proc0(vm_offset_t kstack) 639{ 640 struct pcpu *pcpup = &__pcpu[0]; 641 642 proc_linkup0(&proc0, &thread0); 643 thread0.td_kstack = kstack; 644 thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; 645 thread0.td_pcb->pcb_fpflags = 0; 646 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 647 thread0.td_frame = &proc0_tf; 648 pcpup->pc_curpcb = thread0.td_pcb; 649} 650 651typedef struct { 652 uint32_t type; 653 uint64_t phys_start; 654 uint64_t virt_start; 655 uint64_t num_pages; 656 uint64_t attr; 657} EFI_MEMORY_DESCRIPTOR; 658 659static int 660add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap, 661 u_int *physmap_idxp) 662{ 663 u_int i, insert_idx, _physmap_idx; 664 665 _physmap_idx = *physmap_idxp; 666 667 if (length == 0) 668 return (1); 669 670 /* 671 * Find insertion point while checking for overlap. Start off by 672 * assuming the new entry will be added to the end. 673 */ 674 insert_idx = _physmap_idx; 675 for (i = 0; i <= _physmap_idx; i += 2) { 676 if (base < physmap[i + 1]) { 677 if (base + length <= physmap[i]) { 678 insert_idx = i; 679 break; 680 } 681 if (boothowto & RB_VERBOSE) 682 printf( 683 "Overlapping memory regions, ignoring second region\n"); 684 return (1); 685 } 686 } 687 688 /* See if we can prepend to the next entry. */ 689 if (insert_idx <= _physmap_idx && 690 base + length == physmap[insert_idx]) { 691 physmap[insert_idx] = base; 692 return (1); 693 } 694 695 /* See if we can append to the previous entry. */ 696 if (insert_idx > 0 && base == physmap[insert_idx - 1]) { 697 physmap[insert_idx - 1] += length; 698 return (1); 699 } 700 701 _physmap_idx += 2; 702 *physmap_idxp = _physmap_idx; 703 if (_physmap_idx == PHYSMAP_SIZE) { 704 printf( 705 "Too many segments in the physical address map, giving up\n"); 706 return (0); 707 } 708 709 /* 710 * Move the last 'N' entries down to make room for the new 711 * entry if needed. 712 */ 713 for (i = _physmap_idx; i > insert_idx; i -= 2) { 714 physmap[i] = physmap[i - 2]; 715 physmap[i + 1] = physmap[i - 1]; 716 } 717 718 /* Insert the new entry. */ 719 physmap[insert_idx] = base; 720 physmap[insert_idx + 1] = base + length; 721 return (1); 722} 723 724#ifdef FDT 725static void 726add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap, 727 u_int *physmap_idxp) 728{ 729 730 for (int i = 0; i < mrcnt; i++) { 731 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap, 732 physmap_idxp)) 733 break; 734 } 735} 736#endif 737 738static void 739add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap, 740 u_int *physmap_idxp) 741{ 742 struct efi_md *map, *p; 743 const char *type; 744 size_t efisz; 745 int ndesc, i; 746 747 static const char *types[] = { 748 "Reserved", 749 "LoaderCode", 750 "LoaderData", 751 "BootServicesCode", 752 "BootServicesData", 753 "RuntimeServicesCode", 754 "RuntimeServicesData", 755 "ConventionalMemory", 756 "UnusableMemory", 757 "ACPIReclaimMemory", 758 "ACPIMemoryNVS", 759 "MemoryMappedIO", 760 "MemoryMappedIOPortSpace", 761 "PalCode", 762 "PersistentMemory" 763 }; 764 765 /* 766 * Memory map data provided by UEFI via the GetMemoryMap 767 * Boot Services API. 768 */ 769 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 770 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 771 772 if (efihdr->descriptor_size == 0) 773 return; 774 ndesc = efihdr->memory_size / efihdr->descriptor_size; 775 776 if (boothowto & RB_VERBOSE) 777 printf("%23s %12s %12s %8s %4s\n", 778 "Type", "Physical", "Virtual", "#Pages", "Attr"); 779 780 for (i = 0, p = map; i < ndesc; i++, 781 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 782 if (boothowto & RB_VERBOSE) { 783 if (p->md_type < nitems(types)) 784 type = types[p->md_type]; 785 else 786 type = "<INVALID>"; 787 printf("%23s %012lx %12p %08lx ", type, p->md_phys, 788 p->md_virt, p->md_pages); 789 if (p->md_attr & EFI_MD_ATTR_UC) 790 printf("UC "); 791 if (p->md_attr & EFI_MD_ATTR_WC) 792 printf("WC "); 793 if (p->md_attr & EFI_MD_ATTR_WT) 794 printf("WT "); 795 if (p->md_attr & EFI_MD_ATTR_WB) 796 printf("WB "); 797 if (p->md_attr & EFI_MD_ATTR_UCE) 798 printf("UCE "); 799 if (p->md_attr & EFI_MD_ATTR_WP) 800 printf("WP "); 801 if (p->md_attr & EFI_MD_ATTR_RP) 802 printf("RP "); 803 if (p->md_attr & EFI_MD_ATTR_XP) 804 printf("XP "); 805 if (p->md_attr & EFI_MD_ATTR_NV) 806 printf("NV "); 807 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 808 printf("MORE_RELIABLE "); 809 if (p->md_attr & EFI_MD_ATTR_RO) 810 printf("RO "); 811 if (p->md_attr & EFI_MD_ATTR_RT) 812 printf("RUNTIME"); 813 printf("\n"); 814 } 815 816 switch (p->md_type) { 817 case EFI_MD_TYPE_CODE: 818 case EFI_MD_TYPE_DATA: 819 case EFI_MD_TYPE_BS_CODE: 820 case EFI_MD_TYPE_BS_DATA: 821 case EFI_MD_TYPE_FREE: 822 /* 823 * We're allowed to use any entry with these types. 824 */ 825 break; 826 default: 827 continue; 828 } 829 830 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE), 831 physmap, physmap_idxp)) 832 break; 833 } 834} 835 836#ifdef FDT 837static void 838try_load_dtb(caddr_t kmdp) 839{ 840 vm_offset_t dtbp; 841 842 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 843 if (dtbp == (vm_offset_t)NULL) { 844 printf("ERROR loading DTB\n"); 845 return; 846 } 847 848 if (OF_install(OFW_FDT, 0) == FALSE) 849 panic("Cannot install FDT"); 850 851 if (OF_init((void *)dtbp) != 0) 852 panic("OF_init failed with the found device tree"); 853} 854#endif 855 856static void 857cache_setup(void) 858{ 859 int dcache_line_shift, icache_line_shift, dczva_line_shift; 860 uint32_t ctr_el0; 861 uint32_t dczid_el0; 862 863 ctr_el0 = READ_SPECIALREG(ctr_el0); 864 865 /* Read the log2 words in each D cache line */ 866 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); 867 /* Get the D cache line size */ 868 dcache_line_size = sizeof(int) << dcache_line_shift; 869 870 /* And the same for the I cache */ 871 icache_line_shift = CTR_ILINE_SIZE(ctr_el0); 872 icache_line_size = sizeof(int) << icache_line_shift; 873 874 idcache_line_size = MIN(dcache_line_size, icache_line_size); 875 876 dczid_el0 = READ_SPECIALREG(dczid_el0); 877 878 /* Check if dc zva is not prohibited */ 879 if (dczid_el0 & DCZID_DZP) 880 dczva_line_size = 0; 881 else { 882 /* Same as with above calculations */ 883 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 884 dczva_line_size = sizeof(int) << dczva_line_shift; 885 886 /* Change pagezero function */ 887 pagezero = pagezero_cache; 888 } 889} 890 891void 892initarm(struct arm64_bootparams *abp) 893{ 894 struct efi_map_header *efihdr; 895 struct pcpu *pcpup; 896#ifdef FDT 897 struct mem_region mem_regions[FDT_MEM_REGIONS]; 898 int mem_regions_sz; 899#endif 900 vm_offset_t lastaddr; 901 caddr_t kmdp; 902 vm_paddr_t mem_len; 903 int i; 904 905 /* Set the module data location */ 906 preload_metadata = (caddr_t)(uintptr_t)(abp->modulep); 907 908 /* Find the kernel address */ 909 kmdp = preload_search_by_type("elf kernel"); 910 if (kmdp == NULL) 911 kmdp = preload_search_by_type("elf64 kernel"); 912 913 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); 914 init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0); 915 916#ifdef FDT 917 try_load_dtb(kmdp); 918#endif 919 920 efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); 921 922 /* Find the address to start allocating from */ 923 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); 924 925 /* Load the physical memory ranges */ 926 physmap_idx = 0; 927 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 928 MODINFO_METADATA | MODINFOMD_EFI_MAP); 929 if (efihdr != NULL) 930 add_efi_map_entries(efihdr, physmap, &physmap_idx); 931#ifdef FDT 932 else { 933 /* Grab physical memory regions information from device tree. */ 934 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 935 NULL) != 0) 936 panic("Cannot get physical memory regions"); 937 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap, 938 &physmap_idx); 939 } 940#endif 941 942 /* Print the memory map */ 943 mem_len = 0; 944 for (i = 0; i < physmap_idx; i += 2) { 945 dump_avail[i] = physmap[i]; 946 dump_avail[i + 1] = physmap[i + 1]; 947 mem_len += physmap[i + 1] - physmap[i]; 948 } 949 dump_avail[i] = 0; 950 dump_avail[i + 1] = 0; 951 952 /* Set the pcpu data, this is needed by pmap_bootstrap */ 953 pcpup = &__pcpu[0]; 954 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 955 956 /* 957 * Set the pcpu pointer with a backup in tpidr_el1 to be 958 * loaded when entering the kernel from userland. 959 */ 960 __asm __volatile( 961 "mov x18, %0 \n" 962 "msr tpidr_el1, %0" :: "r"(pcpup)); 963 964 PCPU_SET(curthread, &thread0); 965 966 /* Do basic tuning, hz etc */ 967 init_param1(); 968 969 cache_setup(); 970 pan_setup(); 971 972 /* Bootstrap enough of pmap to enter the kernel proper */ 973 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 974 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 975 976 devmap_bootstrap(0, NULL); 977 978 cninit(); 979 980 init_proc0(abp->kern_stack); 981 msgbufinit(msgbufp, msgbufsize); 982 mutex_init(); 983 init_param2(physmem); 984 985 dbg_init(); 986 kdb_init(); 987 pan_enable(); 988 989 early_boot = 0; 990} 991 992void 993dbg_init(void) 994{ 995 996 /* Clear OS lock */ 997 WRITE_SPECIALREG(OSLAR_EL1, 0); 998 999 /* This permits DDB to use debug registers for watchpoints. */ 1000 dbg_monitor_init(); 1001 1002 /* TODO: Eventually will need to initialize debug registers here. */ 1003} 1004 1005#ifdef DDB 1006#include <ddb/ddb.h> 1007 1008DB_SHOW_COMMAND(specialregs, db_show_spregs) 1009{ 1010#define PRINT_REG(reg) \ 1011 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 1012 1013 PRINT_REG(actlr_el1); 1014 PRINT_REG(afsr0_el1); 1015 PRINT_REG(afsr1_el1); 1016 PRINT_REG(aidr_el1); 1017 PRINT_REG(amair_el1); 1018 PRINT_REG(ccsidr_el1); 1019 PRINT_REG(clidr_el1); 1020 PRINT_REG(contextidr_el1); 1021 PRINT_REG(cpacr_el1); 1022 PRINT_REG(csselr_el1); 1023 PRINT_REG(ctr_el0); 1024 PRINT_REG(currentel); 1025 PRINT_REG(daif); 1026 PRINT_REG(dczid_el0); 1027 PRINT_REG(elr_el1); 1028 PRINT_REG(esr_el1); 1029 PRINT_REG(far_el1); 1030#if 0 1031 /* ARM64TODO: Enable VFP before reading floating-point registers */ 1032 PRINT_REG(fpcr); 1033 PRINT_REG(fpsr); 1034#endif 1035 PRINT_REG(id_aa64afr0_el1); 1036 PRINT_REG(id_aa64afr1_el1); 1037 PRINT_REG(id_aa64dfr0_el1); 1038 PRINT_REG(id_aa64dfr1_el1); 1039 PRINT_REG(id_aa64isar0_el1); 1040 PRINT_REG(id_aa64isar1_el1); 1041 PRINT_REG(id_aa64pfr0_el1); 1042 PRINT_REG(id_aa64pfr1_el1); 1043 PRINT_REG(id_afr0_el1); 1044 PRINT_REG(id_dfr0_el1); 1045 PRINT_REG(id_isar0_el1); 1046 PRINT_REG(id_isar1_el1); 1047 PRINT_REG(id_isar2_el1); 1048 PRINT_REG(id_isar3_el1); 1049 PRINT_REG(id_isar4_el1); 1050 PRINT_REG(id_isar5_el1); 1051 PRINT_REG(id_mmfr0_el1); 1052 PRINT_REG(id_mmfr1_el1); 1053 PRINT_REG(id_mmfr2_el1); 1054 PRINT_REG(id_mmfr3_el1); 1055#if 0 1056 /* Missing from llvm */ 1057 PRINT_REG(id_mmfr4_el1); 1058#endif 1059 PRINT_REG(id_pfr0_el1); 1060 PRINT_REG(id_pfr1_el1); 1061 PRINT_REG(isr_el1); 1062 PRINT_REG(mair_el1); 1063 PRINT_REG(midr_el1); 1064 PRINT_REG(mpidr_el1); 1065 PRINT_REG(mvfr0_el1); 1066 PRINT_REG(mvfr1_el1); 1067 PRINT_REG(mvfr2_el1); 1068 PRINT_REG(revidr_el1); 1069 PRINT_REG(sctlr_el1); 1070 PRINT_REG(sp_el0); 1071 PRINT_REG(spsel); 1072 PRINT_REG(spsr_el1); 1073 PRINT_REG(tcr_el1); 1074 PRINT_REG(tpidr_el0); 1075 PRINT_REG(tpidr_el1); 1076 PRINT_REG(tpidrro_el0); 1077 PRINT_REG(ttbr0_el1); 1078 PRINT_REG(ttbr1_el1); 1079 PRINT_REG(vbar_el1); 1080#undef PRINT_REG 1081} 1082 1083DB_SHOW_COMMAND(vtop, db_show_vtop) 1084{ 1085 uint64_t phys; 1086 1087 if (have_addr) { 1088 phys = arm64_address_translate_s1e1r(addr); 1089 db_printf("Physical address reg (read): 0x%016lx\n", phys); 1090 phys = arm64_address_translate_s1e1w(addr); 1091 db_printf("Physical address reg (write): 0x%016lx\n", phys); 1092 } else 1093 db_printf("show vtop <virt_addr>\n"); 1094} 1095#endif 1096