machdep.c revision 331023
1139790Simp/*- 2117267Smarcel * Copyright (c) 2014 Andrew Turner 385685Sdfr * All rights reserved. 485685Sdfr * 585685Sdfr * Redistribution and use in source and binary forms, with or without 685685Sdfr * modification, are permitted provided that the following conditions 785685Sdfr * are met: 8117267Smarcel * 1. Redistributions of source code must retain the above copyright 985685Sdfr * notice, this list of conditions and the following disclaimer. 1085685Sdfr * 2. Redistributions in binary form must reproduce the above copyright 1185685Sdfr * notice, this list of conditions and the following disclaimer in the 1285685Sdfr * documentation and/or other materials provided with the distribution. 1385685Sdfr * 1485685Sdfr * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15117267Smarcel * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16117267Smarcel * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17117267Smarcel * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18117267Smarcel * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19117267Smarcel * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20117267Smarcel * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21117267Smarcel * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22117267Smarcel * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23117267Smarcel * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24117267Smarcel * SUCH DAMAGE. 2585685Sdfr * 2685685Sdfr */ 2785685Sdfr 2885685Sdfr#include "opt_platform.h" 29115084Smarcel#include "opt_ddb.h" 30115084Smarcel 3185685Sdfr#include <sys/cdefs.h> 32131945Smarcel__FBSDID("$FreeBSD: stable/11/sys/arm64/arm64/machdep.c 331023 2018-03-15 20:09:24Z kevans $"); 33131945Smarcel 34115084Smarcel#include <sys/param.h> 35115084Smarcel#include <sys/systm.h> 36115084Smarcel#include <sys/buf.h> 37131945Smarcel#include <sys/bus.h> 38115084Smarcel#include <sys/cons.h> 39115084Smarcel#include <sys/cpu.h> 40115084Smarcel#include <sys/devmap.h> 41115084Smarcel#include <sys/efi.h> 42115084Smarcel#include <sys/exec.h> 43131945Smarcel#include <sys/imgact.h> 44131945Smarcel#include <sys/kdb.h> 45117267Smarcel#include <sys/kernel.h> 46115084Smarcel#include <sys/limits.h> 47115084Smarcel#include <sys/linker.h> 48115084Smarcel#include <sys/msgbuf.h> 49115084Smarcel#include <sys/pcpu.h> 50115084Smarcel#include <sys/proc.h> 51117467Smarcel#include <sys/ptrace.h> 52115084Smarcel#include <sys/reboot.h> 53115084Smarcel#include <sys/rwlock.h> 54115084Smarcel#include <sys/sched.h> 55115084Smarcel#include <sys/signalvar.h> 56115084Smarcel#include <sys/syscallsubr.h> 57#include <sys/sysent.h> 58#include <sys/sysproto.h> 59#include <sys/ucontext.h> 60#include <sys/vdso.h> 61 62#include <vm/vm.h> 63#include <vm/vm_kern.h> 64#include <vm/vm_object.h> 65#include <vm/vm_page.h> 66#include <vm/pmap.h> 67#include <vm/vm_map.h> 68#include <vm/vm_pager.h> 69 70#include <machine/armreg.h> 71#include <machine/cpu.h> 72#include <machine/debug_monitor.h> 73#include <machine/kdb.h> 74#include <machine/machdep.h> 75#include <machine/metadata.h> 76#include <machine/md_var.h> 77#include <machine/pcb.h> 78#include <machine/reg.h> 79#include <machine/vmparam.h> 80 81#ifdef VFP 82#include <machine/vfp.h> 83#endif 84 85#ifdef FDT 86#include <dev/fdt/fdt_common.h> 87#include <dev/ofw/openfirm.h> 88#endif 89 90struct pcpu __pcpu[MAXCPU]; 91 92static struct trapframe proc0_tf; 93 94vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2]; 95vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2]; 96 97int early_boot = 1; 98int cold = 1; 99long realmem = 0; 100long Maxmem = 0; 101 102#define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) 103vm_paddr_t physmap[PHYSMAP_SIZE]; 104u_int physmap_idx; 105 106struct kva_md_info kmi; 107 108int64_t dcache_line_size; /* The minimum D cache line size */ 109int64_t icache_line_size; /* The minimum I cache line size */ 110int64_t idcache_line_size; /* The minimum cache line size */ 111int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 112int has_pan; 113 114/* pagezero_* implementations are provided in support.S */ 115void pagezero_simple(void *); 116void pagezero_cache(void *); 117 118/* pagezero_simple is default pagezero */ 119void (*pagezero)(void *p) = pagezero_simple; 120 121static void 122pan_setup(void) 123{ 124 uint64_t id_aa64mfr1; 125 126 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 127 if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 128 has_pan = 1; 129} 130 131void 132pan_enable(void) 133{ 134 135 /* 136 * The LLVM integrated assembler doesn't understand the PAN 137 * PSTATE field. Because of this we need to manually create 138 * the instruction in an asm block. This is equivalent to: 139 * msr pan, #1 140 * 141 * This sets the PAN bit, stopping the kernel from accessing 142 * memory when userspace can also access it unless the kernel 143 * uses the userspace load/store instructions. 144 */ 145 if (has_pan) { 146 WRITE_SPECIALREG(sctlr_el1, 147 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 148 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 149 } 150} 151 152static void 153cpu_startup(void *dummy) 154{ 155 156 identify_cpu(); 157 158 vm_ksubmap_init(&kmi); 159 bufinit(); 160 vm_pager_bufferinit(); 161} 162 163SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 164 165int 166cpu_idle_wakeup(int cpu) 167{ 168 169 return (0); 170} 171 172int 173fill_regs(struct thread *td, struct reg *regs) 174{ 175 struct trapframe *frame; 176 177 frame = td->td_frame; 178 regs->sp = frame->tf_sp; 179 regs->lr = frame->tf_lr; 180 regs->elr = frame->tf_elr; 181 regs->spsr = frame->tf_spsr; 182 183 memcpy(regs->x, frame->tf_x, sizeof(regs->x)); 184 185 return (0); 186} 187 188int 189set_regs(struct thread *td, struct reg *regs) 190{ 191 struct trapframe *frame; 192 193 frame = td->td_frame; 194 frame->tf_sp = regs->sp; 195 frame->tf_lr = regs->lr; 196 frame->tf_elr = regs->elr; 197 frame->tf_spsr &= ~PSR_FLAGS; 198 frame->tf_spsr |= regs->spsr & PSR_FLAGS; 199 200 memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); 201 202 return (0); 203} 204 205int 206fill_fpregs(struct thread *td, struct fpreg *regs) 207{ 208#ifdef VFP 209 struct pcb *pcb; 210 211 pcb = td->td_pcb; 212 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 213 /* 214 * If we have just been running VFP instructions we will 215 * need to save the state to memcpy it below. 216 */ 217 if (td == curthread) 218 vfp_save_state(td, pcb); 219 220 memcpy(regs->fp_q, pcb->pcb_vfp, sizeof(regs->fp_q)); 221 regs->fp_cr = pcb->pcb_fpcr; 222 regs->fp_sr = pcb->pcb_fpsr; 223 } else 224#endif 225 memset(regs->fp_q, 0, sizeof(regs->fp_q)); 226 return (0); 227} 228 229int 230set_fpregs(struct thread *td, struct fpreg *regs) 231{ 232#ifdef VFP 233 struct pcb *pcb; 234 235 pcb = td->td_pcb; 236 memcpy(pcb->pcb_vfp, regs->fp_q, sizeof(regs->fp_q)); 237 pcb->pcb_fpcr = regs->fp_cr; 238 pcb->pcb_fpsr = regs->fp_sr; 239#endif 240 return (0); 241} 242 243int 244fill_dbregs(struct thread *td, struct dbreg *regs) 245{ 246 247 printf("ARM64TODO: fill_dbregs"); 248 return (EDOOFUS); 249} 250 251int 252set_dbregs(struct thread *td, struct dbreg *regs) 253{ 254 255 printf("ARM64TODO: set_dbregs"); 256 return (EDOOFUS); 257} 258 259int 260ptrace_set_pc(struct thread *td, u_long addr) 261{ 262 263 printf("ARM64TODO: ptrace_set_pc"); 264 return (EDOOFUS); 265} 266 267int 268ptrace_single_step(struct thread *td) 269{ 270 271 td->td_frame->tf_spsr |= PSR_SS; 272 td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; 273 return (0); 274} 275 276int 277ptrace_clear_single_step(struct thread *td) 278{ 279 280 td->td_frame->tf_spsr &= ~PSR_SS; 281 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 282 return (0); 283} 284 285void 286exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) 287{ 288 struct trapframe *tf = td->td_frame; 289 290 memset(tf, 0, sizeof(struct trapframe)); 291 292 /* 293 * We need to set x0 for init as it doesn't call 294 * cpu_set_syscall_retval to copy the value. We also 295 * need to set td_retval for the cases where we do. 296 */ 297 tf->tf_x[0] = td->td_retval[0] = stack; 298 tf->tf_sp = STACKALIGN(stack); 299 tf->tf_lr = imgp->entry_addr; 300 tf->tf_elr = imgp->entry_addr; 301} 302 303/* Sanity check these are the same size, they will be memcpy'd to and fro */ 304CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 305 sizeof((struct gpregs *)0)->gp_x); 306CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 307 sizeof((struct reg *)0)->x); 308 309int 310get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) 311{ 312 struct trapframe *tf = td->td_frame; 313 314 if (clear_ret & GET_MC_CLEAR_RET) { 315 mcp->mc_gpregs.gp_x[0] = 0; 316 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; 317 } else { 318 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; 319 mcp->mc_gpregs.gp_spsr = tf->tf_spsr; 320 } 321 322 memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], 323 sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); 324 325 mcp->mc_gpregs.gp_sp = tf->tf_sp; 326 mcp->mc_gpregs.gp_lr = tf->tf_lr; 327 mcp->mc_gpregs.gp_elr = tf->tf_elr; 328 329 return (0); 330} 331 332int 333set_mcontext(struct thread *td, mcontext_t *mcp) 334{ 335 struct trapframe *tf = td->td_frame; 336 uint32_t spsr; 337 338 spsr = mcp->mc_gpregs.gp_spsr; 339 if ((spsr & PSR_M_MASK) != PSR_M_EL0t || 340 (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0) 341 return (EINVAL); 342 343 memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); 344 345 tf->tf_sp = mcp->mc_gpregs.gp_sp; 346 tf->tf_lr = mcp->mc_gpregs.gp_lr; 347 tf->tf_elr = mcp->mc_gpregs.gp_elr; 348 tf->tf_spsr = mcp->mc_gpregs.gp_spsr; 349 350 return (0); 351} 352 353static void 354get_fpcontext(struct thread *td, mcontext_t *mcp) 355{ 356#ifdef VFP 357 struct pcb *curpcb; 358 359 critical_enter(); 360 361 curpcb = curthread->td_pcb; 362 363 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 364 /* 365 * If we have just been running VFP instructions we will 366 * need to save the state to memcpy it below. 367 */ 368 vfp_save_state(td, curpcb); 369 370 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_vfp, 371 sizeof(mcp->mc_fpregs)); 372 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpcr; 373 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpsr; 374 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; 375 mcp->mc_flags |= _MC_FP_VALID; 376 } 377 378 critical_exit(); 379#endif 380} 381 382static void 383set_fpcontext(struct thread *td, mcontext_t *mcp) 384{ 385#ifdef VFP 386 struct pcb *curpcb; 387 388 critical_enter(); 389 390 if ((mcp->mc_flags & _MC_FP_VALID) != 0) { 391 curpcb = curthread->td_pcb; 392 393 /* 394 * Discard any vfp state for the current thread, we 395 * are about to override it. 396 */ 397 vfp_discard(td); 398 399 memcpy(curpcb->pcb_vfp, mcp->mc_fpregs.fp_q, 400 sizeof(mcp->mc_fpregs)); 401 curpcb->pcb_fpcr = mcp->mc_fpregs.fp_cr; 402 curpcb->pcb_fpsr = mcp->mc_fpregs.fp_sr; 403 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags; 404 } 405 406 critical_exit(); 407#endif 408} 409 410void 411cpu_idle(int busy) 412{ 413 414 spinlock_enter(); 415 if (!busy) 416 cpu_idleclock(); 417 if (!sched_runnable()) 418 __asm __volatile( 419 "dsb sy \n" 420 "wfi \n"); 421 if (!busy) 422 cpu_activeclock(); 423 spinlock_exit(); 424} 425 426void 427cpu_halt(void) 428{ 429 430 /* We should have shutdown by now, if not enter a low power sleep */ 431 intr_disable(); 432 while (1) { 433 __asm __volatile("wfi"); 434 } 435} 436 437/* 438 * Flush the D-cache for non-DMA I/O so that the I-cache can 439 * be made coherent later. 440 */ 441void 442cpu_flush_dcache(void *ptr, size_t len) 443{ 444 445 /* ARM64TODO TBD */ 446} 447 448/* Get current clock frequency for the given CPU ID. */ 449int 450cpu_est_clockrate(int cpu_id, uint64_t *rate) 451{ 452 struct pcpu *pc; 453 454 pc = pcpu_find(cpu_id); 455 if (pc == NULL || rate == NULL) 456 return (EINVAL); 457 458 if (pc->pc_clock == 0) 459 return (EOPNOTSUPP); 460 461 *rate = pc->pc_clock; 462 return (0); 463} 464 465void 466cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 467{ 468 469 pcpu->pc_acpi_id = 0xffffffff; 470} 471 472void 473spinlock_enter(void) 474{ 475 struct thread *td; 476 register_t daif; 477 478 td = curthread; 479 if (td->td_md.md_spinlock_count == 0) { 480 daif = intr_disable(); 481 td->td_md.md_spinlock_count = 1; 482 td->td_md.md_saved_daif = daif; 483 } else 484 td->td_md.md_spinlock_count++; 485 critical_enter(); 486} 487 488void 489spinlock_exit(void) 490{ 491 struct thread *td; 492 register_t daif; 493 494 td = curthread; 495 critical_exit(); 496 daif = td->td_md.md_saved_daif; 497 td->td_md.md_spinlock_count--; 498 if (td->td_md.md_spinlock_count == 0) 499 intr_restore(daif); 500} 501 502#ifndef _SYS_SYSPROTO_H_ 503struct sigreturn_args { 504 ucontext_t *ucp; 505}; 506#endif 507 508int 509sys_sigreturn(struct thread *td, struct sigreturn_args *uap) 510{ 511 ucontext_t uc; 512 int error; 513 514 if (uap == NULL) 515 return (EFAULT); 516 if (copyin(uap->sigcntxp, &uc, sizeof(uc))) 517 return (EFAULT); 518 519 error = set_mcontext(td, &uc.uc_mcontext); 520 if (error != 0) 521 return (error); 522 set_fpcontext(td, &uc.uc_mcontext); 523 524 /* Restore signal mask. */ 525 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); 526 527 return (EJUSTRETURN); 528} 529 530/* 531 * Construct a PCB from a trapframe. This is called from kdb_trap() where 532 * we want to start a backtrace from the function that caused us to enter 533 * the debugger. We have the context in the trapframe, but base the trace 534 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 535 * enough for a backtrace. 536 */ 537void 538makectx(struct trapframe *tf, struct pcb *pcb) 539{ 540 int i; 541 542 for (i = 0; i < PCB_LR; i++) 543 pcb->pcb_x[i] = tf->tf_x[i]; 544 545 pcb->pcb_x[PCB_LR] = tf->tf_lr; 546 pcb->pcb_pc = tf->tf_elr; 547 pcb->pcb_sp = tf->tf_sp; 548} 549 550void 551sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) 552{ 553 struct thread *td; 554 struct proc *p; 555 struct trapframe *tf; 556 struct sigframe *fp, frame; 557 struct sigacts *psp; 558 struct sysentvec *sysent; 559 int code, onstack, sig; 560 561 td = curthread; 562 p = td->td_proc; 563 PROC_LOCK_ASSERT(p, MA_OWNED); 564 565 sig = ksi->ksi_signo; 566 code = ksi->ksi_code; 567 psp = p->p_sigacts; 568 mtx_assert(&psp->ps_mtx, MA_OWNED); 569 570 tf = td->td_frame; 571 onstack = sigonstack(tf->tf_sp); 572 573 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, 574 catcher, sig); 575 576 /* Allocate and validate space for the signal handler context. */ 577 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && 578 SIGISMEMBER(psp->ps_sigonstack, sig)) { 579 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + 580 td->td_sigstk.ss_size); 581#if defined(COMPAT_43) 582 td->td_sigstk.ss_flags |= SS_ONSTACK; 583#endif 584 } else { 585 fp = (struct sigframe *)td->td_frame->tf_sp; 586 } 587 588 /* Make room, keeping the stack aligned */ 589 fp--; 590 fp = (struct sigframe *)STACKALIGN(fp); 591 592 /* Fill in the frame to copy out */ 593 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); 594 get_fpcontext(td, &frame.sf_uc.uc_mcontext); 595 frame.sf_si = ksi->ksi_info; 596 frame.sf_uc.uc_sigmask = *mask; 597 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? 598 ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; 599 frame.sf_uc.uc_stack = td->td_sigstk; 600 mtx_unlock(&psp->ps_mtx); 601 PROC_UNLOCK(td->td_proc); 602 603 /* Copy the sigframe out to the user's stack. */ 604 if (copyout(&frame, fp, sizeof(*fp)) != 0) { 605 /* Process has trashed its stack. Kill it. */ 606 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); 607 PROC_LOCK(p); 608 sigexit(td, SIGILL); 609 } 610 611 tf->tf_x[0]= sig; 612 tf->tf_x[1] = (register_t)&fp->sf_si; 613 tf->tf_x[2] = (register_t)&fp->sf_uc; 614 615 tf->tf_elr = (register_t)catcher; 616 tf->tf_sp = (register_t)fp; 617 sysent = p->p_sysent; 618 if (sysent->sv_sigcode_base != 0) 619 tf->tf_lr = (register_t)sysent->sv_sigcode_base; 620 else 621 tf->tf_lr = (register_t)(sysent->sv_psstrings - 622 *(sysent->sv_szsigcode)); 623 624 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, 625 tf->tf_sp); 626 627 PROC_LOCK(p); 628 mtx_lock(&psp->ps_mtx); 629} 630 631static void 632init_proc0(vm_offset_t kstack) 633{ 634 struct pcpu *pcpup = &__pcpu[0]; 635 636 proc_linkup0(&proc0, &thread0); 637 thread0.td_kstack = kstack; 638 thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; 639 thread0.td_pcb->pcb_fpflags = 0; 640 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 641 thread0.td_frame = &proc0_tf; 642 pcpup->pc_curpcb = thread0.td_pcb; 643} 644 645typedef struct { 646 uint32_t type; 647 uint64_t phys_start; 648 uint64_t virt_start; 649 uint64_t num_pages; 650 uint64_t attr; 651} EFI_MEMORY_DESCRIPTOR; 652 653static int 654add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap, 655 u_int *physmap_idxp) 656{ 657 u_int i, insert_idx, _physmap_idx; 658 659 _physmap_idx = *physmap_idxp; 660 661 if (length == 0) 662 return (1); 663 664 /* 665 * Find insertion point while checking for overlap. Start off by 666 * assuming the new entry will be added to the end. 667 */ 668 insert_idx = _physmap_idx; 669 for (i = 0; i <= _physmap_idx; i += 2) { 670 if (base < physmap[i + 1]) { 671 if (base + length <= physmap[i]) { 672 insert_idx = i; 673 break; 674 } 675 if (boothowto & RB_VERBOSE) 676 printf( 677 "Overlapping memory regions, ignoring second region\n"); 678 return (1); 679 } 680 } 681 682 /* See if we can prepend to the next entry. */ 683 if (insert_idx <= _physmap_idx && 684 base + length == physmap[insert_idx]) { 685 physmap[insert_idx] = base; 686 return (1); 687 } 688 689 /* See if we can append to the previous entry. */ 690 if (insert_idx > 0 && base == physmap[insert_idx - 1]) { 691 physmap[insert_idx - 1] += length; 692 return (1); 693 } 694 695 _physmap_idx += 2; 696 *physmap_idxp = _physmap_idx; 697 if (_physmap_idx == PHYSMAP_SIZE) { 698 printf( 699 "Too many segments in the physical address map, giving up\n"); 700 return (0); 701 } 702 703 /* 704 * Move the last 'N' entries down to make room for the new 705 * entry if needed. 706 */ 707 for (i = _physmap_idx; i > insert_idx; i -= 2) { 708 physmap[i] = physmap[i - 2]; 709 physmap[i + 1] = physmap[i - 1]; 710 } 711 712 /* Insert the new entry. */ 713 physmap[insert_idx] = base; 714 physmap[insert_idx + 1] = base + length; 715 return (1); 716} 717 718#ifdef FDT 719static void 720add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap, 721 u_int *physmap_idxp) 722{ 723 724 for (int i = 0; i < mrcnt; i++) { 725 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap, 726 physmap_idxp)) 727 break; 728 } 729} 730#endif 731 732static void 733add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap, 734 u_int *physmap_idxp) 735{ 736 struct efi_md *map, *p; 737 const char *type; 738 size_t efisz; 739 int ndesc, i; 740 741 static const char *types[] = { 742 "Reserved", 743 "LoaderCode", 744 "LoaderData", 745 "BootServicesCode", 746 "BootServicesData", 747 "RuntimeServicesCode", 748 "RuntimeServicesData", 749 "ConventionalMemory", 750 "UnusableMemory", 751 "ACPIReclaimMemory", 752 "ACPIMemoryNVS", 753 "MemoryMappedIO", 754 "MemoryMappedIOPortSpace", 755 "PalCode", 756 "PersistentMemory" 757 }; 758 759 /* 760 * Memory map data provided by UEFI via the GetMemoryMap 761 * Boot Services API. 762 */ 763 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 764 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 765 766 if (efihdr->descriptor_size == 0) 767 return; 768 ndesc = efihdr->memory_size / efihdr->descriptor_size; 769 770 if (boothowto & RB_VERBOSE) 771 printf("%23s %12s %12s %8s %4s\n", 772 "Type", "Physical", "Virtual", "#Pages", "Attr"); 773 774 for (i = 0, p = map; i < ndesc; i++, 775 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 776 if (boothowto & RB_VERBOSE) { 777 if (p->md_type < nitems(types)) 778 type = types[p->md_type]; 779 else 780 type = "<INVALID>"; 781 printf("%23s %012lx %12p %08lx ", type, p->md_phys, 782 p->md_virt, p->md_pages); 783 if (p->md_attr & EFI_MD_ATTR_UC) 784 printf("UC "); 785 if (p->md_attr & EFI_MD_ATTR_WC) 786 printf("WC "); 787 if (p->md_attr & EFI_MD_ATTR_WT) 788 printf("WT "); 789 if (p->md_attr & EFI_MD_ATTR_WB) 790 printf("WB "); 791 if (p->md_attr & EFI_MD_ATTR_UCE) 792 printf("UCE "); 793 if (p->md_attr & EFI_MD_ATTR_WP) 794 printf("WP "); 795 if (p->md_attr & EFI_MD_ATTR_RP) 796 printf("RP "); 797 if (p->md_attr & EFI_MD_ATTR_XP) 798 printf("XP "); 799 if (p->md_attr & EFI_MD_ATTR_NV) 800 printf("NV "); 801 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 802 printf("MORE_RELIABLE "); 803 if (p->md_attr & EFI_MD_ATTR_RO) 804 printf("RO "); 805 if (p->md_attr & EFI_MD_ATTR_RT) 806 printf("RUNTIME"); 807 printf("\n"); 808 } 809 810 switch (p->md_type) { 811 case EFI_MD_TYPE_CODE: 812 case EFI_MD_TYPE_DATA: 813 case EFI_MD_TYPE_BS_CODE: 814 case EFI_MD_TYPE_BS_DATA: 815 case EFI_MD_TYPE_FREE: 816 /* 817 * We're allowed to use any entry with these types. 818 */ 819 break; 820 default: 821 continue; 822 } 823 824 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE), 825 physmap, physmap_idxp)) 826 break; 827 } 828} 829 830#ifdef FDT 831static void 832try_load_dtb(caddr_t kmdp) 833{ 834 vm_offset_t dtbp; 835 836 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 837 if (dtbp == (vm_offset_t)NULL) { 838 printf("ERROR loading DTB\n"); 839 return; 840 } 841 842 if (OF_install(OFW_FDT, 0) == FALSE) 843 panic("Cannot install FDT"); 844 845 if (OF_init((void *)dtbp) != 0) 846 panic("OF_init failed with the found device tree"); 847} 848#endif 849 850static void 851cache_setup(void) 852{ 853 int dcache_line_shift, icache_line_shift, dczva_line_shift; 854 uint32_t ctr_el0; 855 uint32_t dczid_el0; 856 857 ctr_el0 = READ_SPECIALREG(ctr_el0); 858 859 /* Read the log2 words in each D cache line */ 860 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); 861 /* Get the D cache line size */ 862 dcache_line_size = sizeof(int) << dcache_line_shift; 863 864 /* And the same for the I cache */ 865 icache_line_shift = CTR_ILINE_SIZE(ctr_el0); 866 icache_line_size = sizeof(int) << icache_line_shift; 867 868 idcache_line_size = MIN(dcache_line_size, icache_line_size); 869 870 dczid_el0 = READ_SPECIALREG(dczid_el0); 871 872 /* Check if dc zva is not prohibited */ 873 if (dczid_el0 & DCZID_DZP) 874 dczva_line_size = 0; 875 else { 876 /* Same as with above calculations */ 877 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 878 dczva_line_size = sizeof(int) << dczva_line_shift; 879 880 /* Change pagezero function */ 881 pagezero = pagezero_cache; 882 } 883} 884 885void 886initarm(struct arm64_bootparams *abp) 887{ 888 struct efi_map_header *efihdr; 889 struct pcpu *pcpup; 890#ifdef FDT 891 struct mem_region mem_regions[FDT_MEM_REGIONS]; 892 int mem_regions_sz; 893#endif 894 vm_offset_t lastaddr; 895 caddr_t kmdp; 896 vm_paddr_t mem_len; 897 int i; 898 899 /* Set the module data location */ 900 preload_metadata = (caddr_t)(uintptr_t)(abp->modulep); 901 902 /* Find the kernel address */ 903 kmdp = preload_search_by_type("elf kernel"); 904 if (kmdp == NULL) 905 kmdp = preload_search_by_type("elf64 kernel"); 906 907 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); 908 init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0); 909 910#ifdef FDT 911 try_load_dtb(kmdp); 912#endif 913 914 /* Find the address to start allocating from */ 915 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); 916 917 /* Load the physical memory ranges */ 918 physmap_idx = 0; 919 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 920 MODINFO_METADATA | MODINFOMD_EFI_MAP); 921 if (efihdr != NULL) 922 add_efi_map_entries(efihdr, physmap, &physmap_idx); 923#ifdef FDT 924 else { 925 /* Grab physical memory regions information from device tree. */ 926 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 927 NULL) != 0) 928 panic("Cannot get physical memory regions"); 929 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap, 930 &physmap_idx); 931 } 932#endif 933 934 /* Print the memory map */ 935 mem_len = 0; 936 for (i = 0; i < physmap_idx; i += 2) { 937 dump_avail[i] = physmap[i]; 938 dump_avail[i + 1] = physmap[i + 1]; 939 mem_len += physmap[i + 1] - physmap[i]; 940 } 941 dump_avail[i] = 0; 942 dump_avail[i + 1] = 0; 943 944 /* Set the pcpu data, this is needed by pmap_bootstrap */ 945 pcpup = &__pcpu[0]; 946 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 947 948 /* 949 * Set the pcpu pointer with a backup in tpidr_el1 to be 950 * loaded when entering the kernel from userland. 951 */ 952 __asm __volatile( 953 "mov x18, %0 \n" 954 "msr tpidr_el1, %0" :: "r"(pcpup)); 955 956 PCPU_SET(curthread, &thread0); 957 958 /* Do basic tuning, hz etc */ 959 init_param1(); 960 961 cache_setup(); 962 pan_setup(); 963 964 /* Bootstrap enough of pmap to enter the kernel proper */ 965 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 966 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 967 968 devmap_bootstrap(0, NULL); 969 970 cninit(); 971 972 init_proc0(abp->kern_stack); 973 msgbufinit(msgbufp, msgbufsize); 974 mutex_init(); 975 init_param2(physmem); 976 977 dbg_init(); 978 kdb_init(); 979 pan_enable(); 980 981 early_boot = 0; 982} 983 984void 985dbg_init(void) 986{ 987 988 /* Clear OS lock */ 989 WRITE_SPECIALREG(OSLAR_EL1, 0); 990 991 /* This permits DDB to use debug registers for watchpoints. */ 992 dbg_monitor_init(); 993 994 /* TODO: Eventually will need to initialize debug registers here. */ 995} 996 997#ifdef DDB 998#include <ddb/ddb.h> 999 1000DB_SHOW_COMMAND(specialregs, db_show_spregs) 1001{ 1002#define PRINT_REG(reg) \ 1003 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 1004 1005 PRINT_REG(actlr_el1); 1006 PRINT_REG(afsr0_el1); 1007 PRINT_REG(afsr1_el1); 1008 PRINT_REG(aidr_el1); 1009 PRINT_REG(amair_el1); 1010 PRINT_REG(ccsidr_el1); 1011 PRINT_REG(clidr_el1); 1012 PRINT_REG(contextidr_el1); 1013 PRINT_REG(cpacr_el1); 1014 PRINT_REG(csselr_el1); 1015 PRINT_REG(ctr_el0); 1016 PRINT_REG(currentel); 1017 PRINT_REG(daif); 1018 PRINT_REG(dczid_el0); 1019 PRINT_REG(elr_el1); 1020 PRINT_REG(esr_el1); 1021 PRINT_REG(far_el1); 1022#if 0 1023 /* ARM64TODO: Enable VFP before reading floating-point registers */ 1024 PRINT_REG(fpcr); 1025 PRINT_REG(fpsr); 1026#endif 1027 PRINT_REG(id_aa64afr0_el1); 1028 PRINT_REG(id_aa64afr1_el1); 1029 PRINT_REG(id_aa64dfr0_el1); 1030 PRINT_REG(id_aa64dfr1_el1); 1031 PRINT_REG(id_aa64isar0_el1); 1032 PRINT_REG(id_aa64isar1_el1); 1033 PRINT_REG(id_aa64pfr0_el1); 1034 PRINT_REG(id_aa64pfr1_el1); 1035 PRINT_REG(id_afr0_el1); 1036 PRINT_REG(id_dfr0_el1); 1037 PRINT_REG(id_isar0_el1); 1038 PRINT_REG(id_isar1_el1); 1039 PRINT_REG(id_isar2_el1); 1040 PRINT_REG(id_isar3_el1); 1041 PRINT_REG(id_isar4_el1); 1042 PRINT_REG(id_isar5_el1); 1043 PRINT_REG(id_mmfr0_el1); 1044 PRINT_REG(id_mmfr1_el1); 1045 PRINT_REG(id_mmfr2_el1); 1046 PRINT_REG(id_mmfr3_el1); 1047#if 0 1048 /* Missing from llvm */ 1049 PRINT_REG(id_mmfr4_el1); 1050#endif 1051 PRINT_REG(id_pfr0_el1); 1052 PRINT_REG(id_pfr1_el1); 1053 PRINT_REG(isr_el1); 1054 PRINT_REG(mair_el1); 1055 PRINT_REG(midr_el1); 1056 PRINT_REG(mpidr_el1); 1057 PRINT_REG(mvfr0_el1); 1058 PRINT_REG(mvfr1_el1); 1059 PRINT_REG(mvfr2_el1); 1060 PRINT_REG(revidr_el1); 1061 PRINT_REG(sctlr_el1); 1062 PRINT_REG(sp_el0); 1063 PRINT_REG(spsel); 1064 PRINT_REG(spsr_el1); 1065 PRINT_REG(tcr_el1); 1066 PRINT_REG(tpidr_el0); 1067 PRINT_REG(tpidr_el1); 1068 PRINT_REG(tpidrro_el0); 1069 PRINT_REG(ttbr0_el1); 1070 PRINT_REG(ttbr1_el1); 1071 PRINT_REG(vbar_el1); 1072#undef PRINT_REG 1073} 1074 1075DB_SHOW_COMMAND(vtop, db_show_vtop) 1076{ 1077 uint64_t phys; 1078 1079 if (have_addr) { 1080 phys = arm64_address_translate_s1e1r(addr); 1081 db_printf("Physical address reg (read): 0x%016lx\n", phys); 1082 phys = arm64_address_translate_s1e1w(addr); 1083 db_printf("Physical address reg (write): 0x%016lx\n", phys); 1084 } else 1085 db_printf("show vtop <virt_addr>\n"); 1086} 1087#endif 1088