arm32_kvminit.c revision 1.34
1/* $NetBSD: arm32_kvminit.c,v 1.34 2015/05/30 23:59:33 matt Exp $ */ 2 3/* 4 * Copyright (c) 2002, 2003, 2005 Genetec Corporation. All rights reserved. 5 * Written by Hiroyuki Bessho for Genetec Corporation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of Genetec Corporation may not be used to endorse or 16 * promote products derived from this software without specific prior 17 * written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY GENETEC CORPORATION ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GENETEC CORPORATION 23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29 * POSSIBILITY OF SUCH DAMAGE. 30 * 31 * Copyright (c) 2001 Wasabi Systems, Inc. 32 * All rights reserved. 33 * 34 * Written by Jason R. Thorpe for Wasabi Systems, Inc. 35 * 36 * Redistribution and use in source and binary forms, with or without 37 * modification, are permitted provided that the following conditions 38 * are met: 39 * 1. Redistributions of source code must retain the above copyright 40 * notice, this list of conditions and the following disclaimer. 41 * 2. Redistributions in binary form must reproduce the above copyright 42 * notice, this list of conditions and the following disclaimer in the 43 * documentation and/or other materials provided with the distribution. 44 * 3. All advertising materials mentioning features or use of this software 45 * must display the following acknowledgement: 46 * This product includes software developed for the NetBSD Project by 47 * Wasabi Systems, Inc. 48 * 4. The name of Wasabi Systems, Inc. may not be used to endorse 49 * or promote products derived from this software without specific prior 50 * written permission. 51 * 52 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND 53 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 54 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 55 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC 56 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 57 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 58 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 59 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 60 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 61 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 62 * POSSIBILITY OF SUCH DAMAGE. 63 * 64 * Copyright (c) 1997,1998 Mark Brinicombe. 65 * Copyright (c) 1997,1998 Causality Limited. 66 * All rights reserved. 67 * 68 * Redistribution and use in source and binary forms, with or without 69 * modification, are permitted provided that the following conditions 70 * are met: 71 * 1. Redistributions of source code must retain the above copyright 72 * notice, this list of conditions and the following disclaimer. 73 * 2. Redistributions in binary form must reproduce the above copyright 74 * notice, this list of conditions and the following disclaimer in the 75 * documentation and/or other materials provided with the distribution. 76 * 3. All advertising materials mentioning features or use of this software 77 * must display the following acknowledgement: 78 * This product includes software developed by Mark Brinicombe 79 * for the NetBSD Project. 80 * 4. The name of the company nor the name of the author may be used to 81 * endorse or promote products derived from this software without specific 82 * prior written permission. 83 * 84 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 85 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 86 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 87 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 88 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 89 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 90 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 91 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 92 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 93 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 94 * SUCH DAMAGE. 95 * 96 * Copyright (c) 2007 Microsoft 97 * All rights reserved. 98 * 99 * Redistribution and use in source and binary forms, with or without 100 * modification, are permitted provided that the following conditions 101 * are met: 102 * 1. Redistributions of source code must retain the above copyright 103 * notice, this list of conditions and the following disclaimer. 104 * 2. Redistributions in binary form must reproduce the above copyright 105 * notice, this list of conditions and the following disclaimer in the 106 * documentation and/or other materials provided with the distribution. 107 * 3. All advertising materials mentioning features or use of this software 108 * must display the following acknowledgement: 109 * This product includes software developed by Microsoft 110 * 111 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 112 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 113 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 114 * IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTERS BE LIABLE FOR ANY DIRECT, 115 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 116 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 117 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 118 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 119 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 120 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 121 * SUCH DAMAGE. 122 */ 123 124#include "opt_multiprocessor.h" 125 126#include <sys/cdefs.h> 127__KERNEL_RCSID(0, "$NetBSD: arm32_kvminit.c,v 1.34 2015/05/30 23:59:33 matt Exp $"); 128 129#include <sys/param.h> 130#include <sys/device.h> 131#include <sys/kernel.h> 132#include <sys/reboot.h> 133#include <sys/bus.h> 134 135#include <dev/cons.h> 136 137#include <uvm/uvm_extern.h> 138 139#include <arm/locore.h> 140#include <arm/db_machdep.h> 141#include <arm/undefined.h> 142#include <arm/bootconfig.h> 143#include <arm/arm32/machdep.h> 144 145struct bootmem_info bootmem_info; 146 147extern void *msgbufaddr; 148paddr_t msgbufphys; 149paddr_t physical_start; 150paddr_t physical_end; 151 152extern char etext[]; 153extern char __data_start[], _edata[]; 154extern char __bss_start[], __bss_end__[]; 155extern char _end[]; 156 157/* Page tables for mapping kernel VM */ 158#define KERNEL_L2PT_VMDATA_NUM 8 /* start with 32MB of KVM */ 159 160/* 161 * Macros to translate between physical and virtual for a subset of the 162 * kernel address space. *Not* for general use. 163 */ 164#if defined(KERNEL_BASE_VOFFSET) 165#define KERN_VTOPHYS(bmi, va) \ 166 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE_VOFFSET)) 167#define KERN_PHYSTOV(bmi, pa) \ 168 ((vaddr_t)((paddr_t)(pa) + KERNEL_BASE_VOFFSET)) 169#elif defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 170#define KERN_VTOPHYS(bmi, va) \ 171 ((paddr_t)((vaddr_t)(va) - pmap_directbase + (bmi)->bmi_start)) 172#define KERN_PHYSTOV(bmi, pa) \ 173 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + pmap_directbase)) 174#else 175#define KERN_VTOPHYS(bmi, va) \ 176 ((paddr_t)((vaddr_t)(va) - KERNEL_BASE + (bmi)->bmi_start)) 177#define KERN_PHYSTOV(bmi, pa) \ 178 ((vaddr_t)((paddr_t)(pa) - (bmi)->bmi_start + KERNEL_BASE)) 179#endif 180 181void 182arm32_bootmem_init(paddr_t memstart, psize_t memsize, vsize_t kernelstart) 183{ 184 struct bootmem_info * const bmi = &bootmem_info; 185 pv_addr_t *pv = bmi->bmi_freeblocks; 186 187#ifdef VERBOSE_INIT_ARM 188 printf("%s: memstart=%#lx, memsize=%#lx, kernelstart=%#lx\n", 189 __func__, memstart, memsize, kernelstart); 190#endif 191 192 physical_start = bmi->bmi_start = memstart; 193 physical_end = bmi->bmi_end = memstart + memsize; 194#ifndef ARM_HAS_LPAE 195 if (physical_end == 0) { 196 physical_end = -PAGE_SIZE; 197 memsize -= PAGE_SIZE; 198 bmi->bmi_end -= PAGE_SIZE; 199#ifdef VERBOSE_INIT_ARM 200 printf("%s: memsize shrunk by a page to avoid ending at 4GB\n", 201 __func__); 202#endif 203 } 204#endif 205 physmem = memsize / PAGE_SIZE; 206 207 /* 208 * Let's record where the kernel lives. 209 */ 210 bmi->bmi_kernelstart = kernelstart; 211 bmi->bmi_kernelend = KERN_VTOPHYS(bmi, round_page((vaddr_t)_end)); 212 213#ifdef VERBOSE_INIT_ARM 214 printf("%s: kernelend=%#lx\n", __func__, bmi->bmi_kernelend); 215#endif 216 217 /* 218 * Now the rest of the free memory must be after the kernel. 219 */ 220 pv->pv_pa = bmi->bmi_kernelend; 221 pv->pv_va = KERN_PHYSTOV(bmi, pv->pv_pa); 222 pv->pv_size = bmi->bmi_end - bmi->bmi_kernelend; 223 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 224#ifdef VERBOSE_INIT_ARM 225 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 226 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 227 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 228#endif 229 pv++; 230 231 /* 232 * Add a free block for any memory before the kernel. 233 */ 234 if (bmi->bmi_start < bmi->bmi_kernelstart) { 235 pv->pv_pa = bmi->bmi_start; 236#if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 237 pv->pv_va = pmap_directbase; 238#else 239 /* 240 * If there's lots of memory the kernel could be placed far 241 * from the start of RAM. If that's the case, don't map the 242 * RAM that would have virtual addresses below KERNEL_BASE. 243 */ 244 if (pv->pv_pa < KERN_VTOPHYS(bmi, KERNEL_BASE)) { 245 psize_t size = KERN_VTOPHYS(bmi, KERNEL_BASE) - pv->pv_pa; 246 bmi->bmi_freepages += size / PAGE_SIZE; 247#ifdef VERBOSE_INIT_ARM 248 printf("%s: adding %lu free pages: [%#lx..%#lx]\n", 249 __func__, size / PAGE_SIZE, pv->pv_va, 250 pv->pv_pa + size - 1); 251#endif 252 pv->pv_pa = KERN_VTOPHYS(bmi, KERNEL_BASE); 253 } 254 pv->pv_va = KERNEL_BASE; 255#endif 256 pv->pv_size = bmi->bmi_kernelstart - pv->pv_pa; 257 bmi->bmi_freepages += pv->pv_size / PAGE_SIZE; 258#ifdef VERBOSE_INIT_ARM 259 printf("%s: adding %lu free pages: [%#lx..%#lx] (VA %#lx)\n", 260 __func__, pv->pv_size / PAGE_SIZE, pv->pv_pa, 261 pv->pv_pa + pv->pv_size - 1, pv->pv_va); 262#endif 263 pv++; 264 } 265 266 bmi->bmi_nfreeblocks = pv - bmi->bmi_freeblocks; 267 268 SLIST_INIT(&bmi->bmi_freechunks); 269 SLIST_INIT(&bmi->bmi_chunks); 270} 271 272static bool 273concat_pvaddr(pv_addr_t *acc_pv, pv_addr_t *pv) 274{ 275 if (acc_pv->pv_pa + acc_pv->pv_size == pv->pv_pa 276 && acc_pv->pv_va + acc_pv->pv_size == pv->pv_va 277 && acc_pv->pv_prot == pv->pv_prot 278 && acc_pv->pv_cache == pv->pv_cache) { 279#ifdef VERBOSE_INIT_ARMX 280 printf("%s: appending pv %p (%#lx..%#lx) to %#lx..%#lx\n", 281 __func__, pv, pv->pv_pa, pv->pv_pa + pv->pv_size + 1, 282 acc_pv->pv_pa, acc_pv->pv_pa + acc_pv->pv_size + 1); 283#endif 284 acc_pv->pv_size += pv->pv_size; 285 return true; 286 } 287 288 return false; 289} 290 291static void 292add_pages(struct bootmem_info *bmi, pv_addr_t *pv) 293{ 294 pv_addr_t **pvp = &SLIST_FIRST(&bmi->bmi_chunks); 295 while ((*pvp) != NULL && (*pvp)->pv_va <= pv->pv_va) { 296 pv_addr_t * const pv0 = (*pvp); 297 KASSERT(SLIST_NEXT(pv0, pv_list) == NULL || pv0->pv_pa < SLIST_NEXT(pv0, pv_list)->pv_pa); 298 if (concat_pvaddr(pv0, pv)) { 299#ifdef VERBOSE_INIT_ARM 300 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 301 __func__, "appending", pv, 302 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 303 pv0->pv_pa, pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 304#endif 305 pv = SLIST_NEXT(pv0, pv_list); 306 if (pv != NULL && concat_pvaddr(pv0, pv)) { 307#ifdef VERBOSE_INIT_ARM 308 printf("%s: %s pv %p (%#lx..%#lx) to %#lx..%#lx\n", 309 __func__, "merging", pv, 310 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 311 pv0->pv_pa, 312 pv0->pv_pa + pv0->pv_size - pv->pv_size - 1); 313#endif 314 SLIST_REMOVE_AFTER(pv0, pv_list); 315 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, pv, pv_list); 316 } 317 return; 318 } 319 KASSERT(pv->pv_va != (*pvp)->pv_va); 320 pvp = &SLIST_NEXT(*pvp, pv_list); 321 } 322 KASSERT((*pvp) == NULL || pv->pv_va < (*pvp)->pv_va); 323 pv_addr_t * const new_pv = SLIST_FIRST(&bmi->bmi_freechunks); 324 KASSERT(new_pv != NULL); 325 SLIST_REMOVE_HEAD(&bmi->bmi_freechunks, pv_list); 326 *new_pv = *pv; 327 SLIST_NEXT(new_pv, pv_list) = *pvp; 328 (*pvp) = new_pv; 329#ifdef VERBOSE_INIT_ARM 330 printf("%s: adding pv %p (pa %#lx, va %#lx, %lu pages) ", 331 __func__, new_pv, new_pv->pv_pa, new_pv->pv_va, 332 new_pv->pv_size / PAGE_SIZE); 333 if (SLIST_NEXT(new_pv, pv_list)) 334 printf("before pa %#lx\n", SLIST_NEXT(new_pv, pv_list)->pv_pa); 335 else 336 printf("at tail\n"); 337#endif 338} 339 340static void 341valloc_pages(struct bootmem_info *bmi, pv_addr_t *pv, size_t npages, 342 int prot, int cache, bool zero_p) 343{ 344 size_t nbytes = npages * PAGE_SIZE; 345 pv_addr_t *free_pv = bmi->bmi_freeblocks; 346 size_t free_idx = 0; 347 static bool l1pt_found; 348 349 KASSERT(npages > 0); 350 351 /* 352 * If we haven't allocated the kernel L1 page table and we are aligned 353 * at a L1 table boundary, alloc the memory for it. 354 */ 355 if (!l1pt_found 356 && (free_pv->pv_pa & (L1_TABLE_SIZE - 1)) == 0 357 && free_pv->pv_size >= L1_TABLE_SIZE) { 358 l1pt_found = true; 359 valloc_pages(bmi, &kernel_l1pt, L1_TABLE_SIZE / PAGE_SIZE, 360 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 361 add_pages(bmi, &kernel_l1pt); 362 } 363 364 while (nbytes > free_pv->pv_size) { 365 free_pv++; 366 free_idx++; 367 if (free_idx == bmi->bmi_nfreeblocks) { 368 panic("%s: could not allocate %zu bytes", 369 __func__, nbytes); 370 } 371 } 372 373 /* 374 * As we allocate the memory, make sure that we don't walk over 375 * our current first level translation table. 376 */ 377 KASSERT((armreg_ttbr_read() & ~(L1_TABLE_SIZE - 1)) != free_pv->pv_pa); 378 379 pv->pv_pa = free_pv->pv_pa; 380 pv->pv_va = free_pv->pv_va; 381 pv->pv_size = nbytes; 382 pv->pv_prot = prot; 383 pv->pv_cache = cache; 384 385 /* 386 * If PTE_PAGETABLE uses the same cache modes as PTE_CACHE 387 * just use PTE_CACHE. 388 */ 389 if (cache == PTE_PAGETABLE 390 && pte_l1_s_cache_mode == pte_l1_s_cache_mode_pt 391 && pte_l2_l_cache_mode == pte_l2_l_cache_mode_pt 392 && pte_l2_s_cache_mode == pte_l2_s_cache_mode_pt) 393 pv->pv_cache = PTE_CACHE; 394 395 free_pv->pv_pa += nbytes; 396 free_pv->pv_va += nbytes; 397 free_pv->pv_size -= nbytes; 398 if (free_pv->pv_size == 0) { 399 --bmi->bmi_nfreeblocks; 400 for (; free_idx < bmi->bmi_nfreeblocks; free_idx++) { 401 free_pv[0] = free_pv[1]; 402 } 403 } 404 405 bmi->bmi_freepages -= npages; 406 407 if (zero_p) 408 memset((void *)pv->pv_va, 0, nbytes); 409} 410 411void 412arm32_kernel_vm_init(vaddr_t kernel_vm_base, vaddr_t vectors, vaddr_t iovbase, 413 const struct pmap_devmap *devmap, bool mapallmem_p) 414{ 415 struct bootmem_info * const bmi = &bootmem_info; 416#ifdef MULTIPROCESSOR 417 const size_t cpu_num = arm_cpu_max; 418#else 419 const size_t cpu_num = 1; 420#endif 421#ifdef ARM_HAS_VBAR 422 const bool map_vectors_p = false; 423#elif defined(CPU_ARMV7) || defined(CPU_ARM11) 424 const bool map_vectors_p = vectors == ARM_VECTORS_HIGH 425 || (armreg_pfr1_read() & ARM_PFR1_SEC_MASK) == 0; 426#else 427 const bool map_vectors_p = true; 428#endif 429 430#ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS 431 KASSERT(mapallmem_p); 432#ifdef ARM_MMU_EXTENDED 433 /* 434 * We can only use address beneath kernel_vm_base to map physical 435 * memory. 436 */ 437 const psize_t physical_size = 438 roundup(physical_end - physical_start, L1_SS_SIZE); 439 KASSERT(kernel_vm_base >= physical_size); 440 /* 441 * If we don't have enough memory via TTBR1, we have use addresses 442 * from TTBR0 to map some of the physical memory. But try to use as 443 * much high memory space as possible. 444 */ 445 pmap_directlimit = kernel_vm_base; 446 if (kernel_vm_base - KERNEL_BASE < physical_size 447 && kernel_vm_base - physical_size >= physical_start) { 448 pmap_directbase -= KERNEL_BASE_VOFFSET; 449 printf("%s: changing pmap_directbase to %#lx\n", __func__, 450 pmap_directbase); 451 } 452#else 453 KASSERT(kernel_vm_base - KERNEL_BASE >= physical_end - physical_start); 454#endif /* ARM_MMU_EXTENDED */ 455#endif /* __HAVE_MM_MD_DIRECT_MAPPED_PHYS */ 456 457 /* 458 * Calculate the number of L2 pages needed for mapping the 459 * kernel + data + stuff. Assume 2 L2 pages for kernel, 1 for vectors, 460 * and 1 for IO 461 */ 462 size_t kernel_size = bmi->bmi_kernelend; 463 kernel_size -= (bmi->bmi_kernelstart & -L2_S_SEGSIZE); 464 kernel_size += L1_TABLE_SIZE_REAL; 465 kernel_size += PAGE_SIZE * KERNEL_L2PT_VMDATA_NUM; 466 if (map_vectors_p) { 467 kernel_size += PAGE_SIZE; /* L2PT for VECTORS */ 468 } 469 if (iovbase) { 470 kernel_size += PAGE_SIZE; /* L2PT for IO */ 471 } 472 kernel_size += 473 cpu_num * (ABT_STACK_SIZE + FIQ_STACK_SIZE + IRQ_STACK_SIZE 474 + UND_STACK_SIZE + UPAGES) * PAGE_SIZE; 475 kernel_size += round_page(MSGBUFSIZE); 476 kernel_size += 0x10000; /* slop */ 477 if (!mapallmem_p) { 478 kernel_size += PAGE_SIZE 479 * ((kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE); 480 } 481 kernel_size = round_page(kernel_size); 482 483 /* 484 * Now we know how many L2 pages it will take. If we've mapped 485 * all of memory, then it won't take any. 486 */ 487 const size_t KERNEL_L2PT_KERNEL_NUM = mapallmem_p 488 ? 0 : round_page(kernel_size + L2_S_SEGSIZE - 1) / L2_S_SEGSIZE; 489 490#ifdef VERBOSE_INIT_ARM 491 printf("%s: %zu L2 pages are needed to map %#zx kernel bytes\n", 492 __func__, KERNEL_L2PT_KERNEL_NUM, kernel_size); 493#endif 494 495 KASSERT(KERNEL_L2PT_KERNEL_NUM + KERNEL_L2PT_VMDATA_NUM < __arraycount(bmi->bmi_l2pts)); 496 pv_addr_t * const kernel_l2pt = bmi->bmi_l2pts; 497 pv_addr_t * const vmdata_l2pt = kernel_l2pt + KERNEL_L2PT_KERNEL_NUM; 498 pv_addr_t msgbuf; 499 pv_addr_t text; 500 pv_addr_t data; 501 pv_addr_t chunks[KERNEL_L2PT_KERNEL_NUM+KERNEL_L2PT_VMDATA_NUM+11]; 502#if ARM_MMU_XSCALE == 1 503 pv_addr_t minidataclean; 504#endif 505 506 /* 507 * We need to allocate some fixed page tables to get the kernel going. 508 * 509 * We are going to allocate our bootstrap pages from the beginning of 510 * the free space that we just calculated. We allocate one page 511 * directory and a number of page tables and store the physical 512 * addresses in the bmi_l2pts array in bootmem_info. 513 * 514 * The kernel page directory must be on a 16K boundary. The page 515 * tables must be on 4K boundaries. What we do is allocate the 516 * page directory on the first 16K boundary that we encounter, and 517 * the page tables on 4K boundaries otherwise. Since we allocate 518 * at least 3 L2 page tables, we are guaranteed to encounter at 519 * least one 16K aligned region. 520 */ 521 522#ifdef VERBOSE_INIT_ARM 523 printf("%s: allocating page tables for", __func__); 524#endif 525 for (size_t i = 0; i < __arraycount(chunks); i++) { 526 SLIST_INSERT_HEAD(&bmi->bmi_freechunks, &chunks[i], pv_list); 527 } 528 529 kernel_l1pt.pv_pa = 0; 530 kernel_l1pt.pv_va = 0; 531 532 /* 533 * Allocate the L2 pages, but if we get to a page that is aligned for 534 * an L1 page table, we will allocate the pages for it first and then 535 * allocate the L2 page. 536 */ 537 538 if (map_vectors_p) { 539 /* 540 * First allocate L2 page for the vectors. 541 */ 542#ifdef VERBOSE_INIT_ARM 543 printf(" vector"); 544#endif 545 valloc_pages(bmi, &bmi->bmi_vector_l2pt, 1, 546 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 547 add_pages(bmi, &bmi->bmi_vector_l2pt); 548 } 549 550 /* 551 * Now allocate L2 pages for the kernel 552 */ 553#ifdef VERBOSE_INIT_ARM 554 printf(" kernel"); 555#endif 556 KASSERT(mapallmem_p || KERNEL_L2PT_KERNEL_NUM > 0); 557 KASSERT(!mapallmem_p || KERNEL_L2PT_KERNEL_NUM == 0); 558 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; ++idx) { 559 valloc_pages(bmi, &kernel_l2pt[idx], 1, 560 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 561 add_pages(bmi, &kernel_l2pt[idx]); 562 } 563 564 /* 565 * Now allocate L2 pages for the initial kernel VA space. 566 */ 567#ifdef VERBOSE_INIT_ARM 568 printf(" vm"); 569#endif 570 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; ++idx) { 571 valloc_pages(bmi, &vmdata_l2pt[idx], 1, 572 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 573 add_pages(bmi, &vmdata_l2pt[idx]); 574 } 575 576 /* 577 * If someone wanted a L2 page for I/O, allocate it now. 578 */ 579 if (iovbase) { 580#ifdef VERBOSE_INIT_ARM 581 printf(" io"); 582#endif 583 valloc_pages(bmi, &bmi->bmi_io_l2pt, 1, 584 VM_PROT_READ|VM_PROT_WRITE, PTE_PAGETABLE, true); 585 add_pages(bmi, &bmi->bmi_io_l2pt); 586 } 587 588#ifdef VERBOSE_INIT_ARM 589 printf("%s: allocating stacks\n", __func__); 590#endif 591 592 /* Allocate stacks for all modes and CPUs */ 593 valloc_pages(bmi, &abtstack, ABT_STACK_SIZE * cpu_num, 594 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 595 add_pages(bmi, &abtstack); 596 valloc_pages(bmi, &fiqstack, FIQ_STACK_SIZE * cpu_num, 597 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 598 add_pages(bmi, &fiqstack); 599 valloc_pages(bmi, &irqstack, IRQ_STACK_SIZE * cpu_num, 600 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 601 add_pages(bmi, &irqstack); 602 valloc_pages(bmi, &undstack, UND_STACK_SIZE * cpu_num, 603 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 604 add_pages(bmi, &undstack); 605 valloc_pages(bmi, &idlestack, UPAGES * cpu_num, /* SVC32 */ 606 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 607 add_pages(bmi, &idlestack); 608 valloc_pages(bmi, &kernelstack, UPAGES, /* SVC32 */ 609 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, true); 610 add_pages(bmi, &kernelstack); 611 612 /* Allocate the message buffer from the end of memory. */ 613 const size_t msgbuf_pgs = round_page(MSGBUFSIZE) / PAGE_SIZE; 614 valloc_pages(bmi, &msgbuf, msgbuf_pgs, 615 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE, false); 616 add_pages(bmi, &msgbuf); 617 msgbufphys = msgbuf.pv_pa; 618 msgbufaddr = (void *)msgbuf.pv_va; 619 620 if (map_vectors_p) { 621 /* 622 * Allocate a page for the system vector page. 623 * This page will just contain the system vectors and can be 624 * shared by all processes. 625 */ 626 valloc_pages(bmi, &systempage, 1, VM_PROT_READ|VM_PROT_WRITE, 627 PTE_CACHE, true); 628 } 629 systempage.pv_va = vectors; 630 631 /* 632 * If the caller needed a few extra pages for some reason, allocate 633 * them now. 634 */ 635#if ARM_MMU_XSCALE == 1 636#if (ARM_NMMUS > 1) 637 if (xscale_use_minidata) 638#endif 639 valloc_pages(bmi, &minidataclean, 1, 640 VM_PROT_READ|VM_PROT_WRITE, 0, true); 641#endif 642 643 /* 644 * Ok we have allocated physical pages for the primary kernel 645 * page tables and stacks. Let's just confirm that. 646 */ 647 if (kernel_l1pt.pv_va == 0 648 && (!kernel_l1pt.pv_pa || (kernel_l1pt.pv_pa & (L1_TABLE_SIZE - 1)) != 0)) 649 panic("%s: Failed to allocate or align the kernel " 650 "page directory", __func__); 651 652 653#ifdef VERBOSE_INIT_ARM 654 printf("Creating L1 page table at 0x%08lx\n", kernel_l1pt.pv_pa); 655#endif 656 657 /* 658 * Now we start construction of the L1 page table 659 * We start by mapping the L2 page tables into the L1. 660 * This means that we can replace L1 mappings later on if necessary 661 */ 662 vaddr_t l1pt_va = kernel_l1pt.pv_va; 663 paddr_t l1pt_pa = kernel_l1pt.pv_pa; 664 665 if (map_vectors_p) { 666 /* Map the L2 pages tables in the L1 page table */ 667 pmap_link_l2pt(l1pt_va, systempage.pv_va & -L2_S_SEGSIZE, 668 &bmi->bmi_vector_l2pt); 669#ifdef VERBOSE_INIT_ARM 670 printf("%s: adding L2 pt (VA %#lx, PA %#lx) " 671 "for VA %#lx\n (vectors)", 672 __func__, bmi->bmi_vector_l2pt.pv_va, 673 bmi->bmi_vector_l2pt.pv_pa, systempage.pv_va); 674#endif 675 } 676 677 const vaddr_t kernel_base = 678 KERN_PHYSTOV(bmi, bmi->bmi_kernelstart & -L2_S_SEGSIZE); 679 for (size_t idx = 0; idx < KERNEL_L2PT_KERNEL_NUM; idx++) { 680 pmap_link_l2pt(l1pt_va, kernel_base + idx * L2_S_SEGSIZE, 681 &kernel_l2pt[idx]); 682#ifdef VERBOSE_INIT_ARM 683 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (kernel)\n", 684 __func__, kernel_l2pt[idx].pv_va, 685 kernel_l2pt[idx].pv_pa, kernel_base + idx * L2_S_SEGSIZE); 686#endif 687 } 688 689 for (size_t idx = 0; idx < KERNEL_L2PT_VMDATA_NUM; idx++) { 690 pmap_link_l2pt(l1pt_va, kernel_vm_base + idx * L2_S_SEGSIZE, 691 &vmdata_l2pt[idx]); 692#ifdef VERBOSE_INIT_ARM 693 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (vm)\n", 694 __func__, vmdata_l2pt[idx].pv_va, vmdata_l2pt[idx].pv_pa, 695 kernel_vm_base + idx * L2_S_SEGSIZE); 696#endif 697 } 698 if (iovbase) { 699 pmap_link_l2pt(l1pt_va, iovbase & -L2_S_SEGSIZE, &bmi->bmi_io_l2pt); 700#ifdef VERBOSE_INIT_ARM 701 printf("%s: adding L2 pt (VA %#lx, PA %#lx) for VA %#lx (io)\n", 702 __func__, bmi->bmi_io_l2pt.pv_va, bmi->bmi_io_l2pt.pv_pa, 703 iovbase & -L2_S_SEGSIZE); 704#endif 705 } 706 707 /* update the top of the kernel VM */ 708 pmap_curmaxkvaddr = 709 kernel_vm_base + (KERNEL_L2PT_VMDATA_NUM * L2_S_SEGSIZE); 710 711#ifdef VERBOSE_INIT_ARM 712 printf("Mapping kernel\n"); 713#endif 714 715 extern char etext[], _end[]; 716 size_t totalsize = bmi->bmi_kernelend - bmi->bmi_kernelstart; 717 size_t textsize = KERN_VTOPHYS(bmi, (uintptr_t)etext) - bmi->bmi_kernelstart; 718 719 textsize = (textsize + PGOFSET) & ~PGOFSET; 720 721 /* start at offset of kernel in RAM */ 722 723 text.pv_pa = bmi->bmi_kernelstart; 724 text.pv_va = KERN_PHYSTOV(bmi, bmi->bmi_kernelstart); 725 text.pv_size = textsize; 726 text.pv_prot = VM_PROT_READ|VM_PROT_WRITE; /* XXX VM_PROT_EXECUTE */ 727 text.pv_cache = PTE_CACHE; 728 729#ifdef VERBOSE_INIT_ARM 730 printf("%s: adding chunk for kernel text %#lx..%#lx (VA %#lx)\n", 731 __func__, text.pv_pa, text.pv_pa + text.pv_size - 1, text.pv_va); 732#endif 733 734 add_pages(bmi, &text); 735 736 data.pv_pa = text.pv_pa + textsize; 737 data.pv_va = text.pv_va + textsize; 738 data.pv_size = totalsize - textsize; 739 data.pv_prot = VM_PROT_READ|VM_PROT_WRITE; 740 data.pv_cache = PTE_CACHE; 741 742#ifdef VERBOSE_INIT_ARM 743 printf("%s: adding chunk for kernel data/bss %#lx..%#lx (VA %#lx)\n", 744 __func__, data.pv_pa, data.pv_pa + data.pv_size - 1, data.pv_va); 745#endif 746 747 add_pages(bmi, &data); 748 749#ifdef VERBOSE_INIT_ARM 750 printf("Listing Chunks\n"); 751 752 pv_addr_t *lpv; 753 SLIST_FOREACH(lpv, &bmi->bmi_chunks, pv_list) { 754 printf("%s: pv %p: chunk VA %#lx..%#lx " 755 "(PA %#lx, prot %d, cache %d)\n", 756 __func__, lpv, lpv->pv_va, lpv->pv_va + lpv->pv_size - 1, 757 lpv->pv_pa, lpv->pv_prot, lpv->pv_cache); 758 } 759 printf("\nMapping Chunks\n"); 760#endif 761 762 pv_addr_t cur_pv; 763 pv_addr_t *pv = SLIST_FIRST(&bmi->bmi_chunks); 764 if (!mapallmem_p || pv->pv_pa == bmi->bmi_start) { 765 cur_pv = *pv; 766 pv = SLIST_NEXT(pv, pv_list); 767 } else { 768#if defined(ARM_MMU_EXTENDED) && defined(__HAVE_MM_MD_DIRECT_MAPPED_PHYS) 769 cur_pv.pv_va = pmap_directbase; 770#else 771 cur_pv.pv_va = KERNEL_BASE; 772#endif 773 cur_pv.pv_pa = bmi->bmi_start; 774 cur_pv.pv_size = pv->pv_pa - bmi->bmi_start; 775 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 776 cur_pv.pv_cache = PTE_CACHE; 777 } 778 while (pv != NULL) { 779 if (mapallmem_p) { 780 if (concat_pvaddr(&cur_pv, pv)) { 781 pv = SLIST_NEXT(pv, pv_list); 782 continue; 783 } 784 if (cur_pv.pv_pa + cur_pv.pv_size < pv->pv_pa) { 785 /* 786 * See if we can extend the current pv to emcompass the 787 * hole, and if so do it and retry the concatenation. 788 */ 789 if (cur_pv.pv_prot == (VM_PROT_READ|VM_PROT_WRITE) 790 && cur_pv.pv_cache == PTE_CACHE) { 791 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 792 continue; 793 } 794 795 /* 796 * We couldn't so emit the current chunk and then 797 */ 798#ifdef VERBOSE_INIT_ARM 799 printf("%s: mapping chunk VA %#lx..%#lx " 800 "(PA %#lx, prot %d, cache %d)\n", 801 __func__, 802 cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 803 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 804#endif 805 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 806 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 807 808 /* 809 * set the current chunk to the hole and try again. 810 */ 811 cur_pv.pv_pa += cur_pv.pv_size; 812 cur_pv.pv_va += cur_pv.pv_size; 813 cur_pv.pv_size = pv->pv_pa - cur_pv.pv_va; 814 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 815 cur_pv.pv_cache = PTE_CACHE; 816 continue; 817 } 818 } 819 820 /* 821 * The new pv didn't concatenate so emit the current one 822 * and use the new pv as the current pv. 823 */ 824#ifdef VERBOSE_INIT_ARM 825 printf("%s: mapping chunk VA %#lx..%#lx " 826 "(PA %#lx, prot %d, cache %d)\n", 827 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 828 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 829#endif 830 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 831 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 832 cur_pv = *pv; 833 pv = SLIST_NEXT(pv, pv_list); 834 } 835 836 /* 837 * If we are mapping all of memory, let's map the rest of memory. 838 */ 839 if (mapallmem_p && cur_pv.pv_pa + cur_pv.pv_size < bmi->bmi_end) { 840 if (cur_pv.pv_prot == (VM_PROT_READ | VM_PROT_WRITE) 841 && cur_pv.pv_cache == PTE_CACHE) { 842 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 843 } else { 844 KASSERTMSG(cur_pv.pv_va + cur_pv.pv_size <= kernel_vm_base, 845 "%#lx >= %#lx", cur_pv.pv_va + cur_pv.pv_size, 846 kernel_vm_base); 847#ifdef VERBOSE_INIT_ARM 848 printf("%s: mapping chunk VA %#lx..%#lx " 849 "(PA %#lx, prot %d, cache %d)\n", 850 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 851 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 852#endif 853 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 854 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 855 cur_pv.pv_pa += cur_pv.pv_size; 856 cur_pv.pv_va += cur_pv.pv_size; 857 cur_pv.pv_size = bmi->bmi_end - cur_pv.pv_pa; 858 cur_pv.pv_prot = VM_PROT_READ | VM_PROT_WRITE; 859 cur_pv.pv_cache = PTE_CACHE; 860 } 861 } 862 863 // The amount we can direct is limited by the start of the 864 // virtual part of the kernel address space. Don't overrun 865 // into it. 866 if (mapallmem_p && cur_pv.pv_va + cur_pv.pv_size > kernel_vm_base) { 867 cur_pv.pv_size = kernel_vm_base - cur_pv.pv_va; 868 } 869 870 /* 871 * Now we map the final chunk. 872 */ 873#ifdef VERBOSE_INIT_ARM 874 printf("%s: mapping last chunk VA %#lx..%#lx (PA %#lx, prot %d, cache %d)\n", 875 __func__, cur_pv.pv_va, cur_pv.pv_va + cur_pv.pv_size - 1, 876 cur_pv.pv_pa, cur_pv.pv_prot, cur_pv.pv_cache); 877#endif 878 pmap_map_chunk(l1pt_va, cur_pv.pv_va, cur_pv.pv_pa, 879 cur_pv.pv_size, cur_pv.pv_prot, cur_pv.pv_cache); 880 881 /* 882 * Now we map the stuff that isn't directly after the kernel 883 */ 884 885 if (map_vectors_p) { 886 /* Map the vector page. */ 887 pmap_map_entry(l1pt_va, systempage.pv_va, systempage.pv_pa, 888 VM_PROT_READ|VM_PROT_WRITE, PTE_CACHE); 889 } 890 891 /* Map the Mini-Data cache clean area. */ 892#if ARM_MMU_XSCALE == 1 893#if (ARM_NMMUS > 1) 894 if (xscale_use_minidata) 895#endif 896 xscale_setup_minidata(l1pt_va, minidataclean.pv_va, 897 minidataclean.pv_pa); 898#endif 899 900 /* 901 * Map integrated peripherals at same address in first level page 902 * table so that we can continue to use console. 903 */ 904 if (devmap) 905 pmap_devmap_bootstrap(l1pt_va, devmap); 906 907#ifdef VERBOSE_INIT_ARM 908 /* Tell the user about where all the bits and pieces live. */ 909 printf("%22s Physical Virtual Num\n", " "); 910 printf("%22s Starting Ending Starting Ending Pages\n", " "); 911 912 static const char mem_fmt[] = 913 "%20s: 0x%08lx 0x%08lx 0x%08lx 0x%08lx %u\n"; 914 static const char mem_fmt_nov[] = 915 "%20s: 0x%08lx 0x%08lx %zu\n"; 916 917 printf(mem_fmt, "SDRAM", bmi->bmi_start, bmi->bmi_end - 1, 918 KERN_PHYSTOV(bmi, bmi->bmi_start), KERN_PHYSTOV(bmi, bmi->bmi_end - 1), 919 physmem); 920 printf(mem_fmt, "text section", 921 text.pv_pa, text.pv_pa + text.pv_size - 1, 922 text.pv_va, text.pv_va + text.pv_size - 1, 923 (int)(text.pv_size / PAGE_SIZE)); 924 printf(mem_fmt, "data section", 925 KERN_VTOPHYS(bmi, __data_start), KERN_VTOPHYS(bmi, _edata), 926 (vaddr_t)__data_start, (vaddr_t)_edata, 927 (int)((round_page((vaddr_t)_edata) 928 - trunc_page((vaddr_t)__data_start)) / PAGE_SIZE)); 929 printf(mem_fmt, "bss section", 930 KERN_VTOPHYS(bmi, __bss_start), KERN_VTOPHYS(bmi, __bss_end__), 931 (vaddr_t)__bss_start, (vaddr_t)__bss_end__, 932 (int)((round_page((vaddr_t)__bss_end__) 933 - trunc_page((vaddr_t)__bss_start)) / PAGE_SIZE)); 934 printf(mem_fmt, "L1 page directory", 935 kernel_l1pt.pv_pa, kernel_l1pt.pv_pa + L1_TABLE_SIZE - 1, 936 kernel_l1pt.pv_va, kernel_l1pt.pv_va + L1_TABLE_SIZE - 1, 937 L1_TABLE_SIZE / PAGE_SIZE); 938 printf(mem_fmt, "ABT stack (CPU 0)", 939 abtstack.pv_pa, abtstack.pv_pa + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 940 abtstack.pv_va, abtstack.pv_va + (ABT_STACK_SIZE * PAGE_SIZE) - 1, 941 ABT_STACK_SIZE); 942 printf(mem_fmt, "FIQ stack (CPU 0)", 943 fiqstack.pv_pa, fiqstack.pv_pa + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 944 fiqstack.pv_va, fiqstack.pv_va + (FIQ_STACK_SIZE * PAGE_SIZE) - 1, 945 FIQ_STACK_SIZE); 946 printf(mem_fmt, "IRQ stack (CPU 0)", 947 irqstack.pv_pa, irqstack.pv_pa + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 948 irqstack.pv_va, irqstack.pv_va + (IRQ_STACK_SIZE * PAGE_SIZE) - 1, 949 IRQ_STACK_SIZE); 950 printf(mem_fmt, "UND stack (CPU 0)", 951 undstack.pv_pa, undstack.pv_pa + (UND_STACK_SIZE * PAGE_SIZE) - 1, 952 undstack.pv_va, undstack.pv_va + (UND_STACK_SIZE * PAGE_SIZE) - 1, 953 UND_STACK_SIZE); 954 printf(mem_fmt, "IDLE stack (CPU 0)", 955 idlestack.pv_pa, idlestack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 956 idlestack.pv_va, idlestack.pv_va + (UPAGES * PAGE_SIZE) - 1, 957 UPAGES); 958 printf(mem_fmt, "SVC stack", 959 kernelstack.pv_pa, kernelstack.pv_pa + (UPAGES * PAGE_SIZE) - 1, 960 kernelstack.pv_va, kernelstack.pv_va + (UPAGES * PAGE_SIZE) - 1, 961 UPAGES); 962 printf(mem_fmt, "Message Buffer", 963 msgbuf.pv_pa, msgbuf.pv_pa + (msgbuf_pgs * PAGE_SIZE) - 1, 964 msgbuf.pv_va, msgbuf.pv_va + (msgbuf_pgs * PAGE_SIZE) - 1, 965 (int)msgbuf_pgs); 966 if (map_vectors_p) { 967 printf(mem_fmt, "Exception Vectors", 968 systempage.pv_pa, systempage.pv_pa + PAGE_SIZE - 1, 969 systempage.pv_va, systempage.pv_va + PAGE_SIZE - 1, 970 1); 971 } 972 for (size_t i = 0; i < bmi->bmi_nfreeblocks; i++) { 973 pv = &bmi->bmi_freeblocks[i]; 974 975 printf(mem_fmt_nov, "Free Memory", 976 pv->pv_pa, pv->pv_pa + pv->pv_size - 1, 977 pv->pv_size / PAGE_SIZE); 978 } 979#endif 980 /* 981 * Now we have the real page tables in place so we can switch to them. 982 * Once this is done we will be running with the REAL kernel page 983 * tables. 984 */ 985 986#if defined(VERBOSE_INIT_ARM) 987 printf("TTBR0=%#x", armreg_ttbr_read()); 988#ifdef _ARM_ARCH_6 989 printf(" TTBR1=%#x TTBCR=%#x CONTEXTIDR=%#x", 990 armreg_ttbr1_read(), armreg_ttbcr_read(), 991 armreg_contextidr_read()); 992#endif 993 printf("\n"); 994#endif 995 996 /* Switch tables */ 997#ifdef VERBOSE_INIT_ARM 998 printf("switching to new L1 page table @%#lx...", l1pt_pa); 999#endif 1000 1001#ifdef ARM_MMU_EXTENDED 1002 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) 1003 | (DOMAIN_CLIENT << (PMAP_DOMAIN_USER*2))); 1004#else 1005 cpu_domains((DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)) | DOMAIN_CLIENT); 1006#endif 1007 cpu_idcache_wbinv_all(); 1008#ifdef VERBOSE_INIT_ARM 1009 printf(" ttb"); 1010#endif 1011#ifdef ARM_MMU_EXTENDED 1012 /* 1013 * TTBCR should have been initialized by the MD start code. 1014 */ 1015 KASSERT((armreg_contextidr_read() & 0xff) == 0); 1016 KASSERT(armreg_ttbcr_read() == __SHIFTIN(1, TTBCR_S_N)); 1017 /* 1018 * Disable lookups via TTBR0 until there is an activated pmap. 1019 */ 1020 armreg_ttbcr_write(armreg_ttbcr_read() | TTBCR_S_PD0); 1021 cpu_setttb(l1pt_pa, KERNEL_PID); 1022 arm_isb(); 1023#else 1024 cpu_setttb(l1pt_pa, true); 1025 cpu_domains(DOMAIN_CLIENT << (PMAP_DOMAIN_KERNEL*2)); 1026#endif 1027 cpu_tlb_flushID(); 1028 1029#ifdef VERBOSE_INIT_ARM 1030#ifdef ARM_MMU_EXTENDED 1031 printf(" (TTBCR=%#x TTBR0=%#x TTBR1=%#x)", 1032 armreg_ttbcr_read(), armreg_ttbr_read(), armreg_ttbr1_read()); 1033#else 1034 printf(" (TTBR0=%#x)", armreg_ttbr_read()); 1035#endif 1036#endif 1037 1038#ifdef MULTIPROCESSOR 1039 /* 1040 * Kick the secondaries to load the TTB. After which they'll go 1041 * back to sleep to wait for the final kick so they will hatch. 1042 */ 1043#ifdef VERBOSE_INIT_ARM 1044 printf(" hatchlings"); 1045#endif 1046 cpu_boot_secondary_processors(); 1047#endif 1048 1049#ifdef VERBOSE_INIT_ARM 1050 printf(" OK\n"); 1051#endif 1052} 1053