1/* $NetBSD: pmapboot.c,v 1.19 2024/02/07 04:20:26 msaitoh Exp $ */ 2 3/* 4 * Copyright (c) 2018 Ryo Shimizu 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 18 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 19 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, 20 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 21 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 22 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, 24 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 25 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 26 * POSSIBILITY OF SUCH DAMAGE. 27 */ 28 29#include <sys/cdefs.h> 30__KERNEL_RCSID(0, "$NetBSD: pmapboot.c,v 1.19 2024/02/07 04:20:26 msaitoh Exp $"); 31 32#include "opt_arm_debug.h" 33#include "opt_ddb.h" 34#include "opt_multiprocessor.h" 35#include "opt_pmap.h" 36#include "opt_pmapboot.h" 37 38#include <sys/param.h> 39#include <sys/types.h> 40 41#include <uvm/uvm.h> 42 43#include <arm/cpufunc.h> 44 45#include <aarch64/armreg.h> 46#ifdef DDB 47#include <aarch64/db_machdep.h> 48#endif 49#include <aarch64/machdep.h> 50#include <aarch64/pmap.h> 51#include <aarch64/pte.h> 52 53#include <arm/cpufunc.h> 54 55#define OPTIMIZE_TLB_CONTIG 56 57static void 58pmapboot_protect_entry(pt_entry_t *pte, vm_prot_t clrprot) 59{ 60 extern uint64_t pmap_attr_gp; 61 62 if (clrprot & VM_PROT_READ) 63 *pte &= ~LX_BLKPAG_AF; 64 if (clrprot & VM_PROT_WRITE) { 65 *pte &= ~LX_BLKPAG_AP; 66 *pte |= LX_BLKPAG_AP_RO; 67 *pte |= pmap_attr_gp; 68 } 69 if (clrprot & VM_PROT_EXECUTE) 70 *pte |= LX_BLKPAG_UXN|LX_BLKPAG_PXN; 71} 72 73/* 74 * like pmap_protect(), but not depend on struct pmap. 75 * this work before pmap_bootstrap(). 76 * 'clrprot' specified by bit of VM_PROT_{READ,WRITE,EXECUTE} 77 * will be dropped from a pte entry. 78 * 79 * require direct (cached) mappings because TLB entries are already cached on. 80 */ 81int 82pmapboot_protect(vaddr_t sva, vaddr_t eva, vm_prot_t clrprot) 83{ 84 int idx; 85 vaddr_t va; 86 paddr_t pa; 87 pd_entry_t *l0, *l1, *l2, *l3; 88 89 switch (aarch64_addressspace(sva)) { 90 case AARCH64_ADDRSPACE_LOWER: 91 /* 0x0000xxxxxxxxxxxx */ 92 pa = (reg_ttbr0_el1_read() & TTBR_BADDR); 93 break; 94 case AARCH64_ADDRSPACE_UPPER: 95 /* 0xFFFFxxxxxxxxxxxx */ 96 pa = (reg_ttbr1_el1_read() & TTBR_BADDR); 97 break; 98 default: 99 return -1; 100 } 101 l0 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa); 102 103 for (va = sva; va < eva;) { 104 idx = l0pde_index(va); 105 if (!l0pde_valid(l0[idx])) 106 return -1; 107 pa = l0pde_pa(l0[idx]); 108 l1 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa); 109 110 idx = l1pde_index(va); 111 if (!l1pde_valid(l1[idx])) 112 return -1; 113 if (l1pde_is_block(l1[idx])) { 114 pmapboot_protect_entry(&l1[idx], clrprot); 115 va += L1_SIZE; 116 continue; 117 } 118 pa = l1pde_pa(l1[idx]); 119 l2 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa); 120 121 idx = l2pde_index(va); 122 if (!l2pde_valid(l2[idx])) 123 return -1; 124 if (l2pde_is_block(l2[idx])) { 125 pmapboot_protect_entry(&l2[idx], clrprot); 126 va += L2_SIZE; 127 continue; 128 } 129 pa = l2pde_pa(l2[idx]); 130 l3 = (pd_entry_t *)AARCH64_PA_TO_KVA(pa); 131 132 idx = l3pte_index(va); 133 if (!l3pte_valid(l3[idx])) 134 return -1; 135 if (!l3pte_is_page(l3[idx])) 136 return -1; 137 138 pmapboot_protect_entry(&l3[idx], clrprot); 139 va += L3_SIZE; 140 } 141 142 return 0; 143} 144 145 146/* 147 * these function will be called from locore without MMU. 148 * the load address varies depending on the bootloader. 149 * cannot use absolute addressing to refer text/data/bss. 150 * 151 * (*pr) function may be minimal printf. (when provided from locore) 152 * it supports only maximum 7 argument, and only '%d', '%x', and '%s' formats. 153 */ 154 155#ifdef VERBOSE_INIT_ARM 156#define VPRINTF(fmt, args...) \ 157 while (pr != NULL) { pr(fmt, ## args); break; } 158#else 159#define VPRINTF(fmt, args...) __nothing 160#endif 161 162#ifdef PMAPBOOT_DEBUG 163static void 164pmapboot_pte_print(pt_entry_t pte, int level, 165 void (*pr)(const char *, ...) __printflike(1, 2)) 166{ 167#ifdef DDB 168 db_pte_print(pte, level, pr); 169#else 170 __USE(level); 171 pr(" %s PA=%016lx\n", 172 l0pde_valid(pte) ? "VALID" : "INVALID", 173 l0pde_pa(pte)); 174#endif 175} 176#define PMAPBOOT_DPRINTF(fmt, args...) \ 177 while (pr != NULL) { pr(fmt, ## args); break; } 178#define PMAPBOOT_DPRINT_PTE(pte, l) \ 179 while (pr != NULL) { pmapboot_pte_print((pte), (l), pr); break; } 180#else /* PMAPBOOT_DEBUG */ 181#define PMAPBOOT_DPRINTF(fmt, args...) __nothing 182#define PMAPBOOT_DPRINT_PTE(pte, l) __nothing 183#endif /* PMAPBOOT_DEBUG */ 184 185 186#ifdef OPTIMIZE_TLB_CONTIG 187static inline bool 188tlb_contiguous_p(vaddr_t va, paddr_t pa, vaddr_t start, vaddr_t end, 189 vsize_t blocksize) 190{ 191 /* 192 * when using 4KB granule, 16 adjacent and aligned entries can be 193 * unified to one TLB cache entry. 194 * in other size of granule, not supported. 195 */ 196 const vaddr_t mask = (blocksize << 4) - 1; 197 198 /* if the output address doesn't align it can't be contiguous */ 199 if ((va & mask) != (pa & mask)) 200 return false; 201 202 if ((va & ~mask) >= start && (va | mask) <= end) 203 return true; 204 205 return false; 206} 207#endif /* OPTIMIZE_TLB_CONTIG */ 208 209/* 210 * pmapboot_enter() accesses pagetables by physical address. 211 * this should be called while identity mapping (VA=PA) available. 212 */ 213void 214pmapboot_enter(vaddr_t va, paddr_t pa, psize_t size, psize_t blocksize, 215 pt_entry_t attr, void (*pr)(const char *, ...) __printflike(1, 2)) 216{ 217 int level, idx0, idx1, idx2, idx3, nskip = 0; 218 int ttbr __unused; 219 vaddr_t va_end; 220 pd_entry_t *l0, *l1, *l2, *l3, pte; 221#ifdef OPTIMIZE_TLB_CONTIG 222 vaddr_t va_start; 223 pd_entry_t *ll; 224 int i, llidx; 225#endif 226 227 switch (blocksize) { 228 case L1_SIZE: 229 level = 1; 230 break; 231 case L2_SIZE: 232 level = 2; 233 break; 234 case L3_SIZE: 235 level = 3; 236 break; 237 default: 238 panic("%s: bad blocksize (%" PRIxPSIZE ")", __func__, blocksize); 239 } 240 241 VPRINTF("pmapboot_enter: va=0x%lx, pa=0x%lx, size=0x%lx, " 242 "blocksize=0x%lx, attr=0x%016lx\n", 243 va, pa, size, blocksize, attr); 244 245 pa &= ~(blocksize - 1); 246 va_end = (va + size + blocksize - 1) & ~(blocksize - 1); 247 va &= ~(blocksize - 1); 248#ifdef OPTIMIZE_TLB_CONTIG 249 va_start = va; 250#endif 251 252 attr |= LX_BLKPAG_OS_BOOT; 253 254 switch (aarch64_addressspace(va)) { 255 case AARCH64_ADDRSPACE_LOWER: 256 /* 0x0000xxxxxxxxxxxx */ 257 l0 = (pd_entry_t *)(reg_ttbr0_el1_read() & TTBR_BADDR); 258 ttbr = 0; 259 break; 260 case AARCH64_ADDRSPACE_UPPER: 261 /* 0xFFFFxxxxxxxxxxxx */ 262 l0 = (pd_entry_t *)(reg_ttbr1_el1_read() & TTBR_BADDR); 263 ttbr = 1; 264 break; 265 default: 266 panic("%s: unknown address space (%d/%" PRIxVADDR ")", __func__, 267 aarch64_addressspace(va), va); 268 } 269 270 while (va < va_end) { 271#ifdef OPTIMIZE_TLB_CONTIG 272 ll = NULL; 273 llidx = -1; 274#endif 275 276 idx0 = l0pde_index(va); 277 if (l0[idx0] == 0) { 278 l1 = pmapboot_pagealloc(); 279 if (l1 == NULL) { 280 VPRINTF("pmapboot_enter: " 281 "cannot allocate L1 page\n"); 282 panic("%s: can't allocate memory", __func__); 283 } 284 285 pte = (uint64_t)l1 | L0_TABLE; 286 l0[idx0] = pte; 287 PMAPBOOT_DPRINTF("TTBR%d[%d] (new)\t= %016lx:", 288 ttbr, idx0, pte); 289 PMAPBOOT_DPRINT_PTE(pte, 0); 290 } else { 291 l1 = (uint64_t *)(l0[idx0] & LX_TBL_PA); 292 } 293 294 idx1 = l1pde_index(va); 295 if (level == 1) { 296 pte = pa | 297 L1_BLOCK | 298 LX_BLKPAG_AF | 299#ifdef MULTIPROCESSOR 300 LX_BLKPAG_SH_IS | 301#endif 302 attr; 303#ifdef OPTIMIZE_TLB_CONTIG 304 if (tlb_contiguous_p(va, pa, va_start, va_end, blocksize)) 305 pte |= LX_BLKPAG_CONTIG; 306 ll = l1; 307 llidx = idx1; 308#endif 309 310 if (l1pde_valid(l1[idx1]) && l1[idx1] != pte) { 311 nskip++; 312 goto nextblk; 313 } 314 315 l1[idx1] = pte; 316 PMAPBOOT_DPRINTF("TTBR%d[%d][%d]\t= %016lx:", ttbr, 317 idx0, idx1, pte); 318 PMAPBOOT_DPRINT_PTE(pte, 1); 319 goto nextblk; 320 } 321 322 if (!l1pde_valid(l1[idx1])) { 323 l2 = pmapboot_pagealloc(); 324 if (l2 == NULL) { 325 VPRINTF("pmapboot_enter: " 326 "cannot allocate L2 page\n"); 327 panic("%s: can't allocate memory", __func__); 328 } 329 330 pte = (uint64_t)l2 | L1_TABLE; 331 l1[idx1] = pte; 332 PMAPBOOT_DPRINTF("TTBR%d[%d][%d] (new)\t= %016lx:", 333 ttbr, idx0, idx1, pte); 334 PMAPBOOT_DPRINT_PTE(pte, 1); 335 } else { 336 l2 = (uint64_t *)(l1[idx1] & LX_TBL_PA); 337 } 338 339 idx2 = l2pde_index(va); 340 if (level == 2) { 341 pte = pa | 342 L2_BLOCK | 343 LX_BLKPAG_AF | 344#ifdef MULTIPROCESSOR 345 LX_BLKPAG_SH_IS | 346#endif 347 attr; 348#ifdef OPTIMIZE_TLB_CONTIG 349 if (tlb_contiguous_p(va, pa, va_start, va_end, blocksize)) 350 pte |= LX_BLKPAG_CONTIG; 351 ll = l2; 352 llidx = idx2; 353#endif 354 if (l2pde_valid(l2[idx2]) && l2[idx2] != pte) { 355 nskip++; 356 goto nextblk; 357 } 358 359 l2[idx2] = pte; 360 PMAPBOOT_DPRINTF("TTBR%d[%d][%d][%d]\t= %016lx:", ttbr, 361 idx0, idx1, idx2, pte); 362 PMAPBOOT_DPRINT_PTE(pte, 2); 363 goto nextblk; 364 } 365 366 if (!l2pde_valid(l2[idx2])) { 367 l3 = pmapboot_pagealloc(); 368 if (l3 == NULL) { 369 VPRINTF("pmapboot_enter: " 370 "cannot allocate L3 page\n"); 371 panic("%s: can't allocate memory", __func__); 372 } 373 374 pte = (uint64_t)l3 | L2_TABLE; 375 l2[idx2] = pte; 376 PMAPBOOT_DPRINTF("TTBR%d[%d][%d][%d] (new)\t= %016lx:", 377 ttbr, idx0, idx1, idx2, pte); 378 PMAPBOOT_DPRINT_PTE(pte, 2); 379 } else { 380 l3 = (uint64_t *)(l2[idx2] & LX_TBL_PA); 381 } 382 383 idx3 = l3pte_index(va); 384 385 pte = pa | 386 L3_PAGE | 387 LX_BLKPAG_AF | 388#ifdef MULTIPROCESSOR 389 LX_BLKPAG_SH_IS | 390#endif 391 attr; 392#ifdef OPTIMIZE_TLB_CONTIG 393 if (tlb_contiguous_p(va, pa, va_start, va_end, blocksize)) 394 pte |= LX_BLKPAG_CONTIG; 395 ll = l3; 396 llidx = idx3; 397#endif 398 if (l3pte_valid(l3[idx3]) && l3[idx3] != pte) { 399 nskip++; 400 goto nextblk; 401 } 402 403 l3[idx3] = pte; 404 PMAPBOOT_DPRINTF("TTBR%d[%d][%d][%d][%d]\t= %lx:", ttbr, 405 idx0, idx1, idx2, idx3, pte); 406 PMAPBOOT_DPRINT_PTE(pte, 3); 407 nextblk: 408#ifdef OPTIMIZE_TLB_CONTIG 409 /* 410 * when overwriting a pte entry the contiguous bit in entries 411 * before/after the entry should be cleared. 412 */ 413 if (ll != NULL) { 414 if (va == va_start && (llidx & 15) != 0) { 415 /* clear CONTIG flag before this pte entry */ 416 for (i = (llidx & ~15); i < llidx; i++) { 417 ll[i] &= ~LX_BLKPAG_CONTIG; 418 } 419 } 420 if (va == va_end && (llidx & 15) != 15) { 421 /* clear CONTIG flag after this pte entry */ 422 for (i = (llidx + 1); i < ((llidx + 16) & ~15); 423 i++) { 424 ll[i] &= ~LX_BLKPAG_CONTIG; 425 } 426 } 427 } 428#endif 429 switch (level) { 430 case 1: 431 va += L1_SIZE; 432 pa += L1_SIZE; 433 break; 434 case 2: 435 va += L2_SIZE; 436 pa += L2_SIZE; 437 break; 438 case 3: 439 va += L3_SIZE; 440 pa += L3_SIZE; 441 break; 442 } 443 } 444 445 dsb(ish); 446 447 if (nskip != 0) 448 panic("%s: overlapping/incompatible mappings (%d)", __func__, nskip); 449} 450 451paddr_t pmapboot_pagebase __attribute__((__section__(".data"))); 452 453pd_entry_t * 454pmapboot_pagealloc(void) 455{ 456 extern long kernend_extra; 457 458 if (kernend_extra < 0) 459 return NULL; 460 461 paddr_t pa = pmapboot_pagebase + kernend_extra; 462 kernend_extra += PAGE_SIZE; 463 464 char *s = (char *)pa; 465 char *e = s + PAGE_SIZE; 466 467 while (s < e) 468 *s++ = 0; 469 470 return (pd_entry_t *)pa; 471} 472 473void 474pmapboot_enter_range(vaddr_t va, paddr_t pa, psize_t size, pt_entry_t attr, 475 void (*pr)(const char *, ...) __printflike(1, 2)) 476{ 477 vaddr_t vend; 478 vsize_t left, mapsize, nblocks; 479 480 vend = round_page(va + size); 481 va = trunc_page(va); 482 left = vend - va; 483 484 /* align the start address to L2 blocksize */ 485 nblocks = ulmin(left / L3_SIZE, 486 Ln_ENTRIES - __SHIFTOUT(va, L3_ADDR_BITS)); 487 if (((va & L3_ADDR_BITS) != 0) && (nblocks > 0)) { 488 mapsize = nblocks * L3_SIZE; 489 VPRINTF("Creating L3 tables: %016lx-%016lx : %016lx-%016lx\n", 490 va, va + mapsize - 1, pa, pa + mapsize - 1); 491 pmapboot_enter(va, pa, mapsize, L3_SIZE, attr, pr); 492 va += mapsize; 493 pa += mapsize; 494 left -= mapsize; 495 } 496 497 /* align the start address to L1 blocksize */ 498 nblocks = ulmin(left / L2_SIZE, 499 Ln_ENTRIES - __SHIFTOUT(va, L2_ADDR_BITS)); 500 if (((va & L2_ADDR_BITS) != 0) && (nblocks > 0)) { 501 mapsize = nblocks * L2_SIZE; 502 VPRINTF("Creating L2 tables: %016lx-%016lx : %016lx-%016lx\n", 503 va, va + mapsize - 1, pa, pa + mapsize - 1); 504 pmapboot_enter(va, pa, mapsize, L2_SIZE, attr, pr); 505 va += mapsize; 506 pa += mapsize; 507 left -= mapsize; 508 } 509 510 nblocks = left / L1_SIZE; 511 if (nblocks > 0) { 512 mapsize = nblocks * L1_SIZE; 513 VPRINTF("Creating L1 tables: %016lx-%016lx : %016lx-%016lx\n", 514 va, va + mapsize - 1, pa, pa + mapsize - 1); 515 pmapboot_enter(va, pa, mapsize, L1_SIZE, attr, pr); 516 va += mapsize; 517 pa += mapsize; 518 left -= mapsize; 519 } 520 521 if ((left & L2_ADDR_BITS) != 0) { 522 nblocks = left / L2_SIZE; 523 mapsize = nblocks * L2_SIZE; 524 VPRINTF("Creating L2 tables: %016lx-%016lx : %016lx-%016lx\n", 525 va, va + mapsize - 1, pa, pa + mapsize - 1); 526 pmapboot_enter(va, pa, mapsize, L2_SIZE, attr, pr); 527 va += mapsize; 528 pa += mapsize; 529 left -= mapsize; 530 } 531 532 if ((left & L3_ADDR_BITS) != 0) { 533 nblocks = left / L3_SIZE; 534 mapsize = nblocks * L3_SIZE; 535 VPRINTF("Creating L3 tables: %016lx-%016lx : %016lx-%016lx\n", 536 va, va + mapsize - 1, pa, pa + mapsize - 1); 537 pmapboot_enter(va, pa, mapsize, L3_SIZE, attr, pr); 538 va += mapsize; 539 pa += mapsize; 540 left -= mapsize; 541 } 542} 543