1/* $NetBSD: uvm_physseg.c,v 1.20 2024/01/13 09:44:42 tnn Exp $ */ 2 3/* 4 * Copyright (c) 1997 Charles D. Cranor and Washington University. 5 * Copyright (c) 1991, 1993, The Regents of the University of California. 6 * 7 * All rights reserved. 8 * 9 * This code is derived from software contributed to Berkeley by 10 * The Mach Operating System project at Carnegie-Mellon University. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions and the following disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 3. Neither the name of the University nor the names of its contributors 21 * may be used to endorse or promote products derived from this software 22 * without specific prior written permission. 23 * 24 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 25 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 26 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 27 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 28 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 29 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 30 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 31 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 32 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 33 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 34 * SUCH DAMAGE. 35 * 36 * @(#)vm_page.h 7.3 (Berkeley) 4/21/91 37 * from: Id: uvm_page.h,v 1.1.2.6 1998/02/04 02:31:42 chuck Exp 38 * 39 * 40 * Copyright (c) 1987, 1990 Carnegie-Mellon University. 41 * All rights reserved. 42 * 43 * Permission to use, copy, modify and distribute this software and 44 * its documentation is hereby granted, provided that both the copyright 45 * notice and this permission notice appear in all copies of the 46 * software, derivative works or modified versions, and any portions 47 * thereof, and that both notices appear in supporting documentation. 48 * 49 * CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS "AS IS" 50 * CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND 51 * FOR ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 52 * 53 * Carnegie Mellon requests users of this software to return to 54 * 55 * Software Distribution Coordinator or Software.Distribution@CS.CMU.EDU 56 * School of Computer Science 57 * Carnegie Mellon University 58 * Pittsburgh PA 15213-3890 59 * 60 * any improvements or extensions that they make and grant Carnegie the 61 * rights to redistribute these changes. 62 */ 63 64/* 65 * Consolidated API from uvm_page.c and others. 66 * Consolidated and designed by Cherry G. Mathew <cherry@zyx.in> 67 * rbtree(3) backing implementation by: 68 * Santhosh N. Raju <santhosh.raju@gmail.com> 69 */ 70 71#ifdef _KERNEL_OPT 72#include "opt_uvm.h" 73#endif 74 75#include <sys/param.h> 76#include <sys/types.h> 77#include <sys/extent.h> 78#include <sys/kmem.h> 79 80#include <uvm/uvm.h> 81#include <uvm/uvm_page.h> 82#include <uvm/uvm_param.h> 83#include <uvm/uvm_pdpolicy.h> 84#include <uvm/uvm_physseg.h> 85 86/* 87 * uvm_physseg: describes one segment of physical memory 88 */ 89struct uvm_physseg { 90 /* used during RB tree lookup for PHYS_TO_VM_PAGE(). */ 91#if defined(UVM_HOTPLUG) 92 struct rb_node rb_node; /* tree information */ 93#endif 94 paddr_t start; /* PF# of first page in segment */ 95 paddr_t end; /* (PF# of last page in segment) + 1 */ 96 struct vm_page *pgs; /* vm_page structures (from start) */ 97 98 /* less performance sensitive fields. */ 99 paddr_t avail_start; /* PF# of first free page in segment */ 100 paddr_t avail_end; /* (PF# of last free page in segment) +1 */ 101 struct extent *ext; /* extent(9) structure to manage pgs[] */ 102 int free_list; /* which free list they belong on */ 103 u_long start_hint; /* start looking for free pages here */ 104#ifdef __HAVE_PMAP_PHYSSEG 105 struct pmap_physseg pmseg; /* pmap specific (MD) data */ 106#endif 107}; 108 109/* 110 * These functions are reserved for uvm(9) internal use and are not 111 * exported in the header file uvm_physseg.h 112 * 113 * Thus they are redefined here. 114 */ 115void uvm_physseg_init_seg(uvm_physseg_t, struct vm_page *); 116void uvm_physseg_seg_chomp_slab(uvm_physseg_t, struct vm_page *, size_t); 117 118/* returns a pgs array */ 119struct vm_page *uvm_physseg_seg_alloc_from_slab(uvm_physseg_t, size_t); 120 121#if defined(UVM_HOTPLUG) /* rbtree impementation */ 122 123#define HANDLE_TO_PHYSSEG_NODE(h) ((struct uvm_physseg *)(h)) 124#define PHYSSEG_NODE_TO_HANDLE(u) ((uvm_physseg_t)(u)) 125 126struct uvm_physseg_graph { 127 struct rb_tree rb_tree; /* Tree for entries */ 128 int nentries; /* Number of entries */ 129} __aligned(COHERENCY_UNIT); 130 131static struct uvm_physseg_graph uvm_physseg_graph __read_mostly; 132 133/* 134 * Note on kmem(9) allocator usage: 135 * We take the conservative approach that plug/unplug are allowed to 136 * fail in high memory stress situations. 137 * 138 * We want to avoid re-entrant situations in which one plug/unplug 139 * operation is waiting on a previous one to complete, since this 140 * makes the design more complicated than necessary. 141 * 142 * We may review this and change its behaviour, once the use cases 143 * become more obvious. 144 */ 145 146/* 147 * Special alloc()/free() functions for boot time support: 148 * We assume that alloc() at boot time is only for new 'vm_physseg's 149 * This allows us to use a static array for memory allocation at boot 150 * time. Thus we avoid using kmem(9) which is not ready at this point 151 * in boot. 152 * 153 * After kmem(9) is ready, we use it. We currently discard any free()s 154 * to this static array, since the size is small enough to be a 155 * trivial waste on all architectures we run on. 156 */ 157 158static size_t nseg = 0; 159static struct uvm_physseg uvm_physseg[VM_PHYSSEG_MAX]; 160 161static void * 162uvm_physseg_alloc(size_t sz) 163{ 164 /* 165 * During boot time, we only support allocating vm_physseg 166 * entries from the static array. 167 * We need to assert for this. 168 */ 169 170 if (__predict_false(uvm.page_init_done == false)) { 171 if (sz % sizeof(struct uvm_physseg)) 172 panic("%s: tried to alloc size other than multiple" 173 " of struct uvm_physseg at boot\n", __func__); 174 175 size_t n = sz / sizeof(struct uvm_physseg); 176 nseg += n; 177 178 KASSERT(nseg > 0); 179 KASSERT(nseg <= VM_PHYSSEG_MAX); 180 181 return &uvm_physseg[nseg - n]; 182 } 183 184 return kmem_zalloc(sz, KM_NOSLEEP); 185} 186 187static void 188uvm_physseg_free(void *p, size_t sz) 189{ 190 /* 191 * This is a bit tricky. We do allow simulation of free() 192 * during boot (for eg: when MD code is "steal"ing memory, 193 * and the segment has been exhausted (and thus needs to be 194 * free() - ed. 195 * free() also complicates things because we leak the 196 * free(). Therefore calling code can't assume that free()-ed 197 * memory is available for alloc() again, at boot time. 198 * 199 * Thus we can't explicitly disallow free()s during 200 * boot time. However, the same restriction for alloc() 201 * applies to free(). We only allow uvm_physseg related free()s 202 * via this function during boot time. 203 */ 204 205 if (__predict_false(uvm.page_init_done == false)) { 206 if (sz % sizeof(struct uvm_physseg)) 207 panic("%s: tried to free size other than struct uvm_physseg" 208 " at boot\n", __func__); 209 210 } 211 212 /* 213 * Could have been in a single if(){} block - split for 214 * clarity 215 */ 216 217 if ((struct uvm_physseg *)p >= uvm_physseg && 218 (struct uvm_physseg *)p < (uvm_physseg + VM_PHYSSEG_MAX)) { 219 if (sz % sizeof(struct uvm_physseg)) 220 panic("%s: tried to free() other than struct uvm_physseg" 221 " from static array\n", __func__); 222 223 if ((sz / sizeof(struct uvm_physseg)) >= VM_PHYSSEG_MAX) 224 panic("%s: tried to free() the entire static array!", __func__); 225 return; /* Nothing to free */ 226 } 227 228 kmem_free(p, sz); 229} 230 231/* XXX: Multi page size */ 232bool 233uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp) 234{ 235 int preload; 236 size_t slabpages; 237 struct uvm_physseg *ps, *current_ps = NULL; 238 struct vm_page *slab = NULL, *pgs = NULL; 239 240#ifdef DEBUG 241 paddr_t off; 242 uvm_physseg_t upm; 243 upm = uvm_physseg_find(pfn, &off); 244 245 ps = HANDLE_TO_PHYSSEG_NODE(upm); 246 247 if (ps != NULL) /* XXX; do we allow "update" plugs ? */ 248 return false; 249#endif 250 251 /* 252 * do we have room? 253 */ 254 255 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg)); 256 if (ps == NULL) { 257 printf("uvm_page_physload: unable to load physical memory " 258 "segment\n"); 259 printf("\t%d segments allocated, ignoring 0x%"PRIxPADDR" -> 0x%"PRIxPADDR"\n", 260 VM_PHYSSEG_MAX, pfn, pfn + pages + 1); 261 printf("\tincrease VM_PHYSSEG_MAX\n"); 262 return false; 263 } 264 265 /* span init */ 266 ps->start = pfn; 267 ps->end = pfn + pages; 268 269 /* 270 * XXX: Ugly hack because uvmexp.npages accounts for only 271 * those pages in the segment included below as well - this 272 * should be legacy and removed. 273 */ 274 275 ps->avail_start = ps->start; 276 ps->avail_end = ps->end; 277 278 /* 279 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 280 * called yet, so kmem is not available). 281 */ 282 283 preload = 1; /* We are going to assume it is a preload */ 284 285 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) { 286 /* If there are non NULL pages then we are not in a preload */ 287 if (current_ps->pgs != NULL) { 288 preload = 0; 289 /* Try to scavenge from earlier unplug()s. */ 290 pgs = uvm_physseg_seg_alloc_from_slab(current_ps, pages); 291 292 if (pgs != NULL) { 293 break; 294 } 295 } 296 } 297 298 299 /* 300 * if VM is already running, attempt to kmem_alloc vm_page structures 301 */ 302 303 if (!preload) { 304 if (pgs == NULL) { /* Brand new */ 305 /* Iteratively try alloc down from uvmexp.npages */ 306 for (slabpages = (size_t) uvmexp.npages; slabpages >= pages; slabpages--) { 307 slab = kmem_zalloc(sizeof *pgs * (long unsigned int)slabpages, KM_NOSLEEP); 308 if (slab != NULL) 309 break; 310 } 311 312 if (slab == NULL) { 313 uvm_physseg_free(ps, sizeof(struct uvm_physseg)); 314 return false; 315 } 316 317 uvm_physseg_seg_chomp_slab(ps, slab, (size_t) slabpages); 318 /* We allocate enough for this plug */ 319 pgs = uvm_physseg_seg_alloc_from_slab(ps, pages); 320 321 if (pgs == NULL) { 322 printf("unable to uvm_physseg_seg_alloc_from_slab() from backend\n"); 323 return false; 324 } 325 } else { 326 /* Reuse scavenged extent */ 327 ps->ext = current_ps->ext; 328 } 329 330 physmem += pages; 331 uvmpdpol_reinit(); 332 } else { /* Boot time - see uvm_page.c:uvm_page_init() */ 333 pgs = NULL; 334 ps->pgs = pgs; 335 } 336 337 /* 338 * now insert us in the proper place in uvm_physseg_graph.rb_tree 339 */ 340 341 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps); 342 if (current_ps != ps) { 343 panic("uvm_page_physload: Duplicate address range detected!"); 344 } 345 uvm_physseg_graph.nentries++; 346 347 /* 348 * uvm_pagefree() requires the PHYS_TO_VM_PAGE(pgs[i]) on the 349 * newly allocated pgs[] to return the correct value. This is 350 * a bit of a chicken and egg problem, since it needs 351 * uvm_physseg_find() to succeed. For this, the node needs to 352 * be inserted *before* uvm_physseg_init_seg() happens. 353 * 354 * During boot, this happens anyway, since 355 * uvm_physseg_init_seg() is called later on and separately 356 * from uvm_page.c:uvm_page_init(). 357 * In the case of hotplug we need to ensure this. 358 */ 359 360 if (__predict_true(!preload)) 361 uvm_physseg_init_seg(ps, pgs); 362 363 if (psp != NULL) 364 *psp = ps; 365 366 return true; 367} 368 369static int 370uvm_physseg_compare_nodes(void *ctx, const void *nnode1, const void *nnode2) 371{ 372 const struct uvm_physseg *enode1 = nnode1; 373 const struct uvm_physseg *enode2 = nnode2; 374 375 KASSERT(enode1->start < enode2->start || enode1->start >= enode2->end); 376 KASSERT(enode2->start < enode1->start || enode2->start >= enode1->end); 377 378 if (enode1->start < enode2->start) 379 return -1; 380 if (enode1->start >= enode2->end) 381 return 1; 382 return 0; 383} 384 385static int 386uvm_physseg_compare_key(void *ctx, const void *nnode, const void *pkey) 387{ 388 const struct uvm_physseg *enode = nnode; 389 const paddr_t pa = *(const paddr_t *) pkey; 390 391 if(enode->start <= pa && pa < enode->end) 392 return 0; 393 if (enode->start < pa) 394 return -1; 395 if (enode->end > pa) 396 return 1; 397 398 return 0; 399} 400 401static const rb_tree_ops_t uvm_physseg_tree_ops = { 402 .rbto_compare_nodes = uvm_physseg_compare_nodes, 403 .rbto_compare_key = uvm_physseg_compare_key, 404 .rbto_node_offset = offsetof(struct uvm_physseg, rb_node), 405 .rbto_context = NULL 406}; 407 408/* 409 * uvm_physseg_init: init the physmem 410 * 411 * => physmem unit should not be in use at this point 412 */ 413 414void 415uvm_physseg_init(void) 416{ 417 rb_tree_init(&(uvm_physseg_graph.rb_tree), &uvm_physseg_tree_ops); 418 uvm_physseg_graph.nentries = 0; 419} 420 421uvm_physseg_t 422uvm_physseg_get_next(uvm_physseg_t upm) 423{ 424 /* next of invalid is invalid, not fatal */ 425 if (uvm_physseg_valid_p(upm) == false) 426 return UVM_PHYSSEG_TYPE_INVALID; 427 428 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm, 429 RB_DIR_RIGHT); 430} 431 432uvm_physseg_t 433uvm_physseg_get_prev(uvm_physseg_t upm) 434{ 435 /* prev of invalid is invalid, not fatal */ 436 if (uvm_physseg_valid_p(upm) == false) 437 return UVM_PHYSSEG_TYPE_INVALID; 438 439 return (uvm_physseg_t) rb_tree_iterate(&(uvm_physseg_graph.rb_tree), upm, 440 RB_DIR_LEFT); 441} 442 443uvm_physseg_t 444uvm_physseg_get_last(void) 445{ 446 return (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree)); 447} 448 449uvm_physseg_t 450uvm_physseg_get_first(void) 451{ 452 return (uvm_physseg_t) RB_TREE_MIN(&(uvm_physseg_graph.rb_tree)); 453} 454 455paddr_t 456uvm_physseg_get_highest_frame(void) 457{ 458 struct uvm_physseg *ps = 459 (uvm_physseg_t) RB_TREE_MAX(&(uvm_physseg_graph.rb_tree)); 460 461 return ps->end - 1; 462} 463 464/* 465 * uvm_page_physunload: unload physical memory and return it to 466 * caller. 467 */ 468bool 469uvm_page_physunload(uvm_physseg_t upm, int freelist, paddr_t *paddrp) 470{ 471 struct uvm_physseg *seg; 472 473 if (__predict_true(uvm.page_init_done == true)) 474 panic("%s: unload attempted after uvm_page_init()\n", __func__); 475 476 seg = HANDLE_TO_PHYSSEG_NODE(upm); 477 478 if (seg->free_list != freelist) { 479 return false; 480 } 481 482 /* 483 * During cold boot, what we're about to unplug hasn't been 484 * put on the uvm freelist, nor has uvmexp.npages been 485 * updated. (This happens in uvm_page.c:uvm_page_init()) 486 * 487 * For hotplug, we assume here that the pages being unloaded 488 * here are completely out of sight of uvm (ie; not on any uvm 489 * lists), and that uvmexp.npages has been suitably 490 * decremented before we're called. 491 * 492 * XXX: will avail_end == start if avail_start < avail_end? 493 */ 494 495 /* try from front */ 496 if (seg->avail_start == seg->start && 497 seg->avail_start < seg->avail_end) { 498 *paddrp = ctob(seg->avail_start); 499 return uvm_physseg_unplug(seg->avail_start, 1); 500 } 501 502 /* try from rear */ 503 if (seg->avail_end == seg->end && 504 seg->avail_start < seg->avail_end) { 505 *paddrp = ctob(seg->avail_end - 1); 506 return uvm_physseg_unplug(seg->avail_end - 1, 1); 507 } 508 509 return false; 510} 511 512bool 513uvm_page_physunload_force(uvm_physseg_t upm, int freelist, paddr_t *paddrp) 514{ 515 struct uvm_physseg *seg; 516 517 seg = HANDLE_TO_PHYSSEG_NODE(upm); 518 519 if (__predict_true(uvm.page_init_done == true)) 520 panic("%s: unload attempted after uvm_page_init()\n", __func__); 521 /* any room in this bank? */ 522 if (seg->avail_start >= seg->avail_end) { 523 return false; /* nope */ 524 } 525 526 *paddrp = ctob(seg->avail_start); 527 528 /* Always unplug from front */ 529 return uvm_physseg_unplug(seg->avail_start, 1); 530} 531 532 533/* 534 * vm_physseg_find: find vm_physseg structure that belongs to a PA 535 */ 536uvm_physseg_t 537uvm_physseg_find(paddr_t pframe, psize_t *offp) 538{ 539 struct uvm_physseg * ps = NULL; 540 541 ps = rb_tree_find_node(&(uvm_physseg_graph.rb_tree), &pframe); 542 543 if(ps != NULL && offp != NULL) 544 *offp = pframe - ps->start; 545 546 return ps; 547} 548 549#else /* UVM_HOTPLUG */ 550 551/* 552 * physical memory config is stored in vm_physmem. 553 */ 554 555#define VM_PHYSMEM_PTR(i) (&vm_physmem[i]) 556#if VM_PHYSSEG_MAX == 1 557#define VM_PHYSMEM_PTR_SWAP(i, j) /* impossible */ 558#else 559#define VM_PHYSMEM_PTR_SWAP(i, j) \ 560 do { vm_physmem[(i)] = vm_physmem[(j)]; } while (0) 561#endif 562 563#define HANDLE_TO_PHYSSEG_NODE(h) (VM_PHYSMEM_PTR((int)h)) 564#define PHYSSEG_NODE_TO_HANDLE(u) ((int)((vsize_t) (u - vm_physmem) / sizeof(struct uvm_physseg))) 565 566/* XXXCDC: uvm.physmem */ 567static struct uvm_physseg vm_physmem[VM_PHYSSEG_MAX] __read_mostly; 568/* XXXCDC: uvm.nphysseg */ 569static int vm_nphysseg __read_mostly = 0; 570#define vm_nphysmem vm_nphysseg 571 572void 573uvm_physseg_init(void) 574{ 575 /* XXX: Provisioning for rb_tree related init(s) */ 576 return; 577} 578 579int 580uvm_physseg_get_next(uvm_physseg_t lcv) 581{ 582 /* next of invalid is invalid, not fatal */ 583 if (uvm_physseg_valid_p(lcv) == false) 584 return UVM_PHYSSEG_TYPE_INVALID; 585 586 return (lcv + 1); 587} 588 589int 590uvm_physseg_get_prev(uvm_physseg_t lcv) 591{ 592 /* prev of invalid is invalid, not fatal */ 593 if (uvm_physseg_valid_p(lcv) == false) 594 return UVM_PHYSSEG_TYPE_INVALID; 595 596 return (lcv - 1); 597} 598 599int 600uvm_physseg_get_last(void) 601{ 602 return (vm_nphysseg - 1); 603} 604 605int 606uvm_physseg_get_first(void) 607{ 608 return 0; 609} 610 611paddr_t 612uvm_physseg_get_highest_frame(void) 613{ 614 int lcv; 615 paddr_t last = 0; 616 struct uvm_physseg *ps; 617 618 for (lcv = 0; lcv < vm_nphysseg; lcv++) { 619 ps = VM_PHYSMEM_PTR(lcv); 620 if (last < ps->end) 621 last = ps->end; 622 } 623 624 return last; 625} 626 627 628static struct vm_page * 629uvm_post_preload_check(void) 630{ 631 int preload, lcv; 632 633 /* 634 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 635 * called yet, so kmem is not available). 636 */ 637 638 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) { 639 if (VM_PHYSMEM_PTR(lcv)->pgs) 640 break; 641 } 642 preload = (lcv == vm_nphysmem); 643 644 /* 645 * if VM is already running, attempt to kmem_alloc vm_page structures 646 */ 647 648 if (!preload) { 649 panic("Tried to add RAM after uvm_page_init"); 650 } 651 652 return NULL; 653} 654 655/* 656 * uvm_page_physunload: unload physical memory and return it to 657 * caller. 658 */ 659bool 660uvm_page_physunload(uvm_physseg_t psi, int freelist, paddr_t *paddrp) 661{ 662 int x; 663 struct uvm_physseg *seg; 664 665 uvm_post_preload_check(); 666 667 seg = VM_PHYSMEM_PTR(psi); 668 669 if (seg->free_list != freelist) { 670 return false; 671 } 672 673 /* try from front */ 674 if (seg->avail_start == seg->start && 675 seg->avail_start < seg->avail_end) { 676 *paddrp = ctob(seg->avail_start); 677 seg->avail_start++; 678 seg->start++; 679 /* nothing left? nuke it */ 680 if (seg->avail_start == seg->end) { 681 if (vm_nphysmem == 1) 682 panic("uvm_page_physget: out of memory!"); 683 vm_nphysmem--; 684 for (x = psi ; x < vm_nphysmem ; x++) 685 /* structure copy */ 686 VM_PHYSMEM_PTR_SWAP(x, x + 1); 687 } 688 return (true); 689 } 690 691 /* try from rear */ 692 if (seg->avail_end == seg->end && 693 seg->avail_start < seg->avail_end) { 694 *paddrp = ctob(seg->avail_end - 1); 695 seg->avail_end--; 696 seg->end--; 697 /* nothing left? nuke it */ 698 if (seg->avail_end == seg->start) { 699 if (vm_nphysmem == 1) 700 panic("uvm_page_physget: out of memory!"); 701 vm_nphysmem--; 702 for (x = psi ; x < vm_nphysmem ; x++) 703 /* structure copy */ 704 VM_PHYSMEM_PTR_SWAP(x, x + 1); 705 } 706 return (true); 707 } 708 709 return false; 710} 711 712bool 713uvm_page_physunload_force(uvm_physseg_t psi, int freelist, paddr_t *paddrp) 714{ 715 int x; 716 struct uvm_physseg *seg; 717 718 uvm_post_preload_check(); 719 720 seg = VM_PHYSMEM_PTR(psi); 721 722 /* any room in this bank? */ 723 if (seg->avail_start >= seg->avail_end) { 724 return false; /* nope */ 725 } 726 727 *paddrp = ctob(seg->avail_start); 728 seg->avail_start++; 729 /* truncate! */ 730 seg->start = seg->avail_start; 731 732 /* nothing left? nuke it */ 733 if (seg->avail_start == seg->end) { 734 if (vm_nphysmem == 1) 735 panic("uvm_page_physget: out of memory!"); 736 vm_nphysmem--; 737 for (x = psi ; x < vm_nphysmem ; x++) 738 /* structure copy */ 739 VM_PHYSMEM_PTR_SWAP(x, x + 1); 740 } 741 return (true); 742} 743 744bool 745uvm_physseg_plug(paddr_t pfn, size_t pages, uvm_physseg_t *psp) 746{ 747 int lcv; 748 struct vm_page *pgs; 749 struct uvm_physseg *ps; 750 751#ifdef DEBUG 752 paddr_t off; 753 uvm_physseg_t upm; 754 upm = uvm_physseg_find(pfn, &off); 755 756 if (uvm_physseg_valid_p(upm)) /* XXX; do we allow "update" plugs ? */ 757 return false; 758#endif 759 760 paddr_t start = pfn; 761 paddr_t end = pfn + pages; 762 paddr_t avail_start = start; 763 paddr_t avail_end = end; 764 765 if (uvmexp.pagesize == 0) 766 panic("uvm_page_physload: page size not set!"); 767 768 /* 769 * do we have room? 770 */ 771 772 if (vm_nphysmem == VM_PHYSSEG_MAX) { 773 printf("uvm_page_physload: unable to load physical memory " 774 "segment\n"); 775 printf("\t%d segments allocated, ignoring 0x%llx -> 0x%llx\n", 776 VM_PHYSSEG_MAX, (long long)start, (long long)end); 777 printf("\tincrease VM_PHYSSEG_MAX\n"); 778 if (psp != NULL) 779 *psp = UVM_PHYSSEG_TYPE_INVALID_OVERFLOW; 780 return false; 781 } 782 783 /* 784 * check to see if this is a "preload" (i.e. uvm_page_init hasn't been 785 * called yet, so kmem is not available). 786 */ 787 pgs = uvm_post_preload_check(); 788 789 /* 790 * now insert us in the proper place in vm_physmem[] 791 */ 792 793#if (VM_PHYSSEG_STRAT == VM_PSTRAT_RANDOM) 794 /* random: put it at the end (easy!) */ 795 ps = VM_PHYSMEM_PTR(vm_nphysmem); 796 lcv = vm_nphysmem; 797#elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 798 { 799 int x; 800 /* sort by address for binary search */ 801 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) 802 if (start < VM_PHYSMEM_PTR(lcv)->start) 803 break; 804 ps = VM_PHYSMEM_PTR(lcv); 805 /* move back other entries, if necessary ... */ 806 for (x = vm_nphysmem ; x > lcv ; x--) 807 /* structure copy */ 808 VM_PHYSMEM_PTR_SWAP(x, x - 1); 809 } 810#elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BIGFIRST) 811 { 812 int x; 813 /* sort by largest segment first */ 814 for (lcv = 0 ; lcv < vm_nphysmem ; lcv++) 815 if ((end - start) > 816 (VM_PHYSMEM_PTR(lcv)->end - VM_PHYSMEM_PTR(lcv)->start)) 817 break; 818 ps = VM_PHYSMEM_PTR(lcv); 819 /* move back other entries, if necessary ... */ 820 for (x = vm_nphysmem ; x > lcv ; x--) 821 /* structure copy */ 822 VM_PHYSMEM_PTR_SWAP(x, x - 1); 823 } 824#else 825 panic("uvm_page_physload: unknown physseg strategy selected!"); 826#endif 827 828 ps->start = start; 829 ps->end = end; 830 ps->avail_start = avail_start; 831 ps->avail_end = avail_end; 832 833 ps->pgs = pgs; 834 835 vm_nphysmem++; 836 837 if (psp != NULL) 838 *psp = lcv; 839 840 return true; 841} 842 843/* 844 * when VM_PHYSSEG_MAX is 1, we can simplify these functions 845 */ 846 847#if VM_PHYSSEG_MAX == 1 848static inline int vm_physseg_find_contig(struct uvm_physseg *, int, paddr_t, psize_t *); 849#elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 850static inline int vm_physseg_find_bsearch(struct uvm_physseg *, int, paddr_t, psize_t *); 851#else 852static inline int vm_physseg_find_linear(struct uvm_physseg *, int, paddr_t, psize_t *); 853#endif 854 855/* 856 * vm_physseg_find: find vm_physseg structure that belongs to a PA 857 */ 858inline int 859uvm_physseg_find(paddr_t pframe, psize_t *offp) 860{ 861 862#if VM_PHYSSEG_MAX == 1 863 return vm_physseg_find_contig(vm_physmem, vm_nphysseg, pframe, offp); 864#elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 865 return vm_physseg_find_bsearch(vm_physmem, vm_nphysseg, pframe, offp); 866#else 867 return vm_physseg_find_linear(vm_physmem, vm_nphysseg, pframe, offp); 868#endif 869} 870 871#if VM_PHYSSEG_MAX == 1 872static inline int 873vm_physseg_find_contig(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 874{ 875 876 /* 'contig' case */ 877 if (pframe >= segs[0].start && pframe < segs[0].end) { 878 if (offp) 879 *offp = pframe - segs[0].start; 880 return(0); 881 } 882 return(-1); 883} 884 885#elif (VM_PHYSSEG_STRAT == VM_PSTRAT_BSEARCH) 886 887static inline int 888vm_physseg_find_bsearch(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 889{ 890 /* binary search for it */ 891 int start, len, guess; 892 893 /* 894 * if try is too large (thus target is less than try) we reduce 895 * the length to trunc(len/2) [i.e. everything smaller than "try"] 896 * 897 * if the try is too small (thus target is greater than try) then 898 * we set the new start to be (try + 1). this means we need to 899 * reduce the length to (round(len/2) - 1). 900 * 901 * note "adjust" below which takes advantage of the fact that 902 * (round(len/2) - 1) == trunc((len - 1) / 2) 903 * for any value of len we may have 904 */ 905 906 for (start = 0, len = nsegs ; len != 0 ; len = len / 2) { 907 guess = start + (len / 2); /* try in the middle */ 908 909 /* start past our try? */ 910 if (pframe >= segs[guess].start) { 911 /* was try correct? */ 912 if (pframe < segs[guess].end) { 913 if (offp) 914 *offp = pframe - segs[guess].start; 915 return guess; /* got it */ 916 } 917 start = guess + 1; /* next time, start here */ 918 len--; /* "adjust" */ 919 } else { 920 /* 921 * pframe before try, just reduce length of 922 * region, done in "for" loop 923 */ 924 } 925 } 926 return(-1); 927} 928 929#else 930 931static inline int 932vm_physseg_find_linear(struct uvm_physseg *segs, int nsegs, paddr_t pframe, psize_t *offp) 933{ 934 /* linear search for it */ 935 int lcv; 936 937 for (lcv = 0; lcv < nsegs; lcv++) { 938 if (pframe >= segs[lcv].start && 939 pframe < segs[lcv].end) { 940 if (offp) 941 *offp = pframe - segs[lcv].start; 942 return(lcv); /* got it */ 943 } 944 } 945 return(-1); 946} 947#endif 948#endif /* UVM_HOTPLUG */ 949 950/* 951 * PHYS_TO_VM_PAGE: find vm_page for a PA. used by MI code to get vm_pages 952 * back from an I/O mapping (ugh!). used in some MD code as well. it can 953 * be prominent in flamegraphs, so optimise it and try to make it easy for 954 * the compiler by including next to the inline lookup routines. 955 */ 956struct vm_page * 957uvm_phys_to_vm_page(paddr_t pa) 958{ 959#if VM_PHYSSEG_STRAT != VM_PSTRAT_BSEARCH 960 /* 'contig' and linear cases */ 961 KASSERT(vm_nphysseg > 0); 962 struct uvm_physseg *ps = &vm_physmem[0]; 963 struct uvm_physseg *end = &vm_physmem[vm_nphysseg]; 964 paddr_t pframe = atop(pa); 965 do { 966 if (pframe >= ps->start && pframe < ps->end) { 967 return &ps->pgs[pframe - ps->start]; 968 } 969 } while (VM_PHYSSEG_MAX > 1 && __predict_false(++ps < end)); 970 return NULL; 971#else 972 /* binary search for it */ 973 paddr_t pf = atop(pa); 974 paddr_t off; 975 uvm_physseg_t upm; 976 977 upm = uvm_physseg_find(pf, &off); 978 if (upm != UVM_PHYSSEG_TYPE_INVALID) 979 return uvm_physseg_get_pg(upm, off); 980 return(NULL); 981#endif 982} 983 984bool 985uvm_physseg_valid_p(uvm_physseg_t upm) 986{ 987 struct uvm_physseg *ps; 988 989 if (upm == UVM_PHYSSEG_TYPE_INVALID || 990 upm == UVM_PHYSSEG_TYPE_INVALID_EMPTY || 991 upm == UVM_PHYSSEG_TYPE_INVALID_OVERFLOW) 992 return false; 993 994 /* 995 * This is the delicate init dance - 996 * needs to go with the dance. 997 */ 998 if (uvm.page_init_done != true) 999 return true; 1000 1001 ps = HANDLE_TO_PHYSSEG_NODE(upm); 1002 1003 /* Extra checks needed only post uvm_page_init() */ 1004 if (ps->pgs == NULL) 1005 return false; 1006 1007 /* XXX: etc. */ 1008 1009 return true; 1010 1011} 1012 1013/* 1014 * Boot protocol dictates that these must be able to return partially 1015 * initialised segments. 1016 */ 1017paddr_t 1018uvm_physseg_get_start(uvm_physseg_t upm) 1019{ 1020 if (uvm_physseg_valid_p(upm) == false) 1021 return (paddr_t) -1; 1022 1023 return HANDLE_TO_PHYSSEG_NODE(upm)->start; 1024} 1025 1026paddr_t 1027uvm_physseg_get_end(uvm_physseg_t upm) 1028{ 1029 if (uvm_physseg_valid_p(upm) == false) 1030 return (paddr_t) -1; 1031 1032 return HANDLE_TO_PHYSSEG_NODE(upm)->end; 1033} 1034 1035paddr_t 1036uvm_physseg_get_avail_start(uvm_physseg_t upm) 1037{ 1038 if (uvm_physseg_valid_p(upm) == false) 1039 return (paddr_t) -1; 1040 1041 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_start; 1042} 1043 1044#if defined(UVM_PHYSSEG_LEGACY) 1045void 1046uvm_physseg_set_avail_start(uvm_physseg_t upm, paddr_t avail_start) 1047{ 1048 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm); 1049 1050#if defined(DIAGNOSTIC) 1051 paddr_t avail_end; 1052 avail_end = uvm_physseg_get_avail_end(upm); 1053 KASSERT(uvm_physseg_valid_p(upm)); 1054 KASSERT(avail_start < avail_end); 1055 KASSERT(avail_start >= ps->start); 1056#endif 1057 1058 ps->avail_start = avail_start; 1059} 1060 1061void 1062uvm_physseg_set_avail_end(uvm_physseg_t upm, paddr_t avail_end) 1063{ 1064 struct uvm_physseg *ps = HANDLE_TO_PHYSSEG_NODE(upm); 1065 1066#if defined(DIAGNOSTIC) 1067 paddr_t avail_start; 1068 avail_start = uvm_physseg_get_avail_start(upm); 1069 KASSERT(uvm_physseg_valid_p(upm)); 1070 KASSERT(avail_end > avail_start); 1071 KASSERT(avail_end <= ps->end); 1072#endif 1073 1074 ps->avail_end = avail_end; 1075} 1076 1077#endif /* UVM_PHYSSEG_LEGACY */ 1078 1079paddr_t 1080uvm_physseg_get_avail_end(uvm_physseg_t upm) 1081{ 1082 if (uvm_physseg_valid_p(upm) == false) 1083 return (paddr_t) -1; 1084 1085 return HANDLE_TO_PHYSSEG_NODE(upm)->avail_end; 1086} 1087 1088inline struct vm_page * 1089uvm_physseg_get_pg(uvm_physseg_t upm, paddr_t idx) 1090{ 1091 KASSERT(uvm_physseg_valid_p(upm)); 1092 return &HANDLE_TO_PHYSSEG_NODE(upm)->pgs[idx]; 1093} 1094 1095#ifdef __HAVE_PMAP_PHYSSEG 1096struct pmap_physseg * 1097uvm_physseg_get_pmseg(uvm_physseg_t upm) 1098{ 1099 KASSERT(uvm_physseg_valid_p(upm)); 1100 return &(HANDLE_TO_PHYSSEG_NODE(upm)->pmseg); 1101} 1102#endif 1103 1104int 1105uvm_physseg_get_free_list(uvm_physseg_t upm) 1106{ 1107 KASSERT(uvm_physseg_valid_p(upm)); 1108 return HANDLE_TO_PHYSSEG_NODE(upm)->free_list; 1109} 1110 1111u_long 1112uvm_physseg_get_start_hint(uvm_physseg_t upm) 1113{ 1114 KASSERT(uvm_physseg_valid_p(upm)); 1115 return HANDLE_TO_PHYSSEG_NODE(upm)->start_hint; 1116} 1117 1118bool 1119uvm_physseg_set_start_hint(uvm_physseg_t upm, u_long start_hint) 1120{ 1121 if (uvm_physseg_valid_p(upm) == false) 1122 return false; 1123 1124 HANDLE_TO_PHYSSEG_NODE(upm)->start_hint = start_hint; 1125 return true; 1126} 1127 1128void 1129uvm_physseg_init_seg(uvm_physseg_t upm, struct vm_page *pgs) 1130{ 1131 psize_t i; 1132 psize_t n; 1133 paddr_t paddr; 1134 struct uvm_physseg *seg; 1135 struct vm_page *pg; 1136 1137 KASSERT(upm != UVM_PHYSSEG_TYPE_INVALID); 1138 KASSERT(pgs != NULL); 1139 1140 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1141 KASSERT(seg != NULL); 1142 KASSERT(seg->pgs == NULL); 1143 1144 n = seg->end - seg->start; 1145 seg->pgs = pgs; 1146 1147 /* init and free vm_pages (we've already zeroed them) */ 1148 paddr = ctob(seg->start); 1149 for (i = 0 ; i < n ; i++, paddr += PAGE_SIZE) { 1150 pg = &seg->pgs[i]; 1151 pg->phys_addr = paddr; 1152#ifdef __HAVE_VM_PAGE_MD 1153 VM_MDPAGE_INIT(pg); 1154#endif 1155 if (atop(paddr) >= seg->avail_start && 1156 atop(paddr) < seg->avail_end) { 1157 uvmexp.npages++; 1158 /* add page to free pool */ 1159 uvm_page_set_freelist(pg, 1160 uvm_page_lookup_freelist(pg)); 1161 /* Disable LOCKDEBUG: too many and too early. */ 1162 mutex_init(&pg->interlock, MUTEX_NODEBUG, IPL_NONE); 1163 uvm_pagefree(pg); 1164 } 1165 } 1166} 1167 1168void 1169uvm_physseg_seg_chomp_slab(uvm_physseg_t upm, struct vm_page *pgs, size_t n) 1170{ 1171 struct uvm_physseg *seg = HANDLE_TO_PHYSSEG_NODE(upm); 1172 1173 /* max number of pre-boot unplug()s allowed */ 1174#define UVM_PHYSSEG_BOOT_UNPLUG_MAX VM_PHYSSEG_MAX 1175 1176 static char btslab_ex_storage[EXTENT_FIXED_STORAGE_SIZE(UVM_PHYSSEG_BOOT_UNPLUG_MAX)]; 1177 1178 if (__predict_false(uvm.page_init_done == false)) { 1179 seg->ext = extent_create("Boot time slab", (u_long) pgs, (u_long) (pgs + n), 1180 (void *)btslab_ex_storage, sizeof(btslab_ex_storage), 0); 1181 } else { 1182 seg->ext = extent_create("Hotplug slab", (u_long) pgs, (u_long) (pgs + n), NULL, 0, 0); 1183 } 1184 1185 KASSERT(seg->ext != NULL); 1186 1187} 1188 1189struct vm_page * 1190uvm_physseg_seg_alloc_from_slab(uvm_physseg_t upm, size_t pages) 1191{ 1192 int err; 1193 struct uvm_physseg *seg; 1194 struct vm_page *pgs = NULL; 1195 1196 KASSERT(pages > 0); 1197 1198 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1199 1200 if (__predict_false(seg->ext == NULL)) { 1201 /* 1202 * This is a situation unique to boot time. 1203 * It shouldn't happen at any point other than from 1204 * the first uvm_page.c:uvm_page_init() call 1205 * Since we're in a loop, we can get away with the 1206 * below. 1207 */ 1208 KASSERT(uvm.page_init_done != true); 1209 1210 uvm_physseg_t upmp = uvm_physseg_get_prev(upm); 1211 KASSERT(upmp != UVM_PHYSSEG_TYPE_INVALID); 1212 1213 seg->ext = HANDLE_TO_PHYSSEG_NODE(upmp)->ext; 1214 1215 KASSERT(seg->ext != NULL); 1216 } 1217 1218 /* We allocate enough for this segment */ 1219 err = extent_alloc(seg->ext, sizeof(*pgs) * pages, 1, 0, EX_BOUNDZERO, (u_long *)&pgs); 1220 1221 if (err != 0) { 1222#ifdef DEBUG 1223 printf("%s: extent_alloc failed with error: %d \n", 1224 __func__, err); 1225#endif 1226 } 1227 1228 return pgs; 1229} 1230 1231/* 1232 * uvm_page_physload: load physical memory into VM system 1233 * 1234 * => all args are PFs 1235 * => all pages in start/end get vm_page structures 1236 * => areas marked by avail_start/avail_end get added to the free page pool 1237 * => we are limited to VM_PHYSSEG_MAX physical memory segments 1238 */ 1239 1240uvm_physseg_t 1241uvm_page_physload(paddr_t start, paddr_t end, paddr_t avail_start, 1242 paddr_t avail_end, int free_list) 1243{ 1244 struct uvm_physseg *ps; 1245 uvm_physseg_t upm; 1246 1247 if (__predict_true(uvm.page_init_done == true)) 1248 panic("%s: unload attempted after uvm_page_init()\n", __func__); 1249 if (uvmexp.pagesize == 0) 1250 panic("uvm_page_physload: page size not set!"); 1251 if (free_list >= VM_NFREELIST || free_list < VM_FREELIST_DEFAULT) 1252 panic("uvm_page_physload: bad free list %d", free_list); 1253 if (start >= end) 1254 panic("uvm_page_physload: start[%" PRIxPADDR "] >= end[%" 1255 PRIxPADDR "]", start, end); 1256 1257 if (uvm_physseg_plug(start, end - start, &upm) == false) { 1258 panic("uvm_physseg_plug() failed at boot."); 1259 /* NOTREACHED */ 1260 return UVM_PHYSSEG_TYPE_INVALID; /* XXX: correct type */ 1261 } 1262 1263 ps = HANDLE_TO_PHYSSEG_NODE(upm); 1264 1265 /* Legacy */ 1266 ps->avail_start = avail_start; 1267 ps->avail_end = avail_end; 1268 1269 ps->free_list = free_list; /* XXX: */ 1270 1271 1272 return upm; 1273} 1274 1275bool 1276uvm_physseg_unplug(paddr_t pfn, size_t pages) 1277{ 1278 uvm_physseg_t upm; 1279 paddr_t off = 0, start __diagused, end; 1280 struct uvm_physseg *seg; 1281 1282 upm = uvm_physseg_find(pfn, &off); 1283 1284 if (!uvm_physseg_valid_p(upm)) { 1285 printf("%s: Tried to unplug from unknown offset\n", __func__); 1286 return false; 1287 } 1288 1289 seg = HANDLE_TO_PHYSSEG_NODE(upm); 1290 1291 start = uvm_physseg_get_start(upm); 1292 end = uvm_physseg_get_end(upm); 1293 1294 if (end < (pfn + pages)) { 1295 printf("%s: Tried to unplug oversized span \n", __func__); 1296 return false; 1297 } 1298 1299 KASSERT(pfn == start + off); /* sanity */ 1300 1301 if (__predict_true(uvm.page_init_done == true)) { 1302 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1303 if (extent_free(seg->ext, (u_long)(seg->pgs + off), sizeof(struct vm_page) * pages, EX_MALLOCOK | EX_NOWAIT) != 0) 1304 return false; 1305 } 1306 1307 if (off == 0 && (pfn + pages) == end) { 1308#if defined(UVM_HOTPLUG) /* rbtree implementation */ 1309 int segcount = 0; 1310 struct uvm_physseg *current_ps; 1311 /* Complete segment */ 1312 if (uvm_physseg_graph.nentries == 1) 1313 panic("%s: out of memory!", __func__); 1314 1315 if (__predict_true(uvm.page_init_done == true)) { 1316 RB_TREE_FOREACH(current_ps, &(uvm_physseg_graph.rb_tree)) { 1317 if (seg->ext == current_ps->ext) 1318 segcount++; 1319 } 1320 KASSERT(segcount > 0); 1321 1322 if (segcount == 1) { 1323 extent_destroy(seg->ext); 1324 } 1325 1326 /* 1327 * We assume that the unplug will succeed from 1328 * this point onwards 1329 */ 1330 uvmexp.npages -= (int) pages; 1331 } 1332 1333 rb_tree_remove_node(&(uvm_physseg_graph.rb_tree), upm); 1334 memset(seg, 0, sizeof(struct uvm_physseg)); 1335 uvm_physseg_free(seg, sizeof(struct uvm_physseg)); 1336 uvm_physseg_graph.nentries--; 1337#else /* UVM_HOTPLUG */ 1338 int x; 1339 if (vm_nphysmem == 1) 1340 panic("uvm_page_physget: out of memory!"); 1341 vm_nphysmem--; 1342 for (x = upm ; x < vm_nphysmem ; x++) 1343 /* structure copy */ 1344 VM_PHYSMEM_PTR_SWAP(x, x + 1); 1345#endif /* UVM_HOTPLUG */ 1346 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1347 return true; 1348 } 1349 1350 if (off > 0 && 1351 (pfn + pages) < end) { 1352#if defined(UVM_HOTPLUG) /* rbtree implementation */ 1353 /* middle chunk - need a new segment */ 1354 struct uvm_physseg *ps, *current_ps; 1355 ps = uvm_physseg_alloc(sizeof (struct uvm_physseg)); 1356 if (ps == NULL) { 1357 printf("%s: Unable to allocated new fragment vm_physseg \n", 1358 __func__); 1359 return false; 1360 } 1361 1362 /* Remove middle chunk */ 1363 if (__predict_true(uvm.page_init_done == true)) { 1364 KASSERT(seg->ext != NULL); 1365 ps->ext = seg->ext; 1366 1367 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1368 /* 1369 * We assume that the unplug will succeed from 1370 * this point onwards 1371 */ 1372 uvmexp.npages -= (int) pages; 1373 } 1374 1375 ps->start = pfn + pages; 1376 ps->avail_start = ps->start; /* XXX: Legacy */ 1377 1378 ps->end = seg->end; 1379 ps->avail_end = ps->end; /* XXX: Legacy */ 1380 1381 seg->end = pfn; 1382 seg->avail_end = seg->end; /* XXX: Legacy */ 1383 1384 1385 /* 1386 * The new pgs array points to the beginning of the 1387 * tail fragment. 1388 */ 1389 if (__predict_true(uvm.page_init_done == true)) 1390 ps->pgs = seg->pgs + off + pages; 1391 1392 current_ps = rb_tree_insert_node(&(uvm_physseg_graph.rb_tree), ps); 1393 if (current_ps != ps) { 1394 panic("uvm_page_physload: Duplicate address range detected!"); 1395 } 1396 uvm_physseg_graph.nentries++; 1397#else /* UVM_HOTPLUG */ 1398 panic("%s: can't unplug() from the middle of a segment without" 1399 " UVM_HOTPLUG\n", __func__); 1400 /* NOTREACHED */ 1401#endif /* UVM_HOTPLUG */ 1402 return true; 1403 } 1404 1405 if (off == 0 && (pfn + pages) < end) { 1406 /* Remove front chunk */ 1407 if (__predict_true(uvm.page_init_done == true)) { 1408 /* XXX: KASSERT() that seg->pgs[] are not on any uvm lists */ 1409 /* 1410 * We assume that the unplug will succeed from 1411 * this point onwards 1412 */ 1413 uvmexp.npages -= (int) pages; 1414 } 1415 1416 /* Truncate */ 1417 seg->start = pfn + pages; 1418 seg->avail_start = seg->start; /* XXX: Legacy */ 1419 1420 /* 1421 * Move the pgs array start to the beginning of the 1422 * tail end. 1423 */ 1424 if (__predict_true(uvm.page_init_done == true)) 1425 seg->pgs += pages; 1426 1427 return true; 1428 } 1429 1430 if (off > 0 && (pfn + pages) == end) { 1431 /* back chunk */ 1432 1433 1434 /* Truncate! */ 1435 seg->end = pfn; 1436 seg->avail_end = seg->end; /* XXX: Legacy */ 1437 1438 uvmexp.npages -= (int) pages; 1439 1440 return true; 1441 } 1442 1443 printf("%s: Tried to unplug unknown range \n", __func__); 1444 1445 return false; 1446} 1447