vm_reserv.c revision 174982
1174982Salc/*- 2174982Salc * Copyright (c) 2002-2006 Rice University 3174982Salc * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4174982Salc * All rights reserved. 5174982Salc * 6174982Salc * This software was developed for the FreeBSD Project by Alan L. Cox, 7174982Salc * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8174982Salc * 9174982Salc * Redistribution and use in source and binary forms, with or without 10174982Salc * modification, are permitted provided that the following conditions 11174982Salc * are met: 12174982Salc * 1. Redistributions of source code must retain the above copyright 13174982Salc * notice, this list of conditions and the following disclaimer. 14174982Salc * 2. Redistributions in binary form must reproduce the above copyright 15174982Salc * notice, this list of conditions and the following disclaimer in the 16174982Salc * documentation and/or other materials provided with the distribution. 17174982Salc * 18174982Salc * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19174982Salc * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20174982Salc * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21174982Salc * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22174982Salc * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23174982Salc * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24174982Salc * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 25174982Salc * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26174982Salc * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27174982Salc * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 28174982Salc * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29174982Salc * POSSIBILITY OF SUCH DAMAGE. 30174982Salc */ 31174982Salc 32174982Salc/* 33174982Salc * Superpage reservation management module 34174982Salc */ 35174982Salc 36174982Salc#include <sys/cdefs.h> 37174982Salc__FBSDID("$FreeBSD: head/sys/vm/vm_reserv.c 174982 2007-12-29 19:53:04Z alc $"); 38174982Salc 39174982Salc#include "opt_vm.h" 40174982Salc 41174982Salc#include <sys/param.h> 42174982Salc#include <sys/kernel.h> 43174982Salc#include <sys/lock.h> 44174982Salc#include <sys/malloc.h> 45174982Salc#include <sys/mutex.h> 46174982Salc#include <sys/queue.h> 47174982Salc#include <sys/sbuf.h> 48174982Salc#include <sys/sysctl.h> 49174982Salc#include <sys/systm.h> 50174982Salc 51174982Salc#include <vm/vm.h> 52174982Salc#include <vm/vm_param.h> 53174982Salc#include <vm/vm_object.h> 54174982Salc#include <vm/vm_page.h> 55174982Salc#include <vm/vm_phys.h> 56174982Salc#include <vm/vm_reserv.h> 57174982Salc 58174982Salc/* 59174982Salc * The reservation system supports the speculative allocation of large physical 60174982Salc * pages ("superpages"). Speculative allocation enables the fully-automatic 61174982Salc * utilization of superpages by the virtual memory system. In other words, no 62174982Salc * programmatic directives are required to use superpages. 63174982Salc */ 64174982Salc 65174982Salc#if VM_NRESERVLEVEL > 0 66174982Salc 67174982Salc/* 68174982Salc * The number of small pages that are contained in a level 0 reservation 69174982Salc */ 70174982Salc#define VM_LEVEL_0_NPAGES (1 << VM_LEVEL_0_ORDER) 71174982Salc 72174982Salc/* 73174982Salc * The number of bits by which a physical address is shifted to obtain the 74174982Salc * reservation number 75174982Salc */ 76174982Salc#define VM_LEVEL_0_SHIFT (VM_LEVEL_0_ORDER + PAGE_SHIFT) 77174982Salc 78174982Salc/* 79174982Salc * The size of a level 0 reservation in bytes 80174982Salc */ 81174982Salc#define VM_LEVEL_0_SIZE (1 << VM_LEVEL_0_SHIFT) 82174982Salc 83174982Salc/* 84174982Salc * Computes the index of the small page underlying the given (object, pindex) 85174982Salc * within the reservation's array of small pages. 86174982Salc */ 87174982Salc#define VM_RESERV_INDEX(object, pindex) \ 88174982Salc (((object)->pg_color + (pindex)) & (VM_LEVEL_0_NPAGES - 1)) 89174982Salc 90174982Salc/* 91174982Salc * The reservation structure 92174982Salc * 93174982Salc * A reservation structure is constructed whenever a large physical page is 94174982Salc * speculatively allocated to an object. The reservation provides the small 95174982Salc * physical pages for the range [pindex, pindex + VM_LEVEL_0_NPAGES) of offsets 96174982Salc * within that object. The reservation's "popcnt" tracks the number of these 97174982Salc * small physical pages that are in use at any given time. When and if the 98174982Salc * reservation is not fully utilized, it appears in the queue of partially- 99174982Salc * populated reservations. The reservation always appears on the containing 100174982Salc * object's list of reservations. 101174982Salc * 102174982Salc * A partially-populated reservation can be broken and reclaimed at any time. 103174982Salc */ 104174982Salcstruct vm_reserv { 105174982Salc TAILQ_ENTRY(vm_reserv) partpopq; 106174982Salc LIST_ENTRY(vm_reserv) objq; 107174982Salc vm_object_t object; /* containing object */ 108174982Salc vm_pindex_t pindex; /* offset within object */ 109174982Salc vm_page_t pages; /* first page of a superpage */ 110174982Salc int popcnt; /* # of pages in use */ 111174982Salc char inpartpopq; 112174982Salc}; 113174982Salc 114174982Salc/* 115174982Salc * The reservation array 116174982Salc * 117174982Salc * This array is analoguous in function to vm_page_array. It differs in the 118174982Salc * respect that it may contain a greater number of useful reservation 119174982Salc * structures than there are (physical) superpages. These "invalid" 120174982Salc * reservation structures exist to trade-off space for time in the 121174982Salc * implementation of vm_reserv_from_page(). Invalid reservation structures are 122174982Salc * distinguishable from "valid" reservation structures by inspecting the 123174982Salc * reservation's "pages" field. Invalid reservation structures have a NULL 124174982Salc * "pages" field. 125174982Salc * 126174982Salc * vm_reserv_from_page() maps a small (physical) page to an element of this 127174982Salc * array by computing a physical reservation number from the page's physical 128174982Salc * address. The physical reservation number is used as the array index. 129174982Salc * 130174982Salc * An "active" reservation is a valid reservation structure that has a non-NULL 131174982Salc * "object" field and a non-zero "popcnt" field. In other words, every active 132174982Salc * reservation belongs to a particular object. Moreover, every active 133174982Salc * reservation has an entry in the containing object's list of reservations. 134174982Salc */ 135174982Salcstatic vm_reserv_t vm_reserv_array; 136174982Salc 137174982Salc/* 138174982Salc * The partially-populated reservation queue 139174982Salc * 140174982Salc * This queue enables the fast recovery of an unused cached or free small page 141174982Salc * from a partially-populated reservation. The head of this queue is either 142174982Salc * the least-recently-populated or most-recently-depopulated reservation. 143174982Salc * 144174982Salc * Access to this queue is synchronized by the free page queue lock. 145174982Salc */ 146174982Salcstatic TAILQ_HEAD(, vm_reserv) vm_rvq_partpop = 147174982Salc TAILQ_HEAD_INITIALIZER(vm_rvq_partpop); 148174982Salc 149174982Salcstatic SYSCTL_NODE(_vm, OID_AUTO, reserv, CTLFLAG_RD, 0, "Reservation Info"); 150174982Salc 151174982Salcstatic long vm_reserv_broken; 152174982SalcSYSCTL_LONG(_vm_reserv, OID_AUTO, broken, CTLFLAG_RD, 153174982Salc &vm_reserv_broken, 0, "Cumulative number of broken reservations"); 154174982Salc 155174982Salcstatic long vm_reserv_freed; 156174982SalcSYSCTL_LONG(_vm_reserv, OID_AUTO, freed, CTLFLAG_RD, 157174982Salc &vm_reserv_freed, 0, "Cumulative number of freed reservations"); 158174982Salc 159174982Salcstatic int sysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS); 160174982Salc 161174982SalcSYSCTL_OID(_vm_reserv, OID_AUTO, partpopq, CTLTYPE_STRING | CTLFLAG_RD, NULL, 0, 162174982Salc sysctl_vm_reserv_partpopq, "A", "Partially-populated reservation queues"); 163174982Salc 164174982Salcstatic long vm_reserv_reclaimed; 165174982SalcSYSCTL_LONG(_vm_reserv, OID_AUTO, reclaimed, CTLFLAG_RD, 166174982Salc &vm_reserv_reclaimed, 0, "Cumulative number of reclaimed reservations"); 167174982Salc 168174982Salcstatic void vm_reserv_depopulate(vm_reserv_t rv); 169174982Salcstatic vm_reserv_t vm_reserv_from_page(vm_page_t m); 170174982Salcstatic boolean_t vm_reserv_has_pindex(vm_reserv_t rv, 171174982Salc vm_pindex_t pindex); 172174982Salcstatic void vm_reserv_populate(vm_reserv_t rv); 173174982Salc 174174982Salc/* 175174982Salc * Describes the current state of the partially-populated reservation queue. 176174982Salc */ 177174982Salcstatic int 178174982Salcsysctl_vm_reserv_partpopq(SYSCTL_HANDLER_ARGS) 179174982Salc{ 180174982Salc struct sbuf sbuf; 181174982Salc vm_reserv_t rv; 182174982Salc char *cbuf; 183174982Salc const int cbufsize = (VM_NRESERVLEVEL + 1) * 81; 184174982Salc int counter, error, level, unused_pages; 185174982Salc 186174982Salc cbuf = malloc(cbufsize, M_TEMP, M_WAITOK | M_ZERO); 187174982Salc sbuf_new(&sbuf, cbuf, cbufsize, SBUF_FIXEDLEN); 188174982Salc sbuf_printf(&sbuf, "\nLEVEL SIZE NUMBER\n\n"); 189174982Salc for (level = -1; level <= VM_NRESERVLEVEL - 2; level++) { 190174982Salc counter = 0; 191174982Salc unused_pages = 0; 192174982Salc mtx_lock(&vm_page_queue_free_mtx); 193174982Salc TAILQ_FOREACH(rv, &vm_rvq_partpop/*[level]*/, partpopq) { 194174982Salc counter++; 195174982Salc unused_pages += VM_LEVEL_0_NPAGES - rv->popcnt; 196174982Salc } 197174982Salc mtx_unlock(&vm_page_queue_free_mtx); 198174982Salc sbuf_printf(&sbuf, "%5.5d: %6.6dK, %6.6d\n", level, 199174982Salc unused_pages * (PAGE_SIZE / 1024), counter); 200174982Salc } 201174982Salc sbuf_finish(&sbuf); 202174982Salc error = SYSCTL_OUT(req, sbuf_data(&sbuf), sbuf_len(&sbuf)); 203174982Salc sbuf_delete(&sbuf); 204174982Salc free(cbuf, M_TEMP); 205174982Salc return (error); 206174982Salc} 207174982Salc 208174982Salc/* 209174982Salc * Reduces the given reservation's population count. If the population count 210174982Salc * becomes zero, the reservation is destroyed. Additionally, moves the 211174982Salc * reservation to the head of the partially-populated reservations queue if the 212174982Salc * population count is non-zero. 213174982Salc * 214174982Salc * The free page queue lock must be held. 215174982Salc */ 216174982Salcstatic void 217174982Salcvm_reserv_depopulate(vm_reserv_t rv) 218174982Salc{ 219174982Salc 220174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 221174982Salc KASSERT(rv->object != NULL, 222174982Salc ("vm_reserv_depopulate: reserv %p is free", rv)); 223174982Salc KASSERT(rv->popcnt > 0, 224174982Salc ("vm_reserv_depopulate: reserv %p's popcnt is corrupted", rv)); 225174982Salc if (rv->inpartpopq) { 226174982Salc TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq); 227174982Salc rv->inpartpopq = FALSE; 228174982Salc } 229174982Salc rv->popcnt--; 230174982Salc if (rv->popcnt == 0) { 231174982Salc LIST_REMOVE(rv, objq); 232174982Salc rv->object = NULL; 233174982Salc vm_phys_free_pages(rv->pages, VM_LEVEL_0_ORDER); 234174982Salc vm_reserv_freed++; 235174982Salc } else { 236174982Salc rv->inpartpopq = TRUE; 237174982Salc TAILQ_INSERT_HEAD(&vm_rvq_partpop, rv, partpopq); 238174982Salc } 239174982Salc} 240174982Salc 241174982Salc/* 242174982Salc * Returns the reservation to which the given page might belong. 243174982Salc */ 244174982Salcstatic __inline vm_reserv_t 245174982Salcvm_reserv_from_page(vm_page_t m) 246174982Salc{ 247174982Salc 248174982Salc return (&vm_reserv_array[VM_PAGE_TO_PHYS(m) >> VM_LEVEL_0_SHIFT]); 249174982Salc} 250174982Salc 251174982Salc/* 252174982Salc * Returns TRUE if the given reservation contains the given page index and 253174982Salc * FALSE otherwise. 254174982Salc */ 255174982Salcstatic __inline boolean_t 256174982Salcvm_reserv_has_pindex(vm_reserv_t rv, vm_pindex_t pindex) 257174982Salc{ 258174982Salc 259174982Salc return (((pindex - rv->pindex) & ~(VM_LEVEL_0_NPAGES - 1)) == 0); 260174982Salc} 261174982Salc 262174982Salc/* 263174982Salc * Increases the given reservation's population count. Moves the reservation 264174982Salc * to the tail of the partially-populated reservation queue. 265174982Salc * 266174982Salc * The free page queue must be locked. 267174982Salc */ 268174982Salcstatic void 269174982Salcvm_reserv_populate(vm_reserv_t rv) 270174982Salc{ 271174982Salc 272174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 273174982Salc KASSERT(rv->object != NULL, 274174982Salc ("vm_reserv_populate: reserv %p is free", rv)); 275174982Salc KASSERT(rv->popcnt < VM_LEVEL_0_NPAGES, 276174982Salc ("vm_reserv_populate: reserv %p is already full", rv)); 277174982Salc if (rv->inpartpopq) { 278174982Salc TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq); 279174982Salc rv->inpartpopq = FALSE; 280174982Salc } 281174982Salc rv->popcnt++; 282174982Salc if (rv->popcnt < VM_LEVEL_0_NPAGES) { 283174982Salc rv->inpartpopq = TRUE; 284174982Salc TAILQ_INSERT_TAIL(&vm_rvq_partpop, rv, partpopq); 285174982Salc } 286174982Salc} 287174982Salc 288174982Salc/* 289174982Salc * Allocates a page from an existing or newly-created reservation. 290174982Salc * 291174982Salc * The object and free page queue must be locked. 292174982Salc */ 293174982Salcvm_page_t 294174982Salcvm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex) 295174982Salc{ 296174982Salc vm_page_t m, mpred, msucc; 297174982Salc vm_pindex_t first, leftcap, rightcap; 298174982Salc vm_reserv_t rv; 299174982Salc 300174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 301174982Salc 302174982Salc /* 303174982Salc * Is a reservation fundamentally not possible? 304174982Salc */ 305174982Salc VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); 306174982Salc if (pindex < VM_RESERV_INDEX(object, pindex) || 307174982Salc pindex >= object->size) 308174982Salc return (NULL); 309174982Salc 310174982Salc /* 311174982Salc * Look for an existing reservation. 312174982Salc */ 313174982Salc msucc = NULL; 314174982Salc mpred = object->root; 315174982Salc while (mpred != NULL) { 316174982Salc KASSERT(mpred->pindex != pindex, 317174982Salc ("vm_reserv_alloc_page: pindex already allocated")); 318174982Salc rv = vm_reserv_from_page(mpred); 319174982Salc if (rv->object == object && vm_reserv_has_pindex(rv, pindex)) { 320174982Salc m = &rv->pages[VM_RESERV_INDEX(object, pindex)]; 321174982Salc /* Handle vm_page_rename(m, new_object, ...). */ 322174982Salc if ((m->flags & (PG_CACHED | PG_FREE)) == 0) 323174982Salc return (NULL); 324174982Salc vm_reserv_populate(rv); 325174982Salc return (m); 326174982Salc } else if (mpred->pindex < pindex) { 327174982Salc if (msucc != NULL || 328174982Salc (msucc = TAILQ_NEXT(mpred, listq)) == NULL) 329174982Salc break; 330174982Salc KASSERT(msucc->pindex != pindex, 331174982Salc ("vm_reserv_alloc_page: pindex already allocated")); 332174982Salc rv = vm_reserv_from_page(msucc); 333174982Salc if (rv->object == object && 334174982Salc vm_reserv_has_pindex(rv, pindex)) { 335174982Salc m = &rv->pages[VM_RESERV_INDEX(object, pindex)]; 336174982Salc /* Handle vm_page_rename(m, new_object, ...). */ 337174982Salc if ((m->flags & (PG_CACHED | PG_FREE)) == 0) 338174982Salc return (NULL); 339174982Salc vm_reserv_populate(rv); 340174982Salc return (m); 341174982Salc } else if (pindex < msucc->pindex) 342174982Salc break; 343174982Salc } else if (msucc == NULL) { 344174982Salc msucc = mpred; 345174982Salc mpred = TAILQ_PREV(msucc, pglist, listq); 346174982Salc continue; 347174982Salc } 348174982Salc msucc = NULL; 349174982Salc mpred = object->root = vm_page_splay(pindex, object->root); 350174982Salc } 351174982Salc 352174982Salc /* 353174982Salc * Determine the first index to the left that can be used. 354174982Salc */ 355174982Salc if (mpred == NULL) 356174982Salc leftcap = 0; 357174982Salc else if ((rv = vm_reserv_from_page(mpred))->object != object) 358174982Salc leftcap = mpred->pindex + 1; 359174982Salc else 360174982Salc leftcap = rv->pindex + VM_LEVEL_0_NPAGES; 361174982Salc 362174982Salc /* 363174982Salc * Determine the first index to the right that cannot be used. 364174982Salc */ 365174982Salc if (msucc == NULL) 366174982Salc rightcap = pindex + VM_LEVEL_0_NPAGES; 367174982Salc else if ((rv = vm_reserv_from_page(msucc))->object != object) 368174982Salc rightcap = msucc->pindex; 369174982Salc else 370174982Salc rightcap = rv->pindex; 371174982Salc 372174982Salc /* 373174982Salc * Determine if a reservation fits between the first index to 374174982Salc * the left that can be used and the first index to the right 375174982Salc * that cannot be used. 376174982Salc */ 377174982Salc first = pindex - VM_RESERV_INDEX(object, pindex); 378174982Salc if (first < leftcap || first + VM_LEVEL_0_NPAGES > rightcap) 379174982Salc return (NULL); 380174982Salc 381174982Salc /* 382174982Salc * Would a new reservation extend past the end of the given object? 383174982Salc */ 384174982Salc if (object->size < first + VM_LEVEL_0_NPAGES) { 385174982Salc /* 386174982Salc * Don't allocate a new reservation if the object is a vnode or 387174982Salc * backed by another object that is a vnode. 388174982Salc */ 389174982Salc if (object->type == OBJT_VNODE || 390174982Salc (object->backing_object != NULL && 391174982Salc object->backing_object->type == OBJT_VNODE)) 392174982Salc return (NULL); 393174982Salc /* Speculate that the object may grow. */ 394174982Salc } 395174982Salc 396174982Salc /* 397174982Salc * Allocate a new reservation. 398174982Salc */ 399174982Salc m = vm_phys_alloc_pages(VM_FREEPOOL_DEFAULT, VM_LEVEL_0_ORDER); 400174982Salc if (m != NULL) { 401174982Salc rv = vm_reserv_from_page(m); 402174982Salc KASSERT(rv->pages == m, 403174982Salc ("vm_reserv_alloc_page: reserv %p's pages is corrupted", 404174982Salc rv)); 405174982Salc KASSERT(rv->object == NULL, 406174982Salc ("vm_reserv_alloc_page: reserv %p isn't free", rv)); 407174982Salc LIST_INSERT_HEAD(&object->rvq, rv, objq); 408174982Salc rv->object = object; 409174982Salc rv->pindex = first; 410174982Salc KASSERT(rv->popcnt == 0, 411174982Salc ("vm_reserv_alloc_page: reserv %p's popcnt is corrupted", 412174982Salc rv)); 413174982Salc KASSERT(!rv->inpartpopq, 414174982Salc ("vm_reserv_alloc_page: reserv %p's inpartpopq is TRUE", 415174982Salc rv)); 416174982Salc vm_reserv_populate(rv); 417174982Salc m = &rv->pages[VM_RESERV_INDEX(object, pindex)]; 418174982Salc } 419174982Salc return (m); 420174982Salc} 421174982Salc 422174982Salc/* 423174982Salc * Breaks all reservations belonging to the given object. 424174982Salc */ 425174982Salcvoid 426174982Salcvm_reserv_break_all(vm_object_t object) 427174982Salc{ 428174982Salc vm_reserv_t rv; 429174982Salc int i; 430174982Salc 431174982Salc mtx_lock(&vm_page_queue_free_mtx); 432174982Salc while ((rv = LIST_FIRST(&object->rvq)) != NULL) { 433174982Salc KASSERT(rv->object == object, 434174982Salc ("vm_reserv_break_all: reserv %p is corrupted", rv)); 435174982Salc if (rv->inpartpopq) { 436174982Salc TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq); 437174982Salc rv->inpartpopq = FALSE; 438174982Salc } 439174982Salc LIST_REMOVE(rv, objq); 440174982Salc rv->object = NULL; 441174982Salc for (i = 0; i < VM_LEVEL_0_NPAGES; i++) { 442174982Salc if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) 443174982Salc vm_phys_free_pages(&rv->pages[i], 0); 444174982Salc else 445174982Salc rv->popcnt--; 446174982Salc } 447174982Salc KASSERT(rv->popcnt == 0, 448174982Salc ("vm_reserv_break_all: reserv %p's popcnt is corrupted", 449174982Salc rv)); 450174982Salc vm_reserv_broken++; 451174982Salc } 452174982Salc mtx_unlock(&vm_page_queue_free_mtx); 453174982Salc} 454174982Salc 455174982Salc/* 456174982Salc * Frees the given page if it belongs to a reservation. Returns TRUE if the 457174982Salc * page is freed and FALSE otherwise. 458174982Salc * 459174982Salc * The free page queue lock must be held. 460174982Salc */ 461174982Salcboolean_t 462174982Salcvm_reserv_free_page(vm_page_t m) 463174982Salc{ 464174982Salc vm_reserv_t rv; 465174982Salc 466174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 467174982Salc rv = vm_reserv_from_page(m); 468174982Salc if (rv->object != NULL) { 469174982Salc vm_reserv_depopulate(rv); 470174982Salc return (TRUE); 471174982Salc } 472174982Salc return (FALSE); 473174982Salc} 474174982Salc 475174982Salc/* 476174982Salc * Initializes the reservation management system. Specifically, initializes 477174982Salc * the reservation array. 478174982Salc * 479174982Salc * Requires that vm_page_array and first_page are initialized! 480174982Salc */ 481174982Salcvoid 482174982Salcvm_reserv_init(void) 483174982Salc{ 484174982Salc vm_paddr_t paddr; 485174982Salc int i; 486174982Salc 487174982Salc /* 488174982Salc * Initialize the reservation array. Specifically, initialize the 489174982Salc * "pages" field for every element that has an underlying superpage. 490174982Salc */ 491174982Salc for (i = 0; phys_avail[i + 1] != 0; i += 2) { 492174982Salc paddr = roundup2(phys_avail[i], VM_LEVEL_0_SIZE); 493174982Salc while (paddr + VM_LEVEL_0_SIZE <= phys_avail[i + 1]) { 494174982Salc vm_reserv_array[paddr >> VM_LEVEL_0_SHIFT].pages = 495174982Salc PHYS_TO_VM_PAGE(paddr); 496174982Salc paddr += VM_LEVEL_0_SIZE; 497174982Salc } 498174982Salc } 499174982Salc} 500174982Salc 501174982Salc/* 502174982Salc * Returns a reservation level if the given page belongs to a fully-populated 503174982Salc * reservation and -1 otherwise. 504174982Salc */ 505174982Salcint 506174982Salcvm_reserv_level_iffullpop(vm_page_t m) 507174982Salc{ 508174982Salc vm_reserv_t rv; 509174982Salc 510174982Salc rv = vm_reserv_from_page(m); 511174982Salc return (rv->popcnt == VM_LEVEL_0_NPAGES ? 0 : -1); 512174982Salc} 513174982Salc 514174982Salc/* 515174982Salc * Prepare for the reactivation of a cached page. 516174982Salc * 517174982Salc * First, suppose that the given page "m" was allocated individually, i.e., not 518174982Salc * as part of a reservation, and cached. Then, suppose a reservation 519174982Salc * containing "m" is allocated by the same object. Although "m" and the 520174982Salc * reservation belong to the same object, "m"'s pindex may not match the 521174982Salc * reservation's. 522174982Salc * 523174982Salc * The free page queue must be locked. 524174982Salc */ 525174982Salcboolean_t 526174982Salcvm_reserv_reactivate_page(vm_page_t m) 527174982Salc{ 528174982Salc vm_reserv_t rv; 529174982Salc int i, m_index; 530174982Salc 531174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 532174982Salc rv = vm_reserv_from_page(m); 533174982Salc if (rv->object == NULL) 534174982Salc return (FALSE); 535174982Salc KASSERT((m->flags & PG_CACHED) != 0, 536174982Salc ("vm_reserv_uncache_page: page %p is not cached", m)); 537174982Salc if (m->object == rv->object && 538174982Salc m->pindex - rv->pindex == VM_RESERV_INDEX(m->object, m->pindex)) 539174982Salc vm_reserv_populate(rv); 540174982Salc else { 541174982Salc KASSERT(rv->inpartpopq, 542174982Salc ("vm_reserv_uncache_page: reserv %p's inpartpopq is FALSE", 543174982Salc rv)); 544174982Salc TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq); 545174982Salc rv->inpartpopq = FALSE; 546174982Salc LIST_REMOVE(rv, objq); 547174982Salc rv->object = NULL; 548174982Salc /* Don't vm_phys_free_pages(m, 0). */ 549174982Salc m_index = m - rv->pages; 550174982Salc for (i = 0; i < m_index; i++) { 551174982Salc if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) 552174982Salc vm_phys_free_pages(&rv->pages[i], 0); 553174982Salc else 554174982Salc rv->popcnt--; 555174982Salc } 556174982Salc for (i++; i < VM_LEVEL_0_NPAGES; i++) { 557174982Salc if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) 558174982Salc vm_phys_free_pages(&rv->pages[i], 0); 559174982Salc else 560174982Salc rv->popcnt--; 561174982Salc } 562174982Salc KASSERT(rv->popcnt == 0, 563174982Salc ("vm_reserv_uncache_page: reserv %p's popcnt is corrupted", 564174982Salc rv)); 565174982Salc vm_reserv_broken++; 566174982Salc } 567174982Salc return (TRUE); 568174982Salc} 569174982Salc 570174982Salc/* 571174982Salc * Breaks the reservation at the head of the partially-populated reservation 572174982Salc * queue, releasing its cached and free pages to the physical memory 573174982Salc * allocator. Returns TRUE if a reservation is broken and FALSE otherwise. 574174982Salc * 575174982Salc * The free page queue lock must be held. 576174982Salc */ 577174982Salcboolean_t 578174982Salcvm_reserv_reclaim(void) 579174982Salc{ 580174982Salc vm_reserv_t rv; 581174982Salc int i; 582174982Salc 583174982Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 584174982Salc if ((rv = TAILQ_FIRST(&vm_rvq_partpop)) != NULL) { 585174982Salc KASSERT(rv->inpartpopq, 586174982Salc ("vm_reserv_reclaim: reserv %p's inpartpopq is corrupted", 587174982Salc rv)); 588174982Salc TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq); 589174982Salc rv->inpartpopq = FALSE; 590174982Salc KASSERT(rv->object != NULL, 591174982Salc ("vm_reserv_reclaim: reserv %p is free", rv)); 592174982Salc LIST_REMOVE(rv, objq); 593174982Salc rv->object = NULL; 594174982Salc for (i = 0; i < VM_LEVEL_0_NPAGES; i++) { 595174982Salc if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) 596174982Salc vm_phys_free_pages(&rv->pages[i], 0); 597174982Salc else 598174982Salc rv->popcnt--; 599174982Salc } 600174982Salc KASSERT(rv->popcnt == 0, 601174982Salc ("vm_reserv_reclaim: reserv %p's popcnt is corrupted", 602174982Salc rv)); 603174982Salc vm_reserv_reclaimed++; 604174982Salc return (TRUE); 605174982Salc } 606174982Salc return (FALSE); 607174982Salc} 608174982Salc 609174982Salc/* 610174982Salc * Transfers the reservation underlying the given page to a new object. 611174982Salc * 612174982Salc * The object must be locked. 613174982Salc */ 614174982Salcvoid 615174982Salcvm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, 616174982Salc vm_pindex_t old_object_offset) 617174982Salc{ 618174982Salc vm_reserv_t rv; 619174982Salc 620174982Salc VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED); 621174982Salc rv = vm_reserv_from_page(m); 622174982Salc if (rv->object == old_object) { 623174982Salc mtx_lock(&vm_page_queue_free_mtx); 624174982Salc if (rv->object == old_object) { 625174982Salc LIST_REMOVE(rv, objq); 626174982Salc LIST_INSERT_HEAD(&new_object->rvq, rv, objq); 627174982Salc rv->object = new_object; 628174982Salc rv->pindex -= old_object_offset; 629174982Salc } 630174982Salc mtx_unlock(&vm_page_queue_free_mtx); 631174982Salc } 632174982Salc} 633174982Salc 634174982Salc/* 635174982Salc * Allocates the virtual and physical memory required by the reservation 636174982Salc * management system's data structures, in particular, the reservation array. 637174982Salc */ 638174982Salcvm_paddr_t 639174982Salcvm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end, vm_paddr_t high_water) 640174982Salc{ 641174982Salc vm_paddr_t new_end; 642174982Salc size_t size; 643174982Salc 644174982Salc /* 645174982Salc * Calculate the size (in bytes) of the reservation array. Round up 646174982Salc * from "high_water" because every small page is mapped to an element 647174982Salc * in the reservation array based on its physical address. Thus, the 648174982Salc * number of elements in the reservation array can be greater than the 649174982Salc * number of superpages. 650174982Salc */ 651174982Salc size = howmany(high_water, VM_LEVEL_0_SIZE) * sizeof(struct vm_reserv); 652174982Salc 653174982Salc /* 654174982Salc * Allocate and map the physical memory for the reservation array. The 655174982Salc * next available virtual address is returned by reference. 656174982Salc */ 657174982Salc new_end = end - round_page(size); 658174982Salc vm_reserv_array = (void *)(uintptr_t)pmap_map(vaddr, new_end, end, 659174982Salc VM_PROT_READ | VM_PROT_WRITE); 660174982Salc bzero(vm_reserv_array, size); 661174982Salc 662174982Salc /* 663174982Salc * Return the next available physical address. 664174982Salc */ 665174982Salc return (new_end); 666174982Salc} 667174982Salc 668174982Salc#endif /* VM_NRESERVLEVEL > 0 */ 669