vm_phys.c revision 217508
1170477Salc/*- 2170477Salc * Copyright (c) 2002-2006 Rice University 3170477Salc * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4170477Salc * All rights reserved. 5170477Salc * 6170477Salc * This software was developed for the FreeBSD Project by Alan L. Cox, 7170477Salc * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8170477Salc * 9170477Salc * Redistribution and use in source and binary forms, with or without 10170477Salc * modification, are permitted provided that the following conditions 11170477Salc * are met: 12170477Salc * 1. Redistributions of source code must retain the above copyright 13170477Salc * notice, this list of conditions and the following disclaimer. 14170477Salc * 2. Redistributions in binary form must reproduce the above copyright 15170477Salc * notice, this list of conditions and the following disclaimer in the 16170477Salc * documentation and/or other materials provided with the distribution. 17170477Salc * 18170477Salc * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 19170477Salc * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 20170477Salc * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 21170477Salc * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 22170477Salc * HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 23170477Salc * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24170477Salc * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS 25170477Salc * OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26170477Salc * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27170477Salc * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY 28170477Salc * WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 29170477Salc * POSSIBILITY OF SUCH DAMAGE. 30170477Salc */ 31170477Salc 32170477Salc#include <sys/cdefs.h> 33170477Salc__FBSDID("$FreeBSD: head/sys/vm/vm_phys.c 217508 2011-01-17 19:17:26Z alc $"); 34170477Salc 35170477Salc#include "opt_ddb.h" 36170477Salc 37170477Salc#include <sys/param.h> 38170477Salc#include <sys/systm.h> 39170477Salc#include <sys/lock.h> 40170477Salc#include <sys/kernel.h> 41170477Salc#include <sys/malloc.h> 42170477Salc#include <sys/mutex.h> 43170477Salc#include <sys/queue.h> 44170477Salc#include <sys/sbuf.h> 45170477Salc#include <sys/sysctl.h> 46170477Salc#include <sys/vmmeter.h> 47172317Salc#include <sys/vnode.h> 48170477Salc 49170477Salc#include <ddb/ddb.h> 50170477Salc 51170477Salc#include <vm/vm.h> 52170477Salc#include <vm/vm_param.h> 53170477Salc#include <vm/vm_kern.h> 54170477Salc#include <vm/vm_object.h> 55170477Salc#include <vm/vm_page.h> 56170477Salc#include <vm/vm_phys.h> 57177956Salc#include <vm/vm_reserv.h> 58170477Salc 59210550Sjhb/* 60210550Sjhb * VM_FREELIST_DEFAULT is split into VM_NDOMAIN lists, one for each 61210550Sjhb * domain. These extra lists are stored at the end of the regular 62210550Sjhb * free lists starting with VM_NFREELIST. 63210550Sjhb */ 64210550Sjhb#define VM_RAW_NFREELIST (VM_NFREELIST + VM_NDOMAIN - 1) 65210550Sjhb 66170477Salcstruct vm_freelist { 67170477Salc struct pglist pl; 68170477Salc int lcnt; 69170477Salc}; 70170477Salc 71170477Salcstruct vm_phys_seg { 72170477Salc vm_paddr_t start; 73170477Salc vm_paddr_t end; 74170477Salc vm_page_t first_page; 75210550Sjhb int domain; 76170477Salc struct vm_freelist (*free_queues)[VM_NFREEPOOL][VM_NFREEORDER]; 77170477Salc}; 78170477Salc 79210550Sjhbstruct mem_affinity *mem_affinity; 80210550Sjhb 81170477Salcstatic struct vm_phys_seg vm_phys_segs[VM_PHYSSEG_MAX]; 82170477Salc 83170477Salcstatic int vm_phys_nsegs; 84170477Salc 85170477Salcstatic struct vm_freelist 86210550Sjhb vm_phys_free_queues[VM_RAW_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 87210550Sjhbstatic struct vm_freelist 88210550Sjhb(*vm_phys_lookup_lists[VM_NDOMAIN][VM_RAW_NFREELIST])[VM_NFREEPOOL][VM_NFREEORDER]; 89170477Salc 90170477Salcstatic int vm_nfreelists = VM_FREELIST_DEFAULT + 1; 91170477Salc 92170477Salcstatic int cnt_prezero; 93170477SalcSYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 94170477Salc &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 95170477Salc 96170477Salcstatic int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 97170477SalcSYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 98170477Salc NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 99170477Salc 100170477Salcstatic int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 101170477SalcSYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 102170477Salc NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 103170477Salc 104210550Sjhb#if VM_NDOMAIN > 1 105210550Sjhbstatic int sysctl_vm_phys_lookup_lists(SYSCTL_HANDLER_ARGS); 106210550SjhbSYSCTL_OID(_vm, OID_AUTO, phys_lookup_lists, CTLTYPE_STRING | CTLFLAG_RD, 107210550Sjhb NULL, 0, sysctl_vm_phys_lookup_lists, "A", "Phys Lookup Lists"); 108210550Sjhb#endif 109210550Sjhb 110210550Sjhbstatic void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, 111210550Sjhb int domain); 112170477Salcstatic void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind); 113170477Salcstatic int vm_phys_paddr_to_segind(vm_paddr_t pa); 114170477Salcstatic void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 115170477Salc int order); 116170477Salc 117170477Salc/* 118170477Salc * Outputs the state of the physical memory allocator, specifically, 119170477Salc * the amount of physical memory in each free list. 120170477Salc */ 121170477Salcstatic int 122170477Salcsysctl_vm_phys_free(SYSCTL_HANDLER_ARGS) 123170477Salc{ 124170477Salc struct sbuf sbuf; 125170477Salc struct vm_freelist *fl; 126170477Salc int error, flind, oind, pind; 127170477Salc 128212750Smdf sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 129170477Salc for (flind = 0; flind < vm_nfreelists; flind++) { 130170477Salc sbuf_printf(&sbuf, "\nFREE LIST %d:\n" 131170477Salc "\n ORDER (SIZE) | NUMBER" 132170477Salc "\n ", flind); 133170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) 134170477Salc sbuf_printf(&sbuf, " | POOL %d", pind); 135170477Salc sbuf_printf(&sbuf, "\n-- "); 136170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) 137170477Salc sbuf_printf(&sbuf, "-- -- "); 138170477Salc sbuf_printf(&sbuf, "--\n"); 139170477Salc for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 140214564Salc sbuf_printf(&sbuf, " %2d (%6dK)", oind, 141170477Salc 1 << (PAGE_SHIFT - 10 + oind)); 142170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) { 143170477Salc fl = vm_phys_free_queues[flind][pind]; 144214564Salc sbuf_printf(&sbuf, " | %6d", fl[oind].lcnt); 145170477Salc } 146170477Salc sbuf_printf(&sbuf, "\n"); 147170477Salc } 148170477Salc } 149212750Smdf error = sbuf_finish(&sbuf); 150170477Salc sbuf_delete(&sbuf); 151170477Salc return (error); 152170477Salc} 153170477Salc 154170477Salc/* 155170477Salc * Outputs the set of physical memory segments. 156170477Salc */ 157170477Salcstatic int 158170477Salcsysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS) 159170477Salc{ 160170477Salc struct sbuf sbuf; 161170477Salc struct vm_phys_seg *seg; 162170477Salc int error, segind; 163170477Salc 164212750Smdf sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 165170477Salc for (segind = 0; segind < vm_phys_nsegs; segind++) { 166170477Salc sbuf_printf(&sbuf, "\nSEGMENT %d:\n\n", segind); 167170477Salc seg = &vm_phys_segs[segind]; 168170477Salc sbuf_printf(&sbuf, "start: %#jx\n", 169170477Salc (uintmax_t)seg->start); 170170477Salc sbuf_printf(&sbuf, "end: %#jx\n", 171170477Salc (uintmax_t)seg->end); 172210550Sjhb sbuf_printf(&sbuf, "domain: %d\n", seg->domain); 173170477Salc sbuf_printf(&sbuf, "free list: %p\n", seg->free_queues); 174170477Salc } 175212750Smdf error = sbuf_finish(&sbuf); 176170477Salc sbuf_delete(&sbuf); 177170477Salc return (error); 178170477Salc} 179170477Salc 180210550Sjhb#if VM_NDOMAIN > 1 181170477Salc/* 182210550Sjhb * Outputs the set of free list lookup lists. 183210550Sjhb */ 184210550Sjhbstatic int 185210550Sjhbsysctl_vm_phys_lookup_lists(SYSCTL_HANDLER_ARGS) 186210550Sjhb{ 187210550Sjhb struct sbuf sbuf; 188210550Sjhb int domain, error, flind, ndomains; 189210550Sjhb 190210550Sjhb ndomains = vm_nfreelists - VM_NFREELIST + 1; 191212750Smdf sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 192210550Sjhb for (domain = 0; domain < ndomains; domain++) { 193210550Sjhb sbuf_printf(&sbuf, "\nDOMAIN %d:\n\n", domain); 194210550Sjhb for (flind = 0; flind < vm_nfreelists; flind++) 195210550Sjhb sbuf_printf(&sbuf, " [%d]:\t%p\n", flind, 196210550Sjhb vm_phys_lookup_lists[domain][flind]); 197210550Sjhb } 198212750Smdf error = sbuf_finish(&sbuf); 199210550Sjhb sbuf_delete(&sbuf); 200210550Sjhb return (error); 201210550Sjhb} 202210550Sjhb#endif 203210550Sjhb 204210550Sjhb/* 205170477Salc * Create a physical memory segment. 206170477Salc */ 207170477Salcstatic void 208210550Sjhb_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain) 209170477Salc{ 210170477Salc struct vm_phys_seg *seg; 211170477Salc#ifdef VM_PHYSSEG_SPARSE 212170477Salc long pages; 213170477Salc int segind; 214170477Salc 215170477Salc pages = 0; 216170477Salc for (segind = 0; segind < vm_phys_nsegs; segind++) { 217170477Salc seg = &vm_phys_segs[segind]; 218170477Salc pages += atop(seg->end - seg->start); 219170477Salc } 220170477Salc#endif 221170477Salc KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 222170477Salc ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 223170477Salc seg = &vm_phys_segs[vm_phys_nsegs++]; 224170477Salc seg->start = start; 225170477Salc seg->end = end; 226210550Sjhb seg->domain = domain; 227170477Salc#ifdef VM_PHYSSEG_SPARSE 228170477Salc seg->first_page = &vm_page_array[pages]; 229170477Salc#else 230170477Salc seg->first_page = PHYS_TO_VM_PAGE(start); 231170477Salc#endif 232210550Sjhb#if VM_NDOMAIN > 1 233210550Sjhb if (flind == VM_FREELIST_DEFAULT && domain != 0) { 234210550Sjhb flind = VM_NFREELIST + (domain - 1); 235210550Sjhb if (flind >= vm_nfreelists) 236210550Sjhb vm_nfreelists = flind + 1; 237210550Sjhb } 238210550Sjhb#endif 239170477Salc seg->free_queues = &vm_phys_free_queues[flind]; 240170477Salc} 241170477Salc 242210550Sjhbstatic void 243210550Sjhbvm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind) 244210550Sjhb{ 245210550Sjhb int i; 246210550Sjhb 247210550Sjhb if (mem_affinity == NULL) { 248210550Sjhb _vm_phys_create_seg(start, end, flind, 0); 249210550Sjhb return; 250210550Sjhb } 251210550Sjhb 252210550Sjhb for (i = 0;; i++) { 253210550Sjhb if (mem_affinity[i].end == 0) 254210550Sjhb panic("Reached end of affinity info"); 255210550Sjhb if (mem_affinity[i].end <= start) 256210550Sjhb continue; 257210550Sjhb if (mem_affinity[i].start > start) 258210550Sjhb panic("No affinity info for start %jx", 259210550Sjhb (uintmax_t)start); 260210550Sjhb if (mem_affinity[i].end >= end) { 261210550Sjhb _vm_phys_create_seg(start, end, flind, 262210550Sjhb mem_affinity[i].domain); 263210550Sjhb break; 264210550Sjhb } 265210550Sjhb _vm_phys_create_seg(start, mem_affinity[i].end, flind, 266210550Sjhb mem_affinity[i].domain); 267210550Sjhb start = mem_affinity[i].end; 268210550Sjhb } 269210550Sjhb} 270210550Sjhb 271170477Salc/* 272170477Salc * Initialize the physical memory allocator. 273170477Salc */ 274170477Salcvoid 275170477Salcvm_phys_init(void) 276170477Salc{ 277170477Salc struct vm_freelist *fl; 278170477Salc int flind, i, oind, pind; 279210550Sjhb#if VM_NDOMAIN > 1 280210550Sjhb int ndomains, j; 281210550Sjhb#endif 282170477Salc 283170477Salc for (i = 0; phys_avail[i + 1] != 0; i += 2) { 284170477Salc#ifdef VM_FREELIST_ISADMA 285170477Salc if (phys_avail[i] < 16777216) { 286170477Salc if (phys_avail[i + 1] > 16777216) { 287170477Salc vm_phys_create_seg(phys_avail[i], 16777216, 288170477Salc VM_FREELIST_ISADMA); 289170477Salc vm_phys_create_seg(16777216, phys_avail[i + 1], 290170477Salc VM_FREELIST_DEFAULT); 291170477Salc } else { 292170477Salc vm_phys_create_seg(phys_avail[i], 293170477Salc phys_avail[i + 1], VM_FREELIST_ISADMA); 294170477Salc } 295170477Salc if (VM_FREELIST_ISADMA >= vm_nfreelists) 296170477Salc vm_nfreelists = VM_FREELIST_ISADMA + 1; 297170477Salc } else 298170477Salc#endif 299170477Salc#ifdef VM_FREELIST_HIGHMEM 300170477Salc if (phys_avail[i + 1] > VM_HIGHMEM_ADDRESS) { 301170477Salc if (phys_avail[i] < VM_HIGHMEM_ADDRESS) { 302170477Salc vm_phys_create_seg(phys_avail[i], 303170477Salc VM_HIGHMEM_ADDRESS, VM_FREELIST_DEFAULT); 304170477Salc vm_phys_create_seg(VM_HIGHMEM_ADDRESS, 305170477Salc phys_avail[i + 1], VM_FREELIST_HIGHMEM); 306170477Salc } else { 307170477Salc vm_phys_create_seg(phys_avail[i], 308170477Salc phys_avail[i + 1], VM_FREELIST_HIGHMEM); 309170477Salc } 310170477Salc if (VM_FREELIST_HIGHMEM >= vm_nfreelists) 311170477Salc vm_nfreelists = VM_FREELIST_HIGHMEM + 1; 312170477Salc } else 313170477Salc#endif 314170477Salc vm_phys_create_seg(phys_avail[i], phys_avail[i + 1], 315170477Salc VM_FREELIST_DEFAULT); 316170477Salc } 317170477Salc for (flind = 0; flind < vm_nfreelists; flind++) { 318170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) { 319170477Salc fl = vm_phys_free_queues[flind][pind]; 320170477Salc for (oind = 0; oind < VM_NFREEORDER; oind++) 321170477Salc TAILQ_INIT(&fl[oind].pl); 322170477Salc } 323170477Salc } 324210550Sjhb#if VM_NDOMAIN > 1 325210550Sjhb /* 326210550Sjhb * Build a free list lookup list for each domain. All of the 327210550Sjhb * memory domain lists are inserted at the VM_FREELIST_DEFAULT 328210550Sjhb * index in a round-robin order starting with the current 329210550Sjhb * domain. 330210550Sjhb */ 331210550Sjhb ndomains = vm_nfreelists - VM_NFREELIST + 1; 332210550Sjhb for (flind = 0; flind < VM_FREELIST_DEFAULT; flind++) 333210550Sjhb for (i = 0; i < ndomains; i++) 334210550Sjhb vm_phys_lookup_lists[i][flind] = 335210550Sjhb &vm_phys_free_queues[flind]; 336210550Sjhb for (i = 0; i < ndomains; i++) 337210550Sjhb for (j = 0; j < ndomains; j++) { 338210550Sjhb flind = (i + j) % ndomains; 339210550Sjhb if (flind == 0) 340210550Sjhb flind = VM_FREELIST_DEFAULT; 341210550Sjhb else 342210550Sjhb flind += VM_NFREELIST - 1; 343210550Sjhb vm_phys_lookup_lists[i][VM_FREELIST_DEFAULT + j] = 344210550Sjhb &vm_phys_free_queues[flind]; 345210550Sjhb } 346210550Sjhb for (flind = VM_FREELIST_DEFAULT + 1; flind < VM_NFREELIST; 347210550Sjhb flind++) 348210550Sjhb for (i = 0; i < ndomains; i++) 349210550Sjhb vm_phys_lookup_lists[i][flind + ndomains - 1] = 350210550Sjhb &vm_phys_free_queues[flind]; 351210550Sjhb#else 352210550Sjhb for (flind = 0; flind < vm_nfreelists; flind++) 353210550Sjhb vm_phys_lookup_lists[0][flind] = &vm_phys_free_queues[flind]; 354210550Sjhb#endif 355170477Salc} 356170477Salc 357170477Salc/* 358170477Salc * Split a contiguous, power of two-sized set of physical pages. 359170477Salc */ 360170477Salcstatic __inline void 361170477Salcvm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) 362170477Salc{ 363170477Salc vm_page_t m_buddy; 364170477Salc 365170477Salc while (oind > order) { 366170477Salc oind--; 367170477Salc m_buddy = &m[1 << oind]; 368170477Salc KASSERT(m_buddy->order == VM_NFREEORDER, 369170477Salc ("vm_phys_split_pages: page %p has unexpected order %d", 370170477Salc m_buddy, m_buddy->order)); 371170477Salc m_buddy->order = oind; 372170477Salc TAILQ_INSERT_HEAD(&fl[oind].pl, m_buddy, pageq); 373170477Salc fl[oind].lcnt++; 374170477Salc } 375170477Salc} 376170477Salc 377170477Salc/* 378170477Salc * Initialize a physical page and add it to the free lists. 379170477Salc */ 380170477Salcvoid 381170477Salcvm_phys_add_page(vm_paddr_t pa) 382170477Salc{ 383170477Salc vm_page_t m; 384170477Salc 385170477Salc cnt.v_page_count++; 386170477Salc m = vm_phys_paddr_to_vm_page(pa); 387170477Salc m->phys_addr = pa; 388217508Salc m->queue = PQ_NONE; 389170477Salc m->segind = vm_phys_paddr_to_segind(pa); 390170477Salc m->flags = PG_FREE; 391170477Salc KASSERT(m->order == VM_NFREEORDER, 392170477Salc ("vm_phys_add_page: page %p has unexpected order %d", 393170477Salc m, m->order)); 394170477Salc m->pool = VM_FREEPOOL_DEFAULT; 395170477Salc pmap_page_init(m); 396171451Salc mtx_lock(&vm_page_queue_free_mtx); 397172317Salc cnt.v_free_count++; 398170477Salc vm_phys_free_pages(m, 0); 399171451Salc mtx_unlock(&vm_page_queue_free_mtx); 400170477Salc} 401170477Salc 402170477Salc/* 403170477Salc * Allocate a contiguous, power of two-sized set of physical pages 404170477Salc * from the free lists. 405171451Salc * 406171451Salc * The free page queues must be locked. 407170477Salc */ 408170477Salcvm_page_t 409170477Salcvm_phys_alloc_pages(int pool, int order) 410170477Salc{ 411210327Sjchandra vm_page_t m; 412210327Sjchandra int flind; 413210327Sjchandra 414210327Sjchandra for (flind = 0; flind < vm_nfreelists; flind++) { 415210327Sjchandra m = vm_phys_alloc_freelist_pages(flind, pool, order); 416210327Sjchandra if (m != NULL) 417210327Sjchandra return (m); 418210327Sjchandra } 419210327Sjchandra return (NULL); 420210327Sjchandra} 421210327Sjchandra 422210327Sjchandra/* 423210327Sjchandra * Find and dequeue a free page on the given free list, with the 424210327Sjchandra * specified pool and order 425210327Sjchandra */ 426210327Sjchandravm_page_t 427210327Sjchandravm_phys_alloc_freelist_pages(int flind, int pool, int order) 428210327Sjchandra{ 429170477Salc struct vm_freelist *fl; 430170477Salc struct vm_freelist *alt; 431210550Sjhb int domain, oind, pind; 432170477Salc vm_page_t m; 433170477Salc 434210327Sjchandra KASSERT(flind < VM_NFREELIST, 435210327Sjchandra ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind)); 436170477Salc KASSERT(pool < VM_NFREEPOOL, 437210327Sjchandra ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 438170477Salc KASSERT(order < VM_NFREEORDER, 439210327Sjchandra ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); 440210550Sjhb 441210550Sjhb#if VM_NDOMAIN > 1 442210550Sjhb domain = PCPU_GET(domain); 443210550Sjhb#else 444210550Sjhb domain = 0; 445210550Sjhb#endif 446170477Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 447210550Sjhb fl = (*vm_phys_lookup_lists[domain][flind])[pool]; 448210327Sjchandra for (oind = order; oind < VM_NFREEORDER; oind++) { 449210327Sjchandra m = TAILQ_FIRST(&fl[oind].pl); 450210327Sjchandra if (m != NULL) { 451210327Sjchandra TAILQ_REMOVE(&fl[oind].pl, m, pageq); 452210327Sjchandra fl[oind].lcnt--; 453210327Sjchandra m->order = VM_NFREEORDER; 454210327Sjchandra vm_phys_split_pages(m, oind, fl, order); 455210327Sjchandra return (m); 456210327Sjchandra } 457210327Sjchandra } 458210327Sjchandra 459210327Sjchandra /* 460210327Sjchandra * The given pool was empty. Find the largest 461210327Sjchandra * contiguous, power-of-two-sized set of pages in any 462210327Sjchandra * pool. Transfer these pages to the given pool, and 463210327Sjchandra * use them to satisfy the allocation. 464210327Sjchandra */ 465210327Sjchandra for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { 466210327Sjchandra for (pind = 0; pind < VM_NFREEPOOL; pind++) { 467210550Sjhb alt = (*vm_phys_lookup_lists[domain][flind])[pind]; 468210327Sjchandra m = TAILQ_FIRST(&alt[oind].pl); 469170477Salc if (m != NULL) { 470210327Sjchandra TAILQ_REMOVE(&alt[oind].pl, m, pageq); 471210327Sjchandra alt[oind].lcnt--; 472170477Salc m->order = VM_NFREEORDER; 473210327Sjchandra vm_phys_set_pool(pool, m, oind); 474170477Salc vm_phys_split_pages(m, oind, fl, order); 475170477Salc return (m); 476170477Salc } 477170477Salc } 478170477Salc } 479170477Salc return (NULL); 480170477Salc} 481170477Salc 482170477Salc/* 483170477Salc * Allocate physical memory from phys_avail[]. 484170477Salc */ 485170477Salcvm_paddr_t 486170477Salcvm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment) 487170477Salc{ 488170477Salc vm_paddr_t pa; 489170477Salc int i; 490170477Salc 491170477Salc size = round_page(size); 492170477Salc for (i = 0; phys_avail[i + 1] != 0; i += 2) { 493170477Salc if (phys_avail[i + 1] - phys_avail[i] < size) 494170477Salc continue; 495170477Salc pa = phys_avail[i]; 496170477Salc phys_avail[i] += size; 497170477Salc return (pa); 498170477Salc } 499170477Salc panic("vm_phys_bootstrap_alloc"); 500170477Salc} 501170477Salc 502170477Salc/* 503170477Salc * Find the vm_page corresponding to the given physical address. 504170477Salc */ 505170477Salcvm_page_t 506170477Salcvm_phys_paddr_to_vm_page(vm_paddr_t pa) 507170477Salc{ 508170477Salc struct vm_phys_seg *seg; 509170477Salc int segind; 510170477Salc 511170477Salc for (segind = 0; segind < vm_phys_nsegs; segind++) { 512170477Salc seg = &vm_phys_segs[segind]; 513170477Salc if (pa >= seg->start && pa < seg->end) 514170477Salc return (&seg->first_page[atop(pa - seg->start)]); 515170477Salc } 516194459Sthompsa return (NULL); 517170477Salc} 518170477Salc 519170477Salc/* 520170477Salc * Find the segment containing the given physical address. 521170477Salc */ 522170477Salcstatic int 523170477Salcvm_phys_paddr_to_segind(vm_paddr_t pa) 524170477Salc{ 525170477Salc struct vm_phys_seg *seg; 526170477Salc int segind; 527170477Salc 528170477Salc for (segind = 0; segind < vm_phys_nsegs; segind++) { 529170477Salc seg = &vm_phys_segs[segind]; 530170477Salc if (pa >= seg->start && pa < seg->end) 531170477Salc return (segind); 532170477Salc } 533170477Salc panic("vm_phys_paddr_to_segind: paddr %#jx is not in any segment" , 534170477Salc (uintmax_t)pa); 535170477Salc} 536170477Salc 537170477Salc/* 538170477Salc * Free a contiguous, power of two-sized set of physical pages. 539171451Salc * 540171451Salc * The free page queues must be locked. 541170477Salc */ 542170477Salcvoid 543170477Salcvm_phys_free_pages(vm_page_t m, int order) 544170477Salc{ 545170477Salc struct vm_freelist *fl; 546170477Salc struct vm_phys_seg *seg; 547170477Salc vm_paddr_t pa, pa_buddy; 548170477Salc vm_page_t m_buddy; 549170477Salc 550170477Salc KASSERT(m->order == VM_NFREEORDER, 551171451Salc ("vm_phys_free_pages: page %p has unexpected order %d", 552170477Salc m, m->order)); 553170477Salc KASSERT(m->pool < VM_NFREEPOOL, 554171451Salc ("vm_phys_free_pages: page %p has unexpected pool %d", 555170477Salc m, m->pool)); 556170477Salc KASSERT(order < VM_NFREEORDER, 557171451Salc ("vm_phys_free_pages: order %d is out of range", order)); 558170477Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 559170477Salc pa = VM_PAGE_TO_PHYS(m); 560170477Salc seg = &vm_phys_segs[m->segind]; 561170477Salc while (order < VM_NFREEORDER - 1) { 562170477Salc pa_buddy = pa ^ (1 << (PAGE_SHIFT + order)); 563170477Salc if (pa_buddy < seg->start || 564170477Salc pa_buddy >= seg->end) 565170477Salc break; 566170477Salc m_buddy = &seg->first_page[atop(pa_buddy - seg->start)]; 567170477Salc if (m_buddy->order != order) 568170477Salc break; 569170477Salc fl = (*seg->free_queues)[m_buddy->pool]; 570170477Salc TAILQ_REMOVE(&fl[m_buddy->order].pl, m_buddy, pageq); 571170477Salc fl[m_buddy->order].lcnt--; 572170477Salc m_buddy->order = VM_NFREEORDER; 573170477Salc if (m_buddy->pool != m->pool) 574170477Salc vm_phys_set_pool(m->pool, m_buddy, order); 575170477Salc order++; 576170477Salc pa &= ~((1 << (PAGE_SHIFT + order)) - 1); 577170477Salc m = &seg->first_page[atop(pa - seg->start)]; 578170477Salc } 579170477Salc m->order = order; 580170477Salc fl = (*seg->free_queues)[m->pool]; 581170477Salc TAILQ_INSERT_TAIL(&fl[order].pl, m, pageq); 582170477Salc fl[order].lcnt++; 583170477Salc} 584170477Salc 585170477Salc/* 586170477Salc * Set the pool for a contiguous, power of two-sized set of physical pages. 587170477Salc */ 588172317Salcvoid 589170477Salcvm_phys_set_pool(int pool, vm_page_t m, int order) 590170477Salc{ 591170477Salc vm_page_t m_tmp; 592170477Salc 593170477Salc for (m_tmp = m; m_tmp < &m[1 << order]; m_tmp++) 594170477Salc m_tmp->pool = pool; 595170477Salc} 596170477Salc 597170477Salc/* 598174825Salc * Search for the given physical page "m" in the free lists. If the search 599174825Salc * succeeds, remove "m" from the free lists and return TRUE. Otherwise, return 600174825Salc * FALSE, indicating that "m" is not in the free lists. 601172317Salc * 602172317Salc * The free page queues must be locked. 603170477Salc */ 604174821Salcboolean_t 605172317Salcvm_phys_unfree_page(vm_page_t m) 606172317Salc{ 607172317Salc struct vm_freelist *fl; 608172317Salc struct vm_phys_seg *seg; 609172317Salc vm_paddr_t pa, pa_half; 610172317Salc vm_page_t m_set, m_tmp; 611172317Salc int order; 612172317Salc 613172317Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 614172317Salc 615172317Salc /* 616172317Salc * First, find the contiguous, power of two-sized set of free 617172317Salc * physical pages containing the given physical page "m" and 618172317Salc * assign it to "m_set". 619172317Salc */ 620172317Salc seg = &vm_phys_segs[m->segind]; 621172317Salc for (m_set = m, order = 0; m_set->order == VM_NFREEORDER && 622174799Salc order < VM_NFREEORDER - 1; ) { 623172317Salc order++; 624172317Salc pa = m->phys_addr & (~(vm_paddr_t)0 << (PAGE_SHIFT + order)); 625177932Salc if (pa >= seg->start) 626174821Salc m_set = &seg->first_page[atop(pa - seg->start)]; 627174821Salc else 628174821Salc return (FALSE); 629172317Salc } 630174821Salc if (m_set->order < order) 631174821Salc return (FALSE); 632174821Salc if (m_set->order == VM_NFREEORDER) 633174821Salc return (FALSE); 634172317Salc KASSERT(m_set->order < VM_NFREEORDER, 635172317Salc ("vm_phys_unfree_page: page %p has unexpected order %d", 636172317Salc m_set, m_set->order)); 637172317Salc 638172317Salc /* 639172317Salc * Next, remove "m_set" from the free lists. Finally, extract 640172317Salc * "m" from "m_set" using an iterative algorithm: While "m_set" 641172317Salc * is larger than a page, shrink "m_set" by returning the half 642172317Salc * of "m_set" that does not contain "m" to the free lists. 643172317Salc */ 644172317Salc fl = (*seg->free_queues)[m_set->pool]; 645172317Salc order = m_set->order; 646172317Salc TAILQ_REMOVE(&fl[order].pl, m_set, pageq); 647172317Salc fl[order].lcnt--; 648172317Salc m_set->order = VM_NFREEORDER; 649172317Salc while (order > 0) { 650172317Salc order--; 651172317Salc pa_half = m_set->phys_addr ^ (1 << (PAGE_SHIFT + order)); 652172317Salc if (m->phys_addr < pa_half) 653172317Salc m_tmp = &seg->first_page[atop(pa_half - seg->start)]; 654172317Salc else { 655172317Salc m_tmp = m_set; 656172317Salc m_set = &seg->first_page[atop(pa_half - seg->start)]; 657172317Salc } 658172317Salc m_tmp->order = order; 659172317Salc TAILQ_INSERT_HEAD(&fl[order].pl, m_tmp, pageq); 660172317Salc fl[order].lcnt++; 661172317Salc } 662172317Salc KASSERT(m_set == m, ("vm_phys_unfree_page: fatal inconsistency")); 663174821Salc return (TRUE); 664172317Salc} 665172317Salc 666172317Salc/* 667172317Salc * Try to zero one physical page. Used by an idle priority thread. 668172317Salc */ 669170477Salcboolean_t 670170477Salcvm_phys_zero_pages_idle(void) 671170477Salc{ 672172317Salc static struct vm_freelist *fl = vm_phys_free_queues[0][0]; 673172317Salc static int flind, oind, pind; 674170477Salc vm_page_t m, m_tmp; 675170477Salc 676170477Salc mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); 677172317Salc for (;;) { 678172317Salc TAILQ_FOREACH_REVERSE(m, &fl[oind].pl, pglist, pageq) { 679172317Salc for (m_tmp = m; m_tmp < &m[1 << oind]; m_tmp++) { 680172317Salc if ((m_tmp->flags & (PG_CACHED | PG_ZERO)) == 0) { 681172317Salc vm_phys_unfree_page(m_tmp); 682172317Salc cnt.v_free_count--; 683172317Salc mtx_unlock(&vm_page_queue_free_mtx); 684172317Salc pmap_zero_page_idle(m_tmp); 685172317Salc m_tmp->flags |= PG_ZERO; 686172317Salc mtx_lock(&vm_page_queue_free_mtx); 687172317Salc cnt.v_free_count++; 688172317Salc vm_phys_free_pages(m_tmp, 0); 689172317Salc vm_page_zero_count++; 690172317Salc cnt_prezero++; 691172317Salc return (TRUE); 692170477Salc } 693170477Salc } 694170477Salc } 695172317Salc oind++; 696172317Salc if (oind == VM_NFREEORDER) { 697172317Salc oind = 0; 698172317Salc pind++; 699172317Salc if (pind == VM_NFREEPOOL) { 700172317Salc pind = 0; 701172317Salc flind++; 702172317Salc if (flind == vm_nfreelists) 703172317Salc flind = 0; 704172317Salc } 705172317Salc fl = vm_phys_free_queues[flind][pind]; 706172317Salc } 707170477Salc } 708170477Salc} 709170477Salc 710170477Salc/* 711170818Salc * Allocate a contiguous set of physical pages of the given size 712170818Salc * "npages" from the free lists. All of the physical pages must be at 713170818Salc * or above the given physical address "low" and below the given 714170818Salc * physical address "high". The given value "alignment" determines the 715170818Salc * alignment of the first physical page in the set. If the given value 716170818Salc * "boundary" is non-zero, then the set of physical pages cannot cross 717170818Salc * any physical address boundary that is a multiple of that value. Both 718170477Salc * "alignment" and "boundary" must be a power of two. 719170477Salc */ 720170477Salcvm_page_t 721170477Salcvm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high, 722195649Salc unsigned long alignment, unsigned long boundary) 723170477Salc{ 724170477Salc struct vm_freelist *fl; 725170477Salc struct vm_phys_seg *seg; 726210327Sjchandra struct vnode *vp; 727170477Salc vm_paddr_t pa, pa_last, size; 728194607Salc vm_page_t deferred_vdrop_list, m, m_ret; 729210550Sjhb int domain, flind, i, oind, order, pind; 730170477Salc 731210550Sjhb#if VM_NDOMAIN > 1 732210550Sjhb domain = PCPU_GET(domain); 733210550Sjhb#else 734210550Sjhb domain = 0; 735210550Sjhb#endif 736170477Salc size = npages << PAGE_SHIFT; 737170477Salc KASSERT(size != 0, 738170477Salc ("vm_phys_alloc_contig: size must not be 0")); 739170477Salc KASSERT((alignment & (alignment - 1)) == 0, 740170477Salc ("vm_phys_alloc_contig: alignment must be a power of 2")); 741170477Salc KASSERT((boundary & (boundary - 1)) == 0, 742170477Salc ("vm_phys_alloc_contig: boundary must be a power of 2")); 743194607Salc deferred_vdrop_list = NULL; 744170477Salc /* Compute the queue that is the best fit for npages. */ 745170477Salc for (order = 0; (1 << order) < npages; order++); 746170477Salc mtx_lock(&vm_page_queue_free_mtx); 747177956Salc#if VM_NRESERVLEVEL > 0 748177956Salcretry: 749177956Salc#endif 750170477Salc for (flind = 0; flind < vm_nfreelists; flind++) { 751170477Salc for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) { 752170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) { 753210550Sjhb fl = (*vm_phys_lookup_lists[domain][flind]) 754210550Sjhb [pind]; 755170477Salc TAILQ_FOREACH(m_ret, &fl[oind].pl, pageq) { 756170477Salc /* 757170477Salc * A free list may contain physical pages 758170477Salc * from one or more segments. 759170477Salc */ 760170477Salc seg = &vm_phys_segs[m_ret->segind]; 761170477Salc if (seg->start > high || 762170477Salc low >= seg->end) 763170477Salc continue; 764170477Salc 765170477Salc /* 766170477Salc * Is the size of this allocation request 767170477Salc * larger than the largest block size? 768170477Salc */ 769170477Salc if (order >= VM_NFREEORDER) { 770170477Salc /* 771170477Salc * Determine if a sufficient number 772170477Salc * of subsequent blocks to satisfy 773170477Salc * the allocation request are free. 774170477Salc */ 775170477Salc pa = VM_PAGE_TO_PHYS(m_ret); 776170477Salc pa_last = pa + size; 777170477Salc for (;;) { 778170477Salc pa += 1 << (PAGE_SHIFT + VM_NFREEORDER - 1); 779170477Salc if (pa >= pa_last) 780170477Salc break; 781170477Salc if (pa < seg->start || 782170477Salc pa >= seg->end) 783170477Salc break; 784170477Salc m = &seg->first_page[atop(pa - seg->start)]; 785170477Salc if (m->order != VM_NFREEORDER - 1) 786170477Salc break; 787170477Salc } 788170477Salc /* If not, continue to the next block. */ 789170477Salc if (pa < pa_last) 790170477Salc continue; 791170477Salc } 792170477Salc 793170477Salc /* 794170477Salc * Determine if the blocks are within the given range, 795170477Salc * satisfy the given alignment, and do not cross the 796170477Salc * given boundary. 797170477Salc */ 798170477Salc pa = VM_PAGE_TO_PHYS(m_ret); 799170477Salc if (pa >= low && 800170477Salc pa + size <= high && 801170477Salc (pa & (alignment - 1)) == 0 && 802170477Salc ((pa ^ (pa + size - 1)) & ~(boundary - 1)) == 0) 803170477Salc goto done; 804170477Salc } 805170477Salc } 806170477Salc } 807170477Salc } 808177956Salc#if VM_NRESERVLEVEL > 0 809177956Salc if (vm_reserv_reclaim_contig(size, low, high, alignment, boundary)) 810177956Salc goto retry; 811177956Salc#endif 812170477Salc mtx_unlock(&vm_page_queue_free_mtx); 813170477Salc return (NULL); 814170477Salcdone: 815170477Salc for (m = m_ret; m < &m_ret[npages]; m = &m[1 << oind]) { 816170477Salc fl = (*seg->free_queues)[m->pool]; 817170477Salc TAILQ_REMOVE(&fl[m->order].pl, m, pageq); 818170477Salc fl[m->order].lcnt--; 819170477Salc m->order = VM_NFREEORDER; 820170477Salc } 821170477Salc if (m_ret->pool != VM_FREEPOOL_DEFAULT) 822170477Salc vm_phys_set_pool(VM_FREEPOOL_DEFAULT, m_ret, oind); 823170477Salc fl = (*seg->free_queues)[m_ret->pool]; 824170477Salc vm_phys_split_pages(m_ret, oind, fl, order); 825170477Salc for (i = 0; i < npages; i++) { 826170477Salc m = &m_ret[i]; 827210327Sjchandra vp = vm_page_alloc_init(m); 828210327Sjchandra if (vp != NULL) { 829210327Sjchandra /* 830210327Sjchandra * Enqueue the vnode for deferred vdrop(). 831210327Sjchandra * 832210327Sjchandra * Unmanaged pages don't use "pageq", so it 833210327Sjchandra * can be safely abused to construct a short- 834210327Sjchandra * lived queue of vnodes. 835210327Sjchandra */ 836210327Sjchandra m->pageq.tqe_prev = (void *)vp; 837210327Sjchandra m->pageq.tqe_next = deferred_vdrop_list; 838210327Sjchandra deferred_vdrop_list = m; 839172317Salc } 840170477Salc } 841170477Salc for (; i < roundup2(npages, 1 << imin(oind, order)); i++) { 842170477Salc m = &m_ret[i]; 843170477Salc KASSERT(m->order == VM_NFREEORDER, 844170477Salc ("vm_phys_alloc_contig: page %p has unexpected order %d", 845170477Salc m, m->order)); 846171451Salc vm_phys_free_pages(m, 0); 847170477Salc } 848170477Salc mtx_unlock(&vm_page_queue_free_mtx); 849194607Salc while (deferred_vdrop_list != NULL) { 850194607Salc vdrop((struct vnode *)deferred_vdrop_list->pageq.tqe_prev); 851194607Salc deferred_vdrop_list = deferred_vdrop_list->pageq.tqe_next; 852194607Salc } 853170477Salc return (m_ret); 854170477Salc} 855170477Salc 856170477Salc#ifdef DDB 857170477Salc/* 858170477Salc * Show the number of physical pages in each of the free lists. 859170477Salc */ 860170477SalcDB_SHOW_COMMAND(freepages, db_show_freepages) 861170477Salc{ 862170477Salc struct vm_freelist *fl; 863170477Salc int flind, oind, pind; 864170477Salc 865170477Salc for (flind = 0; flind < vm_nfreelists; flind++) { 866170477Salc db_printf("FREE LIST %d:\n" 867170477Salc "\n ORDER (SIZE) | NUMBER" 868170477Salc "\n ", flind); 869170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) 870170477Salc db_printf(" | POOL %d", pind); 871170477Salc db_printf("\n-- "); 872170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) 873170477Salc db_printf("-- -- "); 874170477Salc db_printf("--\n"); 875170477Salc for (oind = VM_NFREEORDER - 1; oind >= 0; oind--) { 876170477Salc db_printf(" %2.2d (%6.6dK)", oind, 877170477Salc 1 << (PAGE_SHIFT - 10 + oind)); 878170477Salc for (pind = 0; pind < VM_NFREEPOOL; pind++) { 879170477Salc fl = vm_phys_free_queues[flind][pind]; 880170477Salc db_printf(" | %6.6d", fl[oind].lcnt); 881170477Salc } 882170477Salc db_printf("\n"); 883170477Salc } 884170477Salc db_printf("\n"); 885170477Salc } 886170477Salc} 887170477Salc#endif 888