1/*- 2 * Copyright (c) 2002-2006 Rice University 3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4 * All rights reserved. 5 * 6 * This software was developed for the FreeBSD Project by Alan L. Cox, 7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8 * --- 23 unchanged lines hidden (view full) --- 32/* 33 * Physical memory system implementation 34 * 35 * Any external functions defined by this module are only to be used by the 36 * virtual memory system. 37 */ 38 39#include <sys/cdefs.h> |
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 285634 2015-07-16 14:41:58Z kib $"); |
41 42#include "opt_ddb.h" 43#include "opt_vm.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/lock.h> 48#include <sys/kernel.h> --- 33 unchanged lines hidden (view full) --- 82 vm_page_t first_page; 83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS]; 84static struct mtx vm_phys_fictitious_reg_mtx; 85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); 86 87static struct vm_freelist 88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 89 |
90static int vm_nfreelists; |
91 |
92/* 93 * Provides the mapping from VM_FREELIST_* to free list indices (flind). 94 */ 95static int vm_freelist_to_flind[VM_NFREELIST]; 96 97CTASSERT(VM_FREELIST_DEFAULT == 0); 98 99#ifdef VM_FREELIST_ISADMA 100#define VM_ISADMA_BOUNDARY 16777216 101#endif 102#ifdef VM_FREELIST_DMA32 103#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32) 104#endif 105 106/* 107 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about 108 * the ordering of the free list boundaries. 109 */ 110#if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY) 111CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY); 112#endif 113#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY) 114CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY); 115#endif 116 |
117static int cnt_prezero; 118SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 119 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 120 121static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 122SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 123 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 124 125static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 126SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 127 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 128 129SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, 130 &vm_ndomains, 0, "Number of physical memory domains available."); 131 132static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, 133 int order); |
134static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); 135static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); |
136static int vm_phys_paddr_to_segind(vm_paddr_t pa); 137static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 138 int order); 139 140static __inline int 141vm_rr_selectdomain(void) 142{ 143#if MAXMEMDOM > 1 --- 118 unchanged lines hidden (view full) --- 262 fl[order].lcnt--; 263 m->order = VM_NFREEORDER; 264} 265 266/* 267 * Create a physical memory segment. 268 */ 269static void |
270_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain) |
271{ 272 struct vm_phys_seg *seg; 273 274 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 275 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 276 KASSERT(domain < vm_ndomains, 277 ("vm_phys_create_seg: invalid domain provided")); 278 seg = &vm_phys_segs[vm_phys_nsegs++]; 279 while (seg > vm_phys_segs && (seg - 1)->start >= end) { 280 *seg = *(seg - 1); 281 seg--; 282 } 283 seg->start = start; 284 seg->end = end; 285 seg->domain = domain; |
286} 287 288static void |
289vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end) |
290{ 291 int i; 292 293 if (mem_affinity == NULL) { |
294 _vm_phys_create_seg(start, end, 0); |
295 return; 296 } 297 298 for (i = 0;; i++) { 299 if (mem_affinity[i].end == 0) 300 panic("Reached end of affinity info"); 301 if (mem_affinity[i].end <= start) 302 continue; 303 if (mem_affinity[i].start > start) 304 panic("No affinity info for start %jx", 305 (uintmax_t)start); 306 if (mem_affinity[i].end >= end) { |
307 _vm_phys_create_seg(start, end, |
308 mem_affinity[i].domain); 309 break; 310 } |
311 _vm_phys_create_seg(start, mem_affinity[i].end, |
312 mem_affinity[i].domain); 313 start = mem_affinity[i].end; 314 } 315} 316 317/* 318 * Add a physical memory segment. 319 */ 320void 321vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) 322{ |
323 vm_paddr_t paddr; |
324 325 KASSERT((start & PAGE_MASK) == 0, 326 ("vm_phys_define_seg: start is not page aligned")); 327 KASSERT((end & PAGE_MASK) == 0, 328 ("vm_phys_define_seg: end is not page aligned")); |
329 330 /* 331 * Split the physical memory segment if it spans two or more free 332 * list boundaries. 333 */ 334 paddr = start; |
335#ifdef VM_FREELIST_ISADMA |
336 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) { 337 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY); 338 paddr = VM_ISADMA_BOUNDARY; 339 } |
340#endif |
341#ifdef VM_FREELIST_LOWMEM 342 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) { 343 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY); 344 paddr = VM_LOWMEM_BOUNDARY; 345 } |
346#endif |
347#ifdef VM_FREELIST_DMA32 348 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) { 349 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY); 350 paddr = VM_DMA32_BOUNDARY; 351 } 352#endif 353 vm_phys_create_seg(paddr, end); |
354} 355 356/* 357 * Initialize the physical memory allocator. |
358 * 359 * Requires that vm_page_array is initialized! |
360 */ 361void 362vm_phys_init(void) 363{ 364 struct vm_freelist *fl; 365 struct vm_phys_seg *seg; |
366 u_long npages; 367 int dom, flind, freelist, oind, pind, segind; 368 369 /* 370 * Compute the number of free lists, and generate the mapping from the 371 * manifest constants VM_FREELIST_* to the free list indices. 372 * 373 * Initially, the entries of vm_freelist_to_flind[] are set to either 374 * 0 or 1 to indicate which free lists should be created. 375 */ 376 npages = 0; 377 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { 378 seg = &vm_phys_segs[segind]; 379#ifdef VM_FREELIST_ISADMA 380 if (seg->end <= VM_ISADMA_BOUNDARY) 381 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1; 382 else |
383#endif |
384#ifdef VM_FREELIST_LOWMEM 385 if (seg->end <= VM_LOWMEM_BOUNDARY) 386 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1; 387 else 388#endif 389#ifdef VM_FREELIST_DMA32 390 if ( 391#ifdef VM_DMA32_NPAGES_THRESHOLD 392 /* 393 * Create the DMA32 free list only if the amount of 394 * physical memory above physical address 4G exceeds the 395 * given threshold. 396 */ 397 npages > VM_DMA32_NPAGES_THRESHOLD && 398#endif 399 seg->end <= VM_DMA32_BOUNDARY) 400 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1; 401 else 402#endif 403 { 404 npages += atop(seg->end - seg->start); 405 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1; 406 } 407 } 408 /* Change each entry into a running total of the free lists. */ 409 for (freelist = 1; freelist < VM_NFREELIST; freelist++) { 410 vm_freelist_to_flind[freelist] += 411 vm_freelist_to_flind[freelist - 1]; 412 } 413 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1]; 414 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists")); 415 /* Change each entry into a free list index. */ 416 for (freelist = 0; freelist < VM_NFREELIST; freelist++) 417 vm_freelist_to_flind[freelist]--; |
418 |
419 /* 420 * Initialize the first_page and free_queues fields of each physical 421 * memory segment. 422 */ |
423#ifdef VM_PHYSSEG_SPARSE |
424 npages = 0; |
425#endif 426 for (segind = 0; segind < vm_phys_nsegs; segind++) { 427 seg = &vm_phys_segs[segind]; 428#ifdef VM_PHYSSEG_SPARSE |
429 seg->first_page = &vm_page_array[npages]; 430 npages += atop(seg->end - seg->start); |
431#else 432 seg->first_page = PHYS_TO_VM_PAGE(seg->start); 433#endif |
434#ifdef VM_FREELIST_ISADMA 435 if (seg->end <= VM_ISADMA_BOUNDARY) { 436 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA]; 437 KASSERT(flind >= 0, 438 ("vm_phys_init: ISADMA flind < 0")); 439 } else 440#endif 441#ifdef VM_FREELIST_LOWMEM 442 if (seg->end <= VM_LOWMEM_BOUNDARY) { 443 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM]; 444 KASSERT(flind >= 0, 445 ("vm_phys_init: LOWMEM flind < 0")); 446 } else 447#endif 448#ifdef VM_FREELIST_DMA32 449 if (seg->end <= VM_DMA32_BOUNDARY) { 450 flind = vm_freelist_to_flind[VM_FREELIST_DMA32]; 451 KASSERT(flind >= 0, 452 ("vm_phys_init: DMA32 flind < 0")); 453 } else 454#endif 455 { 456 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT]; 457 KASSERT(flind >= 0, 458 ("vm_phys_init: DEFAULT flind < 0")); 459 } 460 seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; |
461 } |
462 463 /* 464 * Initialize the free queues. 465 */ |
466 for (dom = 0; dom < vm_ndomains; dom++) { 467 for (flind = 0; flind < vm_nfreelists; flind++) { 468 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 469 fl = vm_phys_free_queues[dom][flind][pind]; 470 for (oind = 0; oind < VM_NFREEORDER; oind++) 471 TAILQ_INIT(&fl[oind].pl); 472 } 473 } --- 73 unchanged lines hidden (view full) --- 547 if (m != NULL) 548 return (m); 549 } 550 } 551 return (NULL); 552} 553 554/* |
555 * Allocate a contiguous, power of two-sized set of physical pages from the 556 * specified free list. The free list must be specified using one of the 557 * manifest constants VM_FREELIST_*. 558 * 559 * The free page queues must be locked. |
560 */ 561vm_page_t |
562vm_phys_alloc_freelist_pages(int freelist, int pool, int order) |
563{ 564 vm_page_t m; 565 int dom, domain; 566 |
567 KASSERT(freelist < VM_NFREELIST, 568 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", 569 freelist)); |
570 KASSERT(pool < VM_NFREEPOOL, 571 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 572 KASSERT(order < VM_NFREEORDER, 573 ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); |
574 for (dom = 0; dom < vm_ndomains; dom++) { 575 domain = vm_rr_selectdomain(); |
576 m = vm_phys_alloc_domain_pages(domain, 577 vm_freelist_to_flind[freelist], pool, order); |
578 if (m != NULL) 579 return (m); 580 } 581 return (NULL); 582} 583 584static vm_page_t 585vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order) --- 537 unchanged lines hidden --- |