vm_phys.c (274556) | vm_phys.c (276439) |
---|---|
1/*- 2 * Copyright (c) 2002-2006 Rice University 3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4 * All rights reserved. 5 * 6 * This software was developed for the FreeBSD Project by Alan L. Cox, 7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8 * --- 23 unchanged lines hidden (view full) --- 32/* 33 * Physical memory system implementation 34 * 35 * Any external functions defined by this module are only to be used by the 36 * virtual memory system. 37 */ 38 39#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 2002-2006 Rice University 3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu> 4 * All rights reserved. 5 * 6 * This software was developed for the FreeBSD Project by Alan L. Cox, 7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro. 8 * --- 23 unchanged lines hidden (view full) --- 32/* 33 * Physical memory system implementation 34 * 35 * Any external functions defined by this module are only to be used by the 36 * virtual memory system. 37 */ 38 39#include <sys/cdefs.h> |
40__FBSDID("$FreeBSD: head/sys/vm/vm_phys.c 274556 2014-11-15 23:40:44Z alc $"); | 40__FBSDID("$FreeBSD: head/sys/vm/vm_phys.c 276439 2014-12-31 00:54:38Z alc $"); |
41 42#include "opt_ddb.h" 43#include "opt_vm.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/lock.h> 48#include <sys/kernel.h> --- 47 unchanged lines hidden (view full) --- 96 vm_phys_fictitious_cmp); 97 98static struct rwlock vm_phys_fictitious_reg_lock; 99MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); 100 101static struct vm_freelist 102 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 103 | 41 42#include "opt_ddb.h" 43#include "opt_vm.h" 44 45#include <sys/param.h> 46#include <sys/systm.h> 47#include <sys/lock.h> 48#include <sys/kernel.h> --- 47 unchanged lines hidden (view full) --- 96 vm_phys_fictitious_cmp); 97 98static struct rwlock vm_phys_fictitious_reg_lock; 99MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages"); 100 101static struct vm_freelist 102 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER]; 103 |
104static int vm_nfreelists = VM_FREELIST_DEFAULT + 1; | 104static int vm_nfreelists; |
105 | 105 |
106/* 107 * Provides the mapping from VM_FREELIST_* to free list indices (flind). 108 */ 109static int vm_freelist_to_flind[VM_NFREELIST]; 110 111CTASSERT(VM_FREELIST_DEFAULT == 0); 112 113#ifdef VM_FREELIST_ISADMA 114#define VM_ISADMA_BOUNDARY 16777216 115#endif 116#ifdef VM_FREELIST_DMA32 117#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32) 118#endif 119 120/* 121 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about 122 * the ordering of the free list boundaries. 123 */ 124#if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY) 125CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY); 126#endif 127#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY) 128CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY); 129#endif 130 |
|
106static int cnt_prezero; 107SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 108 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 109 110static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 111SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 112 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 113 114static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 115SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 116 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 117 118SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, 119 &vm_ndomains, 0, "Number of physical memory domains available."); 120 121static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, 122 int order); | 131static int cnt_prezero; 132SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD, 133 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time"); 134 135static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS); 136SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD, 137 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info"); 138 139static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS); 140SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD, 141 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info"); 142 143SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD, 144 &vm_ndomains, 0, "Number of physical memory domains available."); 145 146static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool, 147 int order); |
123static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, 124 int domain); 125static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind); | 148static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain); 149static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end); |
126static int vm_phys_paddr_to_segind(vm_paddr_t pa); 127static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 128 int order); 129 130/* 131 * Red-black tree helpers for vm fictitious range management. 132 */ 133static inline int --- 159 unchanged lines hidden (view full) --- 293 fl[order].lcnt--; 294 m->order = VM_NFREEORDER; 295} 296 297/* 298 * Create a physical memory segment. 299 */ 300static void | 150static int vm_phys_paddr_to_segind(vm_paddr_t pa); 151static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, 152 int order); 153 154/* 155 * Red-black tree helpers for vm fictitious range management. 156 */ 157static inline int --- 159 unchanged lines hidden (view full) --- 317 fl[order].lcnt--; 318 m->order = VM_NFREEORDER; 319} 320 321/* 322 * Create a physical memory segment. 323 */ 324static void |
301_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain) | 325_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain) |
302{ 303 struct vm_phys_seg *seg; 304 305 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 306 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 307 KASSERT(domain < vm_ndomains, 308 ("vm_phys_create_seg: invalid domain provided")); 309 seg = &vm_phys_segs[vm_phys_nsegs++]; 310 while (seg > vm_phys_segs && (seg - 1)->start >= end) { 311 *seg = *(seg - 1); 312 seg--; 313 } 314 seg->start = start; 315 seg->end = end; 316 seg->domain = domain; | 326{ 327 struct vm_phys_seg *seg; 328 329 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX, 330 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX")); 331 KASSERT(domain < vm_ndomains, 332 ("vm_phys_create_seg: invalid domain provided")); 333 seg = &vm_phys_segs[vm_phys_nsegs++]; 334 while (seg > vm_phys_segs && (seg - 1)->start >= end) { 335 *seg = *(seg - 1); 336 seg--; 337 } 338 seg->start = start; 339 seg->end = end; 340 seg->domain = domain; |
317 seg->free_queues = &vm_phys_free_queues[domain][flind]; | |
318} 319 320static void | 341} 342 343static void |
321vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind) | 344vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end) |
322{ 323 int i; 324 325 if (mem_affinity == NULL) { | 345{ 346 int i; 347 348 if (mem_affinity == NULL) { |
326 _vm_phys_create_seg(start, end, flind, 0); | 349 _vm_phys_create_seg(start, end, 0); |
327 return; 328 } 329 330 for (i = 0;; i++) { 331 if (mem_affinity[i].end == 0) 332 panic("Reached end of affinity info"); 333 if (mem_affinity[i].end <= start) 334 continue; 335 if (mem_affinity[i].start > start) 336 panic("No affinity info for start %jx", 337 (uintmax_t)start); 338 if (mem_affinity[i].end >= end) { | 350 return; 351 } 352 353 for (i = 0;; i++) { 354 if (mem_affinity[i].end == 0) 355 panic("Reached end of affinity info"); 356 if (mem_affinity[i].end <= start) 357 continue; 358 if (mem_affinity[i].start > start) 359 panic("No affinity info for start %jx", 360 (uintmax_t)start); 361 if (mem_affinity[i].end >= end) { |
339 _vm_phys_create_seg(start, end, flind, | 362 _vm_phys_create_seg(start, end, |
340 mem_affinity[i].domain); 341 break; 342 } | 363 mem_affinity[i].domain); 364 break; 365 } |
343 _vm_phys_create_seg(start, mem_affinity[i].end, flind, | 366 _vm_phys_create_seg(start, mem_affinity[i].end, |
344 mem_affinity[i].domain); 345 start = mem_affinity[i].end; 346 } 347} 348 349/* 350 * Add a physical memory segment. 351 */ 352void 353vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) 354{ | 367 mem_affinity[i].domain); 368 start = mem_affinity[i].end; 369 } 370} 371 372/* 373 * Add a physical memory segment. 374 */ 375void 376vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end) 377{ |
378 vm_paddr_t paddr; |
|
355 356 KASSERT((start & PAGE_MASK) == 0, 357 ("vm_phys_define_seg: start is not page aligned")); 358 KASSERT((end & PAGE_MASK) == 0, 359 ("vm_phys_define_seg: end is not page aligned")); | 379 380 KASSERT((start & PAGE_MASK) == 0, 381 ("vm_phys_define_seg: start is not page aligned")); 382 KASSERT((end & PAGE_MASK) == 0, 383 ("vm_phys_define_seg: end is not page aligned")); |
384 385 /* 386 * Split the physical memory segment if it spans two or more free 387 * list boundaries. 388 */ 389 paddr = start; |
|
360#ifdef VM_FREELIST_ISADMA | 390#ifdef VM_FREELIST_ISADMA |
361 if (start < 16777216) { 362 if (end > 16777216) { 363 vm_phys_create_seg(start, 16777216, 364 VM_FREELIST_ISADMA); 365 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT); 366 } else 367 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA); 368 if (VM_FREELIST_ISADMA >= vm_nfreelists) 369 vm_nfreelists = VM_FREELIST_ISADMA + 1; 370 } else | 391 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) { 392 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY); 393 paddr = VM_ISADMA_BOUNDARY; 394 } |
371#endif | 395#endif |
372#ifdef VM_FREELIST_HIGHMEM 373 if (end > VM_HIGHMEM_ADDRESS) { 374 if (start < VM_HIGHMEM_ADDRESS) { 375 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS, 376 VM_FREELIST_DEFAULT); 377 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end, 378 VM_FREELIST_HIGHMEM); 379 } else 380 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM); 381 if (VM_FREELIST_HIGHMEM >= vm_nfreelists) 382 vm_nfreelists = VM_FREELIST_HIGHMEM + 1; 383 } else | 396#ifdef VM_FREELIST_LOWMEM 397 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) { 398 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY); 399 paddr = VM_LOWMEM_BOUNDARY; 400 } |
384#endif | 401#endif |
385 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT); | 402#ifdef VM_FREELIST_DMA32 403 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) { 404 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY); 405 paddr = VM_DMA32_BOUNDARY; 406 } 407#endif 408 vm_phys_create_seg(paddr, end); |
386} 387 388/* 389 * Initialize the physical memory allocator. | 409} 410 411/* 412 * Initialize the physical memory allocator. |
413 * 414 * Requires that vm_page_array is initialized! |
|
390 */ 391void 392vm_phys_init(void) 393{ 394 struct vm_freelist *fl; 395 struct vm_phys_seg *seg; | 415 */ 416void 417vm_phys_init(void) 418{ 419 struct vm_freelist *fl; 420 struct vm_phys_seg *seg; |
396#ifdef VM_PHYSSEG_SPARSE 397 long pages; | 421 u_long npages; 422 int dom, flind, freelist, oind, pind, segind; 423 424 /* 425 * Compute the number of free lists, and generate the mapping from the 426 * manifest constants VM_FREELIST_* to the free list indices. 427 * 428 * Initially, the entries of vm_freelist_to_flind[] are set to either 429 * 0 or 1 to indicate which free lists should be created. 430 */ 431 npages = 0; 432 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) { 433 seg = &vm_phys_segs[segind]; 434#ifdef VM_FREELIST_ISADMA 435 if (seg->end <= VM_ISADMA_BOUNDARY) 436 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1; 437 else |
398#endif | 438#endif |
399 int dom, flind, oind, pind, segind; | 439#ifdef VM_FREELIST_LOWMEM 440 if (seg->end <= VM_LOWMEM_BOUNDARY) 441 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1; 442 else 443#endif 444#ifdef VM_FREELIST_DMA32 445 if ( 446#ifdef VM_DMA32_NPAGES_THRESHOLD 447 /* 448 * Create the DMA32 free list only if the amount of 449 * physical memory above physical address 4G exceeds the 450 * given threshold. 451 */ 452 npages > VM_DMA32_NPAGES_THRESHOLD && 453#endif 454 seg->end <= VM_DMA32_BOUNDARY) 455 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1; 456 else 457#endif 458 { 459 npages += atop(seg->end - seg->start); 460 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1; 461 } 462 } 463 /* Change each entry into a running total of the free lists. */ 464 for (freelist = 1; freelist < VM_NFREELIST; freelist++) { 465 vm_freelist_to_flind[freelist] += 466 vm_freelist_to_flind[freelist - 1]; 467 } 468 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1]; 469 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists")); 470 /* Change each entry into a free list index. */ 471 for (freelist = 0; freelist < VM_NFREELIST; freelist++) 472 vm_freelist_to_flind[freelist]--; |
400 | 473 |
474 /* 475 * Initialize the first_page and free_queues fields of each physical 476 * memory segment. 477 */ |
|
401#ifdef VM_PHYSSEG_SPARSE | 478#ifdef VM_PHYSSEG_SPARSE |
402 pages = 0; | 479 npages = 0; |
403#endif 404 for (segind = 0; segind < vm_phys_nsegs; segind++) { 405 seg = &vm_phys_segs[segind]; 406#ifdef VM_PHYSSEG_SPARSE | 480#endif 481 for (segind = 0; segind < vm_phys_nsegs; segind++) { 482 seg = &vm_phys_segs[segind]; 483#ifdef VM_PHYSSEG_SPARSE |
407 seg->first_page = &vm_page_array[pages]; 408 pages += atop(seg->end - seg->start); | 484 seg->first_page = &vm_page_array[npages]; 485 npages += atop(seg->end - seg->start); |
409#else 410 seg->first_page = PHYS_TO_VM_PAGE(seg->start); 411#endif | 486#else 487 seg->first_page = PHYS_TO_VM_PAGE(seg->start); 488#endif |
489#ifdef VM_FREELIST_ISADMA 490 if (seg->end <= VM_ISADMA_BOUNDARY) { 491 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA]; 492 KASSERT(flind >= 0, 493 ("vm_phys_init: ISADMA flind < 0")); 494 } else 495#endif 496#ifdef VM_FREELIST_LOWMEM 497 if (seg->end <= VM_LOWMEM_BOUNDARY) { 498 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM]; 499 KASSERT(flind >= 0, 500 ("vm_phys_init: LOWMEM flind < 0")); 501 } else 502#endif 503#ifdef VM_FREELIST_DMA32 504 if (seg->end <= VM_DMA32_BOUNDARY) { 505 flind = vm_freelist_to_flind[VM_FREELIST_DMA32]; 506 KASSERT(flind >= 0, 507 ("vm_phys_init: DMA32 flind < 0")); 508 } else 509#endif 510 { 511 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT]; 512 KASSERT(flind >= 0, 513 ("vm_phys_init: DEFAULT flind < 0")); 514 } 515 seg->free_queues = &vm_phys_free_queues[seg->domain][flind]; |
|
412 } | 516 } |
517 518 /* 519 * Initialize the free queues. 520 */ |
|
413 for (dom = 0; dom < vm_ndomains; dom++) { 414 for (flind = 0; flind < vm_nfreelists; flind++) { 415 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 416 fl = vm_phys_free_queues[dom][flind][pind]; 417 for (oind = 0; oind < VM_NFREEORDER; oind++) 418 TAILQ_INIT(&fl[oind].pl); 419 } 420 } 421 } | 521 for (dom = 0; dom < vm_ndomains; dom++) { 522 for (flind = 0; flind < vm_nfreelists; flind++) { 523 for (pind = 0; pind < VM_NFREEPOOL; pind++) { 524 fl = vm_phys_free_queues[dom][flind][pind]; 525 for (oind = 0; oind < VM_NFREEORDER; oind++) 526 TAILQ_INIT(&fl[oind].pl); 527 } 528 } 529 } |
530 |
|
422 rw_init(&vm_phys_fictitious_reg_lock, "vmfctr"); 423} 424 425/* 426 * Split a contiguous, power of two-sized set of physical pages. 427 */ 428static __inline void 429vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) --- 63 unchanged lines hidden (view full) --- 493 if (m != NULL) 494 return (m); 495 } 496 } 497 return (NULL); 498} 499 500/* | 531 rw_init(&vm_phys_fictitious_reg_lock, "vmfctr"); 532} 533 534/* 535 * Split a contiguous, power of two-sized set of physical pages. 536 */ 537static __inline void 538vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order) --- 63 unchanged lines hidden (view full) --- 602 if (m != NULL) 603 return (m); 604 } 605 } 606 return (NULL); 607} 608 609/* |
501 * Find and dequeue a free page on the given free list, with the 502 * specified pool and order | 610 * Allocate a contiguous, power of two-sized set of physical pages from the 611 * specified free list. The free list must be specified using one of the 612 * manifest constants VM_FREELIST_*. 613 * 614 * The free page queues must be locked. |
503 */ 504vm_page_t | 615 */ 616vm_page_t |
505vm_phys_alloc_freelist_pages(int flind, int pool, int order) | 617vm_phys_alloc_freelist_pages(int freelist, int pool, int order) |
506{ 507 vm_page_t m; 508 int dom, domain; 509 | 618{ 619 vm_page_t m; 620 int dom, domain; 621 |
510 KASSERT(flind < VM_NFREELIST, 511 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind)); | 622 KASSERT(freelist < VM_NFREELIST, 623 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", 624 freelist)); |
512 KASSERT(pool < VM_NFREEPOOL, 513 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 514 KASSERT(order < VM_NFREEORDER, 515 ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); | 625 KASSERT(pool < VM_NFREEPOOL, 626 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); 627 KASSERT(order < VM_NFREEORDER, 628 ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); |
516 | |
517 for (dom = 0; dom < vm_ndomains; dom++) { 518 domain = vm_rr_selectdomain(); | 629 for (dom = 0; dom < vm_ndomains; dom++) { 630 domain = vm_rr_selectdomain(); |
519 m = vm_phys_alloc_domain_pages(domain, flind, pool, order); | 631 m = vm_phys_alloc_domain_pages(domain, 632 vm_freelist_to_flind[freelist], pool, order); |
520 if (m != NULL) 521 return (m); 522 } 523 return (NULL); 524} 525 526static vm_page_t 527vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order) --- 611 unchanged lines hidden --- | 633 if (m != NULL) 634 return (m); 635 } 636 return (NULL); 637} 638 639static vm_page_t 640vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order) --- 611 unchanged lines hidden --- |