Deleted Added
sdiff udiff text old ( 274556 ) new ( 276439 )
full compact
vm_phys.c (274556) vm_phys.c (276439)
1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *

--- 23 unchanged lines hidden (view full) ---

32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *

--- 23 unchanged lines hidden (view full) ---

32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
40__FBSDID("$FreeBSD: head/sys/vm/vm_phys.c 274556 2014-11-15 23:40:44Z alc $");
40__FBSDID("$FreeBSD: head/sys/vm/vm_phys.c 276439 2014-12-31 00:54:38Z alc $");
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>

--- 47 unchanged lines hidden (view full) ---

96 vm_phys_fictitious_cmp);
97
98static struct rwlock vm_phys_fictitious_reg_lock;
99MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
100
101static struct vm_freelist
102 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
103
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>

--- 47 unchanged lines hidden (view full) ---

96 vm_phys_fictitious_cmp);
97
98static struct rwlock vm_phys_fictitious_reg_lock;
99MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
100
101static struct vm_freelist
102 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
103
104static int vm_nfreelists = VM_FREELIST_DEFAULT + 1;
104static int vm_nfreelists;
105
105
106/*
107 * Provides the mapping from VM_FREELIST_* to free list indices (flind).
108 */
109static int vm_freelist_to_flind[VM_NFREELIST];
110
111CTASSERT(VM_FREELIST_DEFAULT == 0);
112
113#ifdef VM_FREELIST_ISADMA
114#define VM_ISADMA_BOUNDARY 16777216
115#endif
116#ifdef VM_FREELIST_DMA32
117#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32)
118#endif
119
120/*
121 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about
122 * the ordering of the free list boundaries.
123 */
124#if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY)
125CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY);
126#endif
127#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY)
128CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY);
129#endif
130
106static int cnt_prezero;
107SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
108 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
109
110static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
111SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
112 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
113
114static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
115SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
116 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
117
118SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
119 &vm_ndomains, 0, "Number of physical memory domains available.");
120
121static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
122 int order);
131static int cnt_prezero;
132SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
133 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
134
135static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
136SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
137 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
138
139static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
140SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
141 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
142
143SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
144 &vm_ndomains, 0, "Number of physical memory domains available.");
145
146static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
147 int order);
123static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind,
124 int domain);
125static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind);
148static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain);
149static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end);
126static int vm_phys_paddr_to_segind(vm_paddr_t pa);
127static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
128 int order);
129
130/*
131 * Red-black tree helpers for vm fictitious range management.
132 */
133static inline int

--- 159 unchanged lines hidden (view full) ---

293 fl[order].lcnt--;
294 m->order = VM_NFREEORDER;
295}
296
297/*
298 * Create a physical memory segment.
299 */
300static void
150static int vm_phys_paddr_to_segind(vm_paddr_t pa);
151static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
152 int order);
153
154/*
155 * Red-black tree helpers for vm fictitious range management.
156 */
157static inline int

--- 159 unchanged lines hidden (view full) ---

317 fl[order].lcnt--;
318 m->order = VM_NFREEORDER;
319}
320
321/*
322 * Create a physical memory segment.
323 */
324static void
301_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain)
325_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain)
302{
303 struct vm_phys_seg *seg;
304
305 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
306 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
307 KASSERT(domain < vm_ndomains,
308 ("vm_phys_create_seg: invalid domain provided"));
309 seg = &vm_phys_segs[vm_phys_nsegs++];
310 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
311 *seg = *(seg - 1);
312 seg--;
313 }
314 seg->start = start;
315 seg->end = end;
316 seg->domain = domain;
326{
327 struct vm_phys_seg *seg;
328
329 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
330 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
331 KASSERT(domain < vm_ndomains,
332 ("vm_phys_create_seg: invalid domain provided"));
333 seg = &vm_phys_segs[vm_phys_nsegs++];
334 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
335 *seg = *(seg - 1);
336 seg--;
337 }
338 seg->start = start;
339 seg->end = end;
340 seg->domain = domain;
317 seg->free_queues = &vm_phys_free_queues[domain][flind];
318}
319
320static void
341}
342
343static void
321vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind)
344vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end)
322{
323 int i;
324
325 if (mem_affinity == NULL) {
345{
346 int i;
347
348 if (mem_affinity == NULL) {
326 _vm_phys_create_seg(start, end, flind, 0);
349 _vm_phys_create_seg(start, end, 0);
327 return;
328 }
329
330 for (i = 0;; i++) {
331 if (mem_affinity[i].end == 0)
332 panic("Reached end of affinity info");
333 if (mem_affinity[i].end <= start)
334 continue;
335 if (mem_affinity[i].start > start)
336 panic("No affinity info for start %jx",
337 (uintmax_t)start);
338 if (mem_affinity[i].end >= end) {
350 return;
351 }
352
353 for (i = 0;; i++) {
354 if (mem_affinity[i].end == 0)
355 panic("Reached end of affinity info");
356 if (mem_affinity[i].end <= start)
357 continue;
358 if (mem_affinity[i].start > start)
359 panic("No affinity info for start %jx",
360 (uintmax_t)start);
361 if (mem_affinity[i].end >= end) {
339 _vm_phys_create_seg(start, end, flind,
362 _vm_phys_create_seg(start, end,
340 mem_affinity[i].domain);
341 break;
342 }
363 mem_affinity[i].domain);
364 break;
365 }
343 _vm_phys_create_seg(start, mem_affinity[i].end, flind,
366 _vm_phys_create_seg(start, mem_affinity[i].end,
344 mem_affinity[i].domain);
345 start = mem_affinity[i].end;
346 }
347}
348
349/*
350 * Add a physical memory segment.
351 */
352void
353vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
354{
367 mem_affinity[i].domain);
368 start = mem_affinity[i].end;
369 }
370}
371
372/*
373 * Add a physical memory segment.
374 */
375void
376vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
377{
378 vm_paddr_t paddr;
355
356 KASSERT((start & PAGE_MASK) == 0,
357 ("vm_phys_define_seg: start is not page aligned"));
358 KASSERT((end & PAGE_MASK) == 0,
359 ("vm_phys_define_seg: end is not page aligned"));
379
380 KASSERT((start & PAGE_MASK) == 0,
381 ("vm_phys_define_seg: start is not page aligned"));
382 KASSERT((end & PAGE_MASK) == 0,
383 ("vm_phys_define_seg: end is not page aligned"));
384
385 /*
386 * Split the physical memory segment if it spans two or more free
387 * list boundaries.
388 */
389 paddr = start;
360#ifdef VM_FREELIST_ISADMA
390#ifdef VM_FREELIST_ISADMA
361 if (start < 16777216) {
362 if (end > 16777216) {
363 vm_phys_create_seg(start, 16777216,
364 VM_FREELIST_ISADMA);
365 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT);
366 } else
367 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA);
368 if (VM_FREELIST_ISADMA >= vm_nfreelists)
369 vm_nfreelists = VM_FREELIST_ISADMA + 1;
370 } else
391 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) {
392 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY);
393 paddr = VM_ISADMA_BOUNDARY;
394 }
371#endif
395#endif
372#ifdef VM_FREELIST_HIGHMEM
373 if (end > VM_HIGHMEM_ADDRESS) {
374 if (start < VM_HIGHMEM_ADDRESS) {
375 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS,
376 VM_FREELIST_DEFAULT);
377 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end,
378 VM_FREELIST_HIGHMEM);
379 } else
380 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM);
381 if (VM_FREELIST_HIGHMEM >= vm_nfreelists)
382 vm_nfreelists = VM_FREELIST_HIGHMEM + 1;
383 } else
396#ifdef VM_FREELIST_LOWMEM
397 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) {
398 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY);
399 paddr = VM_LOWMEM_BOUNDARY;
400 }
384#endif
401#endif
385 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT);
402#ifdef VM_FREELIST_DMA32
403 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) {
404 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY);
405 paddr = VM_DMA32_BOUNDARY;
406 }
407#endif
408 vm_phys_create_seg(paddr, end);
386}
387
388/*
389 * Initialize the physical memory allocator.
409}
410
411/*
412 * Initialize the physical memory allocator.
413 *
414 * Requires that vm_page_array is initialized!
390 */
391void
392vm_phys_init(void)
393{
394 struct vm_freelist *fl;
395 struct vm_phys_seg *seg;
415 */
416void
417vm_phys_init(void)
418{
419 struct vm_freelist *fl;
420 struct vm_phys_seg *seg;
396#ifdef VM_PHYSSEG_SPARSE
397 long pages;
421 u_long npages;
422 int dom, flind, freelist, oind, pind, segind;
423
424 /*
425 * Compute the number of free lists, and generate the mapping from the
426 * manifest constants VM_FREELIST_* to the free list indices.
427 *
428 * Initially, the entries of vm_freelist_to_flind[] are set to either
429 * 0 or 1 to indicate which free lists should be created.
430 */
431 npages = 0;
432 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) {
433 seg = &vm_phys_segs[segind];
434#ifdef VM_FREELIST_ISADMA
435 if (seg->end <= VM_ISADMA_BOUNDARY)
436 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1;
437 else
398#endif
438#endif
399 int dom, flind, oind, pind, segind;
439#ifdef VM_FREELIST_LOWMEM
440 if (seg->end <= VM_LOWMEM_BOUNDARY)
441 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1;
442 else
443#endif
444#ifdef VM_FREELIST_DMA32
445 if (
446#ifdef VM_DMA32_NPAGES_THRESHOLD
447 /*
448 * Create the DMA32 free list only if the amount of
449 * physical memory above physical address 4G exceeds the
450 * given threshold.
451 */
452 npages > VM_DMA32_NPAGES_THRESHOLD &&
453#endif
454 seg->end <= VM_DMA32_BOUNDARY)
455 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1;
456 else
457#endif
458 {
459 npages += atop(seg->end - seg->start);
460 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1;
461 }
462 }
463 /* Change each entry into a running total of the free lists. */
464 for (freelist = 1; freelist < VM_NFREELIST; freelist++) {
465 vm_freelist_to_flind[freelist] +=
466 vm_freelist_to_flind[freelist - 1];
467 }
468 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1];
469 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists"));
470 /* Change each entry into a free list index. */
471 for (freelist = 0; freelist < VM_NFREELIST; freelist++)
472 vm_freelist_to_flind[freelist]--;
400
473
474 /*
475 * Initialize the first_page and free_queues fields of each physical
476 * memory segment.
477 */
401#ifdef VM_PHYSSEG_SPARSE
478#ifdef VM_PHYSSEG_SPARSE
402 pages = 0;
479 npages = 0;
403#endif
404 for (segind = 0; segind < vm_phys_nsegs; segind++) {
405 seg = &vm_phys_segs[segind];
406#ifdef VM_PHYSSEG_SPARSE
480#endif
481 for (segind = 0; segind < vm_phys_nsegs; segind++) {
482 seg = &vm_phys_segs[segind];
483#ifdef VM_PHYSSEG_SPARSE
407 seg->first_page = &vm_page_array[pages];
408 pages += atop(seg->end - seg->start);
484 seg->first_page = &vm_page_array[npages];
485 npages += atop(seg->end - seg->start);
409#else
410 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
411#endif
486#else
487 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
488#endif
489#ifdef VM_FREELIST_ISADMA
490 if (seg->end <= VM_ISADMA_BOUNDARY) {
491 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA];
492 KASSERT(flind >= 0,
493 ("vm_phys_init: ISADMA flind < 0"));
494 } else
495#endif
496#ifdef VM_FREELIST_LOWMEM
497 if (seg->end <= VM_LOWMEM_BOUNDARY) {
498 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM];
499 KASSERT(flind >= 0,
500 ("vm_phys_init: LOWMEM flind < 0"));
501 } else
502#endif
503#ifdef VM_FREELIST_DMA32
504 if (seg->end <= VM_DMA32_BOUNDARY) {
505 flind = vm_freelist_to_flind[VM_FREELIST_DMA32];
506 KASSERT(flind >= 0,
507 ("vm_phys_init: DMA32 flind < 0"));
508 } else
509#endif
510 {
511 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT];
512 KASSERT(flind >= 0,
513 ("vm_phys_init: DEFAULT flind < 0"));
514 }
515 seg->free_queues = &vm_phys_free_queues[seg->domain][flind];
412 }
516 }
517
518 /*
519 * Initialize the free queues.
520 */
413 for (dom = 0; dom < vm_ndomains; dom++) {
414 for (flind = 0; flind < vm_nfreelists; flind++) {
415 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
416 fl = vm_phys_free_queues[dom][flind][pind];
417 for (oind = 0; oind < VM_NFREEORDER; oind++)
418 TAILQ_INIT(&fl[oind].pl);
419 }
420 }
421 }
521 for (dom = 0; dom < vm_ndomains; dom++) {
522 for (flind = 0; flind < vm_nfreelists; flind++) {
523 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
524 fl = vm_phys_free_queues[dom][flind][pind];
525 for (oind = 0; oind < VM_NFREEORDER; oind++)
526 TAILQ_INIT(&fl[oind].pl);
527 }
528 }
529 }
530
422 rw_init(&vm_phys_fictitious_reg_lock, "vmfctr");
423}
424
425/*
426 * Split a contiguous, power of two-sized set of physical pages.
427 */
428static __inline void
429vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order)

--- 63 unchanged lines hidden (view full) ---

493 if (m != NULL)
494 return (m);
495 }
496 }
497 return (NULL);
498}
499
500/*
531 rw_init(&vm_phys_fictitious_reg_lock, "vmfctr");
532}
533
534/*
535 * Split a contiguous, power of two-sized set of physical pages.
536 */
537static __inline void
538vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl, int order)

--- 63 unchanged lines hidden (view full) ---

602 if (m != NULL)
603 return (m);
604 }
605 }
606 return (NULL);
607}
608
609/*
501 * Find and dequeue a free page on the given free list, with the
502 * specified pool and order
610 * Allocate a contiguous, power of two-sized set of physical pages from the
611 * specified free list. The free list must be specified using one of the
612 * manifest constants VM_FREELIST_*.
613 *
614 * The free page queues must be locked.
503 */
504vm_page_t
615 */
616vm_page_t
505vm_phys_alloc_freelist_pages(int flind, int pool, int order)
617vm_phys_alloc_freelist_pages(int freelist, int pool, int order)
506{
507 vm_page_t m;
508 int dom, domain;
509
618{
619 vm_page_t m;
620 int dom, domain;
621
510 KASSERT(flind < VM_NFREELIST,
511 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind));
622 KASSERT(freelist < VM_NFREELIST,
623 ("vm_phys_alloc_freelist_pages: freelist %d is out of range",
624 freelist));
512 KASSERT(pool < VM_NFREEPOOL,
513 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
514 KASSERT(order < VM_NFREEORDER,
515 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
625 KASSERT(pool < VM_NFREEPOOL,
626 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
627 KASSERT(order < VM_NFREEORDER,
628 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
516
517 for (dom = 0; dom < vm_ndomains; dom++) {
518 domain = vm_rr_selectdomain();
629 for (dom = 0; dom < vm_ndomains; dom++) {
630 domain = vm_rr_selectdomain();
519 m = vm_phys_alloc_domain_pages(domain, flind, pool, order);
631 m = vm_phys_alloc_domain_pages(domain,
632 vm_freelist_to_flind[freelist], pool, order);
520 if (m != NULL)
521 return (m);
522 }
523 return (NULL);
524}
525
526static vm_page_t
527vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)

--- 611 unchanged lines hidden --- 		633 if (m != NULL)
634 return (m);
635 }
636 return (NULL);
637}
638
639static vm_page_t
640vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)

--- 611 unchanged lines hidden ---