Deleted Added
sdiff udiff text old ( 276546 ) new ( 285634 )
full compact
vm_phys.c (276546) vm_phys.c (285634)
1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *

--- 23 unchanged lines hidden (view full) ---

32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *

--- 23 unchanged lines hidden (view full) ---

32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 276546 2015-01-02 17:45:52Z alc $");
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 285634 2015-07-16 14:41:58Z kib $");
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>

--- 33 unchanged lines hidden (view full) ---

82 vm_page_t first_page;
83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS];
84static struct mtx vm_phys_fictitious_reg_mtx;
85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
86
87static struct vm_freelist
88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
89
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>

--- 33 unchanged lines hidden (view full) ---

82 vm_page_t first_page;
83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS];
84static struct mtx vm_phys_fictitious_reg_mtx;
85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
86
87static struct vm_freelist
88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
89
90static int vm_nfreelists = VM_FREELIST_DEFAULT + 1;
90static int vm_nfreelists;
91
91
92/*
93 * Provides the mapping from VM_FREELIST_* to free list indices (flind).
94 */
95static int vm_freelist_to_flind[VM_NFREELIST];
96
97CTASSERT(VM_FREELIST_DEFAULT == 0);
98
99#ifdef VM_FREELIST_ISADMA
100#define VM_ISADMA_BOUNDARY 16777216
101#endif
102#ifdef VM_FREELIST_DMA32
103#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32)
104#endif
105
106/*
107 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about
108 * the ordering of the free list boundaries.
109 */
110#if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY)
111CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY);
112#endif
113#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY)
114CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY);
115#endif
116
92static int cnt_prezero;
93SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
94 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
95
96static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
97SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
98 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
99
100static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
101SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
102 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
103
104SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
105 &vm_ndomains, 0, "Number of physical memory domains available.");
106
107static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
108 int order);
117static int cnt_prezero;
118SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
119 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
120
121static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
122SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
123 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
124
125static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
126SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
127 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
128
129SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
130 &vm_ndomains, 0, "Number of physical memory domains available.");
131
132static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
133 int order);
109static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind,
110 int domain);
111static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind);
134static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain);
135static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end);
112static int vm_phys_paddr_to_segind(vm_paddr_t pa);
113static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
114 int order);
115
116static __inline int
117vm_rr_selectdomain(void)
118{
119#if MAXMEMDOM > 1

--- 118 unchanged lines hidden (view full) ---

238 fl[order].lcnt--;
239 m->order = VM_NFREEORDER;
240}
241
242/*
243 * Create a physical memory segment.
244 */
245static void
136static int vm_phys_paddr_to_segind(vm_paddr_t pa);
137static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
138 int order);
139
140static __inline int
141vm_rr_selectdomain(void)
142{
143#if MAXMEMDOM > 1

--- 118 unchanged lines hidden (view full) ---

262 fl[order].lcnt--;
263 m->order = VM_NFREEORDER;
264}
265
266/*
267 * Create a physical memory segment.
268 */
269static void
246_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain)
270_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain)
247{
248 struct vm_phys_seg *seg;
249
250 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
251 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
252 KASSERT(domain < vm_ndomains,
253 ("vm_phys_create_seg: invalid domain provided"));
254 seg = &vm_phys_segs[vm_phys_nsegs++];
255 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
256 *seg = *(seg - 1);
257 seg--;
258 }
259 seg->start = start;
260 seg->end = end;
261 seg->domain = domain;
271{
272 struct vm_phys_seg *seg;
273
274 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
275 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
276 KASSERT(domain < vm_ndomains,
277 ("vm_phys_create_seg: invalid domain provided"));
278 seg = &vm_phys_segs[vm_phys_nsegs++];
279 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
280 *seg = *(seg - 1);
281 seg--;
282 }
283 seg->start = start;
284 seg->end = end;
285 seg->domain = domain;
262 seg->free_queues = &vm_phys_free_queues[domain][flind];
263}
264
265static void
286}
287
288static void
266vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind)
289vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end)
267{
268 int i;
269
270 if (mem_affinity == NULL) {
290{
291 int i;
292
293 if (mem_affinity == NULL) {
271 _vm_phys_create_seg(start, end, flind, 0);
294 _vm_phys_create_seg(start, end, 0);
272 return;
273 }
274
275 for (i = 0;; i++) {
276 if (mem_affinity[i].end == 0)
277 panic("Reached end of affinity info");
278 if (mem_affinity[i].end <= start)
279 continue;
280 if (mem_affinity[i].start > start)
281 panic("No affinity info for start %jx",
282 (uintmax_t)start);
283 if (mem_affinity[i].end >= end) {
295 return;
296 }
297
298 for (i = 0;; i++) {
299 if (mem_affinity[i].end == 0)
300 panic("Reached end of affinity info");
301 if (mem_affinity[i].end <= start)
302 continue;
303 if (mem_affinity[i].start > start)
304 panic("No affinity info for start %jx",
305 (uintmax_t)start);
306 if (mem_affinity[i].end >= end) {
284 _vm_phys_create_seg(start, end, flind,
307 _vm_phys_create_seg(start, end,
285 mem_affinity[i].domain);
286 break;
287 }
308 mem_affinity[i].domain);
309 break;
310 }
288 _vm_phys_create_seg(start, mem_affinity[i].end, flind,
311 _vm_phys_create_seg(start, mem_affinity[i].end,
289 mem_affinity[i].domain);
290 start = mem_affinity[i].end;
291 }
292}
293
294/*
295 * Add a physical memory segment.
296 */
297void
298vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
299{
312 mem_affinity[i].domain);
313 start = mem_affinity[i].end;
314 }
315}
316
317/*
318 * Add a physical memory segment.
319 */
320void
321vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
322{
323 vm_paddr_t paddr;
300
301 KASSERT((start & PAGE_MASK) == 0,
302 ("vm_phys_define_seg: start is not page aligned"));
303 KASSERT((end & PAGE_MASK) == 0,
304 ("vm_phys_define_seg: end is not page aligned"));
324
325 KASSERT((start & PAGE_MASK) == 0,
326 ("vm_phys_define_seg: start is not page aligned"));
327 KASSERT((end & PAGE_MASK) == 0,
328 ("vm_phys_define_seg: end is not page aligned"));
329
330 /*
331 * Split the physical memory segment if it spans two or more free
332 * list boundaries.
333 */
334 paddr = start;
305#ifdef VM_FREELIST_ISADMA
335#ifdef VM_FREELIST_ISADMA
306 if (start < 16777216) {
307 if (end > 16777216) {
308 vm_phys_create_seg(start, 16777216,
309 VM_FREELIST_ISADMA);
310 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT);
311 } else
312 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA);
313 if (VM_FREELIST_ISADMA >= vm_nfreelists)
314 vm_nfreelists = VM_FREELIST_ISADMA + 1;
315 } else
336 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) {
337 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY);
338 paddr = VM_ISADMA_BOUNDARY;
339 }
316#endif
340#endif
317#ifdef VM_FREELIST_HIGHMEM
318 if (end > VM_HIGHMEM_ADDRESS) {
319 if (start < VM_HIGHMEM_ADDRESS) {
320 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS,
321 VM_FREELIST_DEFAULT);
322 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end,
323 VM_FREELIST_HIGHMEM);
324 } else
325 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM);
326 if (VM_FREELIST_HIGHMEM >= vm_nfreelists)
327 vm_nfreelists = VM_FREELIST_HIGHMEM + 1;
328 } else
341#ifdef VM_FREELIST_LOWMEM
342 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) {
343 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY);
344 paddr = VM_LOWMEM_BOUNDARY;
345 }
329#endif
346#endif
330 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT);
347#ifdef VM_FREELIST_DMA32
348 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) {
349 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY);
350 paddr = VM_DMA32_BOUNDARY;
351 }
352#endif
353 vm_phys_create_seg(paddr, end);
331}
332
333/*
334 * Initialize the physical memory allocator.
354}
355
356/*
357 * Initialize the physical memory allocator.
358 *
359 * Requires that vm_page_array is initialized!
335 */
336void
337vm_phys_init(void)
338{
339 struct vm_freelist *fl;
340 struct vm_phys_seg *seg;
360 */
361void
362vm_phys_init(void)
363{
364 struct vm_freelist *fl;
365 struct vm_phys_seg *seg;
341#ifdef VM_PHYSSEG_SPARSE
342 long pages;
366 u_long npages;
367 int dom, flind, freelist, oind, pind, segind;
368
369 /*
370 * Compute the number of free lists, and generate the mapping from the
371 * manifest constants VM_FREELIST_* to the free list indices.
372 *
373 * Initially, the entries of vm_freelist_to_flind[] are set to either
374 * 0 or 1 to indicate which free lists should be created.
375 */
376 npages = 0;
377 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) {
378 seg = &vm_phys_segs[segind];
379#ifdef VM_FREELIST_ISADMA
380 if (seg->end <= VM_ISADMA_BOUNDARY)
381 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1;
382 else
343#endif
383#endif
344 int dom, flind, oind, pind, segind;
384#ifdef VM_FREELIST_LOWMEM
385 if (seg->end <= VM_LOWMEM_BOUNDARY)
386 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1;
387 else
388#endif
389#ifdef VM_FREELIST_DMA32
390 if (
391#ifdef VM_DMA32_NPAGES_THRESHOLD
392 /*
393 * Create the DMA32 free list only if the amount of
394 * physical memory above physical address 4G exceeds the
395 * given threshold.
396 */
397 npages > VM_DMA32_NPAGES_THRESHOLD &&
398#endif
399 seg->end <= VM_DMA32_BOUNDARY)
400 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1;
401 else
402#endif
403 {
404 npages += atop(seg->end - seg->start);
405 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1;
406 }
407 }
408 /* Change each entry into a running total of the free lists. */
409 for (freelist = 1; freelist < VM_NFREELIST; freelist++) {
410 vm_freelist_to_flind[freelist] +=
411 vm_freelist_to_flind[freelist - 1];
412 }
413 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1];
414 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists"));
415 /* Change each entry into a free list index. */
416 for (freelist = 0; freelist < VM_NFREELIST; freelist++)
417 vm_freelist_to_flind[freelist]--;
345
418
419 /*
420 * Initialize the first_page and free_queues fields of each physical
421 * memory segment.
422 */
346#ifdef VM_PHYSSEG_SPARSE
423#ifdef VM_PHYSSEG_SPARSE
347 pages = 0;
424 npages = 0;
348#endif
349 for (segind = 0; segind < vm_phys_nsegs; segind++) {
350 seg = &vm_phys_segs[segind];
351#ifdef VM_PHYSSEG_SPARSE
425#endif
426 for (segind = 0; segind < vm_phys_nsegs; segind++) {
427 seg = &vm_phys_segs[segind];
428#ifdef VM_PHYSSEG_SPARSE
352 seg->first_page = &vm_page_array[pages];
353 pages += atop(seg->end - seg->start);
429 seg->first_page = &vm_page_array[npages];
430 npages += atop(seg->end - seg->start);
354#else
355 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
356#endif
431#else
432 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
433#endif
434#ifdef VM_FREELIST_ISADMA
435 if (seg->end <= VM_ISADMA_BOUNDARY) {
436 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA];
437 KASSERT(flind >= 0,
438 ("vm_phys_init: ISADMA flind < 0"));
439 } else
440#endif
441#ifdef VM_FREELIST_LOWMEM
442 if (seg->end <= VM_LOWMEM_BOUNDARY) {
443 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM];
444 KASSERT(flind >= 0,
445 ("vm_phys_init: LOWMEM flind < 0"));
446 } else
447#endif
448#ifdef VM_FREELIST_DMA32
449 if (seg->end <= VM_DMA32_BOUNDARY) {
450 flind = vm_freelist_to_flind[VM_FREELIST_DMA32];
451 KASSERT(flind >= 0,
452 ("vm_phys_init: DMA32 flind < 0"));
453 } else
454#endif
455 {
456 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT];
457 KASSERT(flind >= 0,
458 ("vm_phys_init: DEFAULT flind < 0"));
459 }
460 seg->free_queues = &vm_phys_free_queues[seg->domain][flind];
357 }
461 }
462
463 /*
464 * Initialize the free queues.
465 */
358 for (dom = 0; dom < vm_ndomains; dom++) {
359 for (flind = 0; flind < vm_nfreelists; flind++) {
360 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
361 fl = vm_phys_free_queues[dom][flind][pind];
362 for (oind = 0; oind < VM_NFREEORDER; oind++)
363 TAILQ_INIT(&fl[oind].pl);
364 }
365 }

--- 73 unchanged lines hidden (view full) ---

439 if (m != NULL)
440 return (m);
441 }
442 }
443 return (NULL);
444}
445
446/*
466 for (dom = 0; dom < vm_ndomains; dom++) {
467 for (flind = 0; flind < vm_nfreelists; flind++) {
468 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
469 fl = vm_phys_free_queues[dom][flind][pind];
470 for (oind = 0; oind < VM_NFREEORDER; oind++)
471 TAILQ_INIT(&fl[oind].pl);
472 }
473 }

--- 73 unchanged lines hidden (view full) ---

547 if (m != NULL)
548 return (m);
549 }
550 }
551 return (NULL);
552}
553
554/*
447 * Find and dequeue a free page on the given free list, with the
448 * specified pool and order
555 * Allocate a contiguous, power of two-sized set of physical pages from the
556 * specified free list. The free list must be specified using one of the
557 * manifest constants VM_FREELIST_*.
558 *
559 * The free page queues must be locked.
449 */
450vm_page_t
560 */
561vm_page_t
451vm_phys_alloc_freelist_pages(int flind, int pool, int order)
562vm_phys_alloc_freelist_pages(int freelist, int pool, int order)
452{
453 vm_page_t m;
454 int dom, domain;
455
563{
564 vm_page_t m;
565 int dom, domain;
566
456 KASSERT(flind < VM_NFREELIST,
457 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind));
567 KASSERT(freelist < VM_NFREELIST,
568 ("vm_phys_alloc_freelist_pages: freelist %d is out of range",
569 freelist));
458 KASSERT(pool < VM_NFREEPOOL,
459 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
460 KASSERT(order < VM_NFREEORDER,
461 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
570 KASSERT(pool < VM_NFREEPOOL,
571 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
572 KASSERT(order < VM_NFREEORDER,
573 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
462
463 for (dom = 0; dom < vm_ndomains; dom++) {
464 domain = vm_rr_selectdomain();
574 for (dom = 0; dom < vm_ndomains; dom++) {
575 domain = vm_rr_selectdomain();
465 m = vm_phys_alloc_domain_pages(domain, flind, pool, order);
576 m = vm_phys_alloc_domain_pages(domain,
577 vm_freelist_to_flind[freelist], pool, order);
466 if (m != NULL)
467 return (m);
468 }
469 return (NULL);
470}
471
472static vm_page_t
473vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)

--- 537 unchanged lines hidden --- 		578 if (m != NULL)
579 return (m);
580 }
581 return (NULL);
582}
583
584static vm_page_t
585vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)

--- 537 unchanged lines hidden ---