1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *
--- 23 unchanged lines hidden (view full) ---
32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 276546 2015-01-02 17:45:52Z alc $");
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>
--- 33 unchanged lines hidden (view full) ---
82 vm_page_t first_page;
83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS];
84static struct mtx vm_phys_fictitious_reg_mtx;
85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
86
87static struct vm_freelist
88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
89
90static int vm_nfreelists = VM_FREELIST_DEFAULT + 1;
91
92static int cnt_prezero;
93SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
94 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
95
96static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
97SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
98 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
99
100static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
101SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
102 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
103
104SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
105 &vm_ndomains, 0, "Number of physical memory domains available.");
106
107static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
108 int order);
109static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind,
110 int domain);
111static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind);
112static int vm_phys_paddr_to_segind(vm_paddr_t pa);
113static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
114 int order);
115
116static __inline int
117vm_rr_selectdomain(void)
118{
119#if MAXMEMDOM > 1
--- 118 unchanged lines hidden (view full) ---
238 fl[order].lcnt--;
239 m->order = VM_NFREEORDER;
240}
241
242/*
243 * Create a physical memory segment.
244 */
245static void
246_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain)
247{
248 struct vm_phys_seg *seg;
249
250 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
251 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
252 KASSERT(domain < vm_ndomains,
253 ("vm_phys_create_seg: invalid domain provided"));
254 seg = &vm_phys_segs[vm_phys_nsegs++];
255 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
256 *seg = *(seg - 1);
257 seg--;
258 }
259 seg->start = start;
260 seg->end = end;
261 seg->domain = domain;
262 seg->free_queues = &vm_phys_free_queues[domain][flind];
263}
264
265static void
266vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind)
267{
268 int i;
269
270 if (mem_affinity == NULL) {
271 _vm_phys_create_seg(start, end, flind, 0);
272 return;
273 }
274
275 for (i = 0;; i++) {
276 if (mem_affinity[i].end == 0)
277 panic("Reached end of affinity info");
278 if (mem_affinity[i].end <= start)
279 continue;
280 if (mem_affinity[i].start > start)
281 panic("No affinity info for start %jx",
282 (uintmax_t)start);
283 if (mem_affinity[i].end >= end) {
284 _vm_phys_create_seg(start, end, flind,
285 mem_affinity[i].domain);
286 break;
287 }
288 _vm_phys_create_seg(start, mem_affinity[i].end, flind,
289 mem_affinity[i].domain);
290 start = mem_affinity[i].end;
291 }
292}
293
294/*
295 * Add a physical memory segment.
296 */
297void
298vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
299{
300
301 KASSERT((start & PAGE_MASK) == 0,
302 ("vm_phys_define_seg: start is not page aligned"));
303 KASSERT((end & PAGE_MASK) == 0,
304 ("vm_phys_define_seg: end is not page aligned"));
305#ifdef VM_FREELIST_ISADMA
306 if (start < 16777216) {
307 if (end > 16777216) {
308 vm_phys_create_seg(start, 16777216,
309 VM_FREELIST_ISADMA);
310 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT);
311 } else
312 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA);
313 if (VM_FREELIST_ISADMA >= vm_nfreelists)
314 vm_nfreelists = VM_FREELIST_ISADMA + 1;
315 } else
316#endif
317#ifdef VM_FREELIST_HIGHMEM
318 if (end > VM_HIGHMEM_ADDRESS) {
319 if (start < VM_HIGHMEM_ADDRESS) {
320 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS,
321 VM_FREELIST_DEFAULT);
322 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end,
323 VM_FREELIST_HIGHMEM);
324 } else
325 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM);
326 if (VM_FREELIST_HIGHMEM >= vm_nfreelists)
327 vm_nfreelists = VM_FREELIST_HIGHMEM + 1;
328 } else
329#endif
330 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT);
331}
332
333/*
334 * Initialize the physical memory allocator.
335 */
336void
337vm_phys_init(void)
338{
339 struct vm_freelist *fl;
340 struct vm_phys_seg *seg;
341#ifdef VM_PHYSSEG_SPARSE
342 long pages;
343#endif
344 int dom, flind, oind, pind, segind;
345
346#ifdef VM_PHYSSEG_SPARSE
347 pages = 0;
348#endif
349 for (segind = 0; segind < vm_phys_nsegs; segind++) {
350 seg = &vm_phys_segs[segind];
351#ifdef VM_PHYSSEG_SPARSE
352 seg->first_page = &vm_page_array[pages];
353 pages += atop(seg->end - seg->start);
354#else
355 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
356#endif
357 }
358 for (dom = 0; dom < vm_ndomains; dom++) {
359 for (flind = 0; flind < vm_nfreelists; flind++) {
360 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
361 fl = vm_phys_free_queues[dom][flind][pind];
362 for (oind = 0; oind < VM_NFREEORDER; oind++)
363 TAILQ_INIT(&fl[oind].pl);
364 }
365 }
--- 73 unchanged lines hidden (view full) ---
439 if (m != NULL)
440 return (m);
441 }
442 }
443 return (NULL);
444}
445
446/*
447 * Find and dequeue a free page on the given free list, with the
448 * specified pool and order
449 */
450vm_page_t
451vm_phys_alloc_freelist_pages(int flind, int pool, int order)
452{
453 vm_page_t m;
454 int dom, domain;
455
456 KASSERT(flind < VM_NFREELIST,
457 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind));
458 KASSERT(pool < VM_NFREEPOOL,
459 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
460 KASSERT(order < VM_NFREEORDER,
461 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
462
463 for (dom = 0; dom < vm_ndomains; dom++) {
464 domain = vm_rr_selectdomain();
465 m = vm_phys_alloc_domain_pages(domain, flind, pool, order);
466 if (m != NULL)
467 return (m);
468 }
469 return (NULL);
470}
471
472static vm_page_t
473vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)
--- 537 unchanged lines hidden ---
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *
--- 23 unchanged lines hidden (view full) ---
32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 276546 2015-01-02 17:45:52Z alc $");
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>
--- 33 unchanged lines hidden (view full) ---
82 vm_page_t first_page;
83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS];
84static struct mtx vm_phys_fictitious_reg_mtx;
85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
86
87static struct vm_freelist
88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
89
90static int vm_nfreelists = VM_FREELIST_DEFAULT + 1;
91
92static int cnt_prezero;
93SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
94 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
95
96static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
97SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
98 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
99
100static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
101SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
102 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
103
104SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
105 &vm_ndomains, 0, "Number of physical memory domains available.");
106
107static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
108 int order);
109static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind,
110 int domain);
111static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind);
112static int vm_phys_paddr_to_segind(vm_paddr_t pa);
113static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
114 int order);
115
116static __inline int
117vm_rr_selectdomain(void)
118{
119#if MAXMEMDOM > 1
--- 118 unchanged lines hidden (view full) ---
238 fl[order].lcnt--;
239 m->order = VM_NFREEORDER;
240}
241
242/*
243 * Create a physical memory segment.
244 */
245static void
246_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind, int domain)
247{
248 struct vm_phys_seg *seg;
249
250 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
251 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
252 KASSERT(domain < vm_ndomains,
253 ("vm_phys_create_seg: invalid domain provided"));
254 seg = &vm_phys_segs[vm_phys_nsegs++];
255 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
256 *seg = *(seg - 1);
257 seg--;
258 }
259 seg->start = start;
260 seg->end = end;
261 seg->domain = domain;
262 seg->free_queues = &vm_phys_free_queues[domain][flind];
263}
264
265static void
266vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int flind)
267{
268 int i;
269
270 if (mem_affinity == NULL) {
271 _vm_phys_create_seg(start, end, flind, 0);
272 return;
273 }
274
275 for (i = 0;; i++) {
276 if (mem_affinity[i].end == 0)
277 panic("Reached end of affinity info");
278 if (mem_affinity[i].end <= start)
279 continue;
280 if (mem_affinity[i].start > start)
281 panic("No affinity info for start %jx",
282 (uintmax_t)start);
283 if (mem_affinity[i].end >= end) {
284 _vm_phys_create_seg(start, end, flind,
285 mem_affinity[i].domain);
286 break;
287 }
288 _vm_phys_create_seg(start, mem_affinity[i].end, flind,
289 mem_affinity[i].domain);
290 start = mem_affinity[i].end;
291 }
292}
293
294/*
295 * Add a physical memory segment.
296 */
297void
298vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
299{
300
301 KASSERT((start & PAGE_MASK) == 0,
302 ("vm_phys_define_seg: start is not page aligned"));
303 KASSERT((end & PAGE_MASK) == 0,
304 ("vm_phys_define_seg: end is not page aligned"));
305#ifdef VM_FREELIST_ISADMA
306 if (start < 16777216) {
307 if (end > 16777216) {
308 vm_phys_create_seg(start, 16777216,
309 VM_FREELIST_ISADMA);
310 vm_phys_create_seg(16777216, end, VM_FREELIST_DEFAULT);
311 } else
312 vm_phys_create_seg(start, end, VM_FREELIST_ISADMA);
313 if (VM_FREELIST_ISADMA >= vm_nfreelists)
314 vm_nfreelists = VM_FREELIST_ISADMA + 1;
315 } else
316#endif
317#ifdef VM_FREELIST_HIGHMEM
318 if (end > VM_HIGHMEM_ADDRESS) {
319 if (start < VM_HIGHMEM_ADDRESS) {
320 vm_phys_create_seg(start, VM_HIGHMEM_ADDRESS,
321 VM_FREELIST_DEFAULT);
322 vm_phys_create_seg(VM_HIGHMEM_ADDRESS, end,
323 VM_FREELIST_HIGHMEM);
324 } else
325 vm_phys_create_seg(start, end, VM_FREELIST_HIGHMEM);
326 if (VM_FREELIST_HIGHMEM >= vm_nfreelists)
327 vm_nfreelists = VM_FREELIST_HIGHMEM + 1;
328 } else
329#endif
330 vm_phys_create_seg(start, end, VM_FREELIST_DEFAULT);
331}
332
333/*
334 * Initialize the physical memory allocator.
335 */
336void
337vm_phys_init(void)
338{
339 struct vm_freelist *fl;
340 struct vm_phys_seg *seg;
341#ifdef VM_PHYSSEG_SPARSE
342 long pages;
343#endif
344 int dom, flind, oind, pind, segind;
345
346#ifdef VM_PHYSSEG_SPARSE
347 pages = 0;
348#endif
349 for (segind = 0; segind < vm_phys_nsegs; segind++) {
350 seg = &vm_phys_segs[segind];
351#ifdef VM_PHYSSEG_SPARSE
352 seg->first_page = &vm_page_array[pages];
353 pages += atop(seg->end - seg->start);
354#else
355 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
356#endif
357 }
358 for (dom = 0; dom < vm_ndomains; dom++) {
359 for (flind = 0; flind < vm_nfreelists; flind++) {
360 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
361 fl = vm_phys_free_queues[dom][flind][pind];
362 for (oind = 0; oind < VM_NFREEORDER; oind++)
363 TAILQ_INIT(&fl[oind].pl);
364 }
365 }
--- 73 unchanged lines hidden (view full) ---
439 if (m != NULL)
440 return (m);
441 }
442 }
443 return (NULL);
444}
445
446/*
447 * Find and dequeue a free page on the given free list, with the
448 * specified pool and order
449 */
450vm_page_t
451vm_phys_alloc_freelist_pages(int flind, int pool, int order)
452{
453 vm_page_t m;
454 int dom, domain;
455
456 KASSERT(flind < VM_NFREELIST,
457 ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind));
458 KASSERT(pool < VM_NFREEPOOL,
459 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
460 KASSERT(order < VM_NFREEORDER,
461 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
462
463 for (dom = 0; dom < vm_ndomains; dom++) {
464 domain = vm_rr_selectdomain();
465 m = vm_phys_alloc_domain_pages(domain, flind, pool, order);
466 if (m != NULL)
467 return (m);
468 }
469 return (NULL);
470}
471
472static vm_page_t
473vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)
--- 537 unchanged lines hidden ---