1/*-
2 * Copyright (c) 2002-2006 Rice University
3 * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
4 * All rights reserved.
5 *
6 * This software was developed for the FreeBSD Project by Alan L. Cox,
7 * Olivier Crameri, Peter Druschel, Sitaram Iyer, and Juan Navarro.
8 *

--- 23 unchanged lines hidden (view full) ---

32/*
33 * Physical memory system implementation
34 *
35 * Any external functions defined by this module are only to be used by the
36 * virtual memory system.
37 */
38
39#include <sys/cdefs.h>
40__FBSDID("$FreeBSD: stable/10/sys/vm/vm_phys.c 285634 2015-07-16 14:41:58Z kib $");
41
42#include "opt_ddb.h"
43#include "opt_vm.h"
44
45#include <sys/param.h>
46#include <sys/systm.h>
47#include <sys/lock.h>
48#include <sys/kernel.h>

--- 33 unchanged lines hidden (view full) ---

82 vm_page_t first_page;
83} vm_phys_fictitious_segs[VM_PHYS_FICTITIOUS_NSEGS];
84static struct mtx vm_phys_fictitious_reg_mtx;
85MALLOC_DEFINE(M_FICT_PAGES, "vm_fictitious", "Fictitious VM pages");
86
87static struct vm_freelist
88 vm_phys_free_queues[MAXMEMDOM][VM_NFREELIST][VM_NFREEPOOL][VM_NFREEORDER];
89
90static int vm_nfreelists;
91
92/*
93 * Provides the mapping from VM_FREELIST_* to free list indices (flind).
94 */
95static int vm_freelist_to_flind[VM_NFREELIST];
96
97CTASSERT(VM_FREELIST_DEFAULT == 0);
98
99#ifdef VM_FREELIST_ISADMA
100#define VM_ISADMA_BOUNDARY 16777216
101#endif
102#ifdef VM_FREELIST_DMA32
103#define VM_DMA32_BOUNDARY ((vm_paddr_t)1 << 32)
104#endif
105
106/*
107 * Enforce the assumptions made by vm_phys_add_seg() and vm_phys_init() about
108 * the ordering of the free list boundaries.
109 */
110#if defined(VM_ISADMA_BOUNDARY) && defined(VM_LOWMEM_BOUNDARY)
111CTASSERT(VM_ISADMA_BOUNDARY < VM_LOWMEM_BOUNDARY);
112#endif
113#if defined(VM_LOWMEM_BOUNDARY) && defined(VM_DMA32_BOUNDARY)
114CTASSERT(VM_LOWMEM_BOUNDARY < VM_DMA32_BOUNDARY);
115#endif
116
117static int cnt_prezero;
118SYSCTL_INT(_vm_stats_misc, OID_AUTO, cnt_prezero, CTLFLAG_RD,
119 &cnt_prezero, 0, "The number of physical pages prezeroed at idle time");
120
121static int sysctl_vm_phys_free(SYSCTL_HANDLER_ARGS);
122SYSCTL_OID(_vm, OID_AUTO, phys_free, CTLTYPE_STRING | CTLFLAG_RD,
123 NULL, 0, sysctl_vm_phys_free, "A", "Phys Free Info");
124
125static int sysctl_vm_phys_segs(SYSCTL_HANDLER_ARGS);
126SYSCTL_OID(_vm, OID_AUTO, phys_segs, CTLTYPE_STRING | CTLFLAG_RD,
127 NULL, 0, sysctl_vm_phys_segs, "A", "Phys Seg Info");
128
129SYSCTL_INT(_vm, OID_AUTO, ndomains, CTLFLAG_RD,
130 &vm_ndomains, 0, "Number of physical memory domains available.");
131
132static vm_page_t vm_phys_alloc_domain_pages(int domain, int flind, int pool,
133 int order);
134static void _vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain);
135static void vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end);
136static int vm_phys_paddr_to_segind(vm_paddr_t pa);
137static void vm_phys_split_pages(vm_page_t m, int oind, struct vm_freelist *fl,
138 int order);
139
140static __inline int
141vm_rr_selectdomain(void)
142{
143#if MAXMEMDOM > 1

--- 118 unchanged lines hidden (view full) ---

262 fl[order].lcnt--;
263 m->order = VM_NFREEORDER;
264}
265
266/*
267 * Create a physical memory segment.
268 */
269static void
270_vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end, int domain)
271{
272 struct vm_phys_seg *seg;
273
274 KASSERT(vm_phys_nsegs < VM_PHYSSEG_MAX,
275 ("vm_phys_create_seg: increase VM_PHYSSEG_MAX"));
276 KASSERT(domain < vm_ndomains,
277 ("vm_phys_create_seg: invalid domain provided"));
278 seg = &vm_phys_segs[vm_phys_nsegs++];
279 while (seg > vm_phys_segs && (seg - 1)->start >= end) {
280 *seg = *(seg - 1);
281 seg--;
282 }
283 seg->start = start;
284 seg->end = end;
285 seg->domain = domain;
286}
287
288static void
289vm_phys_create_seg(vm_paddr_t start, vm_paddr_t end)
290{
291 int i;
292
293 if (mem_affinity == NULL) {
294 _vm_phys_create_seg(start, end, 0);
295 return;
296 }
297
298 for (i = 0;; i++) {
299 if (mem_affinity[i].end == 0)
300 panic("Reached end of affinity info");
301 if (mem_affinity[i].end <= start)
302 continue;
303 if (mem_affinity[i].start > start)
304 panic("No affinity info for start %jx",
305 (uintmax_t)start);
306 if (mem_affinity[i].end >= end) {
307 _vm_phys_create_seg(start, end,
308 mem_affinity[i].domain);
309 break;
310 }
311 _vm_phys_create_seg(start, mem_affinity[i].end,
312 mem_affinity[i].domain);
313 start = mem_affinity[i].end;
314 }
315}
316
317/*
318 * Add a physical memory segment.
319 */
320void
321vm_phys_add_seg(vm_paddr_t start, vm_paddr_t end)
322{
323 vm_paddr_t paddr;
324
325 KASSERT((start & PAGE_MASK) == 0,
326 ("vm_phys_define_seg: start is not page aligned"));
327 KASSERT((end & PAGE_MASK) == 0,
328 ("vm_phys_define_seg: end is not page aligned"));
329
330 /*
331 * Split the physical memory segment if it spans two or more free
332 * list boundaries.
333 */
334 paddr = start;
335#ifdef VM_FREELIST_ISADMA
336 if (paddr < VM_ISADMA_BOUNDARY && end > VM_ISADMA_BOUNDARY) {
337 vm_phys_create_seg(paddr, VM_ISADMA_BOUNDARY);
338 paddr = VM_ISADMA_BOUNDARY;
339 }
340#endif
341#ifdef VM_FREELIST_LOWMEM
342 if (paddr < VM_LOWMEM_BOUNDARY && end > VM_LOWMEM_BOUNDARY) {
343 vm_phys_create_seg(paddr, VM_LOWMEM_BOUNDARY);
344 paddr = VM_LOWMEM_BOUNDARY;
345 }
346#endif
347#ifdef VM_FREELIST_DMA32
348 if (paddr < VM_DMA32_BOUNDARY && end > VM_DMA32_BOUNDARY) {
349 vm_phys_create_seg(paddr, VM_DMA32_BOUNDARY);
350 paddr = VM_DMA32_BOUNDARY;
351 }
352#endif
353 vm_phys_create_seg(paddr, end);
354}
355
356/*
357 * Initialize the physical memory allocator.
358 *
359 * Requires that vm_page_array is initialized!
360 */
361void
362vm_phys_init(void)
363{
364 struct vm_freelist *fl;
365 struct vm_phys_seg *seg;
366 u_long npages;
367 int dom, flind, freelist, oind, pind, segind;
368
369 /*
370 * Compute the number of free lists, and generate the mapping from the
371 * manifest constants VM_FREELIST_* to the free list indices.
372 *
373 * Initially, the entries of vm_freelist_to_flind[] are set to either
374 * 0 or 1 to indicate which free lists should be created.
375 */
376 npages = 0;
377 for (segind = vm_phys_nsegs - 1; segind >= 0; segind--) {
378 seg = &vm_phys_segs[segind];
379#ifdef VM_FREELIST_ISADMA
380 if (seg->end <= VM_ISADMA_BOUNDARY)
381 vm_freelist_to_flind[VM_FREELIST_ISADMA] = 1;
382 else
383#endif
384#ifdef VM_FREELIST_LOWMEM
385 if (seg->end <= VM_LOWMEM_BOUNDARY)
386 vm_freelist_to_flind[VM_FREELIST_LOWMEM] = 1;
387 else
388#endif
389#ifdef VM_FREELIST_DMA32
390 if (
391#ifdef VM_DMA32_NPAGES_THRESHOLD
392 /*
393 * Create the DMA32 free list only if the amount of
394 * physical memory above physical address 4G exceeds the
395 * given threshold.
396 */
397 npages > VM_DMA32_NPAGES_THRESHOLD &&
398#endif
399 seg->end <= VM_DMA32_BOUNDARY)
400 vm_freelist_to_flind[VM_FREELIST_DMA32] = 1;
401 else
402#endif
403 {
404 npages += atop(seg->end - seg->start);
405 vm_freelist_to_flind[VM_FREELIST_DEFAULT] = 1;
406 }
407 }
408 /* Change each entry into a running total of the free lists. */
409 for (freelist = 1; freelist < VM_NFREELIST; freelist++) {
410 vm_freelist_to_flind[freelist] +=
411 vm_freelist_to_flind[freelist - 1];
412 }
413 vm_nfreelists = vm_freelist_to_flind[VM_NFREELIST - 1];
414 KASSERT(vm_nfreelists > 0, ("vm_phys_init: no free lists"));
415 /* Change each entry into a free list index. */
416 for (freelist = 0; freelist < VM_NFREELIST; freelist++)
417 vm_freelist_to_flind[freelist]--;
418
419 /*
420 * Initialize the first_page and free_queues fields of each physical
421 * memory segment.
422 */
423#ifdef VM_PHYSSEG_SPARSE
424 npages = 0;
425#endif
426 for (segind = 0; segind < vm_phys_nsegs; segind++) {
427 seg = &vm_phys_segs[segind];
428#ifdef VM_PHYSSEG_SPARSE
429 seg->first_page = &vm_page_array[npages];
430 npages += atop(seg->end - seg->start);
431#else
432 seg->first_page = PHYS_TO_VM_PAGE(seg->start);
433#endif
434#ifdef VM_FREELIST_ISADMA
435 if (seg->end <= VM_ISADMA_BOUNDARY) {
436 flind = vm_freelist_to_flind[VM_FREELIST_ISADMA];
437 KASSERT(flind >= 0,
438 ("vm_phys_init: ISADMA flind < 0"));
439 } else
440#endif
441#ifdef VM_FREELIST_LOWMEM
442 if (seg->end <= VM_LOWMEM_BOUNDARY) {
443 flind = vm_freelist_to_flind[VM_FREELIST_LOWMEM];
444 KASSERT(flind >= 0,
445 ("vm_phys_init: LOWMEM flind < 0"));
446 } else
447#endif
448#ifdef VM_FREELIST_DMA32
449 if (seg->end <= VM_DMA32_BOUNDARY) {
450 flind = vm_freelist_to_flind[VM_FREELIST_DMA32];
451 KASSERT(flind >= 0,
452 ("vm_phys_init: DMA32 flind < 0"));
453 } else
454#endif
455 {
456 flind = vm_freelist_to_flind[VM_FREELIST_DEFAULT];
457 KASSERT(flind >= 0,
458 ("vm_phys_init: DEFAULT flind < 0"));
459 }
460 seg->free_queues = &vm_phys_free_queues[seg->domain][flind];
461 }
462
463 /*
464 * Initialize the free queues.
465 */
466 for (dom = 0; dom < vm_ndomains; dom++) {
467 for (flind = 0; flind < vm_nfreelists; flind++) {
468 for (pind = 0; pind < VM_NFREEPOOL; pind++) {
469 fl = vm_phys_free_queues[dom][flind][pind];
470 for (oind = 0; oind < VM_NFREEORDER; oind++)
471 TAILQ_INIT(&fl[oind].pl);
472 }
473 }

--- 73 unchanged lines hidden (view full) ---

547 if (m != NULL)
548 return (m);
549 }
550 }
551 return (NULL);
552}
553
554/*
555 * Allocate a contiguous, power of two-sized set of physical pages from the
556 * specified free list. The free list must be specified using one of the
557 * manifest constants VM_FREELIST_*.
558 *
559 * The free page queues must be locked.
560 */
561vm_page_t
562vm_phys_alloc_freelist_pages(int freelist, int pool, int order)
563{
564 vm_page_t m;
565 int dom, domain;
566
567 KASSERT(freelist < VM_NFREELIST,
568 ("vm_phys_alloc_freelist_pages: freelist %d is out of range",
569 freelist));
570 KASSERT(pool < VM_NFREEPOOL,
571 ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool));
572 KASSERT(order < VM_NFREEORDER,
573 ("vm_phys_alloc_freelist_pages: order %d is out of range", order));
574 for (dom = 0; dom < vm_ndomains; dom++) {
575 domain = vm_rr_selectdomain();
576 m = vm_phys_alloc_domain_pages(domain,
577 vm_freelist_to_flind[freelist], pool, order);
578 if (m != NULL)
579 return (m);
580 }
581 return (NULL);
582}
583
584static vm_page_t
585vm_phys_alloc_domain_pages(int domain, int flind, int pool, int order)

--- 537 unchanged lines hidden ---