vm_zeroidle.c revision 82314
179265Sdillon/*- 279265Sdillon * Copyright (c) 1994 John Dyson 379265Sdillon * Copyright (c) 2001 Matt Dillon 479265Sdillon * 579265Sdillon * All rights reserved. Terms for use and redistribution 679265Sdillon * are covered by the BSD Copyright as found in /usr/src/COPYRIGHT. 779265Sdillon * 879265Sdillon * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 979265Sdillon * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ 1079265Sdillon * $FreeBSD: head/sys/vm/vm_zeroidle.c 82314 2001-08-25 05:00:44Z peter $ 1179265Sdillon */ 1279265Sdillon 1379265Sdillon#include <sys/param.h> 1479265Sdillon#include <sys/systm.h> 1582314Speter#include <sys/kernel.h> 1679265Sdillon#include <sys/proc.h> 1782314Speter#include <sys/resourcevar.h> 1879265Sdillon#include <sys/vmmeter.h> 1982314Speter#include <sys/lock.h> 2079265Sdillon#include <sys/mutex.h> 2179265Sdillon#include <sys/sysctl.h> 2282314Speter#include <sys/kthread.h> 2379265Sdillon 2479265Sdillon#include <vm/vm.h> 2579265Sdillon#include <vm/vm_page.h> 2679265Sdillon 2779265SdillonSYSCTL_DECL(_vm_stats_misc); 2879265Sdillon 2979265Sdillonstatic int cnt_prezero; 3079265SdillonSYSCTL_INT(_vm_stats_misc, OID_AUTO, 3179265Sdillon cnt_prezero, CTLFLAG_RD, &cnt_prezero, 0, ""); 3279265Sdillon 3382314Speterstatic int idlezero_enable = 0; 3482314SpeterSYSCTL_INT(_vm, OID_AUTO, idlezero_enable, CTLFLAG_RW, &idlezero_enable, 0, ""); 3582314SpeterTUNABLE_INT("vm.idlezero_enable", &idlezero_enable); 3682314Speter 3782314Speterstatic int idlezero_maxrun = 16; 3882314SpeterSYSCTL_INT(_vm, OID_AUTO, idlezero_maxrun, CTLFLAG_RW, &idlezero_maxrun, 0, ""); 3982314SpeterTUNABLE_INT("vm.idlezero_maxrun", &idlezero_maxrun); 4082314Speter 4179265Sdillon/* 4279265Sdillon * Implement the pre-zeroed page mechanism. 4379265Sdillon */ 4479265Sdillon 4579265Sdillon#define ZIDLE_LO(v) ((v) * 2 / 3) 4679265Sdillon#define ZIDLE_HI(v) ((v) * 4 / 5) 4779265Sdillon 4882314Speterstatic int zero_state; 4982314Speter 5082314Speterstatic int 5182314Spetervm_page_zero_check(void) 5279265Sdillon{ 5379265Sdillon 5482314Speter if (!idlezero_enable) 5582314Speter return 0; 5679265Sdillon /* 5779265Sdillon * Attempt to maintain approximately 1/2 of our free pages in a 5879265Sdillon * PG_ZERO'd state. Add some hysteresis to (attempt to) avoid 5979265Sdillon * generally zeroing a page when the system is near steady-state. 6079265Sdillon * Otherwise we might get 'flutter' during disk I/O / IPC or 6179265Sdillon * fast sleeps. We also do not want to be continuously zeroing 6279265Sdillon * pages because doing so may flush our L1 and L2 caches too much. 6379265Sdillon */ 6479265Sdillon if (zero_state && vm_page_zero_count >= ZIDLE_LO(cnt.v_free_count)) 6582314Speter return 0; 6679265Sdillon if (vm_page_zero_count >= ZIDLE_HI(cnt.v_free_count)) 6782314Speter return 0; 6882314Speter return 1; 6982314Speter} 7079265Sdillon 7182314Speterstatic int 7282314Spetervm_page_zero_idle(void) 7382314Speter{ 7482314Speter static int free_rover; 7582314Speter vm_page_t m; 7682314Speter 7782314Speter mtx_lock(&Giant); 7882314Speter zero_state = 0; 7982314Speter m = vm_pageq_find(PQ_FREE, free_rover, FALSE); 8082314Speter if (m != NULL && (m->flags & PG_ZERO) == 0) { 8182314Speter vm_page_queues[m->queue].lcnt--; 8282314Speter TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq); 8382314Speter m->queue = PQ_NONE; 8482314Speter /* maybe drop out of Giant here */ 8582314Speter pmap_zero_page(VM_PAGE_TO_PHYS(m)); 8682314Speter /* and return here */ 8782314Speter vm_page_flag_set(m, PG_ZERO); 8882314Speter m->queue = PQ_FREE + m->pc; 8982314Speter vm_page_queues[m->queue].lcnt++; 9082314Speter TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, 9182314Speter pageq); 9282314Speter ++vm_page_zero_count; 9382314Speter ++cnt_prezero; 9482314Speter if (vm_page_zero_count >= ZIDLE_HI(cnt.v_free_count)) 9582314Speter zero_state = 1; 9682314Speter } 9782314Speter free_rover = (free_rover + PQ_PRIME2) & PQ_L2_MASK; 9882314Speter mtx_unlock(&Giant); 9982314Speter return 1; 10082314Speter} 10182314Speter 10282314Speter 10382314Speter/* Called by vm_page_free to hint that a new page is available */ 10482314Spetervoid 10582314Spetervm_page_zero_idle_wakeup(void) 10682314Speter{ 10782314Speter 10882314Speter if (vm_page_zero_check()) 10982314Speter wakeup(&zero_state); 11082314Speter} 11182314Speter 11282314Speterstatic void 11382314Spetervm_pagezero(void) 11482314Speter{ 11582314Speter struct proc *p = curproc; 11682314Speter struct rtprio rtp; 11782314Speter int pages = 0; 11882314Speter 11982314Speter rtp.prio = RTP_PRIO_MAX; 12082314Speter rtp.type = RTP_PRIO_IDLE; 12182314Speter PROC_LOCK(p); 12282314Speter rtp_to_pri(&rtp, &p->p_pri); 12382314Speter PROC_UNLOCK(p); 12482314Speter 12582314Speter for (;;) { 12682314Speter if (vm_page_zero_check()) { 12782314Speter pages += vm_page_zero_idle(); 12882314Speter if (pages > idlezero_maxrun) { 12982314Speter mtx_lock_spin(&sched_lock); 13082314Speter setrunqueue(p); 13182314Speter p->p_stats->p_ru.ru_nvcsw++; 13282314Speter mi_switch(); 13382314Speter mtx_unlock_spin(&sched_lock); 13482314Speter pages = 0; 13582314Speter } 13682314Speter } else { 13782314Speter tsleep(&zero_state, PPAUSE, "pgzero", hz * 300); 13882314Speter pages = 0; 13979265Sdillon } 14079265Sdillon } 14179265Sdillon} 14279265Sdillon 14382314Speterstatic struct proc *pagezero; 14482314Speterstatic struct kproc_desc pagezero_kp = { 14582314Speter "pagezero", 14682314Speter vm_pagezero, 14782314Speter &pagezero 14882314Speter}; 14982314SpeterSYSINIT(pagezero, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, kproc_start, &pagezero_kp) 150