vm_zeroidle.c revision 104964
179265Sdillon/*- 279265Sdillon * Copyright (c) 1994 John Dyson 379265Sdillon * Copyright (c) 2001 Matt Dillon 479265Sdillon * 579265Sdillon * All rights reserved. Terms for use and redistribution 679265Sdillon * are covered by the BSD Copyright as found in /usr/src/COPYRIGHT. 779265Sdillon * 879265Sdillon * from: @(#)vm_machdep.c 7.3 (Berkeley) 5/13/91 979265Sdillon * Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$ 1079265Sdillon * $FreeBSD: head/sys/vm/vm_zeroidle.c 104964 2002-10-12 05:32:24Z jeff $ 1179265Sdillon */ 1279265Sdillon 1379265Sdillon#include <sys/param.h> 1479265Sdillon#include <sys/systm.h> 1582314Speter#include <sys/kernel.h> 1679265Sdillon#include <sys/proc.h> 1782314Speter#include <sys/resourcevar.h> 1879265Sdillon#include <sys/vmmeter.h> 1982314Speter#include <sys/lock.h> 2079265Sdillon#include <sys/mutex.h> 21104964Sjeff#include <sys/sched.h> 2279265Sdillon#include <sys/sysctl.h> 2382314Speter#include <sys/kthread.h> 2479265Sdillon 2579265Sdillon#include <vm/vm.h> 2679265Sdillon#include <vm/vm_page.h> 2779265Sdillon 2879265SdillonSYSCTL_DECL(_vm_stats_misc); 2979265Sdillon 3079265Sdillonstatic int cnt_prezero; 3179265SdillonSYSCTL_INT(_vm_stats_misc, OID_AUTO, 3279265Sdillon cnt_prezero, CTLFLAG_RD, &cnt_prezero, 0, ""); 3379265Sdillon 3499571Speterstatic int idlezero_enable = 1; 3582314SpeterSYSCTL_INT(_vm, OID_AUTO, idlezero_enable, CTLFLAG_RW, &idlezero_enable, 0, ""); 3682314SpeterTUNABLE_INT("vm.idlezero_enable", &idlezero_enable); 3782314Speter 3882314Speterstatic int idlezero_maxrun = 16; 3982314SpeterSYSCTL_INT(_vm, OID_AUTO, idlezero_maxrun, CTLFLAG_RW, &idlezero_maxrun, 0, ""); 4082314SpeterTUNABLE_INT("vm.idlezero_maxrun", &idlezero_maxrun); 4182314Speter 4279265Sdillon/* 4379265Sdillon * Implement the pre-zeroed page mechanism. 4479265Sdillon */ 4579265Sdillon 4679265Sdillon#define ZIDLE_LO(v) ((v) * 2 / 3) 4779265Sdillon#define ZIDLE_HI(v) ((v) * 4 / 5) 4879265Sdillon 4982314Speterstatic int zero_state; 5082314Speter 5182314Speterstatic int 5282314Spetervm_page_zero_check(void) 5379265Sdillon{ 5479265Sdillon 5582314Speter if (!idlezero_enable) 5682314Speter return 0; 5779265Sdillon /* 5879265Sdillon * Attempt to maintain approximately 1/2 of our free pages in a 5979265Sdillon * PG_ZERO'd state. Add some hysteresis to (attempt to) avoid 6079265Sdillon * generally zeroing a page when the system is near steady-state. 6179265Sdillon * Otherwise we might get 'flutter' during disk I/O / IPC or 6279265Sdillon * fast sleeps. We also do not want to be continuously zeroing 6379265Sdillon * pages because doing so may flush our L1 and L2 caches too much. 6479265Sdillon */ 6579265Sdillon if (zero_state && vm_page_zero_count >= ZIDLE_LO(cnt.v_free_count)) 6682314Speter return 0; 6779265Sdillon if (vm_page_zero_count >= ZIDLE_HI(cnt.v_free_count)) 6882314Speter return 0; 6982314Speter return 1; 7082314Speter} 7179265Sdillon 7282314Speterstatic int 7382314Spetervm_page_zero_idle(void) 7482314Speter{ 7582314Speter static int free_rover; 7682314Speter vm_page_t m; 7782314Speter 7899625Speter mtx_lock_spin(&vm_page_queue_free_mtx); 7982314Speter zero_state = 0; 8082314Speter m = vm_pageq_find(PQ_FREE, free_rover, FALSE); 8182314Speter if (m != NULL && (m->flags & PG_ZERO) == 0) { 82100193Salc vm_pageq_remove_nowakeup(m); 8399625Speter mtx_unlock_spin(&vm_page_queue_free_mtx); 8499571Speter pmap_zero_page_idle(m); 8599625Speter mtx_lock_spin(&vm_page_queue_free_mtx); 86100331Salc m->flags |= PG_ZERO; 87100193Salc vm_pageq_enqueue(PQ_FREE + m->pc, m); 8882314Speter ++vm_page_zero_count; 8982314Speter ++cnt_prezero; 9082314Speter if (vm_page_zero_count >= ZIDLE_HI(cnt.v_free_count)) 9182314Speter zero_state = 1; 9282314Speter } 9382314Speter free_rover = (free_rover + PQ_PRIME2) & PQ_L2_MASK; 9499625Speter mtx_unlock_spin(&vm_page_queue_free_mtx); 9582314Speter return 1; 9682314Speter} 9782314Speter 9882314Speter 9982314Speter/* Called by vm_page_free to hint that a new page is available */ 10082314Spetervoid 10182314Spetervm_page_zero_idle_wakeup(void) 10282314Speter{ 10382314Speter 10499571Speter if (idlezero_enable && vm_page_zero_check()) 10582314Speter wakeup(&zero_state); 10682314Speter} 10782314Speter 10882314Speterstatic void 10982314Spetervm_pagezero(void) 11082314Speter{ 111100379Speter struct thread *td; 112100379Speter struct proc *p; 11382314Speter struct rtprio rtp; 11482314Speter int pages = 0; 11599571Speter int pri; 11682314Speter 117100379Speter td = curthread; 118100379Speter p = td->td_proc; 11982314Speter rtp.prio = RTP_PRIO_MAX; 12082314Speter rtp.type = RTP_PRIO_IDLE; 12182756Sjhb mtx_lock_spin(&sched_lock); 12290538Sjulian rtp_to_pri(&rtp, td->td_ksegrp); 12399571Speter pri = td->td_priority; 12482756Sjhb mtx_unlock_spin(&sched_lock); 125100379Speter PROC_LOCK(p); 126100379Speter p->p_flag |= P_NOLOAD; 127100379Speter PROC_UNLOCK(p); 12882314Speter 12982314Speter for (;;) { 13082314Speter if (vm_page_zero_check()) { 13182314Speter pages += vm_page_zero_idle(); 132104964Sjeff if (pages > idlezero_maxrun || sched_runnable()) { 13382314Speter mtx_lock_spin(&sched_lock); 13483366Sjulian td->td_proc->p_stats->p_ru.ru_nvcsw++; 13582314Speter mi_switch(); 13682314Speter mtx_unlock_spin(&sched_lock); 13782314Speter pages = 0; 13882314Speter } 13982314Speter } else { 14099571Speter tsleep(&zero_state, pri, "pgzero", hz * 300); 14182314Speter pages = 0; 14279265Sdillon } 14379265Sdillon } 14479265Sdillon} 14579265Sdillon 14682314Speterstatic struct proc *pagezero; 14782314Speterstatic struct kproc_desc pagezero_kp = { 14882314Speter "pagezero", 14982314Speter vm_pagezero, 15082314Speter &pagezero 15182314Speter}; 15282314SpeterSYSINIT(pagezero, SI_SUB_KTHREAD_VM, SI_ORDER_ANY, kproc_start, &pagezero_kp) 153