kern_timeout.c (278693) | kern_timeout.c (278694) |
---|---|
1/*- 2 * Copyright (c) 1982, 1986, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * (c) UNIX System Laboratories, Inc. 5 * All or some portions of this file are derived from material licensed 6 * to the University of California by American Telephone and Telegraph 7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 8 * the permission of UNIX System Laboratories, Inc. --- 21 unchanged lines hidden (view full) --- 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 35 */ 36 37#include <sys/cdefs.h> | 1/*- 2 * Copyright (c) 1982, 1986, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * (c) UNIX System Laboratories, Inc. 5 * All or some portions of this file are derived from material licensed 6 * to the University of California by American Telephone and Telegraph 7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with 8 * the permission of UNIX System Laboratories, Inc. --- 21 unchanged lines hidden (view full) --- 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 * 34 * From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94 35 */ 36 37#include <sys/cdefs.h> |
38__FBSDID("$FreeBSD: stable/10/sys/kern/kern_timeout.c 278693 2015-02-13 18:45:44Z sbruno $"); | 38__FBSDID("$FreeBSD: stable/10/sys/kern/kern_timeout.c 278694 2015-02-13 19:06:22Z sbruno $"); |
39 40#include "opt_callout_profiling.h" 41#include "opt_kdtrace.h" 42#if defined(__arm__) 43#include "opt_timer.h" 44#endif 45 46#include <sys/param.h> --- 98 unchanged lines hidden (view full) --- 145 struct callout *cc_callout; 146 struct callout_list *cc_callwheel; 147 struct callout_tailq cc_expireq; 148 struct callout_slist cc_callfree; 149 sbintime_t cc_firstevent; 150 sbintime_t cc_lastscan; 151 void *cc_cookie; 152 u_int cc_bucket; | 39 40#include "opt_callout_profiling.h" 41#include "opt_kdtrace.h" 42#if defined(__arm__) 43#include "opt_timer.h" 44#endif 45 46#include <sys/param.h> --- 98 unchanged lines hidden (view full) --- 145 struct callout *cc_callout; 146 struct callout_list *cc_callwheel; 147 struct callout_tailq cc_expireq; 148 struct callout_slist cc_callfree; 149 sbintime_t cc_firstevent; 150 sbintime_t cc_lastscan; 151 void *cc_cookie; 152 u_int cc_bucket; |
153 char cc_ktr_event_name[20]; |
|
153}; 154 155#define cc_exec_curr cc_exec_entity[0].cc_curr 156#define cc_exec_next cc_exec_entity[0].cc_next 157#define cc_exec_cancel cc_exec_entity[0].cc_cancel 158#define cc_exec_waiting cc_exec_entity[0].cc_waiting 159#define cc_exec_curr_dir cc_exec_entity[1].cc_curr 160#define cc_exec_next_dir cc_exec_entity[1].cc_next --- 22 unchanged lines hidden (view full) --- 183#define CC_SELF() &cc_cpu 184#endif 185#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock) 186#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) 187#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) 188 189static int timeout_cpu; 190 | 154}; 155 156#define cc_exec_curr cc_exec_entity[0].cc_curr 157#define cc_exec_next cc_exec_entity[0].cc_next 158#define cc_exec_cancel cc_exec_entity[0].cc_cancel 159#define cc_exec_waiting cc_exec_entity[0].cc_waiting 160#define cc_exec_curr_dir cc_exec_entity[1].cc_curr 161#define cc_exec_next_dir cc_exec_entity[1].cc_next --- 22 unchanged lines hidden (view full) --- 184#define CC_SELF() &cc_cpu 185#endif 186#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock) 187#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock) 188#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED) 189 190static int timeout_cpu; 191 |
191static void callout_cpu_init(struct callout_cpu *cc); | 192static void callout_cpu_init(struct callout_cpu *cc, int cpu); |
192static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, 193#ifdef CALLOUT_PROFILING 194 int *mpcalls, int *lockcalls, int *gcalls, 195#endif 196 int direct); 197 198static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures"); 199 --- 78 unchanged lines hidden (view full) --- 278 * 279 * XXX: Once all timeout(9) consumers are converted this can 280 * be removed. 281 */ 282 timeout_cpu = PCPU_GET(cpuid); 283 cc = CC_CPU(timeout_cpu); 284 cc->cc_callout = malloc(ncallout * sizeof(struct callout), 285 M_CALLOUT, M_WAITOK); | 193static void softclock_call_cc(struct callout *c, struct callout_cpu *cc, 194#ifdef CALLOUT_PROFILING 195 int *mpcalls, int *lockcalls, int *gcalls, 196#endif 197 int direct); 198 199static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures"); 200 --- 78 unchanged lines hidden (view full) --- 279 * 280 * XXX: Once all timeout(9) consumers are converted this can 281 * be removed. 282 */ 283 timeout_cpu = PCPU_GET(cpuid); 284 cc = CC_CPU(timeout_cpu); 285 cc->cc_callout = malloc(ncallout * sizeof(struct callout), 286 M_CALLOUT, M_WAITOK); |
286 callout_cpu_init(cc); | 287 callout_cpu_init(cc, timeout_cpu); |
287} 288SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL); 289 290/* 291 * Initialize the per-cpu callout structures. 292 */ 293static void | 288} 289SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL); 290 291/* 292 * Initialize the per-cpu callout structures. 293 */ 294static void |
294callout_cpu_init(struct callout_cpu *cc) | 295callout_cpu_init(struct callout_cpu *cc, int cpu) |
295{ 296 struct callout *c; 297 int i; 298 299 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); 300 SLIST_INIT(&cc->cc_callfree); 301 cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize, 302 M_CALLOUT, M_WAITOK); 303 for (i = 0; i < callwheelsize; i++) 304 LIST_INIT(&cc->cc_callwheel[i]); 305 TAILQ_INIT(&cc->cc_expireq); 306 cc->cc_firstevent = INT64_MAX; 307 for (i = 0; i < 2; i++) 308 cc_cce_cleanup(cc, i); | 296{ 297 struct callout *c; 298 int i; 299 300 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE); 301 SLIST_INIT(&cc->cc_callfree); 302 cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize, 303 M_CALLOUT, M_WAITOK); 304 for (i = 0; i < callwheelsize; i++) 305 LIST_INIT(&cc->cc_callwheel[i]); 306 TAILQ_INIT(&cc->cc_expireq); 307 cc->cc_firstevent = INT64_MAX; 308 for (i = 0; i < 2; i++) 309 cc_cce_cleanup(cc, i); |
310 snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name), 311 "callwheel cpu %d", cpu); |
|
309 if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */ 310 return; 311 for (i = 0; i < ncallout; i++) { 312 c = &cc->cc_callout[i]; 313 callout_init(c, 0); 314 c->c_flags = CALLOUT_LOCAL_ALLOC; 315 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); 316 } --- 45 unchanged lines hidden (view full) --- 362 INTR_MPSAFE, &cc->cc_cookie)) 363 panic("died while creating standard software ithreads"); 364#ifdef SMP 365 CPU_FOREACH(cpu) { 366 if (cpu == timeout_cpu) 367 continue; 368 cc = CC_CPU(cpu); 369 cc->cc_callout = NULL; /* Only cpu0 handles timeout(9). */ | 312 if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */ 313 return; 314 for (i = 0; i < ncallout; i++) { 315 c = &cc->cc_callout[i]; 316 callout_init(c, 0); 317 c->c_flags = CALLOUT_LOCAL_ALLOC; 318 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle); 319 } --- 45 unchanged lines hidden (view full) --- 365 INTR_MPSAFE, &cc->cc_cookie)) 366 panic("died while creating standard software ithreads"); 367#ifdef SMP 368 CPU_FOREACH(cpu) { 369 if (cpu == timeout_cpu) 370 continue; 371 cc = CC_CPU(cpu); 372 cc->cc_callout = NULL; /* Only cpu0 handles timeout(9). */ |
370 callout_cpu_init(cc); | 373 callout_cpu_init(cc, cpu); |
371 if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK, 372 INTR_MPSAFE, &cc->cc_cookie)) 373 panic("died while creating standard software ithreads"); 374 } 375#endif 376} 377SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL); 378 --- 290 unchanged lines hidden (view full) --- 669 } 670 } else { 671#ifdef CALLOUT_PROFILING 672 (*mpcalls)++; 673#endif 674 CTR3(KTR_CALLOUT, "callout %p func %p arg %p", 675 c, c_func, c_arg); 676 } | 374 if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK, 375 INTR_MPSAFE, &cc->cc_cookie)) 376 panic("died while creating standard software ithreads"); 377 } 378#endif 379} 380SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL); 381 --- 290 unchanged lines hidden (view full) --- 672 } 673 } else { 674#ifdef CALLOUT_PROFILING 675 (*mpcalls)++; 676#endif 677 CTR3(KTR_CALLOUT, "callout %p func %p arg %p", 678 c, c_func, c_arg); 679 } |
680 KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running", 681 "func:%p", c_func, "arg:%p", c_arg, "direct:%d", direct); |
|
677#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) 678 sbt1 = sbinuptime(); 679#endif 680 THREAD_NO_SLEEPING(); 681 SDT_PROBE(callout_execute, kernel, , callout__start, c, 0, 0, 0, 0); 682 c_func(c_arg); 683 SDT_PROBE(callout_execute, kernel, , callout__end, c, 0, 0, 0, 0); 684 THREAD_SLEEPING_OK(); --- 6 unchanged lines hidden (view full) --- 691 printf( 692 "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", 693 c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); 694 } 695 maxdt = sbt2; 696 lastfunc = c_func; 697 } 698#endif | 682#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING) 683 sbt1 = sbinuptime(); 684#endif 685 THREAD_NO_SLEEPING(); 686 SDT_PROBE(callout_execute, kernel, , callout__start, c, 0, 0, 0, 0); 687 c_func(c_arg); 688 SDT_PROBE(callout_execute, kernel, , callout__end, c, 0, 0, 0, 0); 689 THREAD_SLEEPING_OK(); --- 6 unchanged lines hidden (view full) --- 696 printf( 697 "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n", 698 c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec); 699 } 700 maxdt = sbt2; 701 lastfunc = c_func; 702 } 703#endif |
704 KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle"); |
|
699 CTR1(KTR_CALLOUT, "callout %p finished", c); 700 if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0) 701 class->lc_unlock(c_lock); 702skip: 703 CC_LOCK(cc); 704 KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr")); 705 cc->cc_exec_entity[direct].cc_curr = NULL; 706 if (cc->cc_exec_entity[direct].cc_waiting) { --- 740 unchanged lines hidden --- | 705 CTR1(KTR_CALLOUT, "callout %p finished", c); 706 if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0) 707 class->lc_unlock(c_lock); 708skip: 709 CC_LOCK(cc); 710 KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr")); 711 cc->cc_exec_entity[direct].cc_curr = NULL; 712 if (cc->cc_exec_entity[direct].cc_waiting) { --- 740 unchanged lines hidden --- |