Deleted Added
sdiff udiff text old ( 278693 ) new ( 278694 )
full compact
kern_timeout.c (278693) kern_timeout.c (278694)
1/*-
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 21 unchanged lines hidden (view full) ---

30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
35 */
36
37#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 1982, 1986, 1991, 1993
3 * The Regents of the University of California. All rights reserved.
4 * (c) UNIX System Laboratories, Inc.
5 * All or some portions of this file are derived from material licensed
6 * to the University of California by American Telephone and Telegraph
7 * Co. or Unix System Laboratories, Inc. and are reproduced herein with
8 * the permission of UNIX System Laboratories, Inc.

--- 21 unchanged lines hidden (view full) ---

30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 *
34 * From: @(#)kern_clock.c 8.5 (Berkeley) 1/21/94
35 */
36
37#include <sys/cdefs.h>
38__FBSDID("$FreeBSD: stable/10/sys/kern/kern_timeout.c 278693 2015-02-13 18:45:44Z sbruno $");
38__FBSDID("$FreeBSD: stable/10/sys/kern/kern_timeout.c 278694 2015-02-13 19:06:22Z sbruno $");
39
40#include "opt_callout_profiling.h"
41#include "opt_kdtrace.h"
42#if defined(__arm__)
43#include "opt_timer.h"
44#endif
45
46#include <sys/param.h>

--- 98 unchanged lines hidden (view full) ---

145 struct callout *cc_callout;
146 struct callout_list *cc_callwheel;
147 struct callout_tailq cc_expireq;
148 struct callout_slist cc_callfree;
149 sbintime_t cc_firstevent;
150 sbintime_t cc_lastscan;
151 void *cc_cookie;
152 u_int cc_bucket;
39
40#include "opt_callout_profiling.h"
41#include "opt_kdtrace.h"
42#if defined(__arm__)
43#include "opt_timer.h"
44#endif
45
46#include <sys/param.h>

--- 98 unchanged lines hidden (view full) ---

145 struct callout *cc_callout;
146 struct callout_list *cc_callwheel;
147 struct callout_tailq cc_expireq;
148 struct callout_slist cc_callfree;
149 sbintime_t cc_firstevent;
150 sbintime_t cc_lastscan;
151 void *cc_cookie;
152 u_int cc_bucket;
153 char cc_ktr_event_name[20];
153};
154
155#define cc_exec_curr cc_exec_entity[0].cc_curr
156#define cc_exec_next cc_exec_entity[0].cc_next
157#define cc_exec_cancel cc_exec_entity[0].cc_cancel
158#define cc_exec_waiting cc_exec_entity[0].cc_waiting
159#define cc_exec_curr_dir cc_exec_entity[1].cc_curr
160#define cc_exec_next_dir cc_exec_entity[1].cc_next

--- 22 unchanged lines hidden (view full) ---

183#define CC_SELF() &cc_cpu
184#endif
185#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)
186#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)
187#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED)
188
189static int timeout_cpu;
190
154};
155
156#define cc_exec_curr cc_exec_entity[0].cc_curr
157#define cc_exec_next cc_exec_entity[0].cc_next
158#define cc_exec_cancel cc_exec_entity[0].cc_cancel
159#define cc_exec_waiting cc_exec_entity[0].cc_waiting
160#define cc_exec_curr_dir cc_exec_entity[1].cc_curr
161#define cc_exec_next_dir cc_exec_entity[1].cc_next

--- 22 unchanged lines hidden (view full) ---

184#define CC_SELF() &cc_cpu
185#endif
186#define CC_LOCK(cc) mtx_lock_spin(&(cc)->cc_lock)
187#define CC_UNLOCK(cc) mtx_unlock_spin(&(cc)->cc_lock)
188#define CC_LOCK_ASSERT(cc) mtx_assert(&(cc)->cc_lock, MA_OWNED)
189
190static int timeout_cpu;
191
191static void callout_cpu_init(struct callout_cpu *cc);
192static void callout_cpu_init(struct callout_cpu *cc, int cpu);
192static void softclock_call_cc(struct callout *c, struct callout_cpu *cc,
193#ifdef CALLOUT_PROFILING
194 int *mpcalls, int *lockcalls, int *gcalls,
195#endif
196 int direct);
197
198static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
199

--- 78 unchanged lines hidden (view full) ---

278 *
279 * XXX: Once all timeout(9) consumers are converted this can
280 * be removed.
281 */
282 timeout_cpu = PCPU_GET(cpuid);
283 cc = CC_CPU(timeout_cpu);
284 cc->cc_callout = malloc(ncallout * sizeof(struct callout),
285 M_CALLOUT, M_WAITOK);
193static void softclock_call_cc(struct callout *c, struct callout_cpu *cc,
194#ifdef CALLOUT_PROFILING
195 int *mpcalls, int *lockcalls, int *gcalls,
196#endif
197 int direct);
198
199static MALLOC_DEFINE(M_CALLOUT, "callout", "Callout datastructures");
200

--- 78 unchanged lines hidden (view full) ---

279 *
280 * XXX: Once all timeout(9) consumers are converted this can
281 * be removed.
282 */
283 timeout_cpu = PCPU_GET(cpuid);
284 cc = CC_CPU(timeout_cpu);
285 cc->cc_callout = malloc(ncallout * sizeof(struct callout),
286 M_CALLOUT, M_WAITOK);
286 callout_cpu_init(cc);
287 callout_cpu_init(cc, timeout_cpu);
287}
288SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL);
289
290/*
291 * Initialize the per-cpu callout structures.
292 */
293static void
288}
289SYSINIT(callwheel_init, SI_SUB_CPU, SI_ORDER_ANY, callout_callwheel_init, NULL);
290
291/*
292 * Initialize the per-cpu callout structures.
293 */
294static void
294callout_cpu_init(struct callout_cpu *cc)
295callout_cpu_init(struct callout_cpu *cc, int cpu)
295{
296 struct callout *c;
297 int i;
298
299 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
300 SLIST_INIT(&cc->cc_callfree);
301 cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,
302 M_CALLOUT, M_WAITOK);
303 for (i = 0; i < callwheelsize; i++)
304 LIST_INIT(&cc->cc_callwheel[i]);
305 TAILQ_INIT(&cc->cc_expireq);
306 cc->cc_firstevent = INT64_MAX;
307 for (i = 0; i < 2; i++)
308 cc_cce_cleanup(cc, i);
296{
297 struct callout *c;
298 int i;
299
300 mtx_init(&cc->cc_lock, "callout", NULL, MTX_SPIN | MTX_RECURSE);
301 SLIST_INIT(&cc->cc_callfree);
302 cc->cc_callwheel = malloc(sizeof(struct callout_list) * callwheelsize,
303 M_CALLOUT, M_WAITOK);
304 for (i = 0; i < callwheelsize; i++)
305 LIST_INIT(&cc->cc_callwheel[i]);
306 TAILQ_INIT(&cc->cc_expireq);
307 cc->cc_firstevent = INT64_MAX;
308 for (i = 0; i < 2; i++)
309 cc_cce_cleanup(cc, i);
310 snprintf(cc->cc_ktr_event_name, sizeof(cc->cc_ktr_event_name),
311 "callwheel cpu %d", cpu);
309 if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */
310 return;
311 for (i = 0; i < ncallout; i++) {
312 c = &cc->cc_callout[i];
313 callout_init(c, 0);
314 c->c_flags = CALLOUT_LOCAL_ALLOC;
315 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
316 }

--- 45 unchanged lines hidden (view full) ---

362 INTR_MPSAFE, &cc->cc_cookie))
363 panic("died while creating standard software ithreads");
364#ifdef SMP
365 CPU_FOREACH(cpu) {
366 if (cpu == timeout_cpu)
367 continue;
368 cc = CC_CPU(cpu);
369 cc->cc_callout = NULL; /* Only cpu0 handles timeout(9). */
312 if (cc->cc_callout == NULL) /* Only cpu0 handles timeout(9) */
313 return;
314 for (i = 0; i < ncallout; i++) {
315 c = &cc->cc_callout[i];
316 callout_init(c, 0);
317 c->c_flags = CALLOUT_LOCAL_ALLOC;
318 SLIST_INSERT_HEAD(&cc->cc_callfree, c, c_links.sle);
319 }

--- 45 unchanged lines hidden (view full) ---

365 INTR_MPSAFE, &cc->cc_cookie))
366 panic("died while creating standard software ithreads");
367#ifdef SMP
368 CPU_FOREACH(cpu) {
369 if (cpu == timeout_cpu)
370 continue;
371 cc = CC_CPU(cpu);
372 cc->cc_callout = NULL; /* Only cpu0 handles timeout(9). */
370 callout_cpu_init(cc);
373 callout_cpu_init(cc, cpu);
371 if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK,
372 INTR_MPSAFE, &cc->cc_cookie))
373 panic("died while creating standard software ithreads");
374 }
375#endif
376}
377SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL);
378

--- 290 unchanged lines hidden (view full) ---

669 }
670 } else {
671#ifdef CALLOUT_PROFILING
672 (*mpcalls)++;
673#endif
674 CTR3(KTR_CALLOUT, "callout %p func %p arg %p",
675 c, c_func, c_arg);
676 }
374 if (swi_add(NULL, "clock", softclock, cc, SWI_CLOCK,
375 INTR_MPSAFE, &cc->cc_cookie))
376 panic("died while creating standard software ithreads");
377 }
378#endif
379}
380SYSINIT(start_softclock, SI_SUB_SOFTINTR, SI_ORDER_FIRST, start_softclock, NULL);
381

--- 290 unchanged lines hidden (view full) ---

672 }
673 } else {
674#ifdef CALLOUT_PROFILING
675 (*mpcalls)++;
676#endif
677 CTR3(KTR_CALLOUT, "callout %p func %p arg %p",
678 c, c_func, c_arg);
679 }
680 KTR_STATE3(KTR_SCHED, "callout", cc->cc_ktr_event_name, "running",
681 "func:%p", c_func, "arg:%p", c_arg, "direct:%d", direct);
677#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING)
678 sbt1 = sbinuptime();
679#endif
680 THREAD_NO_SLEEPING();
681 SDT_PROBE(callout_execute, kernel, , callout__start, c, 0, 0, 0, 0);
682 c_func(c_arg);
683 SDT_PROBE(callout_execute, kernel, , callout__end, c, 0, 0, 0, 0);
684 THREAD_SLEEPING_OK();

--- 6 unchanged lines hidden (view full) ---

691 printf(
692 "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
693 c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec);
694 }
695 maxdt = sbt2;
696 lastfunc = c_func;
697 }
698#endif
682#if defined(DIAGNOSTIC) || defined(CALLOUT_PROFILING)
683 sbt1 = sbinuptime();
684#endif
685 THREAD_NO_SLEEPING();
686 SDT_PROBE(callout_execute, kernel, , callout__start, c, 0, 0, 0, 0);
687 c_func(c_arg);
688 SDT_PROBE(callout_execute, kernel, , callout__end, c, 0, 0, 0, 0);
689 THREAD_SLEEPING_OK();

--- 6 unchanged lines hidden (view full) ---

696 printf(
697 "Expensive timeout(9) function: %p(%p) %jd.%09ld s\n",
698 c_func, c_arg, (intmax_t)ts2.tv_sec, ts2.tv_nsec);
699 }
700 maxdt = sbt2;
701 lastfunc = c_func;
702 }
703#endif
704 KTR_STATE0(KTR_SCHED, "callout", cc->cc_ktr_event_name, "idle");
699 CTR1(KTR_CALLOUT, "callout %p finished", c);
700 if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
701 class->lc_unlock(c_lock);
702skip:
703 CC_LOCK(cc);
704 KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr"));
705 cc->cc_exec_entity[direct].cc_curr = NULL;
706 if (cc->cc_exec_entity[direct].cc_waiting) {

--- 740 unchanged lines hidden --- 		705 CTR1(KTR_CALLOUT, "callout %p finished", c);
706 if ((c_flags & CALLOUT_RETURNUNLOCKED) == 0)
707 class->lc_unlock(c_lock);
708skip:
709 CC_LOCK(cc);
710 KASSERT(cc->cc_exec_entity[direct].cc_curr == c, ("mishandled cc_curr"));
711 cc->cc_exec_entity[direct].cc_curr = NULL;
712 if (cc->cc_exec_entity[direct].cc_waiting) {

--- 740 unchanged lines hidden ---