Deleted Added
sdiff udiff text old ( 215317 ) new ( 215701 )
full compact
kern_clocksource.c (215317) kern_clocksource.c (215701)
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer,
10 * without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer,
10 * without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 215317 2010-11-14 20:38:11Z dim $");
28__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 215701 2010-11-22 19:32:54Z dim $");
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_device_polling.h"
38#include "opt_kdtrace.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/lock.h>
44#include <sys/kdb.h>
45#include <sys/ktr.h>
46#include <sys/mutex.h>
47#include <sys/proc.h>
48#include <sys/kernel.h>
49#include <sys/sched.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/timeet.h>
53#include <sys/timetc.h>
54
55#include <machine/atomic.h>
56#include <machine/clock.h>
57#include <machine/cpu.h>
58#include <machine/smp.h>
59
60#ifdef KDTRACE_HOOKS
61#include <sys/dtrace_bsd.h>
62cyclic_clock_func_t cyclic_clock_func[MAXCPU];
63#endif
64
65int cpu_disable_deep_sleep = 0; /* Timer dies in C3. */
66
67static void setuptimer(void);
68static void loadtimer(struct bintime *now, int first);
69static int doconfigtimer(void);
70static void configtimer(int start);
71static int round_freq(struct eventtimer *et, int freq);
72
73static void getnextcpuevent(struct bintime *event, int idle);
74static void getnextevent(struct bintime *event);
75static int handleevents(struct bintime *now, int fake);
76#ifdef SMP
77static void cpu_new_callout(int cpu, int ticks);
78#endif
79
80static struct mtx et_hw_mtx;
81
82#define ET_HW_LOCK(state) \
83 { \
84 if (timer->et_flags & ET_FLAGS_PERCPU) \
85 mtx_lock_spin(&(state)->et_hw_mtx); \
86 else \
87 mtx_lock_spin(&et_hw_mtx); \
88 }
89
90#define ET_HW_UNLOCK(state) \
91 { \
92 if (timer->et_flags & ET_FLAGS_PERCPU) \
93 mtx_unlock_spin(&(state)->et_hw_mtx); \
94 else \
95 mtx_unlock_spin(&et_hw_mtx); \
96 }
97
98static struct eventtimer *timer = NULL;
99static struct bintime timerperiod; /* Timer period for periodic mode. */
100static struct bintime hardperiod; /* hardclock() events period. */
101static struct bintime statperiod; /* statclock() events period. */
102static struct bintime profperiod; /* profclock() events period. */
103static struct bintime nexttick; /* Next global timer tick time. */
104static u_int busy = 0; /* Reconfiguration is in progress. */
105static int profiling = 0; /* Profiling events enabled. */
106
107static char timername[32]; /* Wanted timer. */
108TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
109
110static int singlemul = 0; /* Multiplier for periodic mode. */
111TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
112SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
113 0, "Multiplier for periodic mode");
114
115static u_int idletick = 0; /* Idle mode allowed. */
116TUNABLE_INT("kern.eventtimer.idletick", &idletick);
117SYSCTL_INT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
118 0, "Run periodic events when idle");
119
120static int periodic = 0; /* Periodic or one-shot mode. */
121static int want_periodic = 0; /* What mode to prefer. */
122TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
123
124struct pcpu_state {
125 struct mtx et_hw_mtx; /* Per-CPU timer mutex. */
126 u_int action; /* Reconfiguration requests. */
127 u_int handle; /* Immediate handle resuests. */
128 struct bintime now; /* Last tick time. */
129 struct bintime nextevent; /* Next scheduled event on this CPU. */
130 struct bintime nexttick; /* Next timer tick time. */
131 struct bintime nexthard; /* Next hardlock() event. */
132 struct bintime nextstat; /* Next statclock() event. */
133 struct bintime nextprof; /* Next profclock() event. */
134 int ipi; /* This CPU needs IPI. */
135 int idle; /* This CPU is in idle mode. */
136};
137
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_device_polling.h"
38#include "opt_kdtrace.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/lock.h>
44#include <sys/kdb.h>
45#include <sys/ktr.h>
46#include <sys/mutex.h>
47#include <sys/proc.h>
48#include <sys/kernel.h>
49#include <sys/sched.h>
50#include <sys/smp.h>
51#include <sys/sysctl.h>
52#include <sys/timeet.h>
53#include <sys/timetc.h>
54
55#include <machine/atomic.h>
56#include <machine/clock.h>
57#include <machine/cpu.h>
58#include <machine/smp.h>
59
60#ifdef KDTRACE_HOOKS
61#include <sys/dtrace_bsd.h>
62cyclic_clock_func_t cyclic_clock_func[MAXCPU];
63#endif
64
65int cpu_disable_deep_sleep = 0; /* Timer dies in C3. */
66
67static void setuptimer(void);
68static void loadtimer(struct bintime *now, int first);
69static int doconfigtimer(void);
70static void configtimer(int start);
71static int round_freq(struct eventtimer *et, int freq);
72
73static void getnextcpuevent(struct bintime *event, int idle);
74static void getnextevent(struct bintime *event);
75static int handleevents(struct bintime *now, int fake);
76#ifdef SMP
77static void cpu_new_callout(int cpu, int ticks);
78#endif
79
80static struct mtx et_hw_mtx;
81
82#define ET_HW_LOCK(state) \
83 { \
84 if (timer->et_flags & ET_FLAGS_PERCPU) \
85 mtx_lock_spin(&(state)->et_hw_mtx); \
86 else \
87 mtx_lock_spin(&et_hw_mtx); \
88 }
89
90#define ET_HW_UNLOCK(state) \
91 { \
92 if (timer->et_flags & ET_FLAGS_PERCPU) \
93 mtx_unlock_spin(&(state)->et_hw_mtx); \
94 else \
95 mtx_unlock_spin(&et_hw_mtx); \
96 }
97
98static struct eventtimer *timer = NULL;
99static struct bintime timerperiod; /* Timer period for periodic mode. */
100static struct bintime hardperiod; /* hardclock() events period. */
101static struct bintime statperiod; /* statclock() events period. */
102static struct bintime profperiod; /* profclock() events period. */
103static struct bintime nexttick; /* Next global timer tick time. */
104static u_int busy = 0; /* Reconfiguration is in progress. */
105static int profiling = 0; /* Profiling events enabled. */
106
107static char timername[32]; /* Wanted timer. */
108TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
109
110static int singlemul = 0; /* Multiplier for periodic mode. */
111TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
112SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
113 0, "Multiplier for periodic mode");
114
115static u_int idletick = 0; /* Idle mode allowed. */
116TUNABLE_INT("kern.eventtimer.idletick", &idletick);
117SYSCTL_INT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
118 0, "Run periodic events when idle");
119
120static int periodic = 0; /* Periodic or one-shot mode. */
121static int want_periodic = 0; /* What mode to prefer. */
122TUNABLE_INT("kern.eventtimer.periodic", &want_periodic);
123
124struct pcpu_state {
125 struct mtx et_hw_mtx; /* Per-CPU timer mutex. */
126 u_int action; /* Reconfiguration requests. */
127 u_int handle; /* Immediate handle resuests. */
128 struct bintime now; /* Last tick time. */
129 struct bintime nextevent; /* Next scheduled event on this CPU. */
130 struct bintime nexttick; /* Next timer tick time. */
131 struct bintime nexthard; /* Next hardlock() event. */
132 struct bintime nextstat; /* Next statclock() event. */
133 struct bintime nextprof; /* Next profclock() event. */
134 int ipi; /* This CPU needs IPI. */
135 int idle; /* This CPU is in idle mode. */
136};
137
138STATIC_DPCPU_DEFINE(struct pcpu_state, timerstate);
138static DPCPU_DEFINE(struct pcpu_state, timerstate);
139
140#define FREQ2BT(freq, bt) \
141{ \
142 (bt)->sec = 0; \
143 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \
144}
145#define BT2FREQ(bt) \
146 (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \
147 ((bt)->frac >> 1))
148
149/*
150 * Timer broadcast IPI handler.
151 */
152int
153hardclockintr(void)
154{
155 struct bintime now;
156 struct pcpu_state *state;
157 int done;
158
159 if (doconfigtimer() || busy)
160 return (FILTER_HANDLED);
161 state = DPCPU_PTR(timerstate);
162 now = state->now;
163 CTR4(KTR_SPARE2, "ipi at %d: now %d.%08x%08x",
164 curcpu, now.sec, (unsigned int)(now.frac >> 32),
165 (unsigned int)(now.frac & 0xffffffff));
166 done = handleevents(&now, 0);
167 return (done ? FILTER_HANDLED : FILTER_STRAY);
168}
169
170/*
171 * Handle all events for specified time on this CPU
172 */
173static int
174handleevents(struct bintime *now, int fake)
175{
176 struct bintime t;
177 struct trapframe *frame;
178 struct pcpu_state *state;
179 uintfptr_t pc;
180 int usermode;
181 int done, runs;
182
183 CTR4(KTR_SPARE2, "handle at %d: now %d.%08x%08x",
184 curcpu, now->sec, (unsigned int)(now->frac >> 32),
185 (unsigned int)(now->frac & 0xffffffff));
186 done = 0;
187 if (fake) {
188 frame = NULL;
189 usermode = 0;
190 pc = 0;
191 } else {
192 frame = curthread->td_intr_frame;
193 usermode = TRAPF_USERMODE(frame);
194 pc = TRAPF_PC(frame);
195 }
196#ifdef KDTRACE_HOOKS
197 /*
198 * If the DTrace hooks are configured and a callback function
199 * has been registered, then call it to process the high speed
200 * timers.
201 */
202 if (!fake && cyclic_clock_func[curcpu] != NULL)
203 (*cyclic_clock_func[curcpu])(frame);
204#endif
205 runs = 0;
206 state = DPCPU_PTR(timerstate);
207 while (bintime_cmp(now, &state->nexthard, >=)) {
208 bintime_add(&state->nexthard, &hardperiod);
209 runs++;
210 }
211 if (runs && fake < 2) {
212 hardclock_anycpu(runs, usermode);
213 done = 1;
214 }
215 while (bintime_cmp(now, &state->nextstat, >=)) {
216 if (fake < 2)
217 statclock(usermode);
218 bintime_add(&state->nextstat, &statperiod);
219 done = 1;
220 }
221 if (profiling) {
222 while (bintime_cmp(now, &state->nextprof, >=)) {
223 if (!fake)
224 profclock(usermode, pc);
225 bintime_add(&state->nextprof, &profperiod);
226 done = 1;
227 }
228 } else
229 state->nextprof = state->nextstat;
230 getnextcpuevent(&t, 0);
231 if (fake == 2) {
232 state->nextevent = t;
233 return (done);
234 }
235 ET_HW_LOCK(state);
236 if (!busy) {
237 state->idle = 0;
238 state->nextevent = t;
239 loadtimer(now, 0);
240 }
241 ET_HW_UNLOCK(state);
242 return (done);
243}
244
245/*
246 * Schedule binuptime of the next event on current CPU.
247 */
248static void
249getnextcpuevent(struct bintime *event, int idle)
250{
251 struct bintime tmp;
252 struct pcpu_state *state;
253 int skip;
254
255 state = DPCPU_PTR(timerstate);
256 *event = state->nexthard;
257 if (idle) { /* If CPU is idle - ask callouts for how long. */
258 skip = 4;
259 if (curcpu == CPU_FIRST() && tc_min_ticktock_freq > skip)
260 skip = tc_min_ticktock_freq;
261 skip = callout_tickstofirst(hz / skip) - 1;
262 CTR2(KTR_SPARE2, "skip at %d: %d", curcpu, skip);
263 tmp = hardperiod;
264 bintime_mul(&tmp, skip);
265 bintime_add(event, &tmp);
266 } else { /* If CPU is active - handle all types of events. */
267 if (bintime_cmp(event, &state->nextstat, >))
268 *event = state->nextstat;
269 if (profiling &&
270 bintime_cmp(event, &state->nextprof, >))
271 *event = state->nextprof;
272 }
273}
274
275/*
276 * Schedule binuptime of the next event on all CPUs.
277 */
278static void
279getnextevent(struct bintime *event)
280{
281 struct pcpu_state *state;
282#ifdef SMP
283 int cpu;
284#endif
285 int c;
286
287 state = DPCPU_PTR(timerstate);
288 *event = state->nextevent;
289 c = curcpu;
290#ifdef SMP
291 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
292 CPU_FOREACH(cpu) {
293 if (curcpu == cpu)
294 continue;
295 state = DPCPU_ID_PTR(cpu, timerstate);
296 if (bintime_cmp(event, &state->nextevent, >)) {
297 *event = state->nextevent;
298 c = cpu;
299 }
300 }
301 }
302#endif
303 CTR5(KTR_SPARE2, "next at %d: next %d.%08x%08x by %d",
304 curcpu, event->sec, (unsigned int)(event->frac >> 32),
305 (unsigned int)(event->frac & 0xffffffff), c);
306}
307
308/* Hardware timer callback function. */
309static void
310timercb(struct eventtimer *et, void *arg)
311{
312 struct bintime now;
313 struct bintime *next;
314 struct pcpu_state *state;
315#ifdef SMP
316 int cpu, bcast;
317#endif
318
319 /* Do not touch anything if somebody reconfiguring timers. */
320 if (busy)
321 return;
322 /* Update present and next tick times. */
323 state = DPCPU_PTR(timerstate);
324 if (et->et_flags & ET_FLAGS_PERCPU) {
325 next = &state->nexttick;
326 } else
327 next = &nexttick;
328 if (periodic) {
329 now = *next; /* Ex-next tick time becomes present time. */
330 bintime_add(next, &timerperiod); /* Next tick in 1 period. */
331 } else {
332 binuptime(&now); /* Get present time from hardware. */
333 next->sec = -1; /* Next tick is not scheduled yet. */
334 }
335 state->now = now;
336 CTR4(KTR_SPARE2, "intr at %d: now %d.%08x%08x",
337 curcpu, now.sec, (unsigned int)(now.frac >> 32),
338 (unsigned int)(now.frac & 0xffffffff));
339
340#ifdef SMP
341 /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
342 bcast = 0;
343 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
344 CPU_FOREACH(cpu) {
345 state = DPCPU_ID_PTR(cpu, timerstate);
346 ET_HW_LOCK(state);
347 state->now = now;
348 if (bintime_cmp(&now, &state->nextevent, >=)) {
349 state->nextevent.sec++;
350 if (curcpu != cpu) {
351 state->ipi = 1;
352 bcast = 1;
353 }
354 }
355 ET_HW_UNLOCK(state);
356 }
357 }
358#endif
359
360 /* Handle events for this time on this CPU. */
361 handleevents(&now, 0);
362
363#ifdef SMP
364 /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
365 if (bcast) {
366 CPU_FOREACH(cpu) {
367 if (curcpu == cpu)
368 continue;
369 state = DPCPU_ID_PTR(cpu, timerstate);
370 if (state->ipi) {
371 state->ipi = 0;
372 ipi_cpu(cpu, IPI_HARDCLOCK);
373 }
374 }
375 }
376#endif
377}
378
379/*
380 * Load new value into hardware timer.
381 */
382static void
383loadtimer(struct bintime *now, int start)
384{
385 struct pcpu_state *state;
386 struct bintime new;
387 struct bintime *next;
388 uint64_t tmp;
389 int eq;
390
391 if (timer->et_flags & ET_FLAGS_PERCPU) {
392 state = DPCPU_PTR(timerstate);
393 next = &state->nexttick;
394 } else
395 next = &nexttick;
396 if (periodic) {
397 if (start) {
398 /*
399 * Try to start all periodic timers aligned
400 * to period to make events synchronous.
401 */
402 tmp = ((uint64_t)now->sec << 36) + (now->frac >> 28);
403 tmp = (tmp % (timerperiod.frac >> 28)) << 28;
404 new.sec = 0;
405 new.frac = timerperiod.frac - tmp;
406 if (new.frac < tmp) /* Left less then passed. */
407 bintime_add(&new, &timerperiod);
408 CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x",
409 curcpu, now->sec, (unsigned int)(now->frac >> 32),
410 new.sec, (unsigned int)(new.frac >> 32));
411 *next = new;
412 bintime_add(next, now);
413 et_start(timer, &new, &timerperiod);
414 }
415 } else {
416 getnextevent(&new);
417 eq = bintime_cmp(&new, next, ==);
418 CTR5(KTR_SPARE2, "load at %d: next %d.%08x%08x eq %d",
419 curcpu, new.sec, (unsigned int)(new.frac >> 32),
420 (unsigned int)(new.frac & 0xffffffff),
421 eq);
422 if (!eq) {
423 *next = new;
424 bintime_sub(&new, now);
425 et_start(timer, &new, NULL);
426 }
427 }
428}
429
430/*
431 * Prepare event timer parameters after configuration changes.
432 */
433static void
434setuptimer(void)
435{
436 int freq;
437
438 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
439 periodic = 0;
440 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
441 periodic = 1;
442 singlemul = MIN(MAX(singlemul, 1), 20);
443 freq = hz * singlemul;
444 while (freq < (profiling ? profhz : stathz))
445 freq += hz;
446 freq = round_freq(timer, freq);
447 FREQ2BT(freq, &timerperiod);
448}
449
450/*
451 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
452 */
453static int
454doconfigtimer(void)
455{
456 struct bintime now;
457 struct pcpu_state *state;
458
459 state = DPCPU_PTR(timerstate);
460 switch (atomic_load_acq_int(&state->action)) {
461 case 1:
462 binuptime(&now);
463 ET_HW_LOCK(state);
464 loadtimer(&now, 1);
465 ET_HW_UNLOCK(state);
466 state->handle = 0;
467 atomic_store_rel_int(&state->action, 0);
468 return (1);
469 case 2:
470 ET_HW_LOCK(state);
471 et_stop(timer);
472 ET_HW_UNLOCK(state);
473 state->handle = 0;
474 atomic_store_rel_int(&state->action, 0);
475 return (1);
476 }
477 if (atomic_readandclear_int(&state->handle) && !busy) {
478 binuptime(&now);
479 handleevents(&now, 0);
480 return (1);
481 }
482 return (0);
483}
484
485/*
486 * Reconfigure specified timer.
487 * For per-CPU timers use IPI to make other CPUs to reconfigure.
488 */
489static void
490configtimer(int start)
491{
492 struct bintime now, next;
493 struct pcpu_state *state;
494 int cpu;
495
496 if (start) {
497 setuptimer();
498 binuptime(&now);
499 }
500 critical_enter();
501 ET_HW_LOCK(DPCPU_PTR(timerstate));
502 if (start) {
503 /* Initialize time machine parameters. */
504 next = now;
505 bintime_add(&next, &timerperiod);
506 if (periodic)
507 nexttick = next;
508 else
509 nexttick.sec = -1;
510 CPU_FOREACH(cpu) {
511 state = DPCPU_ID_PTR(cpu, timerstate);
512 state->now = now;
513 state->nextevent = next;
514 if (periodic)
515 state->nexttick = next;
516 else
517 state->nexttick.sec = -1;
518 state->nexthard = next;
519 state->nextstat = next;
520 state->nextprof = next;
521 hardclock_sync(cpu);
522 }
523 busy = 0;
524 /* Start global timer or per-CPU timer of this CPU. */
525 loadtimer(&now, 1);
526 } else {
527 busy = 1;
528 /* Stop global timer or per-CPU timer of this CPU. */
529 et_stop(timer);
530 }
531 ET_HW_UNLOCK(DPCPU_PTR(timerstate));
532#ifdef SMP
533 /* If timer is global or there is no other CPUs yet - we are done. */
534 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
535 critical_exit();
536 return;
537 }
538 /* Set reconfigure flags for other CPUs. */
539 CPU_FOREACH(cpu) {
540 state = DPCPU_ID_PTR(cpu, timerstate);
541 atomic_store_rel_int(&state->action,
542 (cpu == curcpu) ? 0 : ( start ? 1 : 2));
543 }
544 /* Broadcast reconfigure IPI. */
545 ipi_all_but_self(IPI_HARDCLOCK);
546 /* Wait for reconfiguration completed. */
547restart:
548 cpu_spinwait();
549 CPU_FOREACH(cpu) {
550 if (cpu == curcpu)
551 continue;
552 state = DPCPU_ID_PTR(cpu, timerstate);
553 if (atomic_load_acq_int(&state->action))
554 goto restart;
555 }
556#endif
557 critical_exit();
558}
559
560/*
561 * Calculate nearest frequency supported by hardware timer.
562 */
563static int
564round_freq(struct eventtimer *et, int freq)
565{
566 uint64_t div;
567
568 if (et->et_frequency != 0) {
569 div = lmax((et->et_frequency + freq / 2) / freq, 1);
570 if (et->et_flags & ET_FLAGS_POW2DIV)
571 div = 1 << (flsl(div + div / 2) - 1);
572 freq = (et->et_frequency + div / 2) / div;
573 }
574 if (et->et_min_period.sec > 0)
575 freq = 0;
576 else if (et->et_min_period.frac != 0)
577 freq = min(freq, BT2FREQ(&et->et_min_period));
578 if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
579 freq = max(freq, BT2FREQ(&et->et_max_period));
580 return (freq);
581}
582
583/*
584 * Configure and start event timers (BSP part).
585 */
586void
587cpu_initclocks_bsp(void)
588{
589 struct pcpu_state *state;
590 int base, div, cpu;
591
592 mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
593 CPU_FOREACH(cpu) {
594 state = DPCPU_ID_PTR(cpu, timerstate);
595 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
596 }
597#ifdef SMP
598 callout_new_inserted = cpu_new_callout;
599#endif
600 periodic = want_periodic;
601 /* Grab requested timer or the best of present. */
602 if (timername[0])
603 timer = et_find(timername, 0, 0);
604 if (timer == NULL && periodic) {
605 timer = et_find(NULL,
606 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
607 }
608 if (timer == NULL) {
609 timer = et_find(NULL,
610 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
611 }
612 if (timer == NULL && !periodic) {
613 timer = et_find(NULL,
614 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
615 }
616 if (timer == NULL)
617 panic("No usable event timer found!");
618 et_init(timer, timercb, NULL, NULL);
619
620 /* Adapt to timer capabilities. */
621 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
622 periodic = 0;
623 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
624 periodic = 1;
625 if (timer->et_flags & ET_FLAGS_C3STOP)
626 cpu_disable_deep_sleep++;
627
628 /*
629 * We honor the requested 'hz' value.
630 * We want to run stathz in the neighborhood of 128hz.
631 * We would like profhz to run as often as possible.
632 */
633 if (singlemul <= 0 || singlemul > 20) {
634 if (hz >= 1500 || (hz % 128) == 0)
635 singlemul = 1;
636 else if (hz >= 750)
637 singlemul = 2;
638 else
639 singlemul = 4;
640 }
641 if (periodic) {
642 base = round_freq(timer, hz * singlemul);
643 singlemul = max((base + hz / 2) / hz, 1);
644 hz = (base + singlemul / 2) / singlemul;
645 if (base <= 128)
646 stathz = base;
647 else {
648 div = base / 128;
649 if (div >= singlemul && (div % singlemul) == 0)
650 div++;
651 stathz = base / div;
652 }
653 profhz = stathz;
654 while ((profhz + stathz) <= 128 * 64)
655 profhz += stathz;
656 profhz = round_freq(timer, profhz);
657 } else {
658 hz = round_freq(timer, hz);
659 stathz = round_freq(timer, 127);
660 profhz = round_freq(timer, stathz * 64);
661 }
662 tick = 1000000 / hz;
663 FREQ2BT(hz, &hardperiod);
664 FREQ2BT(stathz, &statperiod);
665 FREQ2BT(profhz, &profperiod);
666 ET_LOCK();
667 configtimer(1);
668 ET_UNLOCK();
669}
670
671/*
672 * Start per-CPU event timers on APs.
673 */
674void
675cpu_initclocks_ap(void)
676{
677 struct bintime now;
678 struct pcpu_state *state;
679
680 state = DPCPU_PTR(timerstate);
681 binuptime(&now);
682 ET_HW_LOCK(state);
683 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 && periodic) {
684 state->now = nexttick;
685 bintime_sub(&state->now, &timerperiod);
686 } else
687 state->now = now;
688 hardclock_sync(curcpu);
689 handleevents(&state->now, 2);
690 if (timer->et_flags & ET_FLAGS_PERCPU)
691 loadtimer(&now, 1);
692 ET_HW_UNLOCK(state);
693}
694
695/*
696 * Switch to profiling clock rates.
697 */
698void
699cpu_startprofclock(void)
700{
701
702 ET_LOCK();
703 if (periodic) {
704 configtimer(0);
705 profiling = 1;
706 configtimer(1);
707 } else
708 profiling = 1;
709 ET_UNLOCK();
710}
711
712/*
713 * Switch to regular clock rates.
714 */
715void
716cpu_stopprofclock(void)
717{
718
719 ET_LOCK();
720 if (periodic) {
721 configtimer(0);
722 profiling = 0;
723 configtimer(1);
724 } else
725 profiling = 0;
726 ET_UNLOCK();
727}
728
729/*
730 * Switch to idle mode (all ticks handled).
731 */
732void
733cpu_idleclock(void)
734{
735 struct bintime now, t;
736 struct pcpu_state *state;
737
738 if (idletick || busy ||
739 (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
740#ifdef DEVICE_POLLING
741 || curcpu == CPU_FIRST()
742#endif
743 )
744 return;
745 state = DPCPU_PTR(timerstate);
746 if (periodic)
747 now = state->now;
748 else
749 binuptime(&now);
750 CTR4(KTR_SPARE2, "idle at %d: now %d.%08x%08x",
751 curcpu, now.sec, (unsigned int)(now.frac >> 32),
752 (unsigned int)(now.frac & 0xffffffff));
753 getnextcpuevent(&t, 1);
754 ET_HW_LOCK(state);
755 state->idle = 1;
756 state->nextevent = t;
757 if (!periodic)
758 loadtimer(&now, 0);
759 ET_HW_UNLOCK(state);
760}
761
762/*
763 * Switch to active mode (skip empty ticks).
764 */
765void
766cpu_activeclock(void)
767{
768 struct bintime now;
769 struct pcpu_state *state;
770 struct thread *td;
771
772 state = DPCPU_PTR(timerstate);
773 if (state->idle == 0 || busy)
774 return;
775 if (periodic)
776 now = state->now;
777 else
778 binuptime(&now);
779 CTR4(KTR_SPARE2, "active at %d: now %d.%08x%08x",
780 curcpu, now.sec, (unsigned int)(now.frac >> 32),
781 (unsigned int)(now.frac & 0xffffffff));
782 spinlock_enter();
783 td = curthread;
784 td->td_intr_nesting_level++;
785 handleevents(&now, 1);
786 td->td_intr_nesting_level--;
787 spinlock_exit();
788}
789
790#ifdef SMP
791static void
792cpu_new_callout(int cpu, int ticks)
793{
794 struct bintime tmp;
795 struct pcpu_state *state;
796
797 CTR3(KTR_SPARE2, "new co at %d: on %d in %d",
798 curcpu, cpu, ticks);
799 state = DPCPU_ID_PTR(cpu, timerstate);
800 ET_HW_LOCK(state);
801 if (state->idle == 0 || busy) {
802 ET_HW_UNLOCK(state);
803 return;
804 }
805 /*
806 * If timer is periodic - just update next event time for target CPU.
807 * If timer is global - there is chance it is already programmed.
808 */
809 if (periodic || (timer->et_flags & ET_FLAGS_PERCPU) == 0) {
810 state->nextevent = state->nexthard;
811 tmp = hardperiod;
812 bintime_mul(&tmp, ticks - 1);
813 bintime_add(&state->nextevent, &tmp);
814 if (periodic ||
815 bintime_cmp(&state->nextevent, &nexttick, >=)) {
816 ET_HW_UNLOCK(state);
817 return;
818 }
819 }
820 /*
821 * Otherwise we have to wake that CPU up, as we can't get present
822 * bintime to reprogram global timer from here. If timer is per-CPU,
823 * we by definition can't do it from here.
824 */
825 ET_HW_UNLOCK(state);
826 if (timer->et_flags & ET_FLAGS_PERCPU) {
827 state->handle = 1;
828 ipi_cpu(cpu, IPI_HARDCLOCK);
829 } else {
830 if (!cpu_idle_wakeup(cpu))
831 ipi_cpu(cpu, IPI_AST);
832 }
833}
834#endif
835
836/*
837 * Report or change the active event timers hardware.
838 */
839static int
840sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
841{
842 char buf[32];
843 struct eventtimer *et;
844 int error;
845
846 ET_LOCK();
847 et = timer;
848 snprintf(buf, sizeof(buf), "%s", et->et_name);
849 ET_UNLOCK();
850 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
851 ET_LOCK();
852 et = timer;
853 if (error != 0 || req->newptr == NULL ||
854 strcasecmp(buf, et->et_name) == 0) {
855 ET_UNLOCK();
856 return (error);
857 }
858 et = et_find(buf, 0, 0);
859 if (et == NULL) {
860 ET_UNLOCK();
861 return (ENOENT);
862 }
863 configtimer(0);
864 et_free(timer);
865 if (et->et_flags & ET_FLAGS_C3STOP)
866 cpu_disable_deep_sleep++;
867 if (timer->et_flags & ET_FLAGS_C3STOP)
868 cpu_disable_deep_sleep--;
869 periodic = want_periodic;
870 timer = et;
871 et_init(timer, timercb, NULL, NULL);
872 configtimer(1);
873 ET_UNLOCK();
874 return (error);
875}
876SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
877 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
878 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
879
880/*
881 * Report or change the active event timer periodicity.
882 */
883static int
884sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
885{
886 int error, val;
887
888 val = periodic;
889 error = sysctl_handle_int(oidp, &val, 0, req);
890 if (error != 0 || req->newptr == NULL)
891 return (error);
892 ET_LOCK();
893 configtimer(0);
894 periodic = want_periodic = val;
895 configtimer(1);
896 ET_UNLOCK();
897 return (error);
898}
899SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
900 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
901 0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
902
903#endif
139
140#define FREQ2BT(freq, bt) \
141{ \
142 (bt)->sec = 0; \
143 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \
144}
145#define BT2FREQ(bt) \
146 (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \
147 ((bt)->frac >> 1))
148
149/*
150 * Timer broadcast IPI handler.
151 */
152int
153hardclockintr(void)
154{
155 struct bintime now;
156 struct pcpu_state *state;
157 int done;
158
159 if (doconfigtimer() || busy)
160 return (FILTER_HANDLED);
161 state = DPCPU_PTR(timerstate);
162 now = state->now;
163 CTR4(KTR_SPARE2, "ipi at %d: now %d.%08x%08x",
164 curcpu, now.sec, (unsigned int)(now.frac >> 32),
165 (unsigned int)(now.frac & 0xffffffff));
166 done = handleevents(&now, 0);
167 return (done ? FILTER_HANDLED : FILTER_STRAY);
168}
169
170/*
171 * Handle all events for specified time on this CPU
172 */
173static int
174handleevents(struct bintime *now, int fake)
175{
176 struct bintime t;
177 struct trapframe *frame;
178 struct pcpu_state *state;
179 uintfptr_t pc;
180 int usermode;
181 int done, runs;
182
183 CTR4(KTR_SPARE2, "handle at %d: now %d.%08x%08x",
184 curcpu, now->sec, (unsigned int)(now->frac >> 32),
185 (unsigned int)(now->frac & 0xffffffff));
186 done = 0;
187 if (fake) {
188 frame = NULL;
189 usermode = 0;
190 pc = 0;
191 } else {
192 frame = curthread->td_intr_frame;
193 usermode = TRAPF_USERMODE(frame);
194 pc = TRAPF_PC(frame);
195 }
196#ifdef KDTRACE_HOOKS
197 /*
198 * If the DTrace hooks are configured and a callback function
199 * has been registered, then call it to process the high speed
200 * timers.
201 */
202 if (!fake && cyclic_clock_func[curcpu] != NULL)
203 (*cyclic_clock_func[curcpu])(frame);
204#endif
205 runs = 0;
206 state = DPCPU_PTR(timerstate);
207 while (bintime_cmp(now, &state->nexthard, >=)) {
208 bintime_add(&state->nexthard, &hardperiod);
209 runs++;
210 }
211 if (runs && fake < 2) {
212 hardclock_anycpu(runs, usermode);
213 done = 1;
214 }
215 while (bintime_cmp(now, &state->nextstat, >=)) {
216 if (fake < 2)
217 statclock(usermode);
218 bintime_add(&state->nextstat, &statperiod);
219 done = 1;
220 }
221 if (profiling) {
222 while (bintime_cmp(now, &state->nextprof, >=)) {
223 if (!fake)
224 profclock(usermode, pc);
225 bintime_add(&state->nextprof, &profperiod);
226 done = 1;
227 }
228 } else
229 state->nextprof = state->nextstat;
230 getnextcpuevent(&t, 0);
231 if (fake == 2) {
232 state->nextevent = t;
233 return (done);
234 }
235 ET_HW_LOCK(state);
236 if (!busy) {
237 state->idle = 0;
238 state->nextevent = t;
239 loadtimer(now, 0);
240 }
241 ET_HW_UNLOCK(state);
242 return (done);
243}
244
245/*
246 * Schedule binuptime of the next event on current CPU.
247 */
248static void
249getnextcpuevent(struct bintime *event, int idle)
250{
251 struct bintime tmp;
252 struct pcpu_state *state;
253 int skip;
254
255 state = DPCPU_PTR(timerstate);
256 *event = state->nexthard;
257 if (idle) { /* If CPU is idle - ask callouts for how long. */
258 skip = 4;
259 if (curcpu == CPU_FIRST() && tc_min_ticktock_freq > skip)
260 skip = tc_min_ticktock_freq;
261 skip = callout_tickstofirst(hz / skip) - 1;
262 CTR2(KTR_SPARE2, "skip at %d: %d", curcpu, skip);
263 tmp = hardperiod;
264 bintime_mul(&tmp, skip);
265 bintime_add(event, &tmp);
266 } else { /* If CPU is active - handle all types of events. */
267 if (bintime_cmp(event, &state->nextstat, >))
268 *event = state->nextstat;
269 if (profiling &&
270 bintime_cmp(event, &state->nextprof, >))
271 *event = state->nextprof;
272 }
273}
274
275/*
276 * Schedule binuptime of the next event on all CPUs.
277 */
278static void
279getnextevent(struct bintime *event)
280{
281 struct pcpu_state *state;
282#ifdef SMP
283 int cpu;
284#endif
285 int c;
286
287 state = DPCPU_PTR(timerstate);
288 *event = state->nextevent;
289 c = curcpu;
290#ifdef SMP
291 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
292 CPU_FOREACH(cpu) {
293 if (curcpu == cpu)
294 continue;
295 state = DPCPU_ID_PTR(cpu, timerstate);
296 if (bintime_cmp(event, &state->nextevent, >)) {
297 *event = state->nextevent;
298 c = cpu;
299 }
300 }
301 }
302#endif
303 CTR5(KTR_SPARE2, "next at %d: next %d.%08x%08x by %d",
304 curcpu, event->sec, (unsigned int)(event->frac >> 32),
305 (unsigned int)(event->frac & 0xffffffff), c);
306}
307
308/* Hardware timer callback function. */
309static void
310timercb(struct eventtimer *et, void *arg)
311{
312 struct bintime now;
313 struct bintime *next;
314 struct pcpu_state *state;
315#ifdef SMP
316 int cpu, bcast;
317#endif
318
319 /* Do not touch anything if somebody reconfiguring timers. */
320 if (busy)
321 return;
322 /* Update present and next tick times. */
323 state = DPCPU_PTR(timerstate);
324 if (et->et_flags & ET_FLAGS_PERCPU) {
325 next = &state->nexttick;
326 } else
327 next = &nexttick;
328 if (periodic) {
329 now = *next; /* Ex-next tick time becomes present time. */
330 bintime_add(next, &timerperiod); /* Next tick in 1 period. */
331 } else {
332 binuptime(&now); /* Get present time from hardware. */
333 next->sec = -1; /* Next tick is not scheduled yet. */
334 }
335 state->now = now;
336 CTR4(KTR_SPARE2, "intr at %d: now %d.%08x%08x",
337 curcpu, now.sec, (unsigned int)(now.frac >> 32),
338 (unsigned int)(now.frac & 0xffffffff));
339
340#ifdef SMP
341 /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
342 bcast = 0;
343 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
344 CPU_FOREACH(cpu) {
345 state = DPCPU_ID_PTR(cpu, timerstate);
346 ET_HW_LOCK(state);
347 state->now = now;
348 if (bintime_cmp(&now, &state->nextevent, >=)) {
349 state->nextevent.sec++;
350 if (curcpu != cpu) {
351 state->ipi = 1;
352 bcast = 1;
353 }
354 }
355 ET_HW_UNLOCK(state);
356 }
357 }
358#endif
359
360 /* Handle events for this time on this CPU. */
361 handleevents(&now, 0);
362
363#ifdef SMP
364 /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
365 if (bcast) {
366 CPU_FOREACH(cpu) {
367 if (curcpu == cpu)
368 continue;
369 state = DPCPU_ID_PTR(cpu, timerstate);
370 if (state->ipi) {
371 state->ipi = 0;
372 ipi_cpu(cpu, IPI_HARDCLOCK);
373 }
374 }
375 }
376#endif
377}
378
379/*
380 * Load new value into hardware timer.
381 */
382static void
383loadtimer(struct bintime *now, int start)
384{
385 struct pcpu_state *state;
386 struct bintime new;
387 struct bintime *next;
388 uint64_t tmp;
389 int eq;
390
391 if (timer->et_flags & ET_FLAGS_PERCPU) {
392 state = DPCPU_PTR(timerstate);
393 next = &state->nexttick;
394 } else
395 next = &nexttick;
396 if (periodic) {
397 if (start) {
398 /*
399 * Try to start all periodic timers aligned
400 * to period to make events synchronous.
401 */
402 tmp = ((uint64_t)now->sec << 36) + (now->frac >> 28);
403 tmp = (tmp % (timerperiod.frac >> 28)) << 28;
404 new.sec = 0;
405 new.frac = timerperiod.frac - tmp;
406 if (new.frac < tmp) /* Left less then passed. */
407 bintime_add(&new, &timerperiod);
408 CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x",
409 curcpu, now->sec, (unsigned int)(now->frac >> 32),
410 new.sec, (unsigned int)(new.frac >> 32));
411 *next = new;
412 bintime_add(next, now);
413 et_start(timer, &new, &timerperiod);
414 }
415 } else {
416 getnextevent(&new);
417 eq = bintime_cmp(&new, next, ==);
418 CTR5(KTR_SPARE2, "load at %d: next %d.%08x%08x eq %d",
419 curcpu, new.sec, (unsigned int)(new.frac >> 32),
420 (unsigned int)(new.frac & 0xffffffff),
421 eq);
422 if (!eq) {
423 *next = new;
424 bintime_sub(&new, now);
425 et_start(timer, &new, NULL);
426 }
427 }
428}
429
430/*
431 * Prepare event timer parameters after configuration changes.
432 */
433static void
434setuptimer(void)
435{
436 int freq;
437
438 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
439 periodic = 0;
440 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
441 periodic = 1;
442 singlemul = MIN(MAX(singlemul, 1), 20);
443 freq = hz * singlemul;
444 while (freq < (profiling ? profhz : stathz))
445 freq += hz;
446 freq = round_freq(timer, freq);
447 FREQ2BT(freq, &timerperiod);
448}
449
450/*
451 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
452 */
453static int
454doconfigtimer(void)
455{
456 struct bintime now;
457 struct pcpu_state *state;
458
459 state = DPCPU_PTR(timerstate);
460 switch (atomic_load_acq_int(&state->action)) {
461 case 1:
462 binuptime(&now);
463 ET_HW_LOCK(state);
464 loadtimer(&now, 1);
465 ET_HW_UNLOCK(state);
466 state->handle = 0;
467 atomic_store_rel_int(&state->action, 0);
468 return (1);
469 case 2:
470 ET_HW_LOCK(state);
471 et_stop(timer);
472 ET_HW_UNLOCK(state);
473 state->handle = 0;
474 atomic_store_rel_int(&state->action, 0);
475 return (1);
476 }
477 if (atomic_readandclear_int(&state->handle) && !busy) {
478 binuptime(&now);
479 handleevents(&now, 0);
480 return (1);
481 }
482 return (0);
483}
484
485/*
486 * Reconfigure specified timer.
487 * For per-CPU timers use IPI to make other CPUs to reconfigure.
488 */
489static void
490configtimer(int start)
491{
492 struct bintime now, next;
493 struct pcpu_state *state;
494 int cpu;
495
496 if (start) {
497 setuptimer();
498 binuptime(&now);
499 }
500 critical_enter();
501 ET_HW_LOCK(DPCPU_PTR(timerstate));
502 if (start) {
503 /* Initialize time machine parameters. */
504 next = now;
505 bintime_add(&next, &timerperiod);
506 if (periodic)
507 nexttick = next;
508 else
509 nexttick.sec = -1;
510 CPU_FOREACH(cpu) {
511 state = DPCPU_ID_PTR(cpu, timerstate);
512 state->now = now;
513 state->nextevent = next;
514 if (periodic)
515 state->nexttick = next;
516 else
517 state->nexttick.sec = -1;
518 state->nexthard = next;
519 state->nextstat = next;
520 state->nextprof = next;
521 hardclock_sync(cpu);
522 }
523 busy = 0;
524 /* Start global timer or per-CPU timer of this CPU. */
525 loadtimer(&now, 1);
526 } else {
527 busy = 1;
528 /* Stop global timer or per-CPU timer of this CPU. */
529 et_stop(timer);
530 }
531 ET_HW_UNLOCK(DPCPU_PTR(timerstate));
532#ifdef SMP
533 /* If timer is global or there is no other CPUs yet - we are done. */
534 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
535 critical_exit();
536 return;
537 }
538 /* Set reconfigure flags for other CPUs. */
539 CPU_FOREACH(cpu) {
540 state = DPCPU_ID_PTR(cpu, timerstate);
541 atomic_store_rel_int(&state->action,
542 (cpu == curcpu) ? 0 : ( start ? 1 : 2));
543 }
544 /* Broadcast reconfigure IPI. */
545 ipi_all_but_self(IPI_HARDCLOCK);
546 /* Wait for reconfiguration completed. */
547restart:
548 cpu_spinwait();
549 CPU_FOREACH(cpu) {
550 if (cpu == curcpu)
551 continue;
552 state = DPCPU_ID_PTR(cpu, timerstate);
553 if (atomic_load_acq_int(&state->action))
554 goto restart;
555 }
556#endif
557 critical_exit();
558}
559
560/*
561 * Calculate nearest frequency supported by hardware timer.
562 */
563static int
564round_freq(struct eventtimer *et, int freq)
565{
566 uint64_t div;
567
568 if (et->et_frequency != 0) {
569 div = lmax((et->et_frequency + freq / 2) / freq, 1);
570 if (et->et_flags & ET_FLAGS_POW2DIV)
571 div = 1 << (flsl(div + div / 2) - 1);
572 freq = (et->et_frequency + div / 2) / div;
573 }
574 if (et->et_min_period.sec > 0)
575 freq = 0;
576 else if (et->et_min_period.frac != 0)
577 freq = min(freq, BT2FREQ(&et->et_min_period));
578 if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
579 freq = max(freq, BT2FREQ(&et->et_max_period));
580 return (freq);
581}
582
583/*
584 * Configure and start event timers (BSP part).
585 */
586void
587cpu_initclocks_bsp(void)
588{
589 struct pcpu_state *state;
590 int base, div, cpu;
591
592 mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
593 CPU_FOREACH(cpu) {
594 state = DPCPU_ID_PTR(cpu, timerstate);
595 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
596 }
597#ifdef SMP
598 callout_new_inserted = cpu_new_callout;
599#endif
600 periodic = want_periodic;
601 /* Grab requested timer or the best of present. */
602 if (timername[0])
603 timer = et_find(timername, 0, 0);
604 if (timer == NULL && periodic) {
605 timer = et_find(NULL,
606 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
607 }
608 if (timer == NULL) {
609 timer = et_find(NULL,
610 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
611 }
612 if (timer == NULL && !periodic) {
613 timer = et_find(NULL,
614 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
615 }
616 if (timer == NULL)
617 panic("No usable event timer found!");
618 et_init(timer, timercb, NULL, NULL);
619
620 /* Adapt to timer capabilities. */
621 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
622 periodic = 0;
623 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
624 periodic = 1;
625 if (timer->et_flags & ET_FLAGS_C3STOP)
626 cpu_disable_deep_sleep++;
627
628 /*
629 * We honor the requested 'hz' value.
630 * We want to run stathz in the neighborhood of 128hz.
631 * We would like profhz to run as often as possible.
632 */
633 if (singlemul <= 0 || singlemul > 20) {
634 if (hz >= 1500 || (hz % 128) == 0)
635 singlemul = 1;
636 else if (hz >= 750)
637 singlemul = 2;
638 else
639 singlemul = 4;
640 }
641 if (periodic) {
642 base = round_freq(timer, hz * singlemul);
643 singlemul = max((base + hz / 2) / hz, 1);
644 hz = (base + singlemul / 2) / singlemul;
645 if (base <= 128)
646 stathz = base;
647 else {
648 div = base / 128;
649 if (div >= singlemul && (div % singlemul) == 0)
650 div++;
651 stathz = base / div;
652 }
653 profhz = stathz;
654 while ((profhz + stathz) <= 128 * 64)
655 profhz += stathz;
656 profhz = round_freq(timer, profhz);
657 } else {
658 hz = round_freq(timer, hz);
659 stathz = round_freq(timer, 127);
660 profhz = round_freq(timer, stathz * 64);
661 }
662 tick = 1000000 / hz;
663 FREQ2BT(hz, &hardperiod);
664 FREQ2BT(stathz, &statperiod);
665 FREQ2BT(profhz, &profperiod);
666 ET_LOCK();
667 configtimer(1);
668 ET_UNLOCK();
669}
670
671/*
672 * Start per-CPU event timers on APs.
673 */
674void
675cpu_initclocks_ap(void)
676{
677 struct bintime now;
678 struct pcpu_state *state;
679
680 state = DPCPU_PTR(timerstate);
681 binuptime(&now);
682 ET_HW_LOCK(state);
683 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 && periodic) {
684 state->now = nexttick;
685 bintime_sub(&state->now, &timerperiod);
686 } else
687 state->now = now;
688 hardclock_sync(curcpu);
689 handleevents(&state->now, 2);
690 if (timer->et_flags & ET_FLAGS_PERCPU)
691 loadtimer(&now, 1);
692 ET_HW_UNLOCK(state);
693}
694
695/*
696 * Switch to profiling clock rates.
697 */
698void
699cpu_startprofclock(void)
700{
701
702 ET_LOCK();
703 if (periodic) {
704 configtimer(0);
705 profiling = 1;
706 configtimer(1);
707 } else
708 profiling = 1;
709 ET_UNLOCK();
710}
711
712/*
713 * Switch to regular clock rates.
714 */
715void
716cpu_stopprofclock(void)
717{
718
719 ET_LOCK();
720 if (periodic) {
721 configtimer(0);
722 profiling = 0;
723 configtimer(1);
724 } else
725 profiling = 0;
726 ET_UNLOCK();
727}
728
729/*
730 * Switch to idle mode (all ticks handled).
731 */
732void
733cpu_idleclock(void)
734{
735 struct bintime now, t;
736 struct pcpu_state *state;
737
738 if (idletick || busy ||
739 (periodic && (timer->et_flags & ET_FLAGS_PERCPU))
740#ifdef DEVICE_POLLING
741 || curcpu == CPU_FIRST()
742#endif
743 )
744 return;
745 state = DPCPU_PTR(timerstate);
746 if (periodic)
747 now = state->now;
748 else
749 binuptime(&now);
750 CTR4(KTR_SPARE2, "idle at %d: now %d.%08x%08x",
751 curcpu, now.sec, (unsigned int)(now.frac >> 32),
752 (unsigned int)(now.frac & 0xffffffff));
753 getnextcpuevent(&t, 1);
754 ET_HW_LOCK(state);
755 state->idle = 1;
756 state->nextevent = t;
757 if (!periodic)
758 loadtimer(&now, 0);
759 ET_HW_UNLOCK(state);
760}
761
762/*
763 * Switch to active mode (skip empty ticks).
764 */
765void
766cpu_activeclock(void)
767{
768 struct bintime now;
769 struct pcpu_state *state;
770 struct thread *td;
771
772 state = DPCPU_PTR(timerstate);
773 if (state->idle == 0 || busy)
774 return;
775 if (periodic)
776 now = state->now;
777 else
778 binuptime(&now);
779 CTR4(KTR_SPARE2, "active at %d: now %d.%08x%08x",
780 curcpu, now.sec, (unsigned int)(now.frac >> 32),
781 (unsigned int)(now.frac & 0xffffffff));
782 spinlock_enter();
783 td = curthread;
784 td->td_intr_nesting_level++;
785 handleevents(&now, 1);
786 td->td_intr_nesting_level--;
787 spinlock_exit();
788}
789
790#ifdef SMP
791static void
792cpu_new_callout(int cpu, int ticks)
793{
794 struct bintime tmp;
795 struct pcpu_state *state;
796
797 CTR3(KTR_SPARE2, "new co at %d: on %d in %d",
798 curcpu, cpu, ticks);
799 state = DPCPU_ID_PTR(cpu, timerstate);
800 ET_HW_LOCK(state);
801 if (state->idle == 0 || busy) {
802 ET_HW_UNLOCK(state);
803 return;
804 }
805 /*
806 * If timer is periodic - just update next event time for target CPU.
807 * If timer is global - there is chance it is already programmed.
808 */
809 if (periodic || (timer->et_flags & ET_FLAGS_PERCPU) == 0) {
810 state->nextevent = state->nexthard;
811 tmp = hardperiod;
812 bintime_mul(&tmp, ticks - 1);
813 bintime_add(&state->nextevent, &tmp);
814 if (periodic ||
815 bintime_cmp(&state->nextevent, &nexttick, >=)) {
816 ET_HW_UNLOCK(state);
817 return;
818 }
819 }
820 /*
821 * Otherwise we have to wake that CPU up, as we can't get present
822 * bintime to reprogram global timer from here. If timer is per-CPU,
823 * we by definition can't do it from here.
824 */
825 ET_HW_UNLOCK(state);
826 if (timer->et_flags & ET_FLAGS_PERCPU) {
827 state->handle = 1;
828 ipi_cpu(cpu, IPI_HARDCLOCK);
829 } else {
830 if (!cpu_idle_wakeup(cpu))
831 ipi_cpu(cpu, IPI_AST);
832 }
833}
834#endif
835
836/*
837 * Report or change the active event timers hardware.
838 */
839static int
840sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
841{
842 char buf[32];
843 struct eventtimer *et;
844 int error;
845
846 ET_LOCK();
847 et = timer;
848 snprintf(buf, sizeof(buf), "%s", et->et_name);
849 ET_UNLOCK();
850 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
851 ET_LOCK();
852 et = timer;
853 if (error != 0 || req->newptr == NULL ||
854 strcasecmp(buf, et->et_name) == 0) {
855 ET_UNLOCK();
856 return (error);
857 }
858 et = et_find(buf, 0, 0);
859 if (et == NULL) {
860 ET_UNLOCK();
861 return (ENOENT);
862 }
863 configtimer(0);
864 et_free(timer);
865 if (et->et_flags & ET_FLAGS_C3STOP)
866 cpu_disable_deep_sleep++;
867 if (timer->et_flags & ET_FLAGS_C3STOP)
868 cpu_disable_deep_sleep--;
869 periodic = want_periodic;
870 timer = et;
871 et_init(timer, timercb, NULL, NULL);
872 configtimer(1);
873 ET_UNLOCK();
874 return (error);
875}
876SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
877 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
878 0, 0, sysctl_kern_eventtimer_timer, "A", "Chosen event timer");
879
880/*
881 * Report or change the active event timer periodicity.
882 */
883static int
884sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
885{
886 int error, val;
887
888 val = periodic;
889 error = sysctl_handle_int(oidp, &val, 0, req);
890 if (error != 0 || req->newptr == NULL)
891 return (error);
892 ET_LOCK();
893 configtimer(0);
894 periodic = want_periodic = val;
895 configtimer(1);
896 ET_UNLOCK();
897 return (error);
898}
899SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
900 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
901 0, 0, sysctl_kern_eventtimer_periodic, "I", "Enable event timer periodic mode");
902
903#endif