Deleted Added
sdiff udiff text old ( 210298 ) new ( 212541 )
full compact
kern_clocksource.c (210298) kern_clocksource.c (212541)
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright

--- 11 unchanged lines hidden (view full) ---

20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 210298 2010-07-20 15:48:29Z mav $");
28__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 212541 2010-09-13 07:25:35Z mav $");
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_kdtrace.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/bus.h>
42#include <sys/lock.h>
43#include <sys/kdb.h>
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_kdtrace.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/bus.h>
42#include <sys/lock.h>
43#include <sys/kdb.h>
44#include <sys/ktr.h>
44#include <sys/mutex.h>
45#include <sys/proc.h>
46#include <sys/kernel.h>
47#include <sys/sched.h>
48#include <sys/smp.h>
49#include <sys/sysctl.h>
50#include <sys/timeet.h>
51
52#include <machine/atomic.h>
53#include <machine/clock.h>
54#include <machine/cpu.h>
55#include <machine/smp.h>
56
57#ifdef KDTRACE_HOOKS
58#include <sys/dtrace_bsd.h>
59cyclic_clock_func_t cyclic_clock_func[MAXCPU];
60#endif
61
45#include <sys/mutex.h>
46#include <sys/proc.h>
47#include <sys/kernel.h>
48#include <sys/sched.h>
49#include <sys/smp.h>
50#include <sys/sysctl.h>
51#include <sys/timeet.h>
52
53#include <machine/atomic.h>
54#include <machine/clock.h>
55#include <machine/cpu.h>
56#include <machine/smp.h>
57
58#ifdef KDTRACE_HOOKS
59#include <sys/dtrace_bsd.h>
60cyclic_clock_func_t cyclic_clock_func[MAXCPU];
61#endif
62
62static void cpu_restartclocks(void);
63static void timercheck(void);
64inline static int doconfigtimer(int i);
65static void configtimer(int i);
63int cpu_disable_deep_sleep = 0; /* Timer dies in C3. */
66
64
67static struct eventtimer *timer[2] = { NULL, NULL };
68static int timertest = 0;
69static int timerticks[2] = { 0, 0 };
70static int profiling_on = 0;
71static struct bintime timerperiod[2];
65static void setuptimer(void);
66static void loadtimer(struct bintime *now, int first);
67static int doconfigtimer(void);
68static void configtimer(int start);
69static int round_freq(struct eventtimer *et, int freq);
72
70
73static char timername[2][32];
74TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername));
75TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername));
71static void getnextcpuevent(struct bintime *event, int idle);
72static void getnextevent(struct bintime *event);
73static int handleevents(struct bintime *now, int fake);
74#ifdef SMP
75static void cpu_new_callout(int cpu, int ticks);
76#endif
76
77
77static u_int singlemul = 0;
78static struct mtx et_hw_mtx;
79
80#define ET_HW_LOCK(state) \
81 { \
82 if (timer->et_flags & ET_FLAGS_PERCPU) \
83 mtx_lock_spin(&(state)->et_hw_mtx); \
84 else \
85 mtx_lock_spin(&et_hw_mtx); \
86 }
87
88#define ET_HW_UNLOCK(state) \
89 { \
90 if (timer->et_flags & ET_FLAGS_PERCPU) \
91 mtx_unlock_spin(&(state)->et_hw_mtx); \
92 else \
93 mtx_unlock_spin(&et_hw_mtx); \
94 }
95
96static struct eventtimer *timer = NULL;
97static struct bintime timerperiod; /* Timer period for periodic mode. */
98static struct bintime hardperiod; /* hardclock() events period. */
99static struct bintime statperiod; /* statclock() events period. */
100static struct bintime profperiod; /* profclock() events period. */
101static struct bintime nexttick; /* Next global timer tick time. */
102static u_int busy = 0; /* Reconfiguration is in progress. */
103static int profiling = 0; /* Profiling events enabled. */
104
105static char timername[32]; /* Wanted timer. */
106TUNABLE_STR("kern.eventtimer.timer", timername, sizeof(timername));
107
108static u_int singlemul = 0; /* Multiplier for periodic mode. */
78TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
79SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
109TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
110SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
80 0, "Multiplier, used in single timer mode");
111 0, "Multiplier for periodic mode");
81
112
82typedef u_int tc[2];
83static DPCPU_DEFINE(tc, configtimer);
113static u_int idletick = 0; /* Idle mode allowed. */
114TUNABLE_INT("kern.eventtimer.idletick", &idletick);
115SYSCTL_INT(_kern_eventtimer, OID_AUTO, idletick, CTLFLAG_RW, &idletick,
116 0, "Run periodic events when idle");
84
117
118static int periodic = 0; /* Periodic or one-shot mode. */
119TUNABLE_INT("kern.eventtimer.periodic", &periodic);
120
121struct pcpu_state {
122 struct mtx et_hw_mtx; /* Per-CPU timer mutex. */
123 u_int action; /* Reconfiguration requests. */
124 u_int handle; /* Immediate handle resuests. */
125 struct bintime now; /* Last tick time. */
126 struct bintime nextevent; /* Next scheduled event on this CPU. */
127 struct bintime nexttick; /* Next timer tick time. */
128 struct bintime nexthard; /* Next hardlock() event. */
129 struct bintime nextstat; /* Next statclock() event. */
130 struct bintime nextprof; /* Next profclock() event. */
131 int ipi; /* This CPU needs IPI. */
132 int idle; /* This CPU is in idle mode. */
133};
134
135static DPCPU_DEFINE(struct pcpu_state, timerstate);
136
85#define FREQ2BT(freq, bt) \
86{ \
87 (bt)->sec = 0; \
88 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \
89}
90#define BT2FREQ(bt) \
91 (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \
92 ((bt)->frac >> 1))
93
137#define FREQ2BT(freq, bt) \
138{ \
139 (bt)->sec = 0; \
140 (bt)->frac = ((uint64_t)0x8000000000000000 / (freq)) << 1; \
141}
142#define BT2FREQ(bt) \
143 (((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) / \
144 ((bt)->frac >> 1))
145
94/* Per-CPU timer1 handler. */
146/*
147 * Timer broadcast IPI handler.
148 */
149int
150hardclockintr(void)
151{
152 struct bintime now;
153 struct pcpu_state *state;
154 int done;
155
156 if (doconfigtimer() || busy)
157 return (FILTER_HANDLED);
158 state = DPCPU_PTR(timerstate);
159 now = state->now;
160 CTR4(KTR_SPARE2, "ipi at %d: now %d.%08x%08x",
161 curcpu, now.sec, (unsigned int)(now.frac >> 32),
162 (unsigned int)(now.frac & 0xffffffff));
163 done = handleevents(&now, 0);
164 return (done ? FILTER_HANDLED : FILTER_STRAY);
165}
166
167/*
168 * Handle all events for specified time on this CPU
169 */
95static int
170static int
96hardclockhandler(struct trapframe *frame)
171handleevents(struct bintime *now, int fake)
97{
172{
173 struct bintime t;
174 struct trapframe *frame;
175 struct pcpu_state *state;
176 uintfptr_t pc;
177 int usermode;
178 int done, runs;
98
179
180 CTR4(KTR_SPARE2, "handle at %d: now %d.%08x%08x",
181 curcpu, now->sec, (unsigned int)(now->frac >> 32),
182 (unsigned int)(now->frac & 0xffffffff));
183 done = 0;
184 if (fake) {
185 frame = NULL;
186 usermode = 0;
187 pc = 0;
188 } else {
189 frame = curthread->td_intr_frame;
190 usermode = TRAPF_USERMODE(frame);
191 pc = TRAPF_PC(frame);
192 }
99#ifdef KDTRACE_HOOKS
100 /*
101 * If the DTrace hooks are configured and a callback function
102 * has been registered, then call it to process the high speed
103 * timers.
104 */
193#ifdef KDTRACE_HOOKS
194 /*
195 * If the DTrace hooks are configured and a callback function
196 * has been registered, then call it to process the high speed
197 * timers.
198 */
105 int cpu = curcpu;
106 if (cyclic_clock_func[cpu] != NULL)
107 (*cyclic_clock_func[cpu])(frame);
199 if (!fake && cyclic_clock_func[curcpu] != NULL)
200 (*cyclic_clock_func[curcpu])(frame);
108#endif
201#endif
109
110 timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
111 return (FILTER_HANDLED);
202 runs = 0;
203 state = DPCPU_PTR(timerstate);
204 while (bintime_cmp(now, &state->nexthard, >=)) {
205 bintime_add(&state->nexthard, &hardperiod);
206 runs++;
207 }
208 if (runs) {
209 hardclock_anycpu(runs, usermode);
210 done = 1;
211 }
212 while (bintime_cmp(now, &state->nextstat, >=)) {
213 statclock(usermode);
214 bintime_add(&state->nextstat, &statperiod);
215 done = 1;
216 }
217 if (profiling) {
218 while (bintime_cmp(now, &state->nextprof, >=)) {
219 if (!fake)
220 profclock(usermode, pc);
221 bintime_add(&state->nextprof, &profperiod);
222 done = 1;
223 }
224 } else
225 state->nextprof = state->nextstat;
226 getnextcpuevent(&t, 0);
227 ET_HW_LOCK(state);
228 if (!busy) {
229 state->idle = 0;
230 state->nextevent = t;
231 loadtimer(now, 0);
232 }
233 ET_HW_UNLOCK(state);
234 return (done);
112}
113
235}
236
114/* Per-CPU timer2 handler. */
115static int
116statclockhandler(struct trapframe *frame)
237/*
238 * Schedule binuptime of the next event on current CPU.
239 */
240static void
241getnextcpuevent(struct bintime *event, int idle)
117{
242{
243 struct bintime tmp;
244 struct pcpu_state *state;
245 int skip;
118
246
119 timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
120 return (FILTER_HANDLED);
247 state = DPCPU_PTR(timerstate);
248 *event = state->nexthard;
249 if (idle) { /* If CPU is idle - ask callouts for how long. */
250 skip = callout_tickstofirst() - 1;
251 CTR2(KTR_SPARE2, "skip at %d: %d", curcpu, skip);
252 tmp = hardperiod;
253 bintime_mul(&tmp, skip);
254 bintime_add(event, &tmp);
255 } else { /* If CPU is active - handle all types of events. */
256 if (bintime_cmp(event, &state->nextstat, >))
257 *event = state->nextstat;
258 if (profiling &&
259 bintime_cmp(event, &state->nextprof, >))
260 *event = state->nextprof;
261 }
121}
122
262}
263
123/* timer1 broadcast IPI handler. */
124int
125hardclockintr(struct trapframe *frame)
126{
127
128 if (doconfigtimer(0))
129 return (FILTER_HANDLED);
130 return (hardclockhandler(frame));
131}
132
133/* timer2 broadcast IPI handler. */
134int
135statclockintr(struct trapframe *frame)
136{
137
138 if (doconfigtimer(1))
139 return (FILTER_HANDLED);
140 return (statclockhandler(frame));
141}
142
143/* timer1 callback. */
264/*
265 * Schedule binuptime of the next event on all CPUs.
266 */
144static void
267static void
145timer1cb(struct eventtimer *et, void *arg)
268getnextevent(struct bintime *event)
146{
269{
147
270 struct pcpu_state *state;
148#ifdef SMP
271#ifdef SMP
149 /* Broadcast interrupt to other CPUs for non-per-CPU timers */
150 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
151 ipi_all_but_self(IPI_HARDCLOCK);
272 int cpu;
152#endif
273#endif
153 if (timertest) {
154 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
155 timerticks[0]++;
156 if (timerticks[0] >= timer1hz) {
157 ET_LOCK();
158 timercheck();
159 ET_UNLOCK();
274 int c;
275
276 state = DPCPU_PTR(timerstate);
277 *event = state->nextevent;
278 c = curcpu;
279#ifdef SMP
280 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0) {
281 CPU_FOREACH(cpu) {
282 if (curcpu == cpu)
283 continue;
284 state = DPCPU_ID_PTR(cpu, timerstate);
285 if (bintime_cmp(event, &state->nextevent, >)) {
286 *event = state->nextevent;
287 c = cpu;
160 }
161 }
162 }
288 }
289 }
290 }
163 hardclockhandler(curthread->td_intr_frame);
291#endif
292 CTR5(KTR_SPARE2, "next at %d: next %d.%08x%08x by %d",
293 curcpu, event->sec, (unsigned int)(event->frac >> 32),
294 (unsigned int)(event->frac & 0xffffffff), c);
164}
165
295}
296
166/* timer2 callback. */
297/* Hardware timer callback function. */
167static void
298static void
168timer2cb(struct eventtimer *et, void *arg)
299timercb(struct eventtimer *et, void *arg)
169{
300{
301 struct bintime now;
302 struct bintime *next;
303 struct pcpu_state *state;
304#ifdef SMP
305 int cpu, bcast;
306#endif
170
307
308 /* Do not touch anything if somebody reconfiguring timers. */
309 if (busy)
310 return;
311 /* Update present and next tick times. */
312 state = DPCPU_PTR(timerstate);
313 if (et->et_flags & ET_FLAGS_PERCPU) {
314 next = &state->nexttick;
315 } else
316 next = &nexttick;
317 if (periodic) {
318 now = *next; /* Ex-next tick time becomes present time. */
319 bintime_add(next, &timerperiod); /* Next tick in 1 period. */
320 } else {
321 binuptime(&now); /* Get present time from hardware. */
322 next->sec = -1; /* Next tick is not scheduled yet. */
323 }
324 state->now = now;
325 CTR4(KTR_SPARE2, "intr at %d: now %d.%08x%08x",
326 curcpu, now.sec, (unsigned int)(now.frac >> 32),
327 (unsigned int)(now.frac & 0xffffffff));
328
171#ifdef SMP
329#ifdef SMP
172 /* Broadcast interrupt to other CPUs for non-per-CPU timers */
173 if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
174 ipi_all_but_self(IPI_STATCLOCK);
330 /* Prepare broadcasting to other CPUs for non-per-CPU timers. */
331 bcast = 0;
332 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 && smp_started) {
333 CPU_FOREACH(cpu) {
334 if (curcpu == cpu)
335 continue;
336 state = DPCPU_ID_PTR(cpu, timerstate);
337 ET_HW_LOCK(state);
338 state->now = now;
339 if (bintime_cmp(&now, &state->nextevent, >=)) {
340 state->nextevent.sec++;
341 state->ipi = 1;
342 bcast = 1;
343 }
344 ET_HW_UNLOCK(state);
345 }
346 }
175#endif
347#endif
176 if (timertest) {
177 if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
178 timerticks[1]++;
179 if (timerticks[1] >= timer2hz * 2) {
180 ET_LOCK();
181 timercheck();
182 ET_UNLOCK();
348
349 /* Handle events for this time on this CPU. */
350 handleevents(&now, 0);
351
352#ifdef SMP
353 /* Broadcast interrupt to other CPUs for non-per-CPU timers. */
354 if (bcast) {
355 CPU_FOREACH(cpu) {
356 if (curcpu == cpu)
357 continue;
358 state = DPCPU_ID_PTR(cpu, timerstate);
359 if (state->ipi) {
360 state->ipi = 0;
361 ipi_cpu(cpu, IPI_HARDCLOCK);
183 }
184 }
185 }
362 }
363 }
364 }
186 statclockhandler(curthread->td_intr_frame);
365#endif
187}
188
189/*
366}
367
368/*
190 * Check that both timers are running with at least 1/4 of configured rate.
191 * If not - replace the broken one.
369 * Load new value into hardware timer.
192 */
193static void
370 */
371static void
194timercheck(void)
372loadtimer(struct bintime *now, int start)
195{
373{
374 struct pcpu_state *state;
375 struct bintime new;
376 struct bintime *next;
377 uint64_t tmp;
378 int eq;
196
379
197 if (!timertest)
198 return;
199 timertest = 0;
200 if (timerticks[0] * 4 < timer1hz) {
201 printf("Event timer \"%s\" is dead.\n", timer[0]->et_name);
202 timer1hz = 0;
203 configtimer(0);
204 et_ban(timer[0]);
205 et_free(timer[0]);
206 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
207 if (timer[0] == NULL) {
208 timer2hz = 0;
209 configtimer(1);
210 et_free(timer[1]);
211 timer[1] = NULL;
212 timer[0] = timer[1];
380 if (periodic) {
381 if (start) {
382 /*
383 * Try to start all periodic timers aligned
384 * to period to make events synchronous.
385 */
386 tmp = ((uint64_t)now->sec << 36) + (now->frac >> 28);
387 tmp = (tmp % (timerperiod.frac >> 28)) << 28;
388 tmp = timerperiod.frac - tmp;
389 new = timerperiod;
390 bintime_addx(&new, tmp);
391 CTR5(KTR_SPARE2, "load p at %d: now %d.%08x first in %d.%08x",
392 curcpu, now->sec, (unsigned int)(now->frac >> 32),
393 new.sec, (unsigned int)(new.frac >> 32));
394 et_start(timer, &new, &timerperiod);
213 }
395 }
214 et_init(timer[0], timer1cb, NULL, NULL);
215 cpu_restartclocks();
216 return;
396 } else {
397 if (timer->et_flags & ET_FLAGS_PERCPU) {
398 state = DPCPU_PTR(timerstate);
399 next = &state->nexttick;
400 } else
401 next = &nexttick;
402 getnextevent(&new);
403 eq = bintime_cmp(&new, next, ==);
404 CTR5(KTR_SPARE2, "load at %d: next %d.%08x%08x eq %d",
405 curcpu, new.sec, (unsigned int)(new.frac >> 32),
406 (unsigned int)(new.frac & 0xffffffff),
407 eq);
408 if (!eq) {
409 *next = new;
410 bintime_sub(&new, now);
411 et_start(timer, &new, NULL);
412 }
217 }
413 }
218 if (timerticks[1] * 4 < timer2hz) {
219 printf("Event timer \"%s\" is dead.\n", timer[1]->et_name);
220 timer2hz = 0;
221 configtimer(1);
222 et_ban(timer[1]);
223 et_free(timer[1]);
224 timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
225 if (timer[1] != NULL)
226 et_init(timer[1], timer2cb, NULL, NULL);
227 cpu_restartclocks();
228 return;
229 }
230}
231
232/*
414}
415
416/*
417 * Prepare event timer parameters after configuration changes.
418 */
419static void
420setuptimer(void)
421{
422 int freq;
423
424 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
425 periodic = 0;
426 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
427 periodic = 1;
428 freq = hz * singlemul;
429 while (freq < (profiling ? profhz : stathz))
430 freq += hz;
431 freq = round_freq(timer, freq);
432 FREQ2BT(freq, &timerperiod);
433}
434
435/*
233 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
234 */
436 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
437 */
235inline static int
236doconfigtimer(int i)
438static int
439doconfigtimer(void)
237{
440{
238 tc *conf;
441 struct bintime now;
442 struct pcpu_state *state;
239
443
240 conf = DPCPU_PTR(configtimer);
241 if (atomic_load_acq_int(*conf + i)) {
242 if (i == 0 ? timer1hz : timer2hz)
243 et_start(timer[i], NULL, &timerperiod[i]);
244 else
245 et_stop(timer[i]);
246 atomic_store_rel_int(*conf + i, 0);
444 state = DPCPU_PTR(timerstate);
445 switch (atomic_load_acq_int(&state->action)) {
446 case 1:
447 binuptime(&now);
448 ET_HW_LOCK(state);
449 loadtimer(&now, 1);
450 ET_HW_UNLOCK(state);
451 state->handle = 0;
452 atomic_store_rel_int(&state->action, 0);
247 return (1);
453 return (1);
454 case 2:
455 ET_HW_LOCK(state);
456 et_stop(timer);
457 ET_HW_UNLOCK(state);
458 state->handle = 0;
459 atomic_store_rel_int(&state->action, 0);
460 return (1);
248 }
461 }
462 if (atomic_readandclear_int(&state->handle) && !busy) {
463 binuptime(&now);
464 handleevents(&now, 0);
465 return (1);
466 }
249 return (0);
250}
251
252/*
253 * Reconfigure specified timer.
254 * For per-CPU timers use IPI to make other CPUs to reconfigure.
255 */
256static void
467 return (0);
468}
469
470/*
471 * Reconfigure specified timer.
472 * For per-CPU timers use IPI to make other CPUs to reconfigure.
473 */
474static void
257configtimer(int i)
475configtimer(int start)
258{
476{
259#ifdef SMP
260 tc *conf;
477 struct bintime now, next;
478 struct pcpu_state *state;
261 int cpu;
262
479 int cpu;
480
481 if (start) {
482 setuptimer();
483 binuptime(&now);
484 }
263 critical_enter();
485 critical_enter();
264#endif
265 /* Start/stop global timer or per-CPU timer of this CPU. */
266 if (i == 0 ? timer1hz : timer2hz)
267 et_start(timer[i], NULL, &timerperiod[i]);
268 else
269 et_stop(timer[i]);
486 ET_HW_LOCK(DPCPU_PTR(timerstate));
487 if (start) {
488 /* Initialize time machine parameters. */
489 next = now;
490 bintime_add(&next, &timerperiod);
491 if (periodic)
492 nexttick = next;
493 else
494 nexttick.sec = -1;
495 CPU_FOREACH(cpu) {
496 state = DPCPU_ID_PTR(cpu, timerstate);
497 state->now = now;
498 state->nextevent = next;
499 if (periodic)
500 state->nexttick = next;
501 else
502 state->nexttick.sec = -1;
503 state->nexthard = next;
504 state->nextstat = next;
505 state->nextprof = next;
506 hardclock_sync(cpu);
507 }
508 busy = 0;
509 /* Start global timer or per-CPU timer of this CPU. */
510 loadtimer(&now, 1);
511 } else {
512 busy = 1;
513 /* Stop global timer or per-CPU timer of this CPU. */
514 et_stop(timer);
515 }
516 ET_HW_UNLOCK(DPCPU_PTR(timerstate));
270#ifdef SMP
517#ifdef SMP
271 if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
518 /* If timer is global or there is no other CPUs yet - we are done. */
519 if ((timer->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
272 critical_exit();
273 return;
274 }
275 /* Set reconfigure flags for other CPUs. */
276 CPU_FOREACH(cpu) {
520 critical_exit();
521 return;
522 }
523 /* Set reconfigure flags for other CPUs. */
524 CPU_FOREACH(cpu) {
277 conf = DPCPU_ID_PTR(cpu, configtimer);
278 atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1);
525 state = DPCPU_ID_PTR(cpu, timerstate);
526 atomic_store_rel_int(&state->action,
527 (cpu == curcpu) ? 0 : ( start ? 1 : 2));
279 }
528 }
280 /* Send reconfigure IPI. */
281 ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK);
529 /* Broadcast reconfigure IPI. */
530 ipi_all_but_self(IPI_HARDCLOCK);
282 /* Wait for reconfiguration completed. */
283restart:
284 cpu_spinwait();
285 CPU_FOREACH(cpu) {
286 if (cpu == curcpu)
287 continue;
531 /* Wait for reconfiguration completed. */
532restart:
533 cpu_spinwait();
534 CPU_FOREACH(cpu) {
535 if (cpu == curcpu)
536 continue;
288 conf = DPCPU_ID_PTR(cpu, configtimer);
289 if (atomic_load_acq_int(*conf + i))
537 state = DPCPU_ID_PTR(cpu, timerstate);
538 if (atomic_load_acq_int(&state->action))
290 goto restart;
291 }
539 goto restart;
540 }
292 critical_exit();
293#endif
541#endif
542 critical_exit();
294}
295
543}
544
545/*
546 * Calculate nearest frequency supported by hardware timer.
547 */
296static int
297round_freq(struct eventtimer *et, int freq)
298{
299 uint64_t div;
300
301 if (et->et_frequency != 0) {
302 div = lmax((et->et_frequency + freq / 2) / freq, 1);
303 if (et->et_flags & ET_FLAGS_POW2DIV)

--- 5 unchanged lines hidden (view full) ---

309 else if (et->et_min_period.frac != 0)
310 freq = min(freq, BT2FREQ(&et->et_min_period));
311 if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
312 freq = max(freq, BT2FREQ(&et->et_max_period));
313 return (freq);
314}
315
316/*
548static int
549round_freq(struct eventtimer *et, int freq)
550{
551 uint64_t div;
552
553 if (et->et_frequency != 0) {
554 div = lmax((et->et_frequency + freq / 2) / freq, 1);
555 if (et->et_flags & ET_FLAGS_POW2DIV)

--- 5 unchanged lines hidden (view full) ---

561 else if (et->et_min_period.frac != 0)
562 freq = min(freq, BT2FREQ(&et->et_min_period));
563 if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
564 freq = max(freq, BT2FREQ(&et->et_max_period));
565 return (freq);
566}
567
568/*
317 * Configure and start event timers.
569 * Configure and start event timers (BSP part).
318 */
319void
320cpu_initclocks_bsp(void)
321{
570 */
571void
572cpu_initclocks_bsp(void)
573{
322 int base, div;
574 struct pcpu_state *state;
575 int base, div, cpu;
323
576
324 timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
325 if (timer[0] == NULL)
326 timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
327 if (timer[0] == NULL)
577 mtx_init(&et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
578 CPU_FOREACH(cpu) {
579 state = DPCPU_ID_PTR(cpu, timerstate);
580 mtx_init(&state->et_hw_mtx, "et_hw_mtx", NULL, MTX_SPIN);
581 }
582#ifdef SMP
583 callout_new_inserted = cpu_new_callout;
584#endif
585 /* Grab requested timer or the best of present. */
586 if (timername[0])
587 timer = et_find(timername, 0, 0);
588 if (timer == NULL && periodic) {
589 timer = et_find(NULL,
590 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
591 }
592 if (timer == NULL) {
593 timer = et_find(NULL,
594 ET_FLAGS_ONESHOT, ET_FLAGS_ONESHOT);
595 }
596 if (timer == NULL && !periodic) {
597 timer = et_find(NULL,
598 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
599 }
600 if (timer == NULL)
328 panic("No usable event timer found!");
601 panic("No usable event timer found!");
329 et_init(timer[0], timer1cb, NULL, NULL);
330 timer[1] = et_find(timername[1][0] ? timername[1] : NULL,
331 ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
332 if (timer[1])
333 et_init(timer[1], timer2cb, NULL, NULL);
602 et_init(timer, timercb, NULL, NULL);
603
604 /* Adapt to timer capabilities. */
605 if (periodic && (timer->et_flags & ET_FLAGS_PERIODIC) == 0)
606 periodic = 0;
607 else if (!periodic && (timer->et_flags & ET_FLAGS_ONESHOT) == 0)
608 periodic = 1;
609 if (timer->et_flags & ET_FLAGS_C3STOP)
610 cpu_disable_deep_sleep++;
611
334 /*
335 * We honor the requested 'hz' value.
336 * We want to run stathz in the neighborhood of 128hz.
337 * We would like profhz to run as often as possible.
338 */
339 if (singlemul == 0) {
340 if (hz >= 1500 || (hz % 128) == 0)
341 singlemul = 1;
342 else if (hz >= 750)
343 singlemul = 2;
344 else
345 singlemul = 4;
346 }
612 /*
613 * We honor the requested 'hz' value.
614 * We want to run stathz in the neighborhood of 128hz.
615 * We would like profhz to run as often as possible.
616 */
617 if (singlemul == 0) {
618 if (hz >= 1500 || (hz % 128) == 0)
619 singlemul = 1;
620 else if (hz >= 750)
621 singlemul = 2;
622 else
623 singlemul = 4;
624 }
347 if (timer[1] == NULL) {
348 base = round_freq(timer[0], hz * singlemul);
625 if (periodic) {
626 base = round_freq(timer, hz * singlemul);
349 singlemul = max((base + hz / 2) / hz, 1);
350 hz = (base + singlemul / 2) / singlemul;
351 if (base <= 128)
352 stathz = base;
353 else {
354 div = base / 128;
355 if (div >= singlemul && (div % singlemul) == 0)
356 div++;
357 stathz = base / div;
358 }
359 profhz = stathz;
360 while ((profhz + stathz) <= 128 * 64)
361 profhz += stathz;
627 singlemul = max((base + hz / 2) / hz, 1);
628 hz = (base + singlemul / 2) / singlemul;
629 if (base <= 128)
630 stathz = base;
631 else {
632 div = base / 128;
633 if (div >= singlemul && (div % singlemul) == 0)
634 div++;
635 stathz = base / div;
636 }
637 profhz = stathz;
638 while ((profhz + stathz) <= 128 * 64)
639 profhz += stathz;
362 profhz = round_freq(timer[0], profhz);
640 profhz = round_freq(timer, profhz);
363 } else {
641 } else {
364 hz = round_freq(timer[0], hz);
365 stathz = round_freq(timer[1], 127);
366 profhz = round_freq(timer[1], stathz * 64);
642 hz = round_freq(timer, hz);
643 stathz = round_freq(timer, 127);
644 profhz = round_freq(timer, stathz * 64);
367 }
368 tick = 1000000 / hz;
645 }
646 tick = 1000000 / hz;
647 FREQ2BT(hz, &hardperiod);
648 FREQ2BT(stathz, &statperiod);
649 FREQ2BT(profhz, &profperiod);
369 ET_LOCK();
650 ET_LOCK();
370 cpu_restartclocks();
651 configtimer(1);
371 ET_UNLOCK();
372}
373
652 ET_UNLOCK();
653}
654
374/* Start per-CPU event timers on APs. */
655/*
656 * Start per-CPU event timers on APs.
657 */
375void
376cpu_initclocks_ap(void)
377{
658void
659cpu_initclocks_ap(void)
660{
661 struct bintime now;
662 struct pcpu_state *state;
378
663
379 ET_LOCK();
380 if (timer[0]->et_flags & ET_FLAGS_PERCPU)
381 et_start(timer[0], NULL, &timerperiod[0]);
382 if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU)
383 et_start(timer[1], NULL, &timerperiod[1]);
384 ET_UNLOCK();
664 if (timer->et_flags & ET_FLAGS_PERCPU) {
665 state = DPCPU_PTR(timerstate);
666 binuptime(&now);
667 ET_HW_LOCK(state);
668 loadtimer(&now, 1);
669 ET_HW_UNLOCK(state);
670 }
385}
386
671}
672
387/* Reconfigure and restart event timers after configuration changes. */
388static void
389cpu_restartclocks(void)
673/*
674 * Switch to profiling clock rates.
675 */
676void
677cpu_startprofclock(void)
390{
391
678{
679
392 /* Stop all event timers. */
393 timertest = 0;
394 if (timer1hz) {
395 timer1hz = 0;
680 ET_LOCK();
681 if (periodic) {
396 configtimer(0);
682 configtimer(0);
397 }
398 if (timer[1] && timer2hz) {
399 timer2hz = 0;
683 profiling = 1;
400 configtimer(1);
684 configtimer(1);
401 }
402 /* Calculate new event timers parameters. */
403 if (timer[1] == NULL) {
404 timer1hz = hz * singlemul;
405 while (timer1hz < (profiling_on ? profhz : stathz))
406 timer1hz += hz;
407 timer2hz = 0;
408 } else {
409 timer1hz = hz;
410 timer2hz = profiling_on ? profhz : stathz;
411 timer2hz = round_freq(timer[1], timer2hz);
412 }
413 timer1hz = round_freq(timer[0], timer1hz);
414 printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n",
415 timer[0]->et_name, timer1hz,
416 timer[1] ? timer[1]->et_name : "NONE", timer2hz);
417 /* Restart event timers. */
418 FREQ2BT(timer1hz, &timerperiod[0]);
419 configtimer(0);
420 if (timer[1]) {
421 timerticks[0] = 0;
422 timerticks[1] = 0;
423 FREQ2BT(timer2hz, &timerperiod[1]);
424 configtimer(1);
425 timertest = 1;
426 }
685 } else
686 profiling = 1;
687 ET_UNLOCK();
427}
428
688}
689
429/* Switch to profiling clock rates. */
690/*
691 * Switch to regular clock rates.
692 */
430void
693void
431cpu_startprofclock(void)
694cpu_stopprofclock(void)
432{
433
434 ET_LOCK();
695{
696
697 ET_LOCK();
435 profiling_on = 1;
436 cpu_restartclocks();
698 if (periodic) {
699 configtimer(0);
700 profiling = 0;
701 configtimer(1);
702 } else
703 profiling = 0;
437 ET_UNLOCK();
438}
439
704 ET_UNLOCK();
705}
706
440/* Switch to regular clock rates. */
707/*
708 * Switch to idle mode (all ticks handled).
709 */
441void
710void
442cpu_stopprofclock(void)
711cpu_idleclock(void)
443{
712{
713 struct bintime now, t;
714 struct pcpu_state *state;
444
715
445 ET_LOCK();
446 profiling_on = 0;
447 cpu_restartclocks();
448 ET_UNLOCK();
716 if (idletick || busy ||
717 (periodic && (timer->et_flags & ET_FLAGS_PERCPU)))
718 return;
719 state = DPCPU_PTR(timerstate);
720 if (periodic)
721 now = state->now;
722 else
723 binuptime(&now);
724 CTR4(KTR_SPARE2, "idle at %d: now %d.%08x%08x",
725 curcpu, now.sec, (unsigned int)(now.frac >> 32),
726 (unsigned int)(now.frac & 0xffffffff));
727 getnextcpuevent(&t, 1);
728 ET_HW_LOCK(state);
729 state->idle = 1;
730 state->nextevent = t;
731 if (!periodic)
732 loadtimer(&now, 0);
733 ET_HW_UNLOCK(state);
449}
450
734}
735
451/* Report or change the active event timers hardware. */
736/*
737 * Switch to active mode (skip empty ticks).
738 */
739void
740cpu_activeclock(void)
741{
742 struct bintime now;
743 struct pcpu_state *state;
744 struct thread *td;
745
746 state = DPCPU_PTR(timerstate);
747 if (state->idle == 0 || busy)
748 return;
749 if (periodic)
750 now = state->now;
751 else
752 binuptime(&now);
753 CTR4(KTR_SPARE2, "active at %d: now %d.%08x%08x",
754 curcpu, now.sec, (unsigned int)(now.frac >> 32),
755 (unsigned int)(now.frac & 0xffffffff));
756 spinlock_enter();
757 td = curthread;
758 td->td_intr_nesting_level++;
759 handleevents(&now, 1);
760 td->td_intr_nesting_level--;
761 spinlock_exit();
762}
763
764#ifdef SMP
765static void
766cpu_new_callout(int cpu, int ticks)
767{
768 struct bintime tmp;
769 struct pcpu_state *state;
770
771 CTR3(KTR_SPARE2, "new co at %d: on %d in %d",
772 curcpu, cpu, ticks);
773 state = DPCPU_ID_PTR(cpu, timerstate);
774 ET_HW_LOCK(state);
775 if (state->idle == 0 || busy) {
776 ET_HW_UNLOCK(state);
777 return;
778 }
779 /*
780 * If timer is periodic - just update next event time for target CPU.
781 */
782 if (periodic) {
783 state->nextevent = state->nexthard;
784 tmp = hardperiod;
785 bintime_mul(&tmp, ticks - 1);
786 bintime_add(&state->nextevent, &tmp);
787 ET_HW_UNLOCK(state);
788 return;
789 }
790 /*
791 * Otherwise we have to wake that CPU up, as we can't get present
792 * bintime to reprogram global timer from here. If timer is per-CPU,
793 * we by definition can't do it from here.
794 */
795 ET_HW_UNLOCK(state);
796 if (timer->et_flags & ET_FLAGS_PERCPU) {
797 state->handle = 1;
798 ipi_cpu(cpu, IPI_HARDCLOCK);
799 } else {
800 if (!cpu_idle_wakeup(cpu))
801 ipi_cpu(cpu, IPI_AST);
802 }
803}
804#endif
805
806/*
807 * Report or change the active event timers hardware.
808 */
452static int
809static int
453sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS)
810sysctl_kern_eventtimer_timer(SYSCTL_HANDLER_ARGS)
454{
455 char buf[32];
456 struct eventtimer *et;
457 int error;
458
459 ET_LOCK();
811{
812 char buf[32];
813 struct eventtimer *et;
814 int error;
815
816 ET_LOCK();
460 et = timer[0];
817 et = timer;
461 snprintf(buf, sizeof(buf), "%s", et->et_name);
462 ET_UNLOCK();
463 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
464 ET_LOCK();
818 snprintf(buf, sizeof(buf), "%s", et->et_name);
819 ET_UNLOCK();
820 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
821 ET_LOCK();
465 et = timer[0];
822 et = timer;
466 if (error != 0 || req->newptr == NULL ||
823 if (error != 0 || req->newptr == NULL ||
467 strcmp(buf, et->et_name) == 0) {
824 strcasecmp(buf, et->et_name) == 0) {
468 ET_UNLOCK();
469 return (error);
470 }
825 ET_UNLOCK();
826 return (error);
827 }
471 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
828 et = et_find(buf, 0, 0);
472 if (et == NULL) {
473 ET_UNLOCK();
474 return (ENOENT);
475 }
829 if (et == NULL) {
830 ET_UNLOCK();
831 return (ENOENT);
832 }
476 timer1hz = 0;
477 configtimer(0);
833 configtimer(0);
478 et_free(timer[0]);
479 timer[0] = et;
480 et_init(timer[0], timer1cb, NULL, NULL);
481 cpu_restartclocks();
834 et_free(timer);
835 if (et->et_flags & ET_FLAGS_C3STOP)
836 cpu_disable_deep_sleep++;
837 if (timer->et_flags & ET_FLAGS_C3STOP)
838 cpu_disable_deep_sleep--;
839 timer = et;
840 et_init(timer, timercb, NULL, NULL);
841 configtimer(1);
482 ET_UNLOCK();
483 return (error);
484}
842 ET_UNLOCK();
843 return (error);
844}
485SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1,
845SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer,
486 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
846 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
487 0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer");
847 0, 0, sysctl_kern_eventtimer_timer, "A", "Kernel event timer");
488
848
849/*
850 * Report or change the active event timer periodicity.
851 */
489static int
852static int
490sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS)
853sysctl_kern_eventtimer_periodic(SYSCTL_HANDLER_ARGS)
491{
854{
492 char buf[32];
493 struct eventtimer *et;
494 int error;
855 int error, val;
495
856
857 val = periodic;
858 error = sysctl_handle_int(oidp, &val, 0, req);
859 if (error != 0 || req->newptr == NULL)
860 return (error);
496 ET_LOCK();
861 ET_LOCK();
497 et = timer[1];
498 if (et == NULL)
499 snprintf(buf, sizeof(buf), "NONE");
500 else
501 snprintf(buf, sizeof(buf), "%s", et->et_name);
862 configtimer(0);
863 periodic = val;
864 configtimer(1);
502 ET_UNLOCK();
865 ET_UNLOCK();
503 error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
504 ET_LOCK();
505 et = timer[1];
506 if (error != 0 || req->newptr == NULL ||
507 strcmp(buf, et ? et->et_name : "NONE") == 0) {
508 ET_UNLOCK();
509 return (error);
510 }
511 et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
512 if (et == NULL && strcasecmp(buf, "NONE") != 0) {
513 ET_UNLOCK();
514 return (ENOENT);
515 }
516 if (timer[1] != NULL) {
517 timer2hz = 0;
518 configtimer(1);
519 et_free(timer[1]);
520 }
521 timer[1] = et;
522 if (timer[1] != NULL)
523 et_init(timer[1], timer2cb, NULL, NULL);
524 cpu_restartclocks();
525 ET_UNLOCK();
526 return (error);
527}
866 return (error);
867}
528SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2,
529 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
530 0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer");
868SYSCTL_PROC(_kern_eventtimer, OID_AUTO, periodic,
869 CTLTYPE_INT | CTLFLAG_RW | CTLFLAG_MPSAFE,
870 0, 0, sysctl_kern_eventtimer_periodic, "I", "Kernel event timer periodic");
531
532#endif
871
872#endif
533