kern_clocksource.c revision 210298 
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/kern_clocksource.c 210298 2010-07-20 15:48:29Z mav $");
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_kdtrace.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/bus.h>
42#include <sys/lock.h>
43#include <sys/kdb.h>
44#include <sys/mutex.h>
45#include <sys/proc.h>
46#include <sys/kernel.h>
47#include <sys/sched.h>
48#include <sys/smp.h>
49#include <sys/sysctl.h>
50#include <sys/timeet.h>
51
52#include <machine/atomic.h>
53#include <machine/clock.h>
54#include <machine/cpu.h>
55#include <machine/smp.h>
56
57#ifdef KDTRACE_HOOKS
58#include <sys/dtrace_bsd.h>
59cyclic_clock_func_t	cyclic_clock_func[MAXCPU];
60#endif
61
62static void		cpu_restartclocks(void);
63static void		timercheck(void);
64inline static int	doconfigtimer(int i);
65static void		configtimer(int i);
66
67static struct eventtimer *timer[2] = { NULL, NULL };
68static int		timertest = 0;
69static int		timerticks[2] = { 0, 0 };
70static int		profiling_on = 0;
71static struct bintime	timerperiod[2];
72
73static char		timername[2][32];
74TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername));
75TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername));
76
77static u_int		singlemul = 0;
78TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
79SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
80    0, "Multiplier, used in single timer mode");
81
82typedef u_int tc[2];
83static DPCPU_DEFINE(tc, configtimer);
84
85#define FREQ2BT(freq, bt)						\
86{									\
87	(bt)->sec = 0;							\
88	(bt)->frac = ((uint64_t)0x8000000000000000  / (freq)) << 1;	\
89}
90#define BT2FREQ(bt)							\
91	(((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) /		\
92	    ((bt)->frac >> 1))
93
94/* Per-CPU timer1 handler. */
95static int
96hardclockhandler(struct trapframe *frame)
97{
98
99#ifdef KDTRACE_HOOKS
100	/*
101	 * If the DTrace hooks are configured and a callback function
102	 * has been registered, then call it to process the high speed
103	 * timers.
104	 */
105	int cpu = curcpu;
106	if (cyclic_clock_func[cpu] != NULL)
107		(*cyclic_clock_func[cpu])(frame);
108#endif
109
110	timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
111	return (FILTER_HANDLED);
112}
113
114/* Per-CPU timer2 handler. */
115static int
116statclockhandler(struct trapframe *frame)
117{
118
119	timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
120	return (FILTER_HANDLED);
121}
122
123/* timer1 broadcast IPI handler. */
124int
125hardclockintr(struct trapframe *frame)
126{
127
128	if (doconfigtimer(0))
129		return (FILTER_HANDLED);
130	return (hardclockhandler(frame));
131}
132
133/* timer2 broadcast IPI handler. */
134int
135statclockintr(struct trapframe *frame)
136{
137
138	if (doconfigtimer(1))
139		return (FILTER_HANDLED);
140	return (statclockhandler(frame));
141}
142
143/* timer1 callback. */
144static void
145timer1cb(struct eventtimer *et, void *arg)
146{
147
148#ifdef SMP
149	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
150	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
151		ipi_all_but_self(IPI_HARDCLOCK);
152#endif
153	if (timertest) {
154		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
155			timerticks[0]++;
156			if (timerticks[0] >= timer1hz) {
157				ET_LOCK();
158				timercheck();
159				ET_UNLOCK();
160			}
161		}
162	}
163	hardclockhandler(curthread->td_intr_frame);
164}
165
166/* timer2 callback. */
167static void
168timer2cb(struct eventtimer *et, void *arg)
169{
170
171#ifdef SMP
172	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
173	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
174		ipi_all_but_self(IPI_STATCLOCK);
175#endif
176	if (timertest) {
177		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
178			timerticks[1]++;
179			if (timerticks[1] >= timer2hz * 2) {
180				ET_LOCK();
181				timercheck();
182				ET_UNLOCK();
183			}
184		}
185	}
186	statclockhandler(curthread->td_intr_frame);
187}
188
189/*
190 * Check that both timers are running with at least 1/4 of configured rate.
191 * If not - replace the broken one.
192 */
193static void
194timercheck(void)
195{
196
197	if (!timertest)
198		return;
199	timertest = 0;
200	if (timerticks[0] * 4 < timer1hz) {
201		printf("Event timer \"%s\" is dead.\n", timer[0]->et_name);
202		timer1hz = 0;
203		configtimer(0);
204		et_ban(timer[0]);
205		et_free(timer[0]);
206		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
207		if (timer[0] == NULL) {
208			timer2hz = 0;
209			configtimer(1);
210			et_free(timer[1]);
211			timer[1] = NULL;
212			timer[0] = timer[1];
213		}
214		et_init(timer[0], timer1cb, NULL, NULL);
215		cpu_restartclocks();
216		return;
217	}
218	if (timerticks[1] * 4 < timer2hz) {
219		printf("Event timer \"%s\" is dead.\n", timer[1]->et_name);
220		timer2hz = 0;
221		configtimer(1);
222		et_ban(timer[1]);
223		et_free(timer[1]);
224		timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
225		if (timer[1] != NULL)
226			et_init(timer[1], timer2cb, NULL, NULL);
227		cpu_restartclocks();
228		return;
229	}
230}
231
232/*
233 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
234 */
235inline static int
236doconfigtimer(int i)
237{
238	tc *conf;
239
240	conf = DPCPU_PTR(configtimer);
241	if (atomic_load_acq_int(*conf + i)) {
242		if (i == 0 ? timer1hz : timer2hz)
243			et_start(timer[i], NULL, &timerperiod[i]);
244		else
245			et_stop(timer[i]);
246		atomic_store_rel_int(*conf + i, 0);
247		return (1);
248	}
249	return (0);
250}
251
252/*
253 * Reconfigure specified timer.
254 * For per-CPU timers use IPI to make other CPUs to reconfigure.
255 */
256static void
257configtimer(int i)
258{
259#ifdef SMP
260	tc *conf;
261	int cpu;
262
263	critical_enter();
264#endif
265	/* Start/stop global timer or per-CPU timer of this CPU. */
266	if (i == 0 ? timer1hz : timer2hz)
267		et_start(timer[i], NULL, &timerperiod[i]);
268	else
269		et_stop(timer[i]);
270#ifdef SMP
271	if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
272		critical_exit();
273		return;
274	}
275	/* Set reconfigure flags for other CPUs. */
276	CPU_FOREACH(cpu) {
277		conf = DPCPU_ID_PTR(cpu, configtimer);
278		atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1);
279	}
280	/* Send reconfigure IPI. */
281	ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK);
282	/* Wait for reconfiguration completed. */
283restart:
284	cpu_spinwait();
285	CPU_FOREACH(cpu) {
286		if (cpu == curcpu)
287			continue;
288		conf = DPCPU_ID_PTR(cpu, configtimer);
289		if (atomic_load_acq_int(*conf + i))
290			goto restart;
291	}
292	critical_exit();
293#endif
294}
295
296static int
297round_freq(struct eventtimer *et, int freq)
298{
299	uint64_t div;
300
301	if (et->et_frequency != 0) {
302		div = lmax((et->et_frequency + freq / 2) / freq, 1);
303		if (et->et_flags & ET_FLAGS_POW2DIV)
304			div = 1 << (flsl(div + div / 2) - 1);
305		freq = (et->et_frequency + div / 2) / div;
306	}
307	if (et->et_min_period.sec > 0)
308		freq = 0;
309	else if (et->et_min_period.frac != 0)
310		freq = min(freq, BT2FREQ(&et->et_min_period));
311	if (et->et_max_period.sec == 0 && et->et_max_period.frac != 0)
312		freq = max(freq, BT2FREQ(&et->et_max_period));
313	return (freq);
314}
315
316/*
317 * Configure and start event timers.
318 */
319void
320cpu_initclocks_bsp(void)
321{
322	int base, div;
323
324	timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
325	if (timer[0] == NULL)
326		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
327	if (timer[0] == NULL)
328		panic("No usable event timer found!");
329	et_init(timer[0], timer1cb, NULL, NULL);
330	timer[1] = et_find(timername[1][0] ? timername[1] : NULL,
331	    ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
332	if (timer[1])
333		et_init(timer[1], timer2cb, NULL, NULL);
334	/*
335	 * We honor the requested 'hz' value.
336	 * We want to run stathz in the neighborhood of 128hz.
337	 * We would like profhz to run as often as possible.
338	 */
339	if (singlemul == 0) {
340		if (hz >= 1500 || (hz % 128) == 0)
341			singlemul = 1;
342		else if (hz >= 750)
343			singlemul = 2;
344		else
345			singlemul = 4;
346	}
347	if (timer[1] == NULL) {
348		base = round_freq(timer[0], hz * singlemul);
349		singlemul = max((base + hz / 2) / hz, 1);
350		hz = (base + singlemul / 2) / singlemul;
351		if (base <= 128)
352			stathz = base;
353		else {
354			div = base / 128;
355			if (div >= singlemul && (div % singlemul) == 0)
356				div++;
357			stathz = base / div;
358		}
359		profhz = stathz;
360		while ((profhz + stathz) <= 128 * 64)
361			profhz += stathz;
362		profhz = round_freq(timer[0], profhz);
363	} else {
364		hz = round_freq(timer[0], hz);
365		stathz = round_freq(timer[1], 127);
366		profhz = round_freq(timer[1], stathz * 64);
367	}
368	tick = 1000000 / hz;
369	ET_LOCK();
370	cpu_restartclocks();
371	ET_UNLOCK();
372}
373
374/* Start per-CPU event timers on APs. */
375void
376cpu_initclocks_ap(void)
377{
378
379	ET_LOCK();
380	if (timer[0]->et_flags & ET_FLAGS_PERCPU)
381		et_start(timer[0], NULL, &timerperiod[0]);
382	if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU)
383		et_start(timer[1], NULL, &timerperiod[1]);
384	ET_UNLOCK();
385}
386
387/* Reconfigure and restart event timers after configuration changes. */
388static void
389cpu_restartclocks(void)
390{
391
392	/* Stop all event timers. */
393	timertest = 0;
394	if (timer1hz) {
395		timer1hz = 0;
396		configtimer(0);
397	}
398	if (timer[1] && timer2hz) {
399		timer2hz = 0;
400		configtimer(1);
401	}
402	/* Calculate new event timers parameters. */
403	if (timer[1] == NULL) {
404		timer1hz = hz * singlemul;
405		while (timer1hz < (profiling_on ? profhz : stathz))
406			timer1hz += hz;
407		timer2hz = 0;
408	} else {
409		timer1hz = hz;
410		timer2hz = profiling_on ? profhz : stathz;
411		timer2hz = round_freq(timer[1], timer2hz);
412	}
413	timer1hz = round_freq(timer[0], timer1hz);
414	printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n",
415	    timer[0]->et_name, timer1hz,
416	    timer[1] ? timer[1]->et_name : "NONE", timer2hz);
417	/* Restart event timers. */
418	FREQ2BT(timer1hz, &timerperiod[0]);
419	configtimer(0);
420	if (timer[1]) {
421		timerticks[0] = 0;
422		timerticks[1] = 0;
423		FREQ2BT(timer2hz, &timerperiod[1]);
424		configtimer(1);
425		timertest = 1;
426	}
427}
428
429/* Switch to profiling clock rates. */
430void
431cpu_startprofclock(void)
432{
433
434	ET_LOCK();
435	profiling_on = 1;
436	cpu_restartclocks();
437	ET_UNLOCK();
438}
439
440/* Switch to regular clock rates. */
441void
442cpu_stopprofclock(void)
443{
444
445	ET_LOCK();
446	profiling_on = 0;
447	cpu_restartclocks();
448	ET_UNLOCK();
449}
450
451/* Report or change the active event timers hardware. */
452static int
453sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS)
454{
455	char buf[32];
456	struct eventtimer *et;
457	int error;
458
459	ET_LOCK();
460	et = timer[0];
461	snprintf(buf, sizeof(buf), "%s", et->et_name);
462	ET_UNLOCK();
463	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
464	ET_LOCK();
465	et = timer[0];
466	if (error != 0 || req->newptr == NULL ||
467	    strcmp(buf, et->et_name) == 0) {
468		ET_UNLOCK();
469		return (error);
470	}
471	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
472	if (et == NULL) {
473		ET_UNLOCK();
474		return (ENOENT);
475	}
476	timer1hz = 0;
477	configtimer(0);
478	et_free(timer[0]);
479	timer[0] = et;
480	et_init(timer[0], timer1cb, NULL, NULL);
481	cpu_restartclocks();
482	ET_UNLOCK();
483	return (error);
484}
485SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1,
486    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
487    0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer");
488
489static int
490sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS)
491{
492	char buf[32];
493	struct eventtimer *et;
494	int error;
495
496	ET_LOCK();
497	et = timer[1];
498	if (et == NULL)
499		snprintf(buf, sizeof(buf), "NONE");
500	else
501		snprintf(buf, sizeof(buf), "%s", et->et_name);
502	ET_UNLOCK();
503	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
504	ET_LOCK();
505	et = timer[1];
506	if (error != 0 || req->newptr == NULL ||
507	    strcmp(buf, et ? et->et_name : "NONE") == 0) {
508		ET_UNLOCK();
509		return (error);
510	}
511	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
512	if (et == NULL && strcasecmp(buf, "NONE") != 0) {
513		ET_UNLOCK();
514		return (ENOENT);
515	}
516	if (timer[1] != NULL) {
517		timer2hz = 0;
518		configtimer(1);
519		et_free(timer[1]);
520	}
521	timer[1] = et;
522	if (timer[1] != NULL)
523		et_init(timer[1], timer2cb, NULL, NULL);
524	cpu_restartclocks();
525	ET_UNLOCK();
526	return (error);
527}
528SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2,
529    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
530    0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer");
531
532#endif
533
534