kern_clocksource.c revision 210054 
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/kern/timeevents.c 210054 2010-07-14 13:31:27Z mav $");
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_kdtrace.h"
38
39#include <sys/param.h>
40#include <sys/systm.h>
41#include <sys/bus.h>
42#include <sys/lock.h>
43#include <sys/kdb.h>
44#include <sys/mutex.h>
45#include <sys/proc.h>
46#include <sys/kernel.h>
47#include <sys/sched.h>
48#include <sys/smp.h>
49#include <sys/sysctl.h>
50#include <sys/timeet.h>
51
52#include <machine/atomic.h>
53#include <machine/clock.h>
54#include <machine/cpu.h>
55#include <machine/smp.h>
56
57#ifdef KDTRACE_HOOKS
58#include <sys/dtrace_bsd.h>
59cyclic_clock_func_t	cyclic_clock_func[MAXCPU];
60#endif
61
62static void		cpu_restartclocks(void);
63static void		timercheck(void);
64inline static int	doconfigtimer(int i);
65static void		configtimer(int i);
66
67static struct eventtimer *timer[2] = { NULL, NULL };
68static int		timertest = 0;
69static int		timerticks[2] = { 0, 0 };
70static int		profiling_on = 0;
71static struct bintime	timerperiod[2];
72
73static char		timername[2][32];
74TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername));
75TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername));
76
77static u_int		singlemul = 0;
78TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
79SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
80    0, "Multiplier, used in single timer mode");
81
82typedef u_int tc[2];
83static DPCPU_DEFINE(tc, configtimer);
84
85#define FREQ2BT(freq, bt)						\
86{									\
87	(bt)->sec = 0;							\
88	(bt)->frac = ((uint64_t)0x8000000000000000  / (freq)) << 1;	\
89}
90#define BT2FREQ(bt, freq)						\
91{									\
92	*(freq) = ((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) /	\
93		    ((bt)->frac >> 1);					\
94}
95
96/* Per-CPU timer1 handler. */
97static int
98hardclockhandler(struct trapframe *frame)
99{
100
101#ifdef KDTRACE_HOOKS
102	/*
103	 * If the DTrace hooks are configured and a callback function
104	 * has been registered, then call it to process the high speed
105	 * timers.
106	 */
107	int cpu = curcpu;
108	if (cyclic_clock_func[cpu] != NULL)
109		(*cyclic_clock_func[cpu])(frame);
110#endif
111
112	timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
113	return (FILTER_HANDLED);
114}
115
116/* Per-CPU timer2 handler. */
117static int
118statclockhandler(struct trapframe *frame)
119{
120
121	timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
122	return (FILTER_HANDLED);
123}
124
125/* timer1 broadcast IPI handler. */
126int
127hardclockintr(struct trapframe *frame)
128{
129
130	if (doconfigtimer(0))
131		return (FILTER_HANDLED);
132	return (hardclockhandler(frame));
133}
134
135/* timer2 broadcast IPI handler. */
136int
137statclockintr(struct trapframe *frame)
138{
139
140	if (doconfigtimer(1))
141		return (FILTER_HANDLED);
142	return (statclockhandler(frame));
143}
144
145/* timer1 callback. */
146static void
147timer1cb(struct eventtimer *et, void *arg)
148{
149
150#ifdef SMP
151	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
152	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
153		ipi_all_but_self(IPI_HARDCLOCK);
154#endif
155	if (timertest) {
156		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
157			timerticks[0]++;
158			if (timerticks[0] >= timer1hz) {
159				ET_LOCK();
160				timercheck();
161				ET_UNLOCK();
162			}
163		}
164	}
165	hardclockhandler(curthread->td_intr_frame);
166}
167
168/* timer2 callback. */
169static void
170timer2cb(struct eventtimer *et, void *arg)
171{
172
173#ifdef SMP
174	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
175	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
176		ipi_all_but_self(IPI_STATCLOCK);
177#endif
178	if (timertest) {
179		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
180			timerticks[1]++;
181			if (timerticks[1] >= timer2hz * 2) {
182				ET_LOCK();
183				timercheck();
184				ET_UNLOCK();
185			}
186		}
187	}
188	statclockhandler(curthread->td_intr_frame);
189}
190
191/*
192 * Check that both timers are running with at least 1/4 of configured rate.
193 * If not - replace the broken one.
194 */
195static void
196timercheck(void)
197{
198
199	if (!timertest)
200		return;
201	timertest = 0;
202	if (timerticks[0] * 4 < timer1hz) {
203		printf("Event timer \"%s\" is dead.\n", timer[0]->et_name);
204		timer1hz = 0;
205		configtimer(0);
206		et_ban(timer[0]);
207		et_free(timer[0]);
208		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
209		if (timer[0] == NULL) {
210			timer2hz = 0;
211			configtimer(1);
212			et_free(timer[1]);
213			timer[1] = NULL;
214			timer[0] = timer[1];
215		}
216		et_init(timer[0], timer1cb, NULL, NULL);
217		cpu_restartclocks();
218		return;
219	}
220	if (timerticks[1] * 4 < timer2hz) {
221		printf("Event timer \"%s\" is dead.\n", timer[1]->et_name);
222		timer2hz = 0;
223		configtimer(1);
224		et_ban(timer[1]);
225		et_free(timer[1]);
226		timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
227		if (timer[1] != NULL)
228			et_init(timer[1], timer2cb, NULL, NULL);
229		cpu_restartclocks();
230		return;
231	}
232}
233
234/*
235 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
236 */
237inline static int
238doconfigtimer(int i)
239{
240	tc *conf;
241
242	conf = DPCPU_PTR(configtimer);
243	if (atomic_load_acq_int(*conf + i)) {
244		if (i == 0 ? timer1hz : timer2hz)
245			et_start(timer[i], NULL, &timerperiod[i]);
246		else
247			et_stop(timer[i]);
248		atomic_store_rel_int(*conf + i, 0);
249		return (1);
250	}
251	return (0);
252}
253
254/*
255 * Reconfigure specified timer.
256 * For per-CPU timers use IPI to make other CPUs to reconfigure.
257 */
258static void
259configtimer(int i)
260{
261#ifdef SMP
262	tc *conf;
263	int cpu;
264
265	critical_enter();
266#endif
267	/* Start/stop global timer or per-CPU timer of this CPU. */
268	if (i == 0 ? timer1hz : timer2hz)
269		et_start(timer[i], NULL, &timerperiod[i]);
270	else
271		et_stop(timer[i]);
272#ifdef SMP
273	if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
274		critical_exit();
275		return;
276	}
277	/* Set reconfigure flags for other CPUs. */
278	CPU_FOREACH(cpu) {
279		conf = DPCPU_ID_PTR(cpu, configtimer);
280		atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1);
281	}
282	/* Send reconfigure IPI. */
283	ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK);
284	/* Wait for reconfiguration completed. */
285restart:
286	cpu_spinwait();
287	CPU_FOREACH(cpu) {
288		if (cpu == curcpu)
289			continue;
290		conf = DPCPU_ID_PTR(cpu, configtimer);
291		if (atomic_load_acq_int(*conf + i))
292			goto restart;
293	}
294	critical_exit();
295#endif
296}
297
298/*
299 * Configure and start event timers.
300 */
301void
302cpu_initclocks_bsp(void)
303{
304	int base, div;
305
306	timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
307	if (timer[0] == NULL)
308		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
309	if (timer[0] == NULL)
310		panic("No usable event timer found!");
311	et_init(timer[0], timer1cb, NULL, NULL);
312	timer[1] = et_find(timername[1][0] ? timername[1] : NULL,
313	    ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
314	if (timer[1])
315		et_init(timer[1], timer2cb, NULL, NULL);
316	/*
317	 * We honor the requested 'hz' value.
318	 * We want to run stathz in the neighborhood of 128hz.
319	 * We would like profhz to run as often as possible.
320	 */
321	if (singlemul == 0) {
322		if (hz >= 1500 || (hz % 128) == 0)
323			singlemul = 1;
324		else if (hz >= 750)
325			singlemul = 2;
326		else
327			singlemul = 4;
328	}
329	if (timer[1] == NULL) {
330		base = hz * singlemul;
331		if (base < 128)
332			stathz = base;
333		else {
334			div = base / 128;
335			if (div % 2 == 0)
336				div++;
337			stathz = base / div;
338		}
339		profhz = stathz;
340		while ((profhz + stathz) <= 8192)
341			profhz += stathz;
342	} else {
343		stathz = 128;
344		profhz = stathz * 64;
345	}
346	ET_LOCK();
347	cpu_restartclocks();
348	ET_UNLOCK();
349}
350
351/* Start per-CPU event timers on APs. */
352void
353cpu_initclocks_ap(void)
354{
355
356	ET_LOCK();
357	if (timer[0]->et_flags & ET_FLAGS_PERCPU)
358		et_start(timer[0], NULL, &timerperiod[0]);
359	if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU)
360		et_start(timer[1], NULL, &timerperiod[1]);
361	ET_UNLOCK();
362}
363
364/* Reconfigure and restart event timers after configuration changes. */
365static void
366cpu_restartclocks(void)
367{
368
369	/* Stop all event timers. */
370	timertest = 0;
371	if (timer1hz) {
372		timer1hz = 0;
373		configtimer(0);
374	}
375	if (timer[1] && timer2hz) {
376		timer2hz = 0;
377		configtimer(1);
378	}
379	/* Calculate new event timers parameters. */
380	if (timer[1] == NULL) {
381		timer1hz = hz * singlemul;
382		while (timer1hz < (profiling_on ? profhz : stathz))
383			timer1hz += hz;
384		timer2hz = 0;
385	} else {
386		timer1hz = hz;
387		timer2hz = profiling_on ? profhz : stathz;
388	}
389	printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n",
390	    timer[0]->et_name, timer1hz,
391	    timer[1] ? timer[1]->et_name : "NONE", timer2hz);
392	/* Restart event timers. */
393	FREQ2BT(timer1hz, &timerperiod[0]);
394	configtimer(0);
395	if (timer[1]) {
396		timerticks[0] = 0;
397		timerticks[1] = 0;
398		FREQ2BT(timer2hz, &timerperiod[1]);
399		configtimer(1);
400		timertest = 1;
401	}
402}
403
404/* Switch to profiling clock rates. */
405void
406cpu_startprofclock(void)
407{
408
409	ET_LOCK();
410	profiling_on = 1;
411	cpu_restartclocks();
412	ET_UNLOCK();
413}
414
415/* Switch to regular clock rates. */
416void
417cpu_stopprofclock(void)
418{
419
420	ET_LOCK();
421	profiling_on = 0;
422	cpu_restartclocks();
423	ET_UNLOCK();
424}
425
426/* Report or change the active event timers hardware. */
427static int
428sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS)
429{
430	char buf[32];
431	struct eventtimer *et;
432	int error;
433
434	ET_LOCK();
435	et = timer[0];
436	snprintf(buf, sizeof(buf), "%s", et->et_name);
437	ET_UNLOCK();
438	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
439	ET_LOCK();
440	et = timer[0];
441	if (error != 0 || req->newptr == NULL ||
442	    strcmp(buf, et->et_name) == 0) {
443		ET_UNLOCK();
444		return (error);
445	}
446	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
447	if (et == NULL) {
448		ET_UNLOCK();
449		return (ENOENT);
450	}
451	timer1hz = 0;
452	configtimer(0);
453	et_free(timer[0]);
454	timer[0] = et;
455	et_init(timer[0], timer1cb, NULL, NULL);
456	cpu_restartclocks();
457	ET_UNLOCK();
458	return (error);
459}
460SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1,
461    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
462    0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer");
463
464static int
465sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS)
466{
467	char buf[32];
468	struct eventtimer *et;
469	int error;
470
471	ET_LOCK();
472	et = timer[1];
473	if (et == NULL)
474		snprintf(buf, sizeof(buf), "NONE");
475	else
476		snprintf(buf, sizeof(buf), "%s", et->et_name);
477	ET_UNLOCK();
478	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
479	ET_LOCK();
480	et = timer[1];
481	if (error != 0 || req->newptr == NULL ||
482	    strcmp(buf, et ? et->et_name : "NONE") == 0) {
483		ET_UNLOCK();
484		return (error);
485	}
486	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
487	if (et == NULL && strcasecmp(buf, "NONE") != 0) {
488		ET_UNLOCK();
489		return (ENOENT);
490	}
491	if (timer[1] != NULL) {
492		timer2hz = 0;
493		configtimer(1);
494		et_free(timer[1]);
495	}
496	timer[1] = et;
497	if (timer[1] != NULL)
498		et_init(timer[1], timer2cb, NULL, NULL);
499	cpu_restartclocks();
500	ET_UNLOCK();
501	return (error);
502}
503SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2,
504    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
505    0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer");
506
507#endif
508
509