kern_clocksource.c revision 209371 
1/*-
2 * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 *    notice, this list of conditions and the following disclaimer,
10 *    without modification, immediately at the beginning of the file.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
20 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
21 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
22 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
23 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
24 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 */
26
27#include <sys/cdefs.h>
28__FBSDID("$FreeBSD: head/sys/x86/x86/timeevents.c 209371 2010-06-20 21:33:29Z mav $");
29
30/*
31 * Common routines to manage event timers hardware.
32 */
33
34/* XEN has own timer routines now. */
35#ifndef XEN
36
37#include "opt_clock.h"
38#include "opt_kdtrace.h"
39
40#include <sys/param.h>
41#include <sys/systm.h>
42#include <sys/bus.h>
43#include <sys/lock.h>
44#include <sys/kdb.h>
45#include <sys/mutex.h>
46#include <sys/proc.h>
47#include <sys/kernel.h>
48#include <sys/sched.h>
49#include <sys/smp.h>
50#include <sys/sysctl.h>
51#include <sys/timeet.h>
52
53#include <machine/atomic.h>
54#include <machine/clock.h>
55#include <machine/cpu.h>
56#include <machine/intr_machdep.h>
57#include <machine/md_var.h>
58#include <machine/smp.h>
59
60#ifdef KDTRACE_HOOKS
61#include <sys/dtrace_bsd.h>
62cyclic_clock_func_t	cyclic_clock_func[MAXCPU];
63#endif
64
65static void		cpu_restartclocks(void);
66static void		timercheck(void);
67inline static int	doconfigtimer(int i);
68static void		configtimer(int i);
69
70static struct eventtimer *timer[2] = { NULL, NULL };
71static int		timertest = 0;
72static int		timerticks[2] = { 0, 0 };
73static int		profiling_on = 0;
74static struct bintime	timerperiod[2];
75
76static char		timername[2][32];
77TUNABLE_STR("kern.eventtimer.timer1", timername[0], sizeof(*timername));
78TUNABLE_STR("kern.eventtimer.timer2", timername[1], sizeof(*timername));
79
80static u_int		singlemul = 0;
81TUNABLE_INT("kern.eventtimer.singlemul", &singlemul);
82SYSCTL_INT(_kern_eventtimer, OID_AUTO, singlemul, CTLFLAG_RW, &singlemul,
83    0, "Multiplier, used in single timer mode");
84
85typedef u_int tc[2];
86static DPCPU_DEFINE(tc, configtimer);
87
88#define FREQ2BT(freq, bt)						\
89{									\
90	(bt)->sec = 0;							\
91	(bt)->frac = ((uint64_t)0x8000000000000000  / (freq)) << 1;	\
92}
93#define BT2FREQ(bt, freq)						\
94{									\
95	*(freq) = ((uint64_t)0x8000000000000000 + ((bt)->frac >> 2)) /	\
96		    ((bt)->frac >> 1);					\
97}
98
99/* Per-CPU timer1 handler. */
100static int
101hardclockhandler(struct trapframe *frame)
102{
103
104#ifdef KDTRACE_HOOKS
105	/*
106	 * If the DTrace hooks are configured and a callback function
107	 * has been registered, then call it to process the high speed
108	 * timers.
109	 */
110	int cpu = curcpu;
111	if (cyclic_clock_func[cpu] != NULL)
112		(*cyclic_clock_func[cpu])(frame);
113#endif
114
115	timer1clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
116	return (FILTER_HANDLED);
117}
118
119/* Per-CPU timer2 handler. */
120static int
121statclockhandler(struct trapframe *frame)
122{
123
124	timer2clock(TRAPF_USERMODE(frame), TRAPF_PC(frame));
125	return (FILTER_HANDLED);
126}
127
128/* timer1 broadcast IPI handler. */
129int
130hardclockintr(struct trapframe *frame)
131{
132
133	if (doconfigtimer(0))
134		return (FILTER_HANDLED);
135	return (hardclockhandler(frame));
136}
137
138/* timer2 broadcast IPI handler. */
139int
140statclockintr(struct trapframe *frame)
141{
142
143	if (doconfigtimer(1))
144		return (FILTER_HANDLED);
145	return (statclockhandler(frame));
146}
147
148/* timer1 callback. */
149static void
150timer1cb(struct eventtimer *et, void *arg)
151{
152
153#ifdef SMP
154	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
155	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
156		ipi_all_but_self(IPI_HARDCLOCK);
157#endif
158	if (timertest) {
159		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
160			timerticks[0]++;
161			if (timerticks[0] >= timer1hz) {
162				ET_LOCK();
163				timercheck();
164				ET_UNLOCK();
165			}
166		}
167	}
168	hardclockhandler((struct trapframe *)arg);
169}
170
171/* timer2 callback. */
172static void
173timer2cb(struct eventtimer *et, void *arg)
174{
175
176#ifdef SMP
177	/* Broadcast interrupt to other CPUs for non-per-CPU timers */
178	if (smp_started && (et->et_flags & ET_FLAGS_PERCPU) == 0)
179		ipi_all_but_self(IPI_STATCLOCK);
180#endif
181	if (timertest) {
182		if ((et->et_flags & ET_FLAGS_PERCPU) == 0 || curcpu == 0) {
183			timerticks[1]++;
184			if (timerticks[1] >= timer2hz * 2) {
185				ET_LOCK();
186				timercheck();
187				ET_UNLOCK();
188			}
189		}
190	}
191	statclockhandler((struct trapframe *)arg);
192}
193
194/*
195 * Check that both timers are running with at least 1/4 of configured rate.
196 * If not - replace the broken one.
197 */
198static void
199timercheck(void)
200{
201
202	if (!timertest)
203		return;
204	timertest = 0;
205	if (timerticks[0] * 4 < timer1hz) {
206		printf("Event timer \"%s\" is dead.\n", timer[0]->et_name);
207		timer1hz = 0;
208		configtimer(0);
209		et_ban(timer[0]);
210		et_free(timer[0]);
211		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
212		if (timer[0] == NULL) {
213			timer2hz = 0;
214			configtimer(1);
215			et_free(timer[1]);
216			timer[1] = NULL;
217			timer[0] = timer[1];
218		}
219		et_init(timer[0], timer1cb, NULL, NULL);
220		cpu_restartclocks();
221		return;
222	}
223	if (timerticks[1] * 4 < timer2hz) {
224		printf("Event timer \"%s\" is dead.\n", timer[1]->et_name);
225		timer2hz = 0;
226		configtimer(1);
227		et_ban(timer[1]);
228		et_free(timer[1]);
229		timer[1] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
230		if (timer[1] != NULL)
231			et_init(timer[1], timer2cb, NULL, NULL);
232		cpu_restartclocks();
233		return;
234	}
235}
236
237/*
238 * Reconfigure specified per-CPU timer on other CPU. Called from IPI handler.
239 */
240inline static int
241doconfigtimer(int i)
242{
243	tc *conf;
244
245	conf = DPCPU_PTR(configtimer);
246	if (atomic_load_acq_int(*conf + i)) {
247		if (i == 0 ? timer1hz : timer2hz)
248			et_start(timer[i], NULL, &timerperiod[i]);
249		else
250			et_stop(timer[i]);
251		atomic_store_rel_int(*conf + i, 0);
252		return (1);
253	}
254	return (0);
255}
256
257/*
258 * Reconfigure specified timer.
259 * For per-CPU timers use IPI to make other CPUs to reconfigure.
260 */
261static void
262configtimer(int i)
263{
264#ifdef SMP
265	tc *conf;
266	int cpu;
267
268	critical_enter();
269#endif
270	/* Start/stop global timer or per-CPU timer of this CPU. */
271	if (i == 0 ? timer1hz : timer2hz)
272		et_start(timer[i], NULL, &timerperiod[i]);
273	else
274		et_stop(timer[i]);
275#ifdef SMP
276	if ((timer[i]->et_flags & ET_FLAGS_PERCPU) == 0 || !smp_started) {
277		critical_exit();
278		return;
279	}
280	/* Set reconfigure flags for other CPUs. */
281	CPU_FOREACH(cpu) {
282		conf = DPCPU_ID_PTR(cpu, configtimer);
283		atomic_store_rel_int(*conf + i, (cpu == curcpu) ? 0 : 1);
284	}
285	/* Send reconfigure IPI. */
286	ipi_all_but_self(i == 0 ? IPI_HARDCLOCK : IPI_STATCLOCK);
287	/* Wait for reconfiguration completed. */
288restart:
289	cpu_spinwait();
290	CPU_FOREACH(cpu) {
291		if (cpu == curcpu)
292			continue;
293		conf = DPCPU_ID_PTR(cpu, configtimer);
294		if (atomic_load_acq_int(*conf + i))
295			goto restart;
296	}
297	critical_exit();
298#endif
299}
300
301/*
302 * Configure and start event timers.
303 */
304void
305cpu_initclocks_bsp(void)
306{
307	int base, div;
308
309	timer[0] = et_find(timername[0], ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
310	if (timer[0] == NULL)
311		timer[0] = et_find(NULL, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
312	et_init(timer[0], timer1cb, NULL, NULL);
313	timer[1] = et_find(timername[1][0] ? timername[1] : NULL,
314	    ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
315	if (timer[1])
316		et_init(timer[1], timer2cb, NULL, NULL);
317	/*
318	 * We honor the requested 'hz' value.
319	 * We want to run stathz in the neighborhood of 128hz.
320	 * We would like profhz to run as often as possible.
321	 */
322	if (singlemul == 0) {
323		if (hz >= 1500 || (hz % 128) == 0)
324			singlemul = 1;
325		else if (hz >= 750)
326			singlemul = 2;
327		else
328			singlemul = 4;
329	}
330	if (timer[1] == NULL) {
331		base = hz * singlemul;
332		if (base < 128)
333			stathz = base;
334		else {
335			div = base / 128;
336			if (div % 2 == 0)
337				div++;
338			stathz = base / div;
339		}
340		profhz = stathz;
341		while ((profhz + stathz) <= 8192)
342			profhz += stathz;
343	} else {
344		stathz = 128;
345		profhz = stathz * 64;
346	}
347	ET_LOCK();
348	cpu_restartclocks();
349	ET_UNLOCK();
350}
351
352/* Start per-CPU event timers on APs. */
353void
354cpu_initclocks_ap(void)
355{
356
357	ET_LOCK();
358	if (timer[0]->et_flags & ET_FLAGS_PERCPU)
359		et_start(timer[0], NULL, &timerperiod[0]);
360	if (timer[1] && timer[1]->et_flags & ET_FLAGS_PERCPU)
361		et_start(timer[1], NULL, &timerperiod[1]);
362	ET_UNLOCK();
363}
364
365/* Reconfigure and restart event timers after configuration changes. */
366static void
367cpu_restartclocks(void)
368{
369
370	/* Stop all event timers. */
371	timertest = 0;
372	if (timer1hz) {
373		timer1hz = 0;
374		configtimer(0);
375	}
376	if (timer[1] && timer2hz) {
377		timer2hz = 0;
378		configtimer(1);
379	}
380	/* Calculate new event timers parameters. */
381	if (timer[1] == NULL) {
382		timer1hz = hz * singlemul;
383		while (timer1hz < (profiling_on ? profhz : stathz))
384			timer1hz += hz;
385		timer2hz = 0;
386	} else {
387		timer1hz = hz;
388		timer2hz = profiling_on ? profhz : stathz;
389	}
390	printf("Starting kernel event timers: %s @ %dHz, %s @ %dHz\n",
391	    timer[0]->et_name, timer1hz,
392	    timer[1] ? timer[1]->et_name : "NONE", timer2hz);
393	/* Restart event timers. */
394	FREQ2BT(timer1hz, &timerperiod[0]);
395	configtimer(0);
396	if (timer[1]) {
397		timerticks[0] = 0;
398		timerticks[1] = 0;
399		FREQ2BT(timer2hz, &timerperiod[1]);
400		configtimer(1);
401		timertest = 1;
402	}
403}
404
405/* Switch to profiling clock rates. */
406void
407cpu_startprofclock(void)
408{
409
410	ET_LOCK();
411	profiling_on = 1;
412	cpu_restartclocks();
413	ET_UNLOCK();
414}
415
416/* Switch to regular clock rates. */
417void
418cpu_stopprofclock(void)
419{
420
421	ET_LOCK();
422	profiling_on = 0;
423	cpu_restartclocks();
424	ET_UNLOCK();
425}
426
427/* Report or change the active event timers hardware. */
428static int
429sysctl_kern_eventtimer_timer1(SYSCTL_HANDLER_ARGS)
430{
431	char buf[32];
432	struct eventtimer *et;
433	int error;
434
435	ET_LOCK();
436	et = timer[0];
437	snprintf(buf, sizeof(buf), "%s", et->et_name);
438	ET_UNLOCK();
439	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
440	ET_LOCK();
441	et = timer[0];
442	if (error != 0 || req->newptr == NULL ||
443	    strcmp(buf, et->et_name) == 0) {
444		ET_UNLOCK();
445		return (error);
446	}
447	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
448	if (et == NULL) {
449		ET_UNLOCK();
450		return (ENOENT);
451	}
452	timer1hz = 0;
453	configtimer(0);
454	et_free(timer[0]);
455	timer[0] = et;
456	et_init(timer[0], timer1cb, NULL, NULL);
457	cpu_restartclocks();
458	ET_UNLOCK();
459	return (error);
460}
461SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer1,
462    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
463    0, 0, sysctl_kern_eventtimer_timer1, "A", "Primary event timer");
464
465static int
466sysctl_kern_eventtimer_timer2(SYSCTL_HANDLER_ARGS)
467{
468	char buf[32];
469	struct eventtimer *et;
470	int error;
471
472	ET_LOCK();
473	et = timer[1];
474	if (et == NULL)
475		snprintf(buf, sizeof(buf), "NONE");
476	else
477		snprintf(buf, sizeof(buf), "%s", et->et_name);
478	ET_UNLOCK();
479	error = sysctl_handle_string(oidp, buf, sizeof(buf), req);
480	ET_LOCK();
481	et = timer[1];
482	if (error != 0 || req->newptr == NULL ||
483	    strcmp(buf, et ? et->et_name : "NONE") == 0) {
484		ET_UNLOCK();
485		return (error);
486	}
487	et = et_find(buf, ET_FLAGS_PERIODIC, ET_FLAGS_PERIODIC);
488	if (et == NULL && strcasecmp(buf, "NONE") != 0) {
489		ET_UNLOCK();
490		return (ENOENT);
491	}
492	if (timer[1] != NULL) {
493		timer2hz = 0;
494		configtimer(1);
495		et_free(timer[1]);
496	}
497	timer[1] = et;
498	if (timer[1] != NULL)
499		et_init(timer[1], timer2cb, NULL, NULL);
500	cpu_restartclocks();
501	ET_UNLOCK();
502	return (error);
503}
504SYSCTL_PROC(_kern_eventtimer, OID_AUTO, timer2,
505    CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE,
506    0, 0, sysctl_kern_eventtimer_timer2, "A", "Secondary event timer");
507
508#endif
509
510