1/*-
2 * Copyright (c) 2001 Jake Burkholder.
3 * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD$");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/lock.h>
35#include <sys/mutex.h>
36#include <sys/pcpu.h>
37#include <sys/proc.h>
38#include <sys/sched.h>
39#include <sys/smp.h>
40#include <sys/sysctl.h>
41#include <sys/timeet.h>
42#include <sys/timetc.h>
43
44#include <dev/ofw/openfirm.h>
45
46#include <vm/vm.h>
47#include <vm/pmap.h>
48
49#include <machine/cpu.h>
50#include <machine/frame.h>
51#include <machine/intr_machdep.h>
52#include <machine/smp.h>
53#include <machine/tick.h>
54#include <machine/ver.h>
55
56#define	TICK_QUALITY_MP	10
57#define	TICK_QUALITY_UP	1000
58
59static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
60
61static int adjust_edges = 0;
62SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
63    0, "total number of times tick interrupts got more than 12.5% behind");
64
65static int adjust_excess = 0;
66SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess,
67    0, "total number of ignored tick interrupts");
68
69static int adjust_missed = 0;
70SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed,
71    0, "total number of missed tick interrupts");
72
73static int adjust_ticks = 0;
74SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
75    0, "total number of tick interrupts with adjustment");
76
77u_int tick_et_use_stick = 0;
78SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD,
79    &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK");
80
81typedef uint64_t rd_tick_t(void);
82static rd_tick_t *rd_tick;
83typedef void wr_tick_cmpr_t(uint64_t);
84static wr_tick_cmpr_t *wr_tick_cmpr;
85
86static struct timecounter stick_tc;
87static struct eventtimer tick_et;
88static struct timecounter tick_tc;
89
90#ifdef SMP
91static timecounter_get_t stick_get_timecount_mp;
92#endif
93static timecounter_get_t stick_get_timecount_up;
94static rd_tick_t stick_rd;
95static wr_tick_cmpr_t stick_wr_cmpr;
96static int tick_et_start(struct eventtimer *et, struct bintime *first,
97    struct bintime *period);
98static int tick_et_stop(struct eventtimer *et);
99#ifdef SMP
100static timecounter_get_t tick_get_timecount_mp;
101#endif
102static timecounter_get_t tick_get_timecount_up;
103static void tick_intr(struct trapframe *tf);
104static inline void tick_process(struct trapframe *tf);
105static rd_tick_t tick_rd;
106static wr_tick_cmpr_t tick_wr_cmpr;
107static wr_tick_cmpr_t tick_wr_cmpr_bbwar;
108static uint64_t tick_cputicks(void);
109
110static uint64_t
111stick_rd(void)
112{
113
114	return (rdstick());
115}
116
117static void
118stick_wr_cmpr(uint64_t tick)
119{
120
121	wrstickcmpr(tick, 0);
122}
123
124static uint64_t
125tick_rd(void)
126{
127
128	return (rd(tick));
129}
130
131static void
132tick_wr_cmpr(uint64_t tick_cmpr)
133{
134
135	wrtickcmpr(tick_cmpr, 0);
136}
137
138static void
139tick_wr_cmpr_bbwar(uint64_t tick_cmpr)
140{
141
142	wrtickcmpr_bbwar(tick_cmpr, 0);
143}
144
145static uint64_t
146tick_cputicks(void)
147{
148
149	return (rd(tick));
150}
151
152void
153cpu_initclocks(void)
154{
155	uint32_t clock, sclock;
156
157	clock = PCPU_GET(clock);
158	sclock = 0;
159	if (PCPU_GET(impl) == CPU_IMPL_SPARC64V ||
160	    PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) {
161		if (OF_getprop(OF_peer(0), "stick-frequency", &sclock,
162		    sizeof(sclock)) == -1) {
163			panic("%s: could not determine STICK frequency",
164			    __func__);
165		}
166	}
167	/*
168	 * Given that the STICK timers typically are driven at rather low
169	 * frequencies they shouldn't be used except when really necessary.
170	 */
171	if (tick_et_use_stick != 0) {
172		rd_tick = stick_rd;
173		wr_tick_cmpr = stick_wr_cmpr;
174		/*
175		 * We don't provide a CPU ticker as long as the frequency
176		 * supplied isn't actually used per-CPU.
177		 */
178	} else {
179		rd_tick = tick_rd;
180		if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI &&
181		    PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII)
182			wr_tick_cmpr = tick_wr_cmpr_bbwar;
183		else
184			wr_tick_cmpr = tick_wr_cmpr;
185		set_cputicker(tick_cputicks, clock, 0);
186	}
187	intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL);
188
189	/*
190	 * Initialize the (S)TICK-based timecounter(s).
191	 * Note that we (try to) sync the (S)TICK timers of APs with the BSP
192	 * during their startup but not afterwards.  The resulting drift can
193	 * cause problems when the time is calculated based on (S)TICK values
194	 * read on different CPUs.  Thus we always read the register on the
195	 * BSP (if necessary via an IPI as sched_bind(9) isn't available in
196	 * all circumstances) and use a low quality for the otherwise high
197	 * quality (S)TICK timers in the MP case.
198	 */
199	tick_tc.tc_get_timecount = tick_get_timecount_up;
200	tick_tc.tc_counter_mask = ~0u;
201	tick_tc.tc_frequency = clock;
202	tick_tc.tc_name = "tick";
203	tick_tc.tc_quality = TICK_QUALITY_UP;
204#ifdef SMP
205	if (cpu_mp_probe()) {
206		tick_tc.tc_get_timecount = tick_get_timecount_mp;
207		tick_tc.tc_quality = TICK_QUALITY_MP;
208	}
209#endif
210	tc_init(&tick_tc);
211	if (sclock != 0) {
212		stick_tc.tc_get_timecount = stick_get_timecount_up;
213		stick_tc.tc_counter_mask = ~0u;
214		stick_tc.tc_frequency = sclock;
215		stick_tc.tc_name = "stick";
216		stick_tc.tc_quality = TICK_QUALITY_UP;
217#ifdef SMP
218		if (cpu_mp_probe()) {
219			stick_tc.tc_get_timecount = stick_get_timecount_mp;
220			stick_tc.tc_quality = TICK_QUALITY_MP;
221		}
222#endif
223		tc_init(&stick_tc);
224	}
225	tick_et.et_name = tick_et_use_stick ? "stick" : "tick";
226	tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
227	    ET_FLAGS_PERCPU;
228	tick_et.et_quality = 1000;
229	tick_et.et_frequency = tick_et_use_stick ? sclock : clock;
230	tick_et.et_min_period.sec = 0;
231	tick_et.et_min_period.frac = 0x00010000LLU << 32; /* To be safe. */
232	tick_et.et_max_period.sec = 3600 * 24; /* No practical limit. */
233	tick_et.et_max_period.frac = 0;
234	tick_et.et_start = tick_et_start;
235	tick_et.et_stop = tick_et_stop;
236	tick_et.et_priv = NULL;
237	et_register(&tick_et);
238
239	cpu_initclocks_bsp();
240}
241
242static inline void
243tick_process(struct trapframe *tf)
244{
245	struct trapframe *oldframe;
246	struct thread *td;
247
248	td = curthread;
249	td->td_intr_nesting_level++;
250	critical_enter();
251	if (tick_et.et_active) {
252		oldframe = td->td_intr_frame;
253		td->td_intr_frame = tf;
254		tick_et.et_event_cb(&tick_et, tick_et.et_arg);
255		td->td_intr_frame = oldframe;
256	}
257	td->td_intr_nesting_level--;
258	critical_exit();
259}
260
261static void
262tick_intr(struct trapframe *tf)
263{
264	u_long adj, ref, tick, tick_increment;
265	long delta;
266	register_t s;
267	int count;
268
269	tick_increment = PCPU_GET(tickincrement);
270	if (tick_increment != 0) {
271		/*
272		 * NB: the sequence of reading the (S)TICK register,
273		 * calculating the value of the next tick and writing it to
274		 * the (S)TICK_COMPARE register must not be interrupted, not
275		 * even by an IPI, otherwise a value that is in the past could
276		 * be written in the worst case and thus causing the periodic
277		 * timer to stop.
278		 */
279		s = intr_disable();
280		adj = PCPU_GET(tickadj);
281		tick = rd_tick();
282		wr_tick_cmpr(tick + tick_increment - adj);
283		intr_restore(s);
284		ref = PCPU_GET(tickref);
285		delta = tick - ref;
286		count = 0;
287		while (delta >= tick_increment) {
288			tick_process(tf);
289			delta -= tick_increment;
290			ref += tick_increment;
291			if (adj != 0)
292				adjust_ticks++;
293			count++;
294		}
295		if (count > 0) {
296			adjust_missed += count - 1;
297			if (delta > (tick_increment >> 3)) {
298				if (adj == 0)
299					adjust_edges++;
300				adj = tick_increment >> 4;
301			} else
302				adj = 0;
303		} else {
304			adj = 0;
305			adjust_excess++;
306		}
307		PCPU_SET(tickref, ref);
308		PCPU_SET(tickadj, adj);
309	} else
310		tick_process(tf);
311}
312
313static u_int
314stick_get_timecount_up(struct timecounter *tc)
315{
316
317	return ((u_int)rdstick());
318}
319
320static u_int
321tick_get_timecount_up(struct timecounter *tc)
322{
323
324	return ((u_int)rd(tick));
325}
326
327#ifdef SMP
328static u_int
329stick_get_timecount_mp(struct timecounter *tc)
330{
331	u_long stick;
332
333	sched_pin();
334	if (curcpu == 0)
335		stick = rdstick();
336	else
337		ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
338	sched_unpin();
339	return (stick);
340}
341
342static u_int
343tick_get_timecount_mp(struct timecounter *tc)
344{
345	u_long tick;
346
347	sched_pin();
348	if (curcpu == 0)
349		tick = rd(tick);
350	else
351		ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
352	sched_unpin();
353	return (tick);
354}
355#endif
356
357static int
358tick_et_start(struct eventtimer *et, struct bintime *first,
359    struct bintime *period)
360{
361	u_long base, div, fdiv;
362	register_t s;
363
364	if (period != NULL) {
365		div = (tick_et.et_frequency * (period->frac >> 32)) >> 32;
366		if (period->sec != 0)
367			div += tick_et.et_frequency * period->sec;
368	} else
369		div = 0;
370	if (first != NULL) {
371		fdiv = (tick_et.et_frequency * (first->frac >> 32)) >> 32;
372		if (first->sec != 0)
373			fdiv += tick_et.et_frequency * first->sec;
374	} else
375		fdiv = div;
376	PCPU_SET(tickincrement, div);
377
378	/*
379	 * Try to make the (S)TICK interrupts as synchronously as possible
380	 * on all CPUs to avoid inaccuracies for migrating processes.  Leave
381	 * out one tick to make sure that it is not missed.
382	 */
383	s = intr_disable();
384	base = rd_tick();
385	if (div != 0) {
386		PCPU_SET(tickadj, 0);
387		base = roundup(base, div);
388	}
389	PCPU_SET(tickref, base);
390	wr_tick_cmpr(base + fdiv);
391	intr_restore(s);
392	return (0);
393}
394
395static int
396tick_et_stop(struct eventtimer *et)
397{
398
399	PCPU_SET(tickincrement, 0);
400	tick_stop(PCPU_GET(impl));
401	return (0);
402}
403
404void
405tick_clear(u_int cpu_impl)
406{
407
408	if (cpu_impl == CPU_IMPL_SPARC64V ||
409	    cpu_impl >= CPU_IMPL_ULTRASPARCIII)
410		wrstick(0, 0);
411	wrpr(tick, 0, 0);
412}
413
414void
415tick_stop(u_int cpu_impl)
416{
417
418	if (cpu_impl == CPU_IMPL_SPARC64V ||
419	    cpu_impl >= CPU_IMPL_ULTRASPARCIII)
420		wrstickcmpr(1L << 63, 0);
421	wrtickcmpr(1L << 63, 0);
422}
423