1/*-
2 * Copyright (c) 2001 Jake Burkholder.
3 * Copyright (c) 2005, 2008 Marius Strobl <marius@FreeBSD.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28#include <sys/cdefs.h>
29__FBSDID("$FreeBSD$");
30
31#include <sys/param.h>
32#include <sys/systm.h>
33#include <sys/kernel.h>
34#include <sys/pcpu.h>
35#include <sys/proc.h>
36#include <sys/sched.h>
37#include <sys/smp.h>
38#include <sys/sysctl.h>
39#include <sys/timeet.h>
40#include <sys/timetc.h>
41
42#include <dev/ofw/openfirm.h>
43
44#include <vm/vm.h>
45#include <vm/pmap.h>
46
47#include <machine/frame.h>
48#include <machine/intr_machdep.h>
49#include <machine/smp.h>
50#include <machine/tick.h>
51#include <machine/ver.h>
52
53#define	TICK_QUALITY_MP	10
54#define	TICK_QUALITY_UP	1000
55
56static SYSCTL_NODE(_machdep, OID_AUTO, tick, CTLFLAG_RD, 0, "tick statistics");
57
58static int adjust_edges = 0;
59SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_edges, CTLFLAG_RD, &adjust_edges,
60    0, "total number of times tick interrupts got more than 12.5% behind");
61
62static int adjust_excess = 0;
63SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_excess, CTLFLAG_RD, &adjust_excess,
64    0, "total number of ignored tick interrupts");
65
66static int adjust_missed = 0;
67SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_missed, CTLFLAG_RD, &adjust_missed,
68    0, "total number of missed tick interrupts");
69
70static int adjust_ticks = 0;
71SYSCTL_INT(_machdep_tick, OID_AUTO, adjust_ticks, CTLFLAG_RD, &adjust_ticks,
72    0, "total number of tick interrupts with adjustment");
73
74u_int tick_et_use_stick = 0;
75SYSCTL_INT(_machdep_tick, OID_AUTO, tick_et_use_stick, CTLFLAG_RD,
76    &tick_et_use_stick, 0, "tick event timer uses STICK instead of TICK");
77
78typedef uint64_t rd_tick_t(void);
79static rd_tick_t *rd_tick;
80typedef void wr_tick_cmpr_t(uint64_t);
81static wr_tick_cmpr_t *wr_tick_cmpr;
82
83static struct timecounter stick_tc;
84static struct eventtimer tick_et;
85static struct timecounter tick_tc;
86
87#ifdef SMP
88static timecounter_get_t stick_get_timecount_mp;
89#endif
90static timecounter_get_t stick_get_timecount_up;
91static rd_tick_t stick_rd;
92static wr_tick_cmpr_t stick_wr_cmpr;
93static int tick_et_start(struct eventtimer *et, sbintime_t first,
94    sbintime_t period);
95static int tick_et_stop(struct eventtimer *et);
96#ifdef SMP
97static timecounter_get_t tick_get_timecount_mp;
98#endif
99static timecounter_get_t tick_get_timecount_up;
100static void tick_intr(struct trapframe *tf);
101static inline void tick_process(struct trapframe *tf);
102static rd_tick_t tick_rd;
103static wr_tick_cmpr_t tick_wr_cmpr;
104static wr_tick_cmpr_t tick_wr_cmpr_bbwar;
105static uint64_t tick_cputicks(void);
106
107static uint64_t
108stick_rd(void)
109{
110
111	return (rdstick());
112}
113
114static void
115stick_wr_cmpr(uint64_t tick)
116{
117
118	wrstickcmpr(tick, 0);
119}
120
121static uint64_t
122tick_rd(void)
123{
124
125	return (rd(tick));
126}
127
128static void
129tick_wr_cmpr(uint64_t tick_cmpr)
130{
131
132	wrtickcmpr(tick_cmpr, 0);
133}
134
135static void
136tick_wr_cmpr_bbwar(uint64_t tick_cmpr)
137{
138
139	wrtickcmpr_bbwar(tick_cmpr, 0);
140}
141
142static uint64_t
143tick_cputicks(void)
144{
145
146	return (rd(tick));
147}
148
149void
150cpu_initclocks(void)
151{
152	uint32_t clock, sclock;
153
154	clock = PCPU_GET(clock);
155	sclock = 0;
156	if (PCPU_GET(impl) == CPU_IMPL_SPARC64V ||
157	    PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCIII) {
158		if (OF_getprop(OF_peer(0), "stick-frequency", &sclock,
159		    sizeof(sclock)) == -1) {
160			panic("%s: could not determine STICK frequency",
161			    __func__);
162		}
163	}
164	/*
165	 * Given that the STICK timers typically are driven at rather low
166	 * frequencies they shouldn't be used except when really necessary.
167	 */
168	if (tick_et_use_stick != 0) {
169		rd_tick = stick_rd;
170		wr_tick_cmpr = stick_wr_cmpr;
171		/*
172		 * We don't provide a CPU ticker as long as the frequency
173		 * supplied isn't actually used per-CPU.
174		 */
175	} else {
176		rd_tick = tick_rd;
177		if (PCPU_GET(impl) >= CPU_IMPL_ULTRASPARCI &&
178		    PCPU_GET(impl) < CPU_IMPL_ULTRASPARCIII)
179			wr_tick_cmpr = tick_wr_cmpr_bbwar;
180		else
181			wr_tick_cmpr = tick_wr_cmpr;
182		set_cputicker(tick_cputicks, clock, 0);
183	}
184	intr_setup(PIL_TICK, tick_intr, -1, NULL, NULL);
185
186	/*
187	 * Initialize the (S)TICK-based timecounter(s).
188	 * Note that we (try to) sync the (S)TICK timers of APs with the BSP
189	 * during their startup but not afterwards.  The resulting drift can
190	 * cause problems when the time is calculated based on (S)TICK values
191	 * read on different CPUs.  Thus we always read the register on the
192	 * BSP (if necessary via an IPI as sched_bind(9) isn't available in
193	 * all circumstances) and use a low quality for the otherwise high
194	 * quality (S)TICK timers in the MP case.
195	 */
196	tick_tc.tc_get_timecount = tick_get_timecount_up;
197	tick_tc.tc_counter_mask = ~0u;
198	tick_tc.tc_frequency = clock;
199	tick_tc.tc_name = "tick";
200	tick_tc.tc_quality = TICK_QUALITY_UP;
201#ifdef SMP
202	if (cpu_mp_probe()) {
203		tick_tc.tc_get_timecount = tick_get_timecount_mp;
204		tick_tc.tc_quality = TICK_QUALITY_MP;
205	}
206#endif
207	tc_init(&tick_tc);
208	if (sclock != 0) {
209		stick_tc.tc_get_timecount = stick_get_timecount_up;
210		stick_tc.tc_counter_mask = ~0u;
211		stick_tc.tc_frequency = sclock;
212		stick_tc.tc_name = "stick";
213		stick_tc.tc_quality = TICK_QUALITY_UP;
214#ifdef SMP
215		if (cpu_mp_probe()) {
216			stick_tc.tc_get_timecount = stick_get_timecount_mp;
217			stick_tc.tc_quality = TICK_QUALITY_MP;
218		}
219#endif
220		tc_init(&stick_tc);
221	}
222	tick_et.et_name = tick_et_use_stick ? "stick" : "tick";
223	tick_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT |
224	    ET_FLAGS_PERCPU;
225	tick_et.et_quality = 1000;
226	tick_et.et_frequency = tick_et_use_stick ? sclock : clock;
227	tick_et.et_min_period = 0x00010000LLU; /* To be safe. */
228	tick_et.et_max_period = (0xfffffffeLLU << 32) / tick_et.et_frequency;
229	tick_et.et_start = tick_et_start;
230	tick_et.et_stop = tick_et_stop;
231	tick_et.et_priv = NULL;
232	et_register(&tick_et);
233
234	cpu_initclocks_bsp();
235}
236
237static inline void
238tick_process(struct trapframe *tf)
239{
240	struct trapframe *oldframe;
241	struct thread *td;
242
243	td = curthread;
244	td->td_intr_nesting_level++;
245	critical_enter();
246	if (tick_et.et_active) {
247		oldframe = td->td_intr_frame;
248		td->td_intr_frame = tf;
249		tick_et.et_event_cb(&tick_et, tick_et.et_arg);
250		td->td_intr_frame = oldframe;
251	}
252	td->td_intr_nesting_level--;
253	critical_exit();
254}
255
256static void
257tick_intr(struct trapframe *tf)
258{
259	u_long adj, ref, tick, tick_increment;
260	long delta;
261	register_t s;
262	int count;
263
264	tick_increment = PCPU_GET(tickincrement);
265	if (tick_increment != 0) {
266		/*
267		 * NB: the sequence of reading the (S)TICK register,
268		 * calculating the value of the next tick and writing it to
269		 * the (S)TICK_COMPARE register must not be interrupted, not
270		 * even by an IPI, otherwise a value that is in the past could
271		 * be written in the worst case and thus causing the periodic
272		 * timer to stop.
273		 */
274		s = intr_disable();
275		adj = PCPU_GET(tickadj);
276		tick = rd_tick();
277		wr_tick_cmpr(tick + tick_increment - adj);
278		intr_restore(s);
279		ref = PCPU_GET(tickref);
280		delta = tick - ref;
281		count = 0;
282		while (delta >= tick_increment) {
283			tick_process(tf);
284			delta -= tick_increment;
285			ref += tick_increment;
286			if (adj != 0)
287				adjust_ticks++;
288			count++;
289		}
290		if (count > 0) {
291			adjust_missed += count - 1;
292			if (delta > (tick_increment >> 3)) {
293				if (adj == 0)
294					adjust_edges++;
295				adj = tick_increment >> 4;
296			} else
297				adj = 0;
298		} else {
299			adj = 0;
300			adjust_excess++;
301		}
302		PCPU_SET(tickref, ref);
303		PCPU_SET(tickadj, adj);
304	} else
305		tick_process(tf);
306}
307
308static u_int
309stick_get_timecount_up(struct timecounter *tc)
310{
311
312	return ((u_int)rdstick());
313}
314
315static u_int
316tick_get_timecount_up(struct timecounter *tc)
317{
318
319	return ((u_int)rd(tick));
320}
321
322#ifdef SMP
323static u_int
324stick_get_timecount_mp(struct timecounter *tc)
325{
326	static u_long stick;
327
328	sched_pin();
329	if (curcpu == 0)
330		stick = rdstick();
331	else
332		ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
333	sched_unpin();
334	return (stick);
335}
336
337static u_int
338tick_get_timecount_mp(struct timecounter *tc)
339{
340	static u_long tick;
341
342	sched_pin();
343	if (curcpu == 0)
344		tick = rd(tick);
345	else
346		ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
347	sched_unpin();
348	return (tick);
349}
350#endif
351
352static int
353tick_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period)
354{
355	u_long base, div, fdiv;
356	register_t s;
357
358	if (period != 0)
359		div = (tick_et.et_frequency * period) >> 32;
360	else
361		div = 0;
362	if (first != 0)
363		fdiv = (tick_et.et_frequency * first) >> 32;
364	else
365		fdiv = div;
366	PCPU_SET(tickincrement, div);
367
368	/*
369	 * Try to make the (S)TICK interrupts as synchronously as possible
370	 * on all CPUs to avoid inaccuracies for migrating processes.  Leave
371	 * out one tick to make sure that it is not missed.
372	 */
373	s = intr_disable();
374	base = rd_tick();
375	if (div != 0) {
376		PCPU_SET(tickadj, 0);
377		base = roundup(base, div);
378	}
379	PCPU_SET(tickref, base);
380	wr_tick_cmpr(base + fdiv);
381	intr_restore(s);
382	return (0);
383}
384
385static int
386tick_et_stop(struct eventtimer *et)
387{
388
389	PCPU_SET(tickincrement, 0);
390	tick_stop(PCPU_GET(impl));
391	return (0);
392}
393
394void
395tick_clear(u_int cpu_impl)
396{
397
398	if (cpu_impl == CPU_IMPL_SPARC64V ||
399	    cpu_impl >= CPU_IMPL_ULTRASPARCIII)
400		wrstick(0, 0);
401	wrpr(tick, 0, 0);
402}
403
404void
405tick_stop(u_int cpu_impl)
406{
407
408	if (cpu_impl == CPU_IMPL_SPARC64V ||
409	    cpu_impl >= CPU_IMPL_ULTRASPARCIII)
410		wrstickcmpr(1L << 63, 0);
411	wrtickcmpr(1L << 63, 0);
412}
413