agtimer.c revision 1.18 
1/* $OpenBSD: agtimer.c,v 1.18 2021/03/11 11:16:56 jsg Exp $ */
2/*
3 * Copyright (c) 2011 Dale Rahn <drahn@openbsd.org>
4 * Copyright (c) 2013 Patrick Wildt <patrick@blueri.se>
5 *
6 * Permission to use, copy, modify, and distribute this software for any
7 * purpose with or without fee is hereby granted, provided that the above
8 * copyright notice and this permission notice appear in all copies.
9 *
10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
17 */
18
19#include <sys/param.h>
20#include <sys/systm.h>
21#include <sys/queue.h>
22#include <sys/malloc.h>
23#include <sys/device.h>
24#include <sys/kernel.h>
25#include <sys/timetc.h>
26#include <sys/evcount.h>
27
28#include <machine/intr.h>
29#include <machine/bus.h>
30#include <machine/fdt.h>
31
32#include <dev/ofw/fdt.h>
33#include <dev/ofw/openfirm.h>
34
35/* registers */
36#define GTIMER_CNTV_CTL_ENABLE		(1 << 0)
37#define GTIMER_CNTV_CTL_IMASK		(1 << 1)
38#define GTIMER_CNTV_CTL_ISTATUS		(1 << 2)
39
40#define TIMER_FREQUENCY		24 * 1000 * 1000 /* ARM core clock */
41int32_t agtimer_frequency = TIMER_FREQUENCY;
42
43u_int agtimer_get_timecount(struct timecounter *);
44
45static struct timecounter agtimer_timecounter = {
46	.tc_get_timecount = agtimer_get_timecount,
47	.tc_poll_pps = NULL,
48	.tc_counter_mask = 0xffffffff,
49	.tc_frequency = 0,
50	.tc_name = "agtimer",
51	.tc_quality = 0,
52	.tc_priv = NULL,
53	.tc_user = TC_AGTIMER,
54};
55
56struct agtimer_pcpu_softc {
57	uint64_t 		pc_nexttickevent;
58	uint64_t 		pc_nextstatevent;
59	u_int32_t		pc_ticks_err_sum;
60};
61
62struct agtimer_softc {
63	struct device		sc_dev;
64	int			sc_node;
65
66	struct agtimer_pcpu_softc sc_pstat[MAXCPUS];
67
68	u_int32_t		sc_ticks_err_cnt;
69	u_int32_t		sc_ticks_per_second;
70	u_int32_t		sc_ticks_per_intr;
71	u_int32_t		sc_statvar;
72	u_int32_t		sc_statmin;
73
74#ifdef AMPTIMER_DEBUG
75	struct evcount		sc_clk_count;
76	struct evcount		sc_stat_count;
77#endif
78	void			*sc_ih;
79};
80
81int		agtimer_match(struct device *, void *, void *);
82void		agtimer_attach(struct device *, struct device *, void *);
83uint64_t	agtimer_readcnt64(void);
84int		agtimer_intr(void *);
85void		agtimer_cpu_initclocks(void);
86void		agtimer_delay(u_int);
87void		agtimer_setstatclockrate(int stathz);
88void		agtimer_set_clockrate(int32_t new_frequency);
89void		agtimer_startclock(void);
90
91struct cfattach agtimer_ca = {
92	sizeof (struct agtimer_softc), agtimer_match, agtimer_attach
93};
94
95struct cfdriver agtimer_cd = {
96	NULL, "agtimer", DV_DULL
97};
98
99uint64_t
100agtimer_readcnt64(void)
101{
102	uint64_t val0, val1;
103
104	/*
105	 * Work around Cortex-A73 errata 858921, where there is a
106	 * one-cycle window where the read might return the old value
107	 * for the low 32 bits and the new value for the high 32 bits
108	 * upon roll-over of the low 32 bits.
109	 */
110	__asm volatile("isb" ::: "memory");
111	__asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val0));
112	__asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val1));
113	return ((val0 ^ val1) & 0x100000000ULL) ? val0 : val1;
114}
115
116static inline uint64_t
117agtimer_get_freq(void)
118{
119	uint64_t val;
120
121	__asm volatile("mrs %x0, CNTFRQ_EL0" : "=r" (val));
122
123	return (val);
124}
125
126static inline int
127agtimer_get_ctrl(void)
128{
129	uint32_t val;
130
131	__asm volatile("mrs %x0, CNTV_CTL_EL0" : "=r" (val));
132
133	return (val);
134}
135
136static inline int
137agtimer_set_ctrl(uint32_t val)
138{
139	__asm volatile("msr CNTV_CTL_EL0, %x0" :: "r" (val));
140	__asm volatile("isb" ::: "memory");
141
142	return (0);
143}
144
145static inline int
146agtimer_set_tval(uint32_t val)
147{
148	__asm volatile("msr CNTV_TVAL_EL0, %x0" :: "r" (val));
149	__asm volatile("isb" ::: "memory");
150
151	return (0);
152}
153
154int
155agtimer_match(struct device *parent, void *cfdata, void *aux)
156{
157	struct fdt_attach_args *faa = (struct fdt_attach_args *)aux;
158
159	return (OF_is_compatible(faa->fa_node, "arm,armv7-timer") ||
160	    OF_is_compatible(faa->fa_node, "arm,armv8-timer"));
161}
162
163void
164agtimer_attach(struct device *parent, struct device *self, void *aux)
165{
166	struct agtimer_softc *sc = (struct agtimer_softc *)self;
167	struct fdt_attach_args *faa = aux;
168
169	sc->sc_node = faa->fa_node;
170
171	if (agtimer_get_freq() != 0)
172		agtimer_frequency = agtimer_get_freq();
173	agtimer_frequency =
174	    OF_getpropint(sc->sc_node, "clock-frequency", agtimer_frequency);
175	sc->sc_ticks_per_second = agtimer_frequency;
176
177	printf(": %d kHz\n", sc->sc_ticks_per_second / 1000);
178
179#ifdef AMPTIMER_DEBUG
180	evcount_attach(&sc->sc_clk_count, "clock", NULL);
181	evcount_attach(&sc->sc_stat_count, "stat", NULL);
182#endif
183
184	/*
185	 * private timer and interrupts not enabled until
186	 * timer configures
187	 */
188
189	arm_clock_register(agtimer_cpu_initclocks, agtimer_delay,
190	    agtimer_setstatclockrate, agtimer_startclock);
191
192	agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
193	agtimer_timecounter.tc_priv = sc;
194
195	tc_init(&agtimer_timecounter);
196}
197
198u_int
199agtimer_get_timecount(struct timecounter *tc)
200{
201	uint64_t val;
202
203	/*
204	 * No need to work around Cortex-A73 errata 858921 since we
205	 * only look at the low 32 bits here.
206	 */
207	__asm volatile("isb" ::: "memory");
208	__asm volatile("mrs %x0, CNTVCT_EL0" : "=r" (val));
209	return (val & 0xffffffff);
210}
211
212int
213agtimer_intr(void *frame)
214{
215	struct agtimer_softc	*sc = agtimer_cd.cd_devs[0];
216	struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
217	uint64_t		 now;
218	uint64_t		 nextevent;
219	uint32_t		 r;
220#if defined(USE_GTIMER_CMP)
221	int			 skip = 1;
222#else
223	int64_t			 delay;
224#endif
225	int			 rc = 0;
226
227	/*
228	 * DSR - I know that the tick timer is 64 bits, but the following
229	 * code deals with rollover, so there is no point in dealing
230	 * with the 64 bit math, just let the 32 bit rollover
231	 * do the right thing
232	 */
233
234	now = agtimer_readcnt64();
235
236	while (pc->pc_nexttickevent <= now) {
237		pc->pc_nexttickevent += sc->sc_ticks_per_intr;
238		pc->pc_ticks_err_sum += sc->sc_ticks_err_cnt;
239
240		/* looping a few times is faster than divide */
241		while (pc->pc_ticks_err_sum > hz) {
242			pc->pc_nexttickevent += 1;
243			pc->pc_ticks_err_sum -= hz;
244		}
245
246#ifdef AMPTIMER_DEBUG
247		sc->sc_clk_count.ec_count++;
248#endif
249		rc = 1;
250		hardclock(frame);
251	}
252	while (pc->pc_nextstatevent <= now) {
253		do {
254			r = random() & (sc->sc_statvar -1);
255		} while (r == 0); /* random == 0 not allowed */
256		pc->pc_nextstatevent += sc->sc_statmin + r;
257
258		/* XXX - correct nextstatevent? */
259#ifdef AMPTIMER_DEBUG
260		sc->sc_stat_count.ec_count++;
261#endif
262		rc = 1;
263		statclock(frame);
264	}
265
266	if (pc->pc_nexttickevent < pc->pc_nextstatevent)
267		nextevent = pc->pc_nexttickevent;
268	else
269		nextevent = pc->pc_nextstatevent;
270
271	delay = nextevent - now;
272	if (delay < 0)
273		delay = 1;
274
275	agtimer_set_tval(delay);
276
277	return (rc);
278}
279
280void
281agtimer_set_clockrate(int32_t new_frequency)
282{
283	struct agtimer_softc	*sc = agtimer_cd.cd_devs[0];
284
285	agtimer_frequency = new_frequency;
286
287	if (sc == NULL)
288		return;
289
290	sc->sc_ticks_per_second = agtimer_frequency;
291	agtimer_timecounter.tc_frequency = sc->sc_ticks_per_second;
292	printf("agtimer0: adjusting clock: new tick rate %d kHz\n",
293	    sc->sc_ticks_per_second / 1000);
294}
295
296void
297agtimer_cpu_initclocks(void)
298{
299	struct agtimer_softc	*sc = agtimer_cd.cd_devs[0];
300	struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
301	uint32_t		 reg;
302	uint64_t		 next;
303	uint64_t		 kctl;
304
305	stathz = hz;
306	profhz = hz * 10;
307
308	if (sc->sc_ticks_per_second != agtimer_frequency) {
309		agtimer_set_clockrate(agtimer_frequency);
310	}
311
312	agtimer_setstatclockrate(stathz);
313
314	sc->sc_ticks_per_intr = sc->sc_ticks_per_second / hz;
315	sc->sc_ticks_err_cnt = sc->sc_ticks_per_second % hz;
316	pc->pc_ticks_err_sum = 0;
317
318	/* configure virtual timer interrupt */
319	sc->sc_ih = arm_intr_establish_fdt_idx(sc->sc_node, 2,
320	    IPL_CLOCK|IPL_MPSAFE, agtimer_intr, NULL, "tick");
321
322	next = agtimer_readcnt64() + sc->sc_ticks_per_intr;
323	pc->pc_nexttickevent = pc->pc_nextstatevent = next;
324
325	reg = agtimer_get_ctrl();
326	reg &= ~GTIMER_CNTV_CTL_IMASK;
327	reg |= GTIMER_CNTV_CTL_ENABLE;
328	agtimer_set_tval(sc->sc_ticks_per_second);
329	agtimer_set_ctrl(reg);
330
331	/* enable userland access to virtual counter */
332	kctl = READ_SPECIALREG(CNTKCTL_EL1);
333	WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN);
334}
335
336void
337agtimer_delay(u_int usecs)
338{
339	uint64_t		clock, oclock, delta, delaycnt;
340	uint64_t		csec, usec;
341	volatile int		j;
342
343	if (usecs > (0x80000000 / agtimer_frequency)) {
344		csec = usecs / 10000;
345		usec = usecs % 10000;
346
347		delaycnt = (agtimer_frequency / 100) * csec +
348		    (agtimer_frequency / 100) * usec / 10000;
349	} else {
350		delaycnt = agtimer_frequency * usecs / 1000000;
351	}
352	if (delaycnt <= 1)
353		for (j = 100; j > 0; j--)
354			;
355
356	oclock = agtimer_readcnt64();
357	while (1) {
358		for (j = 100; j > 0; j--)
359			;
360		clock = agtimer_readcnt64();
361		delta = clock - oclock;
362		if (delta > delaycnt)
363			break;
364	}
365}
366
367void
368agtimer_setstatclockrate(int newhz)
369{
370	struct agtimer_softc	*sc = agtimer_cd.cd_devs[0];
371	int			 minint, statint;
372	int			 s;
373
374	s = splclock();
375
376	statint = sc->sc_ticks_per_second / newhz;
377	/* calculate largest 2^n which is smaller that just over half statint */
378	sc->sc_statvar = 0x40000000; /* really big power of two */
379	minint = statint / 2 + 100;
380	while (sc->sc_statvar > minint)
381		sc->sc_statvar >>= 1;
382
383	sc->sc_statmin = statint - (sc->sc_statvar >> 1);
384
385	splx(s);
386
387	/*
388	 * XXX this allows the next stat timer to occur then it switches
389	 * to the new frequency. Rather than switching instantly.
390	 */
391}
392
393void
394agtimer_startclock(void)
395{
396	struct agtimer_softc	*sc = agtimer_cd.cd_devs[0];
397	struct agtimer_pcpu_softc *pc = &sc->sc_pstat[CPU_INFO_UNIT(curcpu())];
398	uint64_t nextevent;
399	uint64_t kctl;
400	uint32_t reg;
401
402	nextevent = agtimer_readcnt64() + sc->sc_ticks_per_intr;
403	pc->pc_nexttickevent = pc->pc_nextstatevent = nextevent;
404
405	arm_intr_route(sc->sc_ih, 1, curcpu());
406
407	reg = agtimer_get_ctrl();
408	reg &= ~GTIMER_CNTV_CTL_IMASK;
409	reg |= GTIMER_CNTV_CTL_ENABLE;
410	agtimer_set_tval(sc->sc_ticks_per_second);
411	agtimer_set_ctrl(reg);
412
413	/* enable userland access to virtual counter */
414	kctl = READ_SPECIALREG(CNTKCTL_EL1);
415	WRITE_SPECIALREG(CNTKCTL_EL1, kctl | CNTKCTL_EL0VCTEN);
416}
417
418void
419agtimer_init(void)
420{
421	uint64_t cntfrq = 0;
422
423	/* XXX: Check for Generic Timer support. */
424	cntfrq = agtimer_get_freq();
425
426	if (cntfrq != 0) {
427		agtimer_frequency = cntfrq;
428		arm_clock_register(NULL, agtimer_delay, NULL, NULL);
429	}
430}
431