1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2017 The FreeBSD Foundation
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 * 3. The name of the company nor the name of the author may be used to
15 *    endorse or promote products derived from this software without specific
16 *    prior written permission.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 */
30
31#include <sys/cdefs.h>
32#include <sys/types.h>
33#include <sys/systm.h>
34#include <sys/bus.h>
35#include <sys/kernel.h>
36#include <sys/module.h>
37#include <sys/mutex.h>
38#include <sys/rman.h>
39#include <sys/time.h>
40#include <sys/timeet.h>
41#include <sys/timetc.h>
42
43#include <machine/bus.h>
44#include <machine/machdep.h>
45#include <machine/vmm.h>
46#include <machine/armreg.h>
47
48#include <arm64/vmm/arm64.h>
49
50#include "vgic.h"
51#include "vtimer.h"
52
53#define	RES1		0xffffffffffffffffUL
54
55#define timer_enabled(ctl)	\
56    (!((ctl) & CNTP_CTL_IMASK) && ((ctl) & CNTP_CTL_ENABLE))
57
58static uint64_t cnthctl_el2_reg;
59static uint32_t tmr_frq;
60
61#define timer_condition_met(ctl)	((ctl) & CNTP_CTL_ISTATUS)
62
63static void vtimer_schedule_irq(struct hypctx *hypctx, bool phys);
64
65static int
66vtimer_virtual_timer_intr(void *arg)
67{
68	struct hypctx *hypctx;
69	uint64_t cntpct_el0;
70	uint32_t cntv_ctl;
71
72	hypctx = arm64_get_active_vcpu();
73	cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
74
75	if (!hypctx) {
76		/* vm_destroy() was called. */
77		eprintf("No active vcpu\n");
78		cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
79		goto out;
80	}
81	if (!timer_enabled(cntv_ctl)) {
82		eprintf("Timer not enabled\n");
83		goto out;
84	}
85	if (!timer_condition_met(cntv_ctl)) {
86		eprintf("Timer condition not met\n");
87		goto out;
88	}
89
90	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
91	    hypctx->hyp->vtimer.cntvoff_el2;
92	if (hypctx->vtimer_cpu.virt_timer.cntx_cval_el0 < cntpct_el0)
93		vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
94		    GT_VIRT_IRQ, true);
95
96	cntv_ctl = hypctx->vtimer_cpu.virt_timer.cntx_ctl_el0;
97
98out:
99	/*
100	 * Disable the timer interrupt. This will prevent the interrupt from
101	 * being reasserted as soon as we exit the handler and getting stuck
102	 * in an infinite loop.
103	 *
104	 * This is safe to do because the guest disabled the timer, and then
105	 * enables it as part of the interrupt handling routine.
106	 */
107	cntv_ctl &= ~CNTP_CTL_ENABLE;
108	WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
109
110	return (FILTER_HANDLED);
111}
112
113int
114vtimer_init(uint64_t cnthctl_el2)
115{
116	cnthctl_el2_reg = cnthctl_el2;
117	/*
118	 * The guest *MUST* use the same timer frequency as the host. The
119	 * register CNTFRQ_EL0 is accessible to the guest and a different value
120	 * in the guest dts file might have unforseen consequences.
121	 */
122	tmr_frq = READ_SPECIALREG(cntfrq_el0);
123
124	return (0);
125}
126
127void
128vtimer_vminit(struct hyp *hyp)
129{
130	uint64_t now;
131
132	/*
133	 * Configure the Counter-timer Hypervisor Control Register for the VM.
134	 *
135	 * CNTHCTL_EL1PCEN: trap access to CNTP_{CTL, CVAL, TVAL}_EL0 from EL1
136	 * CNTHCTL_EL1PCTEN: trap access to CNTPCT_EL0
137	 */
138	hyp->vtimer.cnthctl_el2 = cnthctl_el2_reg & ~CNTHCTL_EL1PCEN;
139	hyp->vtimer.cnthctl_el2 &= ~CNTHCTL_EL1PCTEN;
140
141	now = READ_SPECIALREG(cntpct_el0);
142	hyp->vtimer.cntvoff_el2 = now;
143
144	return;
145}
146
147void
148vtimer_cpuinit(struct hypctx *hypctx)
149{
150	struct vtimer_cpu *vtimer_cpu;
151
152	vtimer_cpu = &hypctx->vtimer_cpu;
153	/*
154	 * Configure physical timer interrupts for the VCPU.
155	 *
156	 * CNTP_CTL_IMASK: mask interrupts
157	 * ~CNTP_CTL_ENABLE: disable the timer
158	 */
159	vtimer_cpu->phys_timer.cntx_ctl_el0 = CNTP_CTL_IMASK & ~CNTP_CTL_ENABLE;
160
161	mtx_init(&vtimer_cpu->phys_timer.mtx, "vtimer phys callout mutex", NULL,
162	    MTX_DEF);
163	callout_init_mtx(&vtimer_cpu->phys_timer.callout,
164	    &vtimer_cpu->phys_timer.mtx, 0);
165	vtimer_cpu->phys_timer.irqid = GT_PHYS_NS_IRQ;
166
167	mtx_init(&vtimer_cpu->virt_timer.mtx, "vtimer virt callout mutex", NULL,
168	    MTX_DEF);
169	callout_init_mtx(&vtimer_cpu->virt_timer.callout,
170	    &vtimer_cpu->virt_timer.mtx, 0);
171	vtimer_cpu->virt_timer.irqid = GT_VIRT_IRQ;
172}
173
174void
175vtimer_cpucleanup(struct hypctx *hypctx)
176{
177	struct vtimer_cpu *vtimer_cpu;
178
179	vtimer_cpu = &hypctx->vtimer_cpu;
180	callout_drain(&vtimer_cpu->phys_timer.callout);
181	callout_drain(&vtimer_cpu->virt_timer.callout);
182	mtx_destroy(&vtimer_cpu->phys_timer.mtx);
183	mtx_destroy(&vtimer_cpu->virt_timer.mtx);
184}
185
186void
187vtimer_vmcleanup(struct hyp *hyp)
188{
189	struct hypctx *hypctx;
190	uint32_t cntv_ctl;
191
192	hypctx = arm64_get_active_vcpu();
193	if (!hypctx) {
194		/* The active VM was destroyed, stop the timer. */
195		cntv_ctl = READ_SPECIALREG(cntv_ctl_el0);
196		cntv_ctl &= ~CNTP_CTL_ENABLE;
197		WRITE_SPECIALREG(cntv_ctl_el0, cntv_ctl);
198	}
199}
200
201void
202vtimer_cleanup(void)
203{
204}
205
206void
207vtimer_sync_hwstate(struct hypctx *hypctx)
208{
209	struct vtimer_timer *timer;
210	uint64_t cntpct_el0;
211
212	timer = &hypctx->vtimer_cpu.virt_timer;
213	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
214	    hypctx->hyp->vtimer.cntvoff_el2;
215	if (!timer_enabled(timer->cntx_ctl_el0)) {
216		vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
217		    timer->irqid, false);
218	} else if (timer->cntx_cval_el0 < cntpct_el0) {
219		vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
220		    timer->irqid, true);
221	} else {
222		vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
223		    timer->irqid, false);
224		vtimer_schedule_irq(hypctx, false);
225	}
226}
227
228static void
229vtimer_inject_irq_callout_phys(void *context)
230{
231	struct hypctx *hypctx;
232
233	hypctx = context;
234	vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
235	    hypctx->vtimer_cpu.phys_timer.irqid, true);
236}
237
238static void
239vtimer_inject_irq_callout_virt(void *context)
240{
241	struct hypctx *hypctx;
242
243	hypctx = context;
244	vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
245	    hypctx->vtimer_cpu.virt_timer.irqid, true);
246}
247
248static void
249vtimer_schedule_irq(struct hypctx *hypctx, bool phys)
250{
251	sbintime_t time;
252	struct vtimer_timer *timer;
253	uint64_t cntpct_el0;
254	uint64_t diff;
255
256	if (phys)
257		timer = &hypctx->vtimer_cpu.phys_timer;
258	else
259		timer = &hypctx->vtimer_cpu.virt_timer;
260	cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
261	    hypctx->hyp->vtimer.cntvoff_el2;
262	if (timer->cntx_cval_el0 < cntpct_el0) {
263		/* Timer set in the past, trigger interrupt */
264		vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu),
265		    timer->irqid, true);
266	} else {
267		diff = timer->cntx_cval_el0 - cntpct_el0;
268		time = diff * SBT_1S / tmr_frq;
269		if (phys)
270			callout_reset_sbt(&timer->callout, time, 0,
271			    vtimer_inject_irq_callout_phys, hypctx, 0);
272		else
273			callout_reset_sbt(&timer->callout, time, 0,
274			    vtimer_inject_irq_callout_virt, hypctx, 0);
275	}
276}
277
278static void
279vtimer_remove_irq(struct hypctx *hypctx, struct vcpu *vcpu)
280{
281	struct vtimer_cpu *vtimer_cpu;
282	struct vtimer_timer *timer;
283
284	vtimer_cpu = &hypctx->vtimer_cpu;
285	timer = &vtimer_cpu->phys_timer;
286
287	callout_drain(&timer->callout);
288	/*
289	 * The interrupt needs to be deactivated here regardless of the callout
290	 * function having been executed. The timer interrupt can be masked with
291	 * the CNTP_CTL_EL0.IMASK bit instead of reading the IAR register.
292	 * Masking the interrupt doesn't remove it from the list registers.
293	 */
294	vgic_inject_irq(hypctx->hyp, vcpu_vcpuid(vcpu), timer->irqid, false);
295}
296
297/*
298 * Timer emulation functions.
299 *
300 * The guest should use the virtual timer, however some software, e.g. u-boot,
301 * used the physical timer. Emulate this in software for the guest to use.
302 *
303 * Adjust for cntvoff_el2 so the physical and virtual timers are at similar
304 * times. This simplifies interrupt handling in the virtual timer as the
305 * adjustment will have already happened.
306 */
307
308int
309vtimer_phys_ctl_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
310{
311	struct hyp *hyp;
312	struct hypctx *hypctx;
313	struct vtimer_cpu *vtimer_cpu;
314	uint64_t cntpct_el0;
315
316	hypctx = vcpu_get_cookie(vcpu);
317	hyp = hypctx->hyp;
318	vtimer_cpu = &hypctx->vtimer_cpu;
319
320	cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
321	if (vtimer_cpu->phys_timer.cntx_cval_el0 < cntpct_el0)
322		/* Timer condition met */
323		*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 | CNTP_CTL_ISTATUS;
324	else
325		*rval = vtimer_cpu->phys_timer.cntx_ctl_el0 & ~CNTP_CTL_ISTATUS;
326
327	return (0);
328}
329
330int
331vtimer_phys_ctl_write(struct vcpu *vcpu, uint64_t wval, void *arg)
332{
333	struct hypctx *hypctx;
334	struct vtimer_cpu *vtimer_cpu;
335	uint64_t ctl_el0;
336	bool timer_toggled_on;
337
338	hypctx = vcpu_get_cookie(vcpu);
339	vtimer_cpu = &hypctx->vtimer_cpu;
340
341	timer_toggled_on = false;
342	ctl_el0 = vtimer_cpu->phys_timer.cntx_ctl_el0;
343
344	if (!timer_enabled(ctl_el0) && timer_enabled(wval))
345		timer_toggled_on = true;
346	else if (timer_enabled(ctl_el0) && !timer_enabled(wval))
347		vtimer_remove_irq(hypctx, vcpu);
348
349	vtimer_cpu->phys_timer.cntx_ctl_el0 = wval;
350
351	if (timer_toggled_on)
352		vtimer_schedule_irq(hypctx, true);
353
354	return (0);
355}
356
357int
358vtimer_phys_cnt_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
359{
360	struct vm *vm;
361	struct hyp *hyp;
362
363	vm = vcpu_vm(vcpu);
364	hyp = vm_get_cookie(vm);
365	*rval = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
366	return (0);
367}
368
369int
370vtimer_phys_cnt_write(struct vcpu *vcpu, uint64_t wval, void *arg)
371{
372	return (0);
373}
374
375int
376vtimer_phys_cval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
377{
378	struct hypctx *hypctx;
379	struct vtimer_cpu *vtimer_cpu;
380
381	hypctx = vcpu_get_cookie(vcpu);
382	vtimer_cpu = &hypctx->vtimer_cpu;
383
384	*rval = vtimer_cpu->phys_timer.cntx_cval_el0;
385
386	return (0);
387}
388
389int
390vtimer_phys_cval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
391{
392	struct hypctx *hypctx;
393	struct vtimer_cpu *vtimer_cpu;
394
395	hypctx = vcpu_get_cookie(vcpu);
396	vtimer_cpu = &hypctx->vtimer_cpu;
397
398	vtimer_cpu->phys_timer.cntx_cval_el0 = wval;
399
400	vtimer_remove_irq(hypctx, vcpu);
401	if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
402		vtimer_schedule_irq(hypctx, true);
403	}
404
405	return (0);
406}
407
408int
409vtimer_phys_tval_read(struct vcpu *vcpu, uint64_t *rval, void *arg)
410{
411	struct hyp *hyp;
412	struct hypctx *hypctx;
413	struct vtimer_cpu *vtimer_cpu;
414	uint32_t cntpct_el0;
415
416	hypctx = vcpu_get_cookie(vcpu);
417	hyp = hypctx->hyp;
418	vtimer_cpu = &hypctx->vtimer_cpu;
419
420	if (!(vtimer_cpu->phys_timer.cntx_ctl_el0 & CNTP_CTL_ENABLE)) {
421		/*
422		 * ARMv8 Architecture Manual, p. D7-2702: the result of reading
423		 * TVAL when the timer is disabled is UNKNOWN. I have chosen to
424		 * return the maximum value possible on 32 bits which means the
425		 * timer will fire very far into the future.
426		 */
427		*rval = (uint32_t)RES1;
428	} else {
429		cntpct_el0 = READ_SPECIALREG(cntpct_el0) -
430		    hyp->vtimer.cntvoff_el2;
431		*rval = vtimer_cpu->phys_timer.cntx_cval_el0 - cntpct_el0;
432	}
433
434	return (0);
435}
436
437int
438vtimer_phys_tval_write(struct vcpu *vcpu, uint64_t wval, void *arg)
439{
440	struct hyp *hyp;
441	struct hypctx *hypctx;
442	struct vtimer_cpu *vtimer_cpu;
443	uint64_t cntpct_el0;
444
445	hypctx = vcpu_get_cookie(vcpu);
446	hyp = hypctx->hyp;
447	vtimer_cpu = &hypctx->vtimer_cpu;
448
449	cntpct_el0 = READ_SPECIALREG(cntpct_el0) - hyp->vtimer.cntvoff_el2;
450	vtimer_cpu->phys_timer.cntx_cval_el0 = (int32_t)wval + cntpct_el0;
451
452	vtimer_remove_irq(hypctx, vcpu);
453	if (timer_enabled(vtimer_cpu->phys_timer.cntx_ctl_el0)) {
454		vtimer_schedule_irq(hypctx, true);
455	}
456
457	return (0);
458}
459
460struct vtimer_softc {
461	struct resource *res;
462	void *ihl;
463	int rid;
464};
465
466static int
467vtimer_probe(device_t dev)
468{
469	device_set_desc(dev, "Virtual timer");
470	return (BUS_PROBE_DEFAULT);
471}
472
473static int
474vtimer_attach(device_t dev)
475{
476	struct vtimer_softc *sc;
477
478	sc = device_get_softc(dev);
479
480	sc->rid = 0;
481	sc->res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &sc->rid, RF_ACTIVE);
482	if (sc->res == NULL)
483		return (ENXIO);
484
485	bus_setup_intr(dev, sc->res, INTR_TYPE_CLK, vtimer_virtual_timer_intr,
486	    NULL, NULL, &sc->ihl);
487
488	return (0);
489}
490
491static device_method_t vtimer_methods[] = {
492	/* Device interface */
493	DEVMETHOD(device_probe,		vtimer_probe),
494	DEVMETHOD(device_attach,	vtimer_attach),
495
496	/* End */
497	DEVMETHOD_END
498};
499
500DEFINE_CLASS_0(vtimer, vtimer_driver, vtimer_methods,
501    sizeof(struct vtimer_softc));
502
503DRIVER_MODULE(vtimer, generic_timer, vtimer_driver, 0, 0);
504