1/*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
5 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
6 * All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 *    notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27 * SUCH DAMAGE.
28 */
29
30#include <sys/cdefs.h>
31#include "opt_bhyve_snapshot.h"
32
33#include <sys/param.h>
34#include <sys/queue.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/kernel.h>
39#include <sys/malloc.h>
40
41#include <x86/apicreg.h>
42#include <machine/vmm.h>
43#include <machine/vmm_snapshot.h>
44
45#include "vmm_ktr.h"
46#include "vmm_lapic.h"
47#include "vlapic.h"
48#include "vioapic.h"
49
50#define	IOREGSEL	0x00
51#define	IOWIN		0x10
52
53#define	REDIR_ENTRIES	32
54#define	RTBL_RO_BITS	((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
55
56struct vioapic {
57	struct vm	*vm;
58	struct mtx	mtx;
59	uint32_t	id;
60	uint32_t	ioregsel;
61	struct {
62		uint64_t reg;
63		int	 acnt;	/* sum of pin asserts (+1) and deasserts (-1) */
64	} rtbl[REDIR_ENTRIES];
65};
66
67#define	VIOAPIC_LOCK(vioapic)		mtx_lock_spin(&((vioapic)->mtx))
68#define	VIOAPIC_UNLOCK(vioapic)		mtx_unlock_spin(&((vioapic)->mtx))
69#define	VIOAPIC_LOCKED(vioapic)		mtx_owned(&((vioapic)->mtx))
70
71static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
72
73#define	VIOAPIC_CTR1(vioapic, fmt, a1)					\
74	VM_CTR1((vioapic)->vm, fmt, a1)
75
76#define	VIOAPIC_CTR2(vioapic, fmt, a1, a2)				\
77	VM_CTR2((vioapic)->vm, fmt, a1, a2)
78
79#define	VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3)				\
80	VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
81
82#define	VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4)			\
83	VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
84
85#ifdef KTR
86static const char *
87pinstate_str(bool asserted)
88{
89
90	if (asserted)
91		return ("asserted");
92	else
93		return ("deasserted");
94}
95#endif
96
97static void
98vioapic_send_intr(struct vioapic *vioapic, int pin)
99{
100	int vector, delmode;
101	uint32_t low, high, dest;
102	bool level, phys;
103
104	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
105	    ("vioapic_set_pinstate: invalid pin number %d", pin));
106
107	KASSERT(VIOAPIC_LOCKED(vioapic),
108	    ("vioapic_set_pinstate: vioapic is not locked"));
109
110	low = vioapic->rtbl[pin].reg;
111	high = vioapic->rtbl[pin].reg >> 32;
112
113	if ((low & IOART_INTMASK) == IOART_INTMSET) {
114		VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
115		return;
116	}
117
118	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
119	delmode = low & IOART_DELMOD;
120	level = low & IOART_TRGRLVL ? true : false;
121	if (level) {
122		if ((low & IOART_REM_IRR) != 0) {
123			VIOAPIC_CTR1(vioapic, "ioapic pin%d: irr pending",
124			    pin);
125			return;
126		}
127		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
128	}
129
130	vector = low & IOART_INTVEC;
131	dest = high >> APIC_ID_SHIFT;
132	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
133}
134
135static void
136vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
137{
138	int oldcnt, newcnt;
139	bool needintr;
140
141	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
142	    ("vioapic_set_pinstate: invalid pin number %d", pin));
143
144	KASSERT(VIOAPIC_LOCKED(vioapic),
145	    ("vioapic_set_pinstate: vioapic is not locked"));
146
147	oldcnt = vioapic->rtbl[pin].acnt;
148	if (newstate)
149		vioapic->rtbl[pin].acnt++;
150	else
151		vioapic->rtbl[pin].acnt--;
152	newcnt = vioapic->rtbl[pin].acnt;
153
154	if (newcnt < 0) {
155		VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
156		    pin, newcnt);
157	}
158
159	needintr = false;
160	if (oldcnt == 0 && newcnt == 1) {
161		needintr = true;
162		VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
163	} else if (oldcnt == 1 && newcnt == 0) {
164		VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
165	} else {
166		VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
167		    pin, pinstate_str(newstate), newcnt);
168	}
169
170	if (needintr)
171		vioapic_send_intr(vioapic, pin);
172}
173
174enum irqstate {
175	IRQSTATE_ASSERT,
176	IRQSTATE_DEASSERT,
177	IRQSTATE_PULSE
178};
179
180static int
181vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
182{
183	struct vioapic *vioapic;
184
185	if (irq < 0 || irq >= REDIR_ENTRIES)
186		return (EINVAL);
187
188	vioapic = vm_ioapic(vm);
189
190	VIOAPIC_LOCK(vioapic);
191	switch (irqstate) {
192	case IRQSTATE_ASSERT:
193		vioapic_set_pinstate(vioapic, irq, true);
194		break;
195	case IRQSTATE_DEASSERT:
196		vioapic_set_pinstate(vioapic, irq, false);
197		break;
198	case IRQSTATE_PULSE:
199		vioapic_set_pinstate(vioapic, irq, true);
200		vioapic_set_pinstate(vioapic, irq, false);
201		break;
202	default:
203		panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
204	}
205	VIOAPIC_UNLOCK(vioapic);
206
207	return (0);
208}
209
210int
211vioapic_assert_irq(struct vm *vm, int irq)
212{
213
214	return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
215}
216
217int
218vioapic_deassert_irq(struct vm *vm, int irq)
219{
220
221	return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
222}
223
224int
225vioapic_pulse_irq(struct vm *vm, int irq)
226{
227
228	return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
229}
230
231/*
232 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
233 * configuration.
234 */
235static void
236vioapic_update_tmr(struct vcpu *vcpu, void *arg)
237{
238	struct vioapic *vioapic;
239	struct vlapic *vlapic;
240	uint32_t low, high, dest;
241	int delmode, pin, vector;
242	bool level, phys;
243
244	vlapic = vm_lapic(vcpu);
245	vioapic = vm_ioapic(vcpu_vm(vcpu));
246
247	VIOAPIC_LOCK(vioapic);
248	/*
249	 * Reset all vectors to be edge-triggered.
250	 */
251	vlapic_reset_tmr(vlapic);
252	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
253		low = vioapic->rtbl[pin].reg;
254		high = vioapic->rtbl[pin].reg >> 32;
255
256		level = low & IOART_TRGRLVL ? true : false;
257		if (!level)
258			continue;
259
260		/*
261		 * For a level-triggered 'pin' let the vlapic figure out if
262		 * an assertion on this 'pin' would result in an interrupt
263		 * being delivered to it. If yes, then it will modify the
264		 * TMR bit associated with this vector to level-triggered.
265		 */
266		phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
267		delmode = low & IOART_DELMOD;
268		vector = low & IOART_INTVEC;
269		dest = high >> APIC_ID_SHIFT;
270		vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
271	}
272	VIOAPIC_UNLOCK(vioapic);
273}
274
275static uint32_t
276vioapic_read(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr)
277{
278	int regnum, pin, rshift;
279
280	regnum = addr & 0xff;
281	switch (regnum) {
282	case IOAPIC_ID:
283		return (vioapic->id);
284		break;
285	case IOAPIC_VER:
286		return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
287		break;
288	case IOAPIC_ARB:
289		return (vioapic->id);
290		break;
291	default:
292		break;
293	}
294
295	/* redirection table entries */
296	if (regnum >= IOAPIC_REDTBL &&
297	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
298		pin = (regnum - IOAPIC_REDTBL) / 2;
299		if ((regnum - IOAPIC_REDTBL) % 2)
300			rshift = 32;
301		else
302			rshift = 0;
303
304		return (vioapic->rtbl[pin].reg >> rshift);
305	}
306
307	return (0);
308}
309
310static void
311vioapic_write(struct vioapic *vioapic, struct vcpu *vcpu, uint32_t addr,
312    uint32_t data)
313{
314	uint64_t data64, mask64;
315	uint64_t last, changed;
316	int regnum, pin, lshift;
317	cpuset_t allvcpus;
318
319	regnum = addr & 0xff;
320	switch (regnum) {
321	case IOAPIC_ID:
322		vioapic->id = data & APIC_ID_MASK;
323		break;
324	case IOAPIC_VER:
325	case IOAPIC_ARB:
326		/* readonly */
327		break;
328	default:
329		break;
330	}
331
332	/* redirection table entries */
333	if (regnum >= IOAPIC_REDTBL &&
334	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
335		pin = (regnum - IOAPIC_REDTBL) / 2;
336		if ((regnum - IOAPIC_REDTBL) % 2)
337			lshift = 32;
338		else
339			lshift = 0;
340
341		last = vioapic->rtbl[pin].reg;
342
343		data64 = (uint64_t)data << lshift;
344		mask64 = (uint64_t)0xffffffff << lshift;
345		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
346		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
347
348		/*
349		 * Switching from level to edge triggering will clear the IRR
350		 * bit. This is what FreeBSD will do in order to EOI an
351		 * interrupt when the IO-APIC doesn't support targeted EOI (see
352		 * _ioapic_eoi_source).
353		 */
354		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGREDG &&
355		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) != 0)
356			vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
357
358		VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
359		    pin, vioapic->rtbl[pin].reg);
360
361		/*
362		 * If any fields in the redirection table entry (except mask
363		 * or polarity) have changed then rendezvous all the vcpus
364		 * to update their vlapic trigger-mode registers.
365		 */
366		changed = last ^ vioapic->rtbl[pin].reg;
367		if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
368			VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
369			    "vlapic trigger-mode register", pin);
370			VIOAPIC_UNLOCK(vioapic);
371			allvcpus = vm_active_cpus(vioapic->vm);
372			(void)vm_smp_rendezvous(vcpu, allvcpus,
373			    vioapic_update_tmr, NULL);
374			VIOAPIC_LOCK(vioapic);
375		}
376
377		/*
378		 * Generate an interrupt if the following conditions are met:
379		 * - pin trigger mode is level
380		 * - pin level is asserted
381		 */
382		if ((vioapic->rtbl[pin].reg & IOART_TRGRMOD) == IOART_TRGRLVL &&
383		    (vioapic->rtbl[pin].acnt > 0)) {
384			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
385			    "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
386			vioapic_send_intr(vioapic, pin);
387		}
388	}
389}
390
391static int
392vioapic_mmio_rw(struct vioapic *vioapic, struct vcpu *vcpu, uint64_t gpa,
393    uint64_t *data, int size, bool doread)
394{
395	uint64_t offset;
396
397	offset = gpa - VIOAPIC_BASE;
398
399	/*
400	 * The IOAPIC specification allows 32-bit wide accesses to the
401	 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
402	 */
403	if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
404		if (doread)
405			*data = 0;
406		return (0);
407	}
408
409	VIOAPIC_LOCK(vioapic);
410	if (offset == IOREGSEL) {
411		if (doread)
412			*data = vioapic->ioregsel;
413		else
414			vioapic->ioregsel = *data;
415	} else {
416		if (doread) {
417			*data = vioapic_read(vioapic, vcpu,
418			    vioapic->ioregsel);
419		} else {
420			vioapic_write(vioapic, vcpu, vioapic->ioregsel,
421			    *data);
422		}
423	}
424	VIOAPIC_UNLOCK(vioapic);
425
426	return (0);
427}
428
429int
430vioapic_mmio_read(struct vcpu *vcpu, uint64_t gpa, uint64_t *rval,
431    int size, void *arg)
432{
433	int error;
434	struct vioapic *vioapic;
435
436	vioapic = vm_ioapic(vcpu_vm(vcpu));
437	error = vioapic_mmio_rw(vioapic, vcpu, gpa, rval, size, true);
438	return (error);
439}
440
441int
442vioapic_mmio_write(struct vcpu *vcpu, uint64_t gpa, uint64_t wval,
443    int size, void *arg)
444{
445	int error;
446	struct vioapic *vioapic;
447
448	vioapic = vm_ioapic(vcpu_vm(vcpu));
449	error = vioapic_mmio_rw(vioapic, vcpu, gpa, &wval, size, false);
450	return (error);
451}
452
453void
454vioapic_process_eoi(struct vm *vm, int vector)
455{
456	struct vioapic *vioapic;
457	int pin;
458
459	KASSERT(vector >= 0 && vector < 256,
460	    ("vioapic_process_eoi: invalid vector %d", vector));
461
462	vioapic = vm_ioapic(vm);
463	VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
464
465	/*
466	 * XXX keep track of the pins associated with this vector instead
467	 * of iterating on every single pin each time.
468	 */
469	VIOAPIC_LOCK(vioapic);
470	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
471		if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
472			continue;
473		if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
474			continue;
475		vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
476		if (vioapic->rtbl[pin].acnt > 0) {
477			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
478			    "acnt %d", pin, vioapic->rtbl[pin].acnt);
479			vioapic_send_intr(vioapic, pin);
480		}
481	}
482	VIOAPIC_UNLOCK(vioapic);
483}
484
485struct vioapic *
486vioapic_init(struct vm *vm)
487{
488	int i;
489	struct vioapic *vioapic;
490
491	vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
492
493	vioapic->vm = vm;
494	mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
495
496	/* Initialize all redirection entries to mask all interrupts */
497	for (i = 0; i < REDIR_ENTRIES; i++)
498		vioapic->rtbl[i].reg = 0x0001000000010000UL;
499
500	return (vioapic);
501}
502
503void
504vioapic_cleanup(struct vioapic *vioapic)
505{
506
507	mtx_destroy(&vioapic->mtx);
508	free(vioapic, M_VIOAPIC);
509}
510
511int
512vioapic_pincount(struct vm *vm)
513{
514
515	return (REDIR_ENTRIES);
516}
517
518#ifdef BHYVE_SNAPSHOT
519int
520vioapic_snapshot(struct vioapic *vioapic, struct vm_snapshot_meta *meta)
521{
522	int ret;
523	int i;
524
525	SNAPSHOT_VAR_OR_LEAVE(vioapic->ioregsel, meta, ret, done);
526
527	for (i = 0; i < nitems(vioapic->rtbl); i++) {
528		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].reg, meta, ret, done);
529		SNAPSHOT_VAR_OR_LEAVE(vioapic->rtbl[i].acnt, meta, ret, done);
530	}
531
532done:
533	return (ret);
534}
535#endif
536