1/*-
2 * Copyright (c) 2013 Tycho Nightingale <tycho.nightingale@pluribusnetworks.com>
3 * Copyright (c) 2013 Neel Natu <neel@freebsd.org>
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 *    notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 *    notice, this list of conditions and the following disclaimer in the
13 *    documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED.  IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 *
27 * $FreeBSD: stable/11/sys/amd64/vmm/io/vioapic.c 321414 2017-07-24 06:49:57Z mav $
28 */
29
30#include <sys/cdefs.h>
31__FBSDID("$FreeBSD: stable/11/sys/amd64/vmm/io/vioapic.c 321414 2017-07-24 06:49:57Z mav $");
32
33#include <sys/param.h>
34#include <sys/queue.h>
35#include <sys/lock.h>
36#include <sys/mutex.h>
37#include <sys/systm.h>
38#include <sys/kernel.h>
39#include <sys/malloc.h>
40
41#include <x86/apicreg.h>
42#include <machine/vmm.h>
43
44#include "vmm_ktr.h"
45#include "vmm_lapic.h"
46#include "vlapic.h"
47#include "vioapic.h"
48
49#define	IOREGSEL	0x00
50#define	IOWIN		0x10
51
52#define	REDIR_ENTRIES	32
53#define	RTBL_RO_BITS	((uint64_t)(IOART_REM_IRR | IOART_DELIVS))
54
55struct vioapic {
56	struct vm	*vm;
57	struct mtx	mtx;
58	uint32_t	id;
59	uint32_t	ioregsel;
60	struct {
61		uint64_t reg;
62		int	 acnt;	/* sum of pin asserts (+1) and deasserts (-1) */
63	} rtbl[REDIR_ENTRIES];
64};
65
66#define	VIOAPIC_LOCK(vioapic)		mtx_lock_spin(&((vioapic)->mtx))
67#define	VIOAPIC_UNLOCK(vioapic)		mtx_unlock_spin(&((vioapic)->mtx))
68#define	VIOAPIC_LOCKED(vioapic)		mtx_owned(&((vioapic)->mtx))
69
70static MALLOC_DEFINE(M_VIOAPIC, "vioapic", "bhyve virtual ioapic");
71
72#define	VIOAPIC_CTR1(vioapic, fmt, a1)					\
73	VM_CTR1((vioapic)->vm, fmt, a1)
74
75#define	VIOAPIC_CTR2(vioapic, fmt, a1, a2)				\
76	VM_CTR2((vioapic)->vm, fmt, a1, a2)
77
78#define	VIOAPIC_CTR3(vioapic, fmt, a1, a2, a3)				\
79	VM_CTR3((vioapic)->vm, fmt, a1, a2, a3)
80
81#define	VIOAPIC_CTR4(vioapic, fmt, a1, a2, a3, a4)			\
82	VM_CTR4((vioapic)->vm, fmt, a1, a2, a3, a4)
83
84#ifdef KTR
85static const char *
86pinstate_str(bool asserted)
87{
88
89	if (asserted)
90		return ("asserted");
91	else
92		return ("deasserted");
93}
94#endif
95
96static void
97vioapic_send_intr(struct vioapic *vioapic, int pin)
98{
99	int vector, delmode;
100	uint32_t low, high, dest;
101	bool level, phys;
102
103	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
104	    ("vioapic_set_pinstate: invalid pin number %d", pin));
105
106	KASSERT(VIOAPIC_LOCKED(vioapic),
107	    ("vioapic_set_pinstate: vioapic is not locked"));
108
109	low = vioapic->rtbl[pin].reg;
110	high = vioapic->rtbl[pin].reg >> 32;
111
112	if ((low & IOART_INTMASK) == IOART_INTMSET) {
113		VIOAPIC_CTR1(vioapic, "ioapic pin%d: masked", pin);
114		return;
115	}
116
117	phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
118	delmode = low & IOART_DELMOD;
119	level = low & IOART_TRGRLVL ? true : false;
120	if (level)
121		vioapic->rtbl[pin].reg |= IOART_REM_IRR;
122
123	vector = low & IOART_INTVEC;
124	dest = high >> APIC_ID_SHIFT;
125	vlapic_deliver_intr(vioapic->vm, level, dest, phys, delmode, vector);
126}
127
128static void
129vioapic_set_pinstate(struct vioapic *vioapic, int pin, bool newstate)
130{
131	int oldcnt, newcnt;
132	bool needintr;
133
134	KASSERT(pin >= 0 && pin < REDIR_ENTRIES,
135	    ("vioapic_set_pinstate: invalid pin number %d", pin));
136
137	KASSERT(VIOAPIC_LOCKED(vioapic),
138	    ("vioapic_set_pinstate: vioapic is not locked"));
139
140	oldcnt = vioapic->rtbl[pin].acnt;
141	if (newstate)
142		vioapic->rtbl[pin].acnt++;
143	else
144		vioapic->rtbl[pin].acnt--;
145	newcnt = vioapic->rtbl[pin].acnt;
146
147	if (newcnt < 0) {
148		VIOAPIC_CTR2(vioapic, "ioapic pin%d: bad acnt %d",
149		    pin, newcnt);
150	}
151
152	needintr = false;
153	if (oldcnt == 0 && newcnt == 1) {
154		needintr = true;
155		VIOAPIC_CTR1(vioapic, "ioapic pin%d: asserted", pin);
156	} else if (oldcnt == 1 && newcnt == 0) {
157		VIOAPIC_CTR1(vioapic, "ioapic pin%d: deasserted", pin);
158	} else {
159		VIOAPIC_CTR3(vioapic, "ioapic pin%d: %s, ignored, acnt %d",
160		    pin, pinstate_str(newstate), newcnt);
161	}
162
163	if (needintr)
164		vioapic_send_intr(vioapic, pin);
165}
166
167enum irqstate {
168	IRQSTATE_ASSERT,
169	IRQSTATE_DEASSERT,
170	IRQSTATE_PULSE
171};
172
173static int
174vioapic_set_irqstate(struct vm *vm, int irq, enum irqstate irqstate)
175{
176	struct vioapic *vioapic;
177
178	if (irq < 0 || irq >= REDIR_ENTRIES)
179		return (EINVAL);
180
181	vioapic = vm_ioapic(vm);
182
183	VIOAPIC_LOCK(vioapic);
184	switch (irqstate) {
185	case IRQSTATE_ASSERT:
186		vioapic_set_pinstate(vioapic, irq, true);
187		break;
188	case IRQSTATE_DEASSERT:
189		vioapic_set_pinstate(vioapic, irq, false);
190		break;
191	case IRQSTATE_PULSE:
192		vioapic_set_pinstate(vioapic, irq, true);
193		vioapic_set_pinstate(vioapic, irq, false);
194		break;
195	default:
196		panic("vioapic_set_irqstate: invalid irqstate %d", irqstate);
197	}
198	VIOAPIC_UNLOCK(vioapic);
199
200	return (0);
201}
202
203int
204vioapic_assert_irq(struct vm *vm, int irq)
205{
206
207	return (vioapic_set_irqstate(vm, irq, IRQSTATE_ASSERT));
208}
209
210int
211vioapic_deassert_irq(struct vm *vm, int irq)
212{
213
214	return (vioapic_set_irqstate(vm, irq, IRQSTATE_DEASSERT));
215}
216
217int
218vioapic_pulse_irq(struct vm *vm, int irq)
219{
220
221	return (vioapic_set_irqstate(vm, irq, IRQSTATE_PULSE));
222}
223
224/*
225 * Reset the vlapic's trigger-mode register to reflect the ioapic pin
226 * configuration.
227 */
228static void
229vioapic_update_tmr(struct vm *vm, int vcpuid, void *arg)
230{
231	struct vioapic *vioapic;
232	struct vlapic *vlapic;
233	uint32_t low, high, dest;
234	int delmode, pin, vector;
235	bool level, phys;
236
237	vlapic = vm_lapic(vm, vcpuid);
238	vioapic = vm_ioapic(vm);
239
240	VIOAPIC_LOCK(vioapic);
241	/*
242	 * Reset all vectors to be edge-triggered.
243	 */
244	vlapic_reset_tmr(vlapic);
245	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
246		low = vioapic->rtbl[pin].reg;
247		high = vioapic->rtbl[pin].reg >> 32;
248
249		level = low & IOART_TRGRLVL ? true : false;
250		if (!level)
251			continue;
252
253		/*
254		 * For a level-triggered 'pin' let the vlapic figure out if
255		 * an assertion on this 'pin' would result in an interrupt
256		 * being delivered to it. If yes, then it will modify the
257		 * TMR bit associated with this vector to level-triggered.
258		 */
259		phys = ((low & IOART_DESTMOD) == IOART_DESTPHY);
260		delmode = low & IOART_DELMOD;
261		vector = low & IOART_INTVEC;
262		dest = high >> APIC_ID_SHIFT;
263		vlapic_set_tmr_level(vlapic, dest, phys, delmode, vector);
264	}
265	VIOAPIC_UNLOCK(vioapic);
266}
267
268static uint32_t
269vioapic_read(struct vioapic *vioapic, int vcpuid, uint32_t addr)
270{
271	int regnum, pin, rshift;
272
273	regnum = addr & 0xff;
274	switch (regnum) {
275	case IOAPIC_ID:
276		return (vioapic->id);
277		break;
278	case IOAPIC_VER:
279		return (((REDIR_ENTRIES - 1) << MAXREDIRSHIFT) | 0x11);
280		break;
281	case IOAPIC_ARB:
282		return (vioapic->id);
283		break;
284	default:
285		break;
286	}
287
288	/* redirection table entries */
289	if (regnum >= IOAPIC_REDTBL &&
290	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
291		pin = (regnum - IOAPIC_REDTBL) / 2;
292		if ((regnum - IOAPIC_REDTBL) % 2)
293			rshift = 32;
294		else
295			rshift = 0;
296
297		return (vioapic->rtbl[pin].reg >> rshift);
298	}
299
300	return (0);
301}
302
303static void
304vioapic_write(struct vioapic *vioapic, int vcpuid, uint32_t addr, uint32_t data)
305{
306	uint64_t data64, mask64;
307	uint64_t last, changed;
308	int regnum, pin, lshift;
309	cpuset_t allvcpus;
310
311	regnum = addr & 0xff;
312	switch (regnum) {
313	case IOAPIC_ID:
314		vioapic->id = data & APIC_ID_MASK;
315		break;
316	case IOAPIC_VER:
317	case IOAPIC_ARB:
318		/* readonly */
319		break;
320	default:
321		break;
322	}
323
324	/* redirection table entries */
325	if (regnum >= IOAPIC_REDTBL &&
326	    regnum < IOAPIC_REDTBL + REDIR_ENTRIES * 2) {
327		pin = (regnum - IOAPIC_REDTBL) / 2;
328		if ((regnum - IOAPIC_REDTBL) % 2)
329			lshift = 32;
330		else
331			lshift = 0;
332
333		last = vioapic->rtbl[pin].reg;
334
335		data64 = (uint64_t)data << lshift;
336		mask64 = (uint64_t)0xffffffff << lshift;
337		vioapic->rtbl[pin].reg &= ~mask64 | RTBL_RO_BITS;
338		vioapic->rtbl[pin].reg |= data64 & ~RTBL_RO_BITS;
339
340		VIOAPIC_CTR2(vioapic, "ioapic pin%d: redir table entry %#lx",
341		    pin, vioapic->rtbl[pin].reg);
342
343		/*
344		 * If any fields in the redirection table entry (except mask
345		 * or polarity) have changed then rendezvous all the vcpus
346		 * to update their vlapic trigger-mode registers.
347		 */
348		changed = last ^ vioapic->rtbl[pin].reg;
349		if (changed & ~(IOART_INTMASK | IOART_INTPOL)) {
350			VIOAPIC_CTR1(vioapic, "ioapic pin%d: recalculate "
351			    "vlapic trigger-mode register", pin);
352			VIOAPIC_UNLOCK(vioapic);
353			allvcpus = vm_active_cpus(vioapic->vm);
354			vm_smp_rendezvous(vioapic->vm, vcpuid, allvcpus,
355			    vioapic_update_tmr, NULL);
356			VIOAPIC_LOCK(vioapic);
357		}
358
359		/*
360		 * Generate an interrupt if the following conditions are met:
361		 * - pin is not masked
362		 * - previous interrupt has been EOIed
363		 * - pin level is asserted
364		 */
365		if ((vioapic->rtbl[pin].reg & IOART_INTMASK) == IOART_INTMCLR &&
366		    (vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0 &&
367		    (vioapic->rtbl[pin].acnt > 0)) {
368			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at rtbl "
369			    "write, acnt %d", pin, vioapic->rtbl[pin].acnt);
370			vioapic_send_intr(vioapic, pin);
371		}
372	}
373}
374
375static int
376vioapic_mmio_rw(struct vioapic *vioapic, int vcpuid, uint64_t gpa,
377    uint64_t *data, int size, bool doread)
378{
379	uint64_t offset;
380
381	offset = gpa - VIOAPIC_BASE;
382
383	/*
384	 * The IOAPIC specification allows 32-bit wide accesses to the
385	 * IOREGSEL (offset 0) and IOWIN (offset 16) registers.
386	 */
387	if (size != 4 || (offset != IOREGSEL && offset != IOWIN)) {
388		if (doread)
389			*data = 0;
390		return (0);
391	}
392
393	VIOAPIC_LOCK(vioapic);
394	if (offset == IOREGSEL) {
395		if (doread)
396			*data = vioapic->ioregsel;
397		else
398			vioapic->ioregsel = *data;
399	} else {
400		if (doread) {
401			*data = vioapic_read(vioapic, vcpuid,
402			    vioapic->ioregsel);
403		} else {
404			vioapic_write(vioapic, vcpuid, vioapic->ioregsel,
405			    *data);
406		}
407	}
408	VIOAPIC_UNLOCK(vioapic);
409
410	return (0);
411}
412
413int
414vioapic_mmio_read(void *vm, int vcpuid, uint64_t gpa, uint64_t *rval,
415    int size, void *arg)
416{
417	int error;
418	struct vioapic *vioapic;
419
420	vioapic = vm_ioapic(vm);
421	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, rval, size, true);
422	return (error);
423}
424
425int
426vioapic_mmio_write(void *vm, int vcpuid, uint64_t gpa, uint64_t wval,
427    int size, void *arg)
428{
429	int error;
430	struct vioapic *vioapic;
431
432	vioapic = vm_ioapic(vm);
433	error = vioapic_mmio_rw(vioapic, vcpuid, gpa, &wval, size, false);
434	return (error);
435}
436
437void
438vioapic_process_eoi(struct vm *vm, int vcpuid, int vector)
439{
440	struct vioapic *vioapic;
441	int pin;
442
443	KASSERT(vector >= 0 && vector < 256,
444	    ("vioapic_process_eoi: invalid vector %d", vector));
445
446	vioapic = vm_ioapic(vm);
447	VIOAPIC_CTR1(vioapic, "ioapic processing eoi for vector %d", vector);
448
449	/*
450	 * XXX keep track of the pins associated with this vector instead
451	 * of iterating on every single pin each time.
452	 */
453	VIOAPIC_LOCK(vioapic);
454	for (pin = 0; pin < REDIR_ENTRIES; pin++) {
455		if ((vioapic->rtbl[pin].reg & IOART_REM_IRR) == 0)
456			continue;
457		if ((vioapic->rtbl[pin].reg & IOART_INTVEC) != vector)
458			continue;
459		vioapic->rtbl[pin].reg &= ~IOART_REM_IRR;
460		if (vioapic->rtbl[pin].acnt > 0) {
461			VIOAPIC_CTR2(vioapic, "ioapic pin%d: asserted at eoi, "
462			    "acnt %d", pin, vioapic->rtbl[pin].acnt);
463			vioapic_send_intr(vioapic, pin);
464		}
465	}
466	VIOAPIC_UNLOCK(vioapic);
467}
468
469struct vioapic *
470vioapic_init(struct vm *vm)
471{
472	int i;
473	struct vioapic *vioapic;
474
475	vioapic = malloc(sizeof(struct vioapic), M_VIOAPIC, M_WAITOK | M_ZERO);
476
477	vioapic->vm = vm;
478	mtx_init(&vioapic->mtx, "vioapic lock", NULL, MTX_SPIN);
479
480	/* Initialize all redirection entries to mask all interrupts */
481	for (i = 0; i < REDIR_ENTRIES; i++)
482		vioapic->rtbl[i].reg = 0x0001000000010000UL;
483
484	return (vioapic);
485}
486
487void
488vioapic_cleanup(struct vioapic *vioapic)
489{
490
491	free(vioapic, M_VIOAPIC);
492}
493
494int
495vioapic_pincount(struct vm *vm)
496{
497
498	return (REDIR_ENTRIES);
499}
500