1/*-
2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3 *
4 * Copyright (c) 2011, Bryan Venteicher <bryanv@FreeBSD.org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 *    notice unmodified, this list of conditions, and the following
12 *    disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 *    notice, this list of conditions and the following disclaimer in the
15 *    documentation and/or other materials provided with the distribution.
16 *
17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
18 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
19 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
20 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
21 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
22 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 */
28
29/* Driver for the VirtIO PCI interface. */
30
31#include <sys/cdefs.h>
32__FBSDID("$FreeBSD$");
33
34#include <sys/param.h>
35#include <sys/systm.h>
36#include <sys/bus.h>
37#include <sys/kernel.h>
38#include <sys/module.h>
39#include <sys/malloc.h>
40
41#include <machine/bus.h>
42#include <machine/resource.h>
43#include <sys/bus.h>
44#include <sys/rman.h>
45
46#include <dev/pci/pcivar.h>
47#include <dev/pci/pcireg.h>
48
49#include <dev/virtio/virtio.h>
50#include <dev/virtio/virtqueue.h>
51#include <dev/virtio/pci/virtio_pci.h>
52
53#include "virtio_bus_if.h"
54#include "virtio_if.h"
55
56struct vtpci_interrupt {
57	struct resource		*vti_irq;
58	int			 vti_rid;
59	void			*vti_handler;
60};
61
62struct vtpci_virtqueue {
63	struct virtqueue	*vtv_vq;
64	int			 vtv_no_intr;
65};
66
67struct vtpci_softc {
68	device_t			 vtpci_dev;
69	struct resource			*vtpci_res;
70	struct resource			*vtpci_msix_res;
71	uint64_t			 vtpci_features;
72	uint32_t			 vtpci_flags;
73#define VTPCI_FLAG_NO_MSI		0x0001
74#define VTPCI_FLAG_NO_MSIX		0x0002
75#define VTPCI_FLAG_LEGACY		0x1000
76#define VTPCI_FLAG_MSI			0x2000
77#define VTPCI_FLAG_MSIX			0x4000
78#define VTPCI_FLAG_SHARED_MSIX		0x8000
79#define VTPCI_FLAG_ITYPE_MASK		0xF000
80
81	/* This "bus" will only ever have one child. */
82	device_t			 vtpci_child_dev;
83	struct virtio_feature_desc	*vtpci_child_feat_desc;
84
85	int				 vtpci_nvqs;
86	struct vtpci_virtqueue		*vtpci_vqs;
87
88	/*
89	 * Ideally, each virtqueue that the driver provides a callback for will
90	 * receive its own MSIX vector. If there are not sufficient vectors
91	 * available, then attempt to have all the VQs share one vector. For
92	 * MSIX, the configuration changed notifications must be on their own
93	 * vector.
94	 *
95	 * If MSIX is not available, we will attempt to have the whole device
96	 * share one MSI vector, and then, finally, one legacy interrupt.
97	 */
98	struct vtpci_interrupt		 vtpci_device_interrupt;
99	struct vtpci_interrupt		*vtpci_msix_vq_interrupts;
100	int				 vtpci_nmsix_resources;
101};
102
103static int	vtpci_probe(device_t);
104static int	vtpci_attach(device_t);
105static int	vtpci_detach(device_t);
106static int	vtpci_suspend(device_t);
107static int	vtpci_resume(device_t);
108static int	vtpci_shutdown(device_t);
109static void	vtpci_driver_added(device_t, driver_t *);
110static void	vtpci_child_detached(device_t, device_t);
111static int	vtpci_read_ivar(device_t, device_t, int, uintptr_t *);
112static int	vtpci_write_ivar(device_t, device_t, int, uintptr_t);
113
114static uint64_t	vtpci_negotiate_features(device_t, uint64_t);
115static int	vtpci_with_feature(device_t, uint64_t);
116static int	vtpci_alloc_virtqueues(device_t, int, int,
117		    struct vq_alloc_info *);
118static int	vtpci_setup_intr(device_t, enum intr_type);
119static void	vtpci_stop(device_t);
120static int	vtpci_reinit(device_t, uint64_t);
121static void	vtpci_reinit_complete(device_t);
122static void	vtpci_notify_virtqueue(device_t, uint16_t);
123static uint8_t	vtpci_get_status(device_t);
124static void	vtpci_set_status(device_t, uint8_t);
125static void	vtpci_read_dev_config(device_t, bus_size_t, void *, int);
126static void	vtpci_write_dev_config(device_t, bus_size_t, void *, int);
127
128static void	vtpci_describe_features(struct vtpci_softc *, const char *,
129		    uint64_t);
130static void	vtpci_probe_and_attach_child(struct vtpci_softc *);
131
132static int	vtpci_alloc_msix(struct vtpci_softc *, int);
133static int	vtpci_alloc_msi(struct vtpci_softc *);
134static int	vtpci_alloc_intr_msix_pervq(struct vtpci_softc *);
135static int	vtpci_alloc_intr_msix_shared(struct vtpci_softc *);
136static int	vtpci_alloc_intr_msi(struct vtpci_softc *);
137static int	vtpci_alloc_intr_legacy(struct vtpci_softc *);
138static int	vtpci_alloc_interrupt(struct vtpci_softc *, int, int,
139		    struct vtpci_interrupt *);
140static int	vtpci_alloc_intr_resources(struct vtpci_softc *);
141
142static int	vtpci_setup_legacy_interrupt(struct vtpci_softc *,
143		    enum intr_type);
144static int	vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *,
145		    enum intr_type);
146static int	vtpci_setup_msix_interrupts(struct vtpci_softc *,
147		    enum intr_type);
148static int	vtpci_setup_interrupts(struct vtpci_softc *, enum intr_type);
149
150static int	vtpci_register_msix_vector(struct vtpci_softc *, int,
151		    struct vtpci_interrupt *);
152static int	vtpci_set_host_msix_vectors(struct vtpci_softc *);
153static int	vtpci_reinit_virtqueue(struct vtpci_softc *, int);
154
155static void	vtpci_free_interrupt(struct vtpci_softc *,
156		    struct vtpci_interrupt *);
157static void	vtpci_free_interrupts(struct vtpci_softc *);
158static void	vtpci_free_virtqueues(struct vtpci_softc *);
159static void	vtpci_release_child_resources(struct vtpci_softc *);
160static void	vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *);
161static void	vtpci_reset(struct vtpci_softc *);
162
163static void	vtpci_select_virtqueue(struct vtpci_softc *, int);
164
165static void	vtpci_legacy_intr(void *);
166static int	vtpci_vq_shared_intr_filter(void *);
167static void	vtpci_vq_shared_intr(void *);
168static int	vtpci_vq_intr_filter(void *);
169static void	vtpci_vq_intr(void *);
170static void	vtpci_config_intr(void *);
171
172#define vtpci_setup_msi_interrupt vtpci_setup_legacy_interrupt
173
174#define VIRTIO_PCI_CONFIG(_sc) \
175    VIRTIO_PCI_CONFIG_OFF((((_sc)->vtpci_flags & VTPCI_FLAG_MSIX)) != 0)
176
177/*
178 * I/O port read/write wrappers.
179 */
180#define vtpci_read_config_1(sc, o)	bus_read_1((sc)->vtpci_res, (o))
181#define vtpci_read_config_2(sc, o)	bus_read_2((sc)->vtpci_res, (o))
182#define vtpci_read_config_4(sc, o)	bus_read_4((sc)->vtpci_res, (o))
183#define vtpci_write_config_1(sc, o, v)	bus_write_1((sc)->vtpci_res, (o), (v))
184#define vtpci_write_config_2(sc, o, v)	bus_write_2((sc)->vtpci_res, (o), (v))
185#define vtpci_write_config_4(sc, o, v)	bus_write_4((sc)->vtpci_res, (o), (v))
186
187/* Tunables. */
188static int vtpci_disable_msix = 0;
189TUNABLE_INT("hw.virtio.pci.disable_msix", &vtpci_disable_msix);
190
191static device_method_t vtpci_methods[] = {
192	/* Device interface. */
193	DEVMETHOD(device_probe,			  vtpci_probe),
194	DEVMETHOD(device_attach,		  vtpci_attach),
195	DEVMETHOD(device_detach,		  vtpci_detach),
196	DEVMETHOD(device_suspend,		  vtpci_suspend),
197	DEVMETHOD(device_resume,		  vtpci_resume),
198	DEVMETHOD(device_shutdown,		  vtpci_shutdown),
199
200	/* Bus interface. */
201	DEVMETHOD(bus_driver_added,		  vtpci_driver_added),
202	DEVMETHOD(bus_child_detached,		  vtpci_child_detached),
203	DEVMETHOD(bus_read_ivar,		  vtpci_read_ivar),
204	DEVMETHOD(bus_write_ivar,		  vtpci_write_ivar),
205
206	/* VirtIO bus interface. */
207	DEVMETHOD(virtio_bus_negotiate_features,  vtpci_negotiate_features),
208	DEVMETHOD(virtio_bus_with_feature,	  vtpci_with_feature),
209	DEVMETHOD(virtio_bus_alloc_virtqueues,	  vtpci_alloc_virtqueues),
210	DEVMETHOD(virtio_bus_setup_intr,	  vtpci_setup_intr),
211	DEVMETHOD(virtio_bus_stop,		  vtpci_stop),
212	DEVMETHOD(virtio_bus_reinit,		  vtpci_reinit),
213	DEVMETHOD(virtio_bus_reinit_complete,	  vtpci_reinit_complete),
214	DEVMETHOD(virtio_bus_notify_vq,		  vtpci_notify_virtqueue),
215	DEVMETHOD(virtio_bus_read_device_config,  vtpci_read_dev_config),
216	DEVMETHOD(virtio_bus_write_device_config, vtpci_write_dev_config),
217
218	DEVMETHOD_END
219};
220
221static driver_t vtpci_driver = {
222	"virtio_pci",
223	vtpci_methods,
224	sizeof(struct vtpci_softc)
225};
226
227devclass_t vtpci_devclass;
228
229DRIVER_MODULE(virtio_pci, pci, vtpci_driver, vtpci_devclass, 0, 0);
230MODULE_VERSION(virtio_pci, 1);
231MODULE_DEPEND(virtio_pci, pci, 1, 1, 1);
232MODULE_DEPEND(virtio_pci, virtio, 1, 1, 1);
233
234static int
235vtpci_probe(device_t dev)
236{
237	char desc[36];
238	const char *name;
239
240	if (pci_get_vendor(dev) != VIRTIO_PCI_VENDORID)
241		return (ENXIO);
242
243	if (pci_get_device(dev) < VIRTIO_PCI_DEVICEID_MIN ||
244	    pci_get_device(dev) > VIRTIO_PCI_DEVICEID_MAX)
245		return (ENXIO);
246
247	if (pci_get_revid(dev) != VIRTIO_PCI_ABI_VERSION)
248		return (ENXIO);
249
250	name = virtio_device_name(pci_get_subdevice(dev));
251	if (name == NULL)
252		name = "Unknown";
253
254	snprintf(desc, sizeof(desc), "VirtIO PCI %s adapter", name);
255	device_set_desc_copy(dev, desc);
256
257	return (BUS_PROBE_DEFAULT);
258}
259
260static int
261vtpci_attach(device_t dev)
262{
263	struct vtpci_softc *sc;
264	device_t child;
265	int rid;
266
267	sc = device_get_softc(dev);
268	sc->vtpci_dev = dev;
269
270	pci_enable_busmaster(dev);
271
272	rid = PCIR_BAR(0);
273	sc->vtpci_res = bus_alloc_resource_any(dev, SYS_RES_IOPORT, &rid,
274	    RF_ACTIVE);
275	if (sc->vtpci_res == NULL) {
276		device_printf(dev, "cannot map I/O space\n");
277		return (ENXIO);
278	}
279
280	if (pci_find_cap(dev, PCIY_MSI, NULL) != 0)
281		sc->vtpci_flags |= VTPCI_FLAG_NO_MSI;
282
283	if (pci_find_cap(dev, PCIY_MSIX, NULL) == 0) {
284		rid = PCIR_BAR(1);
285		sc->vtpci_msix_res = bus_alloc_resource_any(dev,
286		    SYS_RES_MEMORY, &rid, RF_ACTIVE);
287	}
288
289	if (sc->vtpci_msix_res == NULL)
290		sc->vtpci_flags |= VTPCI_FLAG_NO_MSIX;
291
292	vtpci_reset(sc);
293
294	/* Tell the host we've noticed this device. */
295	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
296
297	if ((child = device_add_child(dev, NULL, -1)) == NULL) {
298		device_printf(dev, "cannot create child device\n");
299		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
300		vtpci_detach(dev);
301		return (ENOMEM);
302	}
303
304	sc->vtpci_child_dev = child;
305	vtpci_probe_and_attach_child(sc);
306
307	return (0);
308}
309
310static int
311vtpci_detach(device_t dev)
312{
313	struct vtpci_softc *sc;
314	device_t child;
315	int error;
316
317	sc = device_get_softc(dev);
318
319	if ((child = sc->vtpci_child_dev) != NULL) {
320		error = device_delete_child(dev, child);
321		if (error)
322			return (error);
323		sc->vtpci_child_dev = NULL;
324	}
325
326	vtpci_reset(sc);
327
328	if (sc->vtpci_msix_res != NULL) {
329		bus_release_resource(dev, SYS_RES_MEMORY, PCIR_BAR(1),
330		    sc->vtpci_msix_res);
331		sc->vtpci_msix_res = NULL;
332	}
333
334	if (sc->vtpci_res != NULL) {
335		bus_release_resource(dev, SYS_RES_IOPORT, PCIR_BAR(0),
336		    sc->vtpci_res);
337		sc->vtpci_res = NULL;
338	}
339
340	return (0);
341}
342
343static int
344vtpci_suspend(device_t dev)
345{
346
347	return (bus_generic_suspend(dev));
348}
349
350static int
351vtpci_resume(device_t dev)
352{
353
354	return (bus_generic_resume(dev));
355}
356
357static int
358vtpci_shutdown(device_t dev)
359{
360
361	(void) bus_generic_shutdown(dev);
362	/* Forcibly stop the host device. */
363	vtpci_stop(dev);
364
365	return (0);
366}
367
368static void
369vtpci_driver_added(device_t dev, driver_t *driver)
370{
371	struct vtpci_softc *sc;
372
373	sc = device_get_softc(dev);
374
375	vtpci_probe_and_attach_child(sc);
376}
377
378static void
379vtpci_child_detached(device_t dev, device_t child)
380{
381	struct vtpci_softc *sc;
382
383	sc = device_get_softc(dev);
384
385	vtpci_reset(sc);
386	vtpci_release_child_resources(sc);
387}
388
389static int
390vtpci_read_ivar(device_t dev, device_t child, int index, uintptr_t *result)
391{
392	struct vtpci_softc *sc;
393
394	sc = device_get_softc(dev);
395
396	if (sc->vtpci_child_dev != child)
397		return (ENOENT);
398
399	switch (index) {
400	case VIRTIO_IVAR_DEVTYPE:
401	case VIRTIO_IVAR_SUBDEVICE:
402		*result = pci_get_subdevice(dev);
403		break;
404	case VIRTIO_IVAR_VENDOR:
405		*result = pci_get_vendor(dev);
406		break;
407	case VIRTIO_IVAR_DEVICE:
408		*result = pci_get_device(dev);
409		break;
410	case VIRTIO_IVAR_SUBVENDOR:
411		*result = pci_get_subdevice(dev);
412		break;
413	default:
414		return (ENOENT);
415	}
416
417	return (0);
418}
419
420static int
421vtpci_write_ivar(device_t dev, device_t child, int index, uintptr_t value)
422{
423	struct vtpci_softc *sc;
424
425	sc = device_get_softc(dev);
426
427	if (sc->vtpci_child_dev != child)
428		return (ENOENT);
429
430	switch (index) {
431	case VIRTIO_IVAR_FEATURE_DESC:
432		sc->vtpci_child_feat_desc = (void *) value;
433		break;
434	default:
435		return (ENOENT);
436	}
437
438	return (0);
439}
440
441static uint64_t
442vtpci_negotiate_features(device_t dev, uint64_t child_features)
443{
444	struct vtpci_softc *sc;
445	uint64_t host_features, features;
446
447	sc = device_get_softc(dev);
448
449	host_features = vtpci_read_config_4(sc, VIRTIO_PCI_HOST_FEATURES);
450	vtpci_describe_features(sc, "host", host_features);
451
452	/*
453	 * Limit negotiated features to what the driver, virtqueue, and
454	 * host all support.
455	 */
456	features = host_features & child_features;
457	features = virtqueue_filter_features(features);
458	sc->vtpci_features = features;
459
460	vtpci_describe_features(sc, "negotiated", features);
461	vtpci_write_config_4(sc, VIRTIO_PCI_GUEST_FEATURES, features);
462
463	return (features);
464}
465
466static int
467vtpci_with_feature(device_t dev, uint64_t feature)
468{
469	struct vtpci_softc *sc;
470
471	sc = device_get_softc(dev);
472
473	return ((sc->vtpci_features & feature) != 0);
474}
475
476static int
477vtpci_alloc_virtqueues(device_t dev, int flags, int nvqs,
478    struct vq_alloc_info *vq_info)
479{
480	struct vtpci_softc *sc;
481	struct virtqueue *vq;
482	struct vtpci_virtqueue *vqx;
483	struct vq_alloc_info *info;
484	int idx, error;
485	uint16_t size;
486
487	sc = device_get_softc(dev);
488
489	if (sc->vtpci_nvqs != 0)
490		return (EALREADY);
491	if (nvqs <= 0)
492		return (EINVAL);
493
494	sc->vtpci_vqs = malloc(nvqs * sizeof(struct vtpci_virtqueue),
495	    M_DEVBUF, M_NOWAIT | M_ZERO);
496	if (sc->vtpci_vqs == NULL)
497		return (ENOMEM);
498
499	for (idx = 0; idx < nvqs; idx++) {
500		vqx = &sc->vtpci_vqs[idx];
501		info = &vq_info[idx];
502
503		vtpci_select_virtqueue(sc, idx);
504		size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
505
506		error = virtqueue_alloc(dev, idx, size, VIRTIO_PCI_VRING_ALIGN,
507		    0xFFFFFFFFUL, info, &vq);
508		if (error) {
509			device_printf(dev,
510			    "cannot allocate virtqueue %d: %d\n", idx, error);
511			break;
512		}
513
514		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
515		    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
516
517		vqx->vtv_vq = *info->vqai_vq = vq;
518		vqx->vtv_no_intr = info->vqai_intr == NULL;
519
520		sc->vtpci_nvqs++;
521	}
522
523	if (error)
524		vtpci_free_virtqueues(sc);
525
526	return (error);
527}
528
529static int
530vtpci_setup_intr(device_t dev, enum intr_type type)
531{
532	struct vtpci_softc *sc;
533	int attempt, error;
534
535	sc = device_get_softc(dev);
536
537	for (attempt = 0; attempt < 5; attempt++) {
538		/*
539		 * Start with the most desirable interrupt configuration and
540		 * fallback towards less desirable ones.
541		 */
542		switch (attempt) {
543		case 0:
544			error = vtpci_alloc_intr_msix_pervq(sc);
545			break;
546		case 1:
547			error = vtpci_alloc_intr_msix_shared(sc);
548			break;
549		case 2:
550			error = vtpci_alloc_intr_msi(sc);
551			break;
552		case 3:
553			error = vtpci_alloc_intr_legacy(sc);
554			break;
555		default:
556			device_printf(dev,
557			    "exhausted all interrupt allocation attempts\n");
558			return (ENXIO);
559		}
560
561		if (error == 0 && vtpci_setup_interrupts(sc, type) == 0)
562			break;
563
564		vtpci_cleanup_setup_intr_attempt(sc);
565	}
566
567	if (bootverbose) {
568		if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
569			device_printf(dev, "using legacy interrupt\n");
570		else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
571			device_printf(dev, "using MSI interrupt\n");
572		else if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX)
573			device_printf(dev, "using shared MSIX interrupts\n");
574		else
575			device_printf(dev, "using per VQ MSIX interrupts\n");
576	}
577
578	return (0);
579}
580
581static void
582vtpci_stop(device_t dev)
583{
584
585	vtpci_reset(device_get_softc(dev));
586}
587
588static int
589vtpci_reinit(device_t dev, uint64_t features)
590{
591	struct vtpci_softc *sc;
592	int idx, error;
593
594	sc = device_get_softc(dev);
595
596	/*
597	 * Redrive the device initialization. This is a bit of an abuse of
598	 * the specification, but VirtualBox, QEMU/KVM, and BHyVe seem to
599	 * play nice.
600	 *
601	 * We do not allow the host device to change from what was originally
602	 * negotiated beyond what the guest driver changed. MSIX state should
603	 * not change, number of virtqueues and their size remain the same, etc.
604	 * This will need to be rethought when we want to support migration.
605	 */
606
607	if (vtpci_get_status(dev) != VIRTIO_CONFIG_STATUS_RESET)
608		vtpci_stop(dev);
609
610	/*
611	 * Quickly drive the status through ACK and DRIVER. The device
612	 * does not become usable again until vtpci_reinit_complete().
613	 */
614	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
615	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
616
617	vtpci_negotiate_features(dev, features);
618
619	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
620		error = vtpci_reinit_virtqueue(sc, idx);
621		if (error)
622			return (error);
623	}
624
625	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
626		error = vtpci_set_host_msix_vectors(sc);
627		if (error)
628			return (error);
629	}
630
631	return (0);
632}
633
634static void
635vtpci_reinit_complete(device_t dev)
636{
637
638	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
639}
640
641static void
642vtpci_notify_virtqueue(device_t dev, uint16_t queue)
643{
644	struct vtpci_softc *sc;
645
646	sc = device_get_softc(dev);
647
648	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_NOTIFY, queue);
649}
650
651static uint8_t
652vtpci_get_status(device_t dev)
653{
654	struct vtpci_softc *sc;
655
656	sc = device_get_softc(dev);
657
658	return (vtpci_read_config_1(sc, VIRTIO_PCI_STATUS));
659}
660
661static void
662vtpci_set_status(device_t dev, uint8_t status)
663{
664	struct vtpci_softc *sc;
665
666	sc = device_get_softc(dev);
667
668	if (status != VIRTIO_CONFIG_STATUS_RESET)
669		status |= vtpci_get_status(dev);
670
671	vtpci_write_config_1(sc, VIRTIO_PCI_STATUS, status);
672}
673
674static void
675vtpci_read_dev_config(device_t dev, bus_size_t offset,
676    void *dst, int length)
677{
678	struct vtpci_softc *sc;
679	bus_size_t off;
680	uint8_t *d;
681	int size;
682
683	sc = device_get_softc(dev);
684	off = VIRTIO_PCI_CONFIG(sc) + offset;
685
686	for (d = dst; length > 0; d += size, off += size, length -= size) {
687		if (length >= 4) {
688			size = 4;
689			*(uint32_t *)d = vtpci_read_config_4(sc, off);
690		} else if (length >= 2) {
691			size = 2;
692			*(uint16_t *)d = vtpci_read_config_2(sc, off);
693		} else {
694			size = 1;
695			*d = vtpci_read_config_1(sc, off);
696		}
697	}
698}
699
700static void
701vtpci_write_dev_config(device_t dev, bus_size_t offset,
702    void *src, int length)
703{
704	struct vtpci_softc *sc;
705	bus_size_t off;
706	uint8_t *s;
707	int size;
708
709	sc = device_get_softc(dev);
710	off = VIRTIO_PCI_CONFIG(sc) + offset;
711
712	for (s = src; length > 0; s += size, off += size, length -= size) {
713		if (length >= 4) {
714			size = 4;
715			vtpci_write_config_4(sc, off, *(uint32_t *)s);
716		} else if (length >= 2) {
717			size = 2;
718			vtpci_write_config_2(sc, off, *(uint16_t *)s);
719		} else {
720			size = 1;
721			vtpci_write_config_1(sc, off, *s);
722		}
723	}
724}
725
726static void
727vtpci_describe_features(struct vtpci_softc *sc, const char *msg,
728    uint64_t features)
729{
730	device_t dev, child;
731
732	dev = sc->vtpci_dev;
733	child = sc->vtpci_child_dev;
734
735	if (device_is_attached(child) || bootverbose == 0)
736		return;
737
738	virtio_describe(dev, msg, features, sc->vtpci_child_feat_desc);
739}
740
741static void
742vtpci_probe_and_attach_child(struct vtpci_softc *sc)
743{
744	device_t dev, child;
745
746	dev = sc->vtpci_dev;
747	child = sc->vtpci_child_dev;
748
749	if (child == NULL)
750		return;
751
752	if (device_get_state(child) != DS_NOTPRESENT)
753		return;
754
755	if (device_probe(child) != 0)
756		return;
757
758	vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER);
759	if (device_attach(child) != 0) {
760		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_FAILED);
761		vtpci_reset(sc);
762		vtpci_release_child_resources(sc);
763		/* Reset status for future attempt. */
764		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_ACK);
765	} else {
766		vtpci_set_status(dev, VIRTIO_CONFIG_STATUS_DRIVER_OK);
767		VIRTIO_ATTACH_COMPLETED(child);
768	}
769}
770
771static int
772vtpci_alloc_msix(struct vtpci_softc *sc, int nvectors)
773{
774	device_t dev;
775	int nmsix, cnt, required;
776
777	dev = sc->vtpci_dev;
778
779	/* Allocate an additional vector for the config changes. */
780	required = nvectors + 1;
781
782	nmsix = pci_msix_count(dev);
783	if (nmsix < required)
784		return (1);
785
786	cnt = required;
787	if (pci_alloc_msix(dev, &cnt) == 0 && cnt >= required) {
788		sc->vtpci_nmsix_resources = required;
789		return (0);
790	}
791
792	pci_release_msi(dev);
793
794	return (1);
795}
796
797static int
798vtpci_alloc_msi(struct vtpci_softc *sc)
799{
800	device_t dev;
801	int nmsi, cnt, required;
802
803	dev = sc->vtpci_dev;
804	required = 1;
805
806	nmsi = pci_msi_count(dev);
807	if (nmsi < required)
808		return (1);
809
810	cnt = required;
811	if (pci_alloc_msi(dev, &cnt) == 0 && cnt >= required)
812		return (0);
813
814	pci_release_msi(dev);
815
816	return (1);
817}
818
819static int
820vtpci_alloc_intr_msix_pervq(struct vtpci_softc *sc)
821{
822	int i, nvectors, error;
823
824	if (vtpci_disable_msix != 0 ||
825	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
826		return (ENOTSUP);
827
828	for (nvectors = 0, i = 0; i < sc->vtpci_nvqs; i++) {
829		if (sc->vtpci_vqs[i].vtv_no_intr == 0)
830			nvectors++;
831	}
832
833	error = vtpci_alloc_msix(sc, nvectors);
834	if (error)
835		return (error);
836
837	sc->vtpci_flags |= VTPCI_FLAG_MSIX;
838
839	return (0);
840}
841
842static int
843vtpci_alloc_intr_msix_shared(struct vtpci_softc *sc)
844{
845	int error;
846
847	if (vtpci_disable_msix != 0 ||
848	    sc->vtpci_flags & VTPCI_FLAG_NO_MSIX)
849		return (ENOTSUP);
850
851	error = vtpci_alloc_msix(sc, 1);
852	if (error)
853		return (error);
854
855	sc->vtpci_flags |= VTPCI_FLAG_MSIX | VTPCI_FLAG_SHARED_MSIX;
856
857	return (0);
858}
859
860static int
861vtpci_alloc_intr_msi(struct vtpci_softc *sc)
862{
863	int error;
864
865	/* Only BHyVe supports MSI. */
866	if (sc->vtpci_flags & VTPCI_FLAG_NO_MSI)
867		return (ENOTSUP);
868
869	error = vtpci_alloc_msi(sc);
870	if (error)
871		return (error);
872
873	sc->vtpci_flags |= VTPCI_FLAG_MSI;
874
875	return (0);
876}
877
878static int
879vtpci_alloc_intr_legacy(struct vtpci_softc *sc)
880{
881
882	sc->vtpci_flags |= VTPCI_FLAG_LEGACY;
883
884	return (0);
885}
886
887static int
888vtpci_alloc_interrupt(struct vtpci_softc *sc, int rid, int flags,
889    struct vtpci_interrupt *intr)
890{
891	struct resource *irq;
892
893	irq = bus_alloc_resource_any(sc->vtpci_dev, SYS_RES_IRQ, &rid, flags);
894	if (irq == NULL)
895		return (ENXIO);
896
897	intr->vti_irq = irq;
898	intr->vti_rid = rid;
899
900	return (0);
901}
902
903static int
904vtpci_alloc_intr_resources(struct vtpci_softc *sc)
905{
906	struct vtpci_interrupt *intr;
907	int i, rid, flags, nvq_intrs, error;
908
909	rid = 0;
910	flags = RF_ACTIVE;
911
912	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
913		flags |= RF_SHAREABLE;
914	else
915		rid = 1;
916
917	/*
918	 * For legacy and MSI interrupts, this single resource handles all
919	 * interrupts. For MSIX, this resource is used for the configuration
920	 * changed interrupt.
921	 */
922	intr = &sc->vtpci_device_interrupt;
923	error = vtpci_alloc_interrupt(sc, rid, flags, intr);
924	if (error || sc->vtpci_flags & (VTPCI_FLAG_LEGACY | VTPCI_FLAG_MSI))
925		return (error);
926
927	/* Subtract one for the configuration changed interrupt. */
928	nvq_intrs = sc->vtpci_nmsix_resources - 1;
929
930	intr = sc->vtpci_msix_vq_interrupts = malloc(nvq_intrs *
931	    sizeof(struct vtpci_interrupt), M_DEVBUF, M_NOWAIT | M_ZERO);
932	if (sc->vtpci_msix_vq_interrupts == NULL)
933		return (ENOMEM);
934
935	for (i = 0, rid++; i < nvq_intrs; i++, rid++, intr++) {
936		error = vtpci_alloc_interrupt(sc, rid, flags, intr);
937		if (error)
938			return (error);
939	}
940
941	return (0);
942}
943
944static int
945vtpci_setup_legacy_interrupt(struct vtpci_softc *sc, enum intr_type type)
946{
947	struct vtpci_interrupt *intr;
948	int error;
949
950	intr = &sc->vtpci_device_interrupt;
951	error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type, NULL,
952	    vtpci_legacy_intr, sc, &intr->vti_handler);
953
954	return (error);
955}
956
957static int
958vtpci_setup_pervq_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
959{
960	struct vtpci_virtqueue *vqx;
961	struct vtpci_interrupt *intr;
962	int i, error;
963
964	intr = sc->vtpci_msix_vq_interrupts;
965
966	for (i = 0; i < sc->vtpci_nvqs; i++) {
967		vqx = &sc->vtpci_vqs[i];
968
969		if (vqx->vtv_no_intr)
970			continue;
971
972		error = bus_setup_intr(sc->vtpci_dev, intr->vti_irq, type,
973		    vtpci_vq_intr_filter, vtpci_vq_intr, vqx->vtv_vq,
974		    &intr->vti_handler);
975		if (error)
976			return (error);
977
978		intr++;
979	}
980
981	return (0);
982}
983
984static int
985vtpci_setup_msix_interrupts(struct vtpci_softc *sc, enum intr_type type)
986{
987	device_t dev;
988	struct vtpci_interrupt *intr;
989	int error;
990
991	dev = sc->vtpci_dev;
992	intr = &sc->vtpci_device_interrupt;
993
994	error = bus_setup_intr(dev, intr->vti_irq, type, NULL,
995	    vtpci_config_intr, sc, &intr->vti_handler);
996	if (error)
997		return (error);
998
999	if (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) {
1000		intr = sc->vtpci_msix_vq_interrupts;
1001		error = bus_setup_intr(dev, intr->vti_irq, type,
1002		    vtpci_vq_shared_intr_filter, vtpci_vq_shared_intr, sc,
1003		    &intr->vti_handler);
1004	} else
1005		error = vtpci_setup_pervq_msix_interrupts(sc, type);
1006
1007	return (error ? error : vtpci_set_host_msix_vectors(sc));
1008}
1009
1010static int
1011vtpci_setup_interrupts(struct vtpci_softc *sc, enum intr_type type)
1012{
1013	int error;
1014
1015	type |= INTR_MPSAFE;
1016	KASSERT(sc->vtpci_flags & VTPCI_FLAG_ITYPE_MASK,
1017	    ("%s: no interrupt type selected %#x", __func__, sc->vtpci_flags));
1018
1019	error = vtpci_alloc_intr_resources(sc);
1020	if (error)
1021		return (error);
1022
1023	if (sc->vtpci_flags & VTPCI_FLAG_LEGACY)
1024		error = vtpci_setup_legacy_interrupt(sc, type);
1025	else if (sc->vtpci_flags & VTPCI_FLAG_MSI)
1026		error = vtpci_setup_msi_interrupt(sc, type);
1027	else
1028		error = vtpci_setup_msix_interrupts(sc, type);
1029
1030	return (error);
1031}
1032
1033static int
1034vtpci_register_msix_vector(struct vtpci_softc *sc, int offset,
1035    struct vtpci_interrupt *intr)
1036{
1037	device_t dev;
1038	uint16_t vector;
1039
1040	dev = sc->vtpci_dev;
1041
1042	if (intr != NULL) {
1043		/* Map from guest rid to host vector. */
1044		vector = intr->vti_rid - 1;
1045	} else
1046		vector = VIRTIO_MSI_NO_VECTOR;
1047
1048	vtpci_write_config_2(sc, offset, vector);
1049
1050	/* Read vector to determine if the host had sufficient resources. */
1051	if (vtpci_read_config_2(sc, offset) != vector) {
1052		device_printf(dev,
1053		    "insufficient host resources for MSIX interrupts\n");
1054		return (ENODEV);
1055	}
1056
1057	return (0);
1058}
1059
1060static int
1061vtpci_set_host_msix_vectors(struct vtpci_softc *sc)
1062{
1063	struct vtpci_interrupt *intr, *tintr;
1064	int idx, offset, error;
1065
1066	intr = &sc->vtpci_device_interrupt;
1067	offset = VIRTIO_MSI_CONFIG_VECTOR;
1068
1069	error = vtpci_register_msix_vector(sc, offset, intr);
1070	if (error)
1071		return (error);
1072
1073	intr = sc->vtpci_msix_vq_interrupts;
1074	offset = VIRTIO_MSI_QUEUE_VECTOR;
1075
1076	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1077		vtpci_select_virtqueue(sc, idx);
1078
1079		if (sc->vtpci_vqs[idx].vtv_no_intr)
1080			tintr = NULL;
1081		else
1082			tintr = intr;
1083
1084		error = vtpci_register_msix_vector(sc, offset, tintr);
1085		if (error)
1086			break;
1087
1088		/*
1089		 * For shared MSIX, all the virtqueues share the first
1090		 * interrupt.
1091		 */
1092		if (!sc->vtpci_vqs[idx].vtv_no_intr &&
1093		    (sc->vtpci_flags & VTPCI_FLAG_SHARED_MSIX) == 0)
1094			intr++;
1095	}
1096
1097	return (error);
1098}
1099
1100static int
1101vtpci_reinit_virtqueue(struct vtpci_softc *sc, int idx)
1102{
1103	struct vtpci_virtqueue *vqx;
1104	struct virtqueue *vq;
1105	int error;
1106	uint16_t size;
1107
1108	vqx = &sc->vtpci_vqs[idx];
1109	vq = vqx->vtv_vq;
1110
1111	KASSERT(vq != NULL, ("%s: vq %d not allocated", __func__, idx));
1112
1113	vtpci_select_virtqueue(sc, idx);
1114	size = vtpci_read_config_2(sc, VIRTIO_PCI_QUEUE_NUM);
1115
1116	error = virtqueue_reinit(vq, size);
1117	if (error)
1118		return (error);
1119
1120	vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN,
1121	    virtqueue_paddr(vq) >> VIRTIO_PCI_QUEUE_ADDR_SHIFT);
1122
1123	return (0);
1124}
1125
1126static void
1127vtpci_free_interrupt(struct vtpci_softc *sc, struct vtpci_interrupt *intr)
1128{
1129	device_t dev;
1130
1131	dev = sc->vtpci_dev;
1132
1133	if (intr->vti_handler != NULL) {
1134		bus_teardown_intr(dev, intr->vti_irq, intr->vti_handler);
1135		intr->vti_handler = NULL;
1136	}
1137
1138	if (intr->vti_irq != NULL) {
1139		bus_release_resource(dev, SYS_RES_IRQ, intr->vti_rid,
1140		    intr->vti_irq);
1141		intr->vti_irq = NULL;
1142		intr->vti_rid = -1;
1143	}
1144}
1145
1146static void
1147vtpci_free_interrupts(struct vtpci_softc *sc)
1148{
1149	struct vtpci_interrupt *intr;
1150	int i, nvq_intrs;
1151
1152	vtpci_free_interrupt(sc, &sc->vtpci_device_interrupt);
1153
1154	if (sc->vtpci_nmsix_resources != 0) {
1155		nvq_intrs = sc->vtpci_nmsix_resources - 1;
1156		sc->vtpci_nmsix_resources = 0;
1157
1158		intr = sc->vtpci_msix_vq_interrupts;
1159		if (intr != NULL) {
1160			for (i = 0; i < nvq_intrs; i++, intr++)
1161				vtpci_free_interrupt(sc, intr);
1162
1163			free(sc->vtpci_msix_vq_interrupts, M_DEVBUF);
1164			sc->vtpci_msix_vq_interrupts = NULL;
1165		}
1166	}
1167
1168	if (sc->vtpci_flags & (VTPCI_FLAG_MSI | VTPCI_FLAG_MSIX))
1169		pci_release_msi(sc->vtpci_dev);
1170
1171	sc->vtpci_flags &= ~VTPCI_FLAG_ITYPE_MASK;
1172}
1173
1174static void
1175vtpci_free_virtqueues(struct vtpci_softc *sc)
1176{
1177	struct vtpci_virtqueue *vqx;
1178	int idx;
1179
1180	for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1181		vqx = &sc->vtpci_vqs[idx];
1182
1183		vtpci_select_virtqueue(sc, idx);
1184		vtpci_write_config_4(sc, VIRTIO_PCI_QUEUE_PFN, 0);
1185
1186		virtqueue_free(vqx->vtv_vq);
1187		vqx->vtv_vq = NULL;
1188	}
1189
1190	free(sc->vtpci_vqs, M_DEVBUF);
1191	sc->vtpci_vqs = NULL;
1192	sc->vtpci_nvqs = 0;
1193}
1194
1195static void
1196vtpci_release_child_resources(struct vtpci_softc *sc)
1197{
1198
1199	vtpci_free_interrupts(sc);
1200	vtpci_free_virtqueues(sc);
1201}
1202
1203static void
1204vtpci_cleanup_setup_intr_attempt(struct vtpci_softc *sc)
1205{
1206	int idx;
1207
1208	if (sc->vtpci_flags & VTPCI_FLAG_MSIX) {
1209		vtpci_write_config_2(sc, VIRTIO_MSI_CONFIG_VECTOR,
1210		    VIRTIO_MSI_NO_VECTOR);
1211
1212		for (idx = 0; idx < sc->vtpci_nvqs; idx++) {
1213			vtpci_select_virtqueue(sc, idx);
1214			vtpci_write_config_2(sc, VIRTIO_MSI_QUEUE_VECTOR,
1215			    VIRTIO_MSI_NO_VECTOR);
1216		}
1217	}
1218
1219	vtpci_free_interrupts(sc);
1220}
1221
1222static void
1223vtpci_reset(struct vtpci_softc *sc)
1224{
1225
1226	/*
1227	 * Setting the status to RESET sets the host device to
1228	 * the original, uninitialized state.
1229	 */
1230	vtpci_set_status(sc->vtpci_dev, VIRTIO_CONFIG_STATUS_RESET);
1231}
1232
1233static void
1234vtpci_select_virtqueue(struct vtpci_softc *sc, int idx)
1235{
1236
1237	vtpci_write_config_2(sc, VIRTIO_PCI_QUEUE_SEL, idx);
1238}
1239
1240static void
1241vtpci_legacy_intr(void *xsc)
1242{
1243	struct vtpci_softc *sc;
1244	struct vtpci_virtqueue *vqx;
1245	int i;
1246	uint8_t isr;
1247
1248	sc = xsc;
1249	vqx = &sc->vtpci_vqs[0];
1250
1251	/* Reading the ISR also clears it. */
1252	isr = vtpci_read_config_1(sc, VIRTIO_PCI_ISR);
1253
1254	if (isr & VIRTIO_PCI_ISR_CONFIG)
1255		vtpci_config_intr(sc);
1256
1257	if (isr & VIRTIO_PCI_ISR_INTR) {
1258		for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1259			if (vqx->vtv_no_intr == 0)
1260				virtqueue_intr(vqx->vtv_vq);
1261		}
1262	}
1263}
1264
1265static int
1266vtpci_vq_shared_intr_filter(void *xsc)
1267{
1268	struct vtpci_softc *sc;
1269	struct vtpci_virtqueue *vqx;
1270	int i, rc;
1271
1272	rc = 0;
1273	sc = xsc;
1274	vqx = &sc->vtpci_vqs[0];
1275
1276	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1277		if (vqx->vtv_no_intr == 0)
1278			rc |= virtqueue_intr_filter(vqx->vtv_vq);
1279	}
1280
1281	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1282}
1283
1284static void
1285vtpci_vq_shared_intr(void *xsc)
1286{
1287	struct vtpci_softc *sc;
1288	struct vtpci_virtqueue *vqx;
1289	int i;
1290
1291	sc = xsc;
1292	vqx = &sc->vtpci_vqs[0];
1293
1294	for (i = 0; i < sc->vtpci_nvqs; i++, vqx++) {
1295		if (vqx->vtv_no_intr == 0)
1296			virtqueue_intr(vqx->vtv_vq);
1297	}
1298}
1299
1300static int
1301vtpci_vq_intr_filter(void *xvq)
1302{
1303	struct virtqueue *vq;
1304	int rc;
1305
1306	vq = xvq;
1307	rc = virtqueue_intr_filter(vq);
1308
1309	return (rc ? FILTER_SCHEDULE_THREAD : FILTER_STRAY);
1310}
1311
1312static void
1313vtpci_vq_intr(void *xvq)
1314{
1315	struct virtqueue *vq;
1316
1317	vq = xvq;
1318	virtqueue_intr(vq);
1319}
1320
1321static void
1322vtpci_config_intr(void *xsc)
1323{
1324	struct vtpci_softc *sc;
1325	device_t child;
1326
1327	sc = xsc;
1328	child = sc->vtpci_child_dev;
1329
1330	if (child != NULL)
1331		VIRTIO_CONFIG_CHANGE(child);
1332}
1333